├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .github ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── docs.yml │ ├── landing-page.yml │ ├── python-publish.yml │ ├── run.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── CHANGELOG.rst ├── LICENSE ├── README.md ├── check_quality.bat ├── create_docs.bat ├── create_venv.py ├── dfetch.code-workspace ├── dfetch.yaml ├── dfetch ├── __init__.py ├── __main__.py ├── commands │ ├── __init__.py │ ├── check.py │ ├── command.py │ ├── common.py │ ├── diff.py │ ├── environment.py │ ├── freeze.py │ ├── import_.py │ ├── init.py │ ├── report.py │ ├── update.py │ └── validate.py ├── log.py ├── manifest │ ├── __init__.py │ ├── manifest.py │ ├── project.py │ ├── remote.py │ ├── validate.py │ └── version.py ├── project │ ├── __init__.py │ ├── abstract_check_reporter.py │ ├── git.py │ ├── metadata.py │ ├── svn.py │ └── vcs.py ├── reporting │ ├── __init__.py │ ├── check │ │ ├── __init__.py │ │ ├── code_climate_reporter.py │ │ ├── jenkins_reporter.py │ │ ├── reporter.py │ │ ├── sarif_reporter.py │ │ └── stdout_reporter.py │ ├── reporter.py │ ├── sbom_reporter.py │ └── stdout_reporter.py ├── resources │ ├── __init__.py │ ├── schema.yaml │ └── template.yaml ├── util │ ├── __init__.py │ ├── cmdline.py │ ├── util.py │ └── versions.py └── vcs │ ├── __init__.py │ └── git.py ├── doc ├── Makefile ├── _ext │ └── scenario_directive.py ├── alternatives.rst ├── asciicasts │ ├── basic.cast │ ├── check-ci.cast │ ├── check.cast │ ├── diff.cast │ ├── environment.cast │ ├── freeze.cast │ ├── import.cast │ ├── init.cast │ ├── report.cast │ ├── sbom.cast │ ├── update.cast │ └── validate.cast ├── changelog.rst ├── conf.py ├── contributing.rst ├── generate-casts │ ├── README.md │ ├── basic-demo.sh │ ├── check-ci-demo.sh │ ├── check-demo.sh │ ├── demo-magic │ │ ├── .dfetch_data.yaml │ │ ├── demo-magic.sh │ │ └── license.txt │ ├── diff-demo.sh │ ├── environment-demo.sh │ ├── freeze-demo.sh │ ├── generate-casts.sh │ ├── import-demo.sh │ ├── init-demo.sh │ ├── report-demo.sh │ ├── report-sbom-demo.sh │ ├── strip-setup-from-cast.sh │ ├── update-demo.sh │ └── validate-demo.sh ├── getting_started.rst ├── images │ ├── dfetch_header.png │ ├── dfetch_header.svg │ ├── dfetch_logo.png │ ├── dfetch_logo.svg │ ├── dog.svg │ ├── github-actions-result.png │ ├── gitlab-check-pipeline-result.png │ ├── gitlab-highlighted-manifest.png │ ├── local-change-github-details.png │ ├── local-change-github.png │ ├── out-of-date-jenkins.png │ └── out-of-date-jenkins2.png ├── index.rst ├── internal.rst ├── landing-page │ ├── Makefile │ ├── conf.py │ ├── index.rst │ ├── make.bat │ ├── robots.txt │ └── static │ │ └── css │ │ └── custom.css ├── legal.rst ├── make.bat ├── manifest.rst ├── manual.rst ├── robots.txt ├── static │ ├── css │ │ └── custom.css │ └── uml │ │ ├── c1_dfetch_context.puml │ │ ├── c2_dfetch_containers.puml │ │ ├── c3_dfetch_components_commands.puml │ │ ├── c3_dfetch_components_manifest.puml │ │ ├── c3_dfetch_components_project.puml │ │ ├── check.puml │ │ ├── generate_diagram.py │ │ ├── styles │ │ └── plantuml-c4 │ │ │ ├── .dfetch_data.yaml │ │ │ ├── C4.puml │ │ │ ├── C4_Component.puml │ │ │ ├── C4_Container.puml │ │ │ ├── C4_Context.puml │ │ │ ├── C4_Deployment.puml │ │ │ ├── C4_Dynamic.puml │ │ │ ├── C4_Sequence.puml │ │ │ └── LICENSE │ │ └── update.puml └── troubleshooting.rst ├── example └── dfetch.yaml ├── features ├── check-git-repo.feature ├── check-report-code-climate.feature ├── check-report-jenkins.feature ├── check-report-sarif.feature ├── check-specific-projects.feature ├── check-svn-repo.feature ├── checked-project-has-dependencies.feature ├── diff-in-git.feature ├── diff-in-svn.feature ├── environment.py ├── fetch-checks-destination.feature ├── fetch-file-pattern-git.feature ├── fetch-file-pattern-svn.feature ├── fetch-git-repo.feature ├── fetch-single-file-git.feature ├── fetch-single-file-svn.feature ├── fetch-specific-project.feature ├── fetch-svn-repo.feature ├── fetch-with-ignore-git.feature ├── fetch-with-ignore-svn.feature ├── freeze-projects.feature ├── guard-against-overwriting.feature ├── handle-invalid-metadata.feature ├── import-from-git.feature ├── import-from-svn.feature ├── journey-basic-usage.feature ├── keep-license-in-project.feature ├── list-projects.feature ├── patch-after-fetch-git.feature ├── patch-after-fetch-svn.feature ├── report-sbom.feature ├── steps │ ├── __init__.py │ ├── generic_steps.py │ ├── git_steps.py │ ├── manifest_steps.py │ └── svn_steps.py ├── suggest-project-name.feature ├── updated-project-has-dependencies.feature └── validate-manifest.feature ├── pyproject.toml ├── script └── release.py └── tests ├── __init__.py ├── manifest_mock.py ├── run_tests.bat ├── test_check.py ├── test_cmdline.py ├── test_git_vcs.py ├── test_import.py ├── test_manifest.py ├── test_project_entry.py ├── test_project_version.py ├── test_report.py ├── test_resources.py ├── test_svn.py ├── test_update.py ├── test_util_version.py └── test_vcs.py /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye 2 | 3 | # Install dependencies 4 | # pv is required for asciicasts 5 | RUN apt-get update && apt-get install --no-install-recommends -y \ 6 | pv=1.6.6-1+b1 \ 7 | subversion=1.14.1-3+deb11u1 && \ 8 | rm -rf /var/lib/apt/lists/* 9 | 10 | WORKDIR /workspaces/dfetch 11 | 12 | # Add a non-root user (dev) 13 | RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch 14 | 15 | USER dev 16 | 17 | ENV PATH="/home/dev/.local/bin:${PATH}" 18 | ENV PYTHONPATH="/home/dev/.local/lib/python3.12" 19 | ENV PYTHONUSERBASE="/home/dev/.local" 20 | 21 | COPY --chown=dev:dev . . 22 | 23 | RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.0.1 \ 24 | && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts] \ 25 | && pre-commit install --install-hooks 26 | 27 | # Set bash as the default shell 28 | SHELL ["/bin/bash", "-ec"] 29 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/python 3 | { 4 | "name": "Python 3", 5 | "build": { 6 | "dockerfile": "Dockerfile", 7 | "context": ".." 8 | }, 9 | "postCreateCommand": "pip install -e .[development,docs,casts]", 10 | "customizations": { 11 | "vscode": { 12 | "extensions": [ 13 | "lextudio.restructuredtext", 14 | "alexkrechik.cucumberautocomplete", 15 | "ms-python.python", 16 | "ms-python.isort", 17 | "ms-python.black-formatter", 18 | "ms-python.debugpy", 19 | "mhutchie.git-graph", 20 | "tamasfe.even-better-toml", 21 | "trond-snekvik.simple-rst", 22 | "jebbs.plantuml", 23 | "jimasp.behave-vsc" 24 | ], 25 | "settings": { 26 | "terminal.integrated.profiles.linux": { 27 | "bash": { 28 | "path": "bash", 29 | "icon": "terminal-bash" 30 | } 31 | }, 32 | "terminal.integrated.defaultProfile.linux": "bash" 33 | } 34 | } 35 | }, 36 | "workspaceFolder": "/workspaces/dfetch", 37 | "remoteUser": "dev" 38 | } 39 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | # Raise pull requests for version updates 8 | # to pip against the `develop` branch 9 | target-branch: "main" 10 | # Labels on pull requests for version updates only 11 | labels: 12 | - "dependencies" 13 | - package-ecosystem: "github-actions" 14 | directory: "/" 15 | schedule: 16 | interval: "daily" 17 | - package-ecosystem: "devcontainers" 18 | directory: "/" 19 | schedule: 20 | interval: "daily" 21 | - package-ecosystem: "docker" 22 | directory: "/.devcontainer" 23 | schedule: 24 | interval: "daily" 25 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # ******** NOTE ******** 12 | 13 | name: "CodeQL" 14 | 15 | on: 16 | push: 17 | branches: [ main ] 18 | pull_request: 19 | # The branches below must be a subset of the branches above 20 | branches: [ main ] 21 | schedule: 22 | - cron: '24 22 * * 4' 23 | 24 | jobs: 25 | analyze: 26 | name: Analyze 27 | runs-on: ubuntu-latest 28 | 29 | strategy: 30 | fail-fast: false 31 | matrix: 32 | language: [ 'python' ] 33 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 34 | # Learn more... 35 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 36 | 37 | steps: 38 | - name: Checkout repository 39 | uses: actions/checkout@v4 40 | 41 | # Initializes the CodeQL tools for scanning. 42 | - name: Initialize CodeQL 43 | uses: github/codeql-action/init@v3 44 | with: 45 | languages: ${{ matrix.language }} 46 | # If you wish to specify custom queries, you can do so here or in a config file. 47 | # By default, queries listed here will override any specified in a config file. 48 | # Prefix the list here with "+" to use these queries and those in the config file. 49 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 50 | 51 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 52 | # If this step fails, then you should remove it and run the build manually (see below) 53 | - name: Autobuild 54 | uses: github/codeql-action/autobuild@v3 55 | 56 | # ℹ️ Command-line programs to run using the OS shell. 57 | # 📚 https://git.io/JvXDl 58 | 59 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 60 | # and modify them (or add more) to build your code if your project 61 | # uses a compiled language 62 | 63 | #- run: | 64 | # make bootstrap 65 | # make release 66 | 67 | - name: Perform CodeQL Analysis 68 | uses: github/codeql-action/analyze@v3 69 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: "Docs" 2 | on: 3 | - pull_request 4 | 5 | jobs: 6 | docs: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | 11 | - name: Install Python 12 | uses: actions/setup-python@v5 13 | with: 14 | python-version: '3.x' 15 | 16 | - name: Install documentation requirements 17 | run: "pip install .[docs] && pip install sphinx_design" 18 | 19 | - name: Build docs 20 | run: "make -C doc html" 21 | 22 | - name: Build landing-page 23 | run: "make -C doc/landing-page html" 24 | -------------------------------------------------------------------------------- /.github/workflows/landing-page.yml: -------------------------------------------------------------------------------- 1 | name: Landing-page 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - feature/simplify-landing-page 8 | 9 | jobs: 10 | publish: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - name: Setup Python 16 | uses: actions/setup-python@v5 17 | with: 18 | python-version: "3.12" 19 | 20 | - name: Install dependencies 21 | run: | 22 | pip install .[docs] 23 | pip install sphinx_design 24 | 25 | - name: Build landing-page 26 | run: | 27 | cd doc/landing-page 28 | make html 29 | - name: Publish 30 | uses: tsunematsu21/actions-publish-gh-pages@v1.0.2 31 | with: 32 | dir: doc/landing-page/_build/html 33 | repo: dfetch-org/dfetch-org.github.io 34 | branch: main 35 | token: ${{ secrets.GH_DFETCH_ORG_DEPLOY }} 36 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | # Allows to run this workflow manually 11 | workflow_dispatch: 12 | 13 | jobs: 14 | deploy: 15 | 16 | runs-on: ubuntu-latest 17 | environment: PyPi 18 | permissions: 19 | id-token: write 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | - name: Set up Python 24 | uses: actions/setup-python@v5 25 | with: 26 | python-version: '3.x' 27 | - name: Install dependencies 28 | run: | 29 | python -m pip install --upgrade pip 30 | pip install setuptools build 31 | python -m pip install git+https://github.com/takluyver/twine@ec859fbfd285284e800461c45d22187f7948a275#egg=twine 32 | - name: Build and publish 33 | run: | 34 | python -m build 35 | twine upload dist/* 36 | -------------------------------------------------------------------------------- /.github/workflows/run.yml: -------------------------------------------------------------------------------- 1 | name: Run 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | types: [opened, synchronize, reopened] 9 | 10 | jobs: 11 | 12 | test-cygwin: 13 | runs-on: windows-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v4 17 | 18 | - uses: cygwin/cygwin-install-action@master 19 | 20 | - name: Install Subversion (SVN) on Windows 21 | run: | 22 | choco install subversion -y 23 | svn --version # Verify installation 24 | 25 | - name: Install dfetch 26 | run: pip install . 27 | 28 | - run: dfetch environment 29 | - run: dfetch validate 30 | - run: dfetch check 31 | - run: dfetch update 32 | - run: dfetch update 33 | - run: dfetch check --sarif sarif.json 34 | - name: Upload SARIF file 35 | uses: github/codeql-action/upload-sarif@v3 36 | with: 37 | sarif_file: sarif.json 38 | 39 | - name: Run example 40 | working-directory: ./example 41 | env: 42 | CI: 'false' 43 | run: | 44 | dfetch update 45 | dfetch update 46 | dfetch report 47 | 48 | test: 49 | strategy: 50 | matrix: 51 | platform: [ubuntu-latest, macos-latest, windows-latest] 52 | python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] 53 | runs-on: ${{ matrix.platform }} 54 | 55 | steps: 56 | - uses: actions/checkout@v4 57 | 58 | - name: Setup Python 59 | uses: actions/setup-python@v5 60 | with: 61 | python-version: ${{ matrix.python-version }} 62 | 63 | - name: Install Subversion (SVN) 64 | if: matrix.platform == 'ubuntu-latest' 65 | run: | 66 | sudo apt-get update 67 | sudo apt-get install -y subversion 68 | svn --version # Verify installation 69 | 70 | - name: Install Subversion (SVN) 71 | if: matrix.platform == 'macos-latest' 72 | run: | 73 | brew install svn 74 | svn --version # Verify installation 75 | 76 | - name: Install Subversion (SVN) 77 | if: matrix.platform == 'windows-latest' 78 | run: | 79 | choco install subversion -y 80 | svn --version # Verify installation 81 | 82 | - name: Install dfetch 83 | run: pip install . 84 | 85 | - run: dfetch environment 86 | - run: dfetch validate 87 | - run: dfetch check 88 | - run: dfetch update 89 | - run: dfetch update 90 | - run: dfetch check --sarif sarif.json 91 | - name: Upload SARIF file 92 | uses: github/codeql-action/upload-sarif@v3 93 | with: 94 | sarif_file: sarif.json 95 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | types: [opened, synchronize, reopened] 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Setup Python 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: '3.12' 20 | 21 | - name: Install Subversion (SVN) 22 | run: | 23 | sudo apt-get update 24 | sudo apt-get install -y subversion 25 | svn --version # Verify installation 26 | svnadmin --version # Verify installation 27 | 28 | - name: Install dependencies 29 | run: | 30 | pip install .[development,test] 31 | 32 | - run: codespell # Check for typo's 33 | - run: isort --diff dfetch # Checks import order 34 | - run: black --check dfetch # Checks code style 35 | # - run: flake8 dfetch # Checks pep8 conformance 36 | - run: pylint dfetch # Checks pep8 conformance 37 | - run: ruff check dfetch # Check using ruff 38 | - run: mypy --strict dfetch # Check types 39 | - run: pyright . # Check types 40 | - run: doc8 doc # Checks documentation 41 | - run: pydocstyle dfetch # Checks doc strings 42 | - run: bandit -r dfetch # Checks security issues 43 | - run: xenon -b B -m A -a A dfetch # Check code quality 44 | - run: pytest --cov=dfetch tests # Run tests 45 | - run: coverage run --source=dfetch --append -m behave features # Run features tests 46 | - run: coverage xml -o coverage.xml # Create XML report 47 | - run: pyroma --directory --min=10 . # Check pyproject 48 | 49 | - name: Run codacy-coverage-reporter 50 | uses: codacy/codacy-coverage-reporter-action@master 51 | with: 52 | project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} 53 | coverage-reports: coverage.xml 54 | env: 55 | CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} 56 | if: "${{ (!!env.CODACY_PROJECT_TOKEN) }}" 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .coverage 3 | .mypy_cache 4 | .pytest_cache 5 | .ruff_cache 6 | .vscode 7 | build 8 | coverage.xml 9 | dfetch.egg-info 10 | dist 11 | doc/_build 12 | doc/landing-page/_build 13 | example/Tests/ 14 | venv* 15 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v5.0.0 6 | hooks: 7 | - id: trailing-whitespace 8 | exclude: ^doc/static/uml/styles/plantuml-c4/ 9 | - id: end-of-file-fixer 10 | exclude: ^doc/static/uml/styles/plantuml-c4/ 11 | - id: check-yaml 12 | - id: check-added-large-files 13 | - repo: local 14 | hooks: 15 | - id: isort 16 | name: Sort import 17 | entry: isort 18 | language: system 19 | types: [file, python] 20 | - id: black 21 | name: Black (auto-format) 22 | entry: black 23 | language: system 24 | types: [file, python] 25 | - id: pylint 26 | name: pylint 27 | entry: pylint 28 | language: system 29 | exclude: ^tests/|^doc/static/uml|conf.py 30 | types: [file, python] 31 | # - id: flake8 32 | # name: flake8 33 | # entry: flake8 34 | # language: system 35 | # files: ^dfetch/ 36 | # types: [file, python] 37 | - id: mypy 38 | name: mypy 39 | entry: mypy 40 | language: system 41 | files: ^dfetch/ 42 | types: [file, python] 43 | - id: doc8 44 | name: doc8 45 | entry: doc8 46 | language: system 47 | files: ^doc/ 48 | types: [file] 49 | - id: pydocstyle 50 | name: pydocstyle 51 | entry: pydocstyle 52 | language: system 53 | files: ^dfetch/ 54 | types: [file, python] 55 | - id: bandit (security) 56 | name: bandit 57 | entry: bandit 58 | language: system 59 | files: ^dfetch/ 60 | types: [file, python] 61 | - id: vulture (dead code) 62 | name: vulture 63 | language: python 64 | entry: vulture 65 | args: ["--min-confidence=80"] 66 | types: [python] 67 | files: ^dfetch/ 68 | require_serial: true 69 | - id: pyroma (package friendliness) 70 | name: Pyroma 71 | entry: pyroma 72 | language: python 73 | args: ["-d", "--min=10", "."] 74 | pass_filenames: false 75 | always_run: true 76 | - id: radon (maintenance index) 77 | name: radon (maintenance index) 78 | entry: radon 79 | language: system 80 | files: ^dfetch/ 81 | args: [mi, -nb] 82 | types: [file, python] 83 | - id: radon (cyclomatic complexity) 84 | name: radon (cyclomatic complexity) 85 | entry: radon 86 | language: system 87 | files: ^dfetch/ 88 | args: [cc, -nb] 89 | types: [file, python] 90 | - id: xenon 91 | name: xenon 92 | entry: xenon 93 | language: system 94 | files: ^dfetch/ 95 | args: [-b, B, -m, A, -a, A] 96 | types: [file, python] 97 | - id: codespell 98 | name: codespell 99 | description: Checks for common misspellings in text files. 100 | entry: codespell 101 | language: python 102 | types: [text] 103 | - id: ruff 104 | name: ruff 105 | description: Lint using ruff 106 | entry: ruff 107 | language: python 108 | args: [check] 109 | types: [file, python] 110 | - id: pyright 111 | name: pyright 112 | description: Lint using pyright 113 | entry: pyright 114 | language: python 115 | types: [file, python] 116 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: doc/conf.py 11 | 12 | formats: 13 | - epub 14 | - htmlzip 15 | 16 | build: 17 | os: ubuntu-22.04 18 | tools: 19 | python: "3.12" 20 | 21 | # Optionally set the version of Python and requirements required to build your docs 22 | python: 23 | install: 24 | - method: pip 25 | path: . 26 | extra_requirements: 27 | - docs 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 dfetch-org 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](doc/images/dfetch_header.png) 2 | [![](https://codescene.io/projects/10989/status-badges/code-health)](https://codescene.io/projects/10989) 3 | [![](https://codescene.io/projects/10989/status-badges/system-mastery)](https://codescene.io/projects/10989) 4 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/431474d43db0420a92ebc10c1886df8d)](https://app.codacy.com/gh/dfetch-org/dfetch?utm_source=github.com&utm_medium=referral&utm_content=dfetch-org/dfetch&utm_campaign=Badge_Grade) 5 | [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/503c21c8e46b4baca0b4519bcc9fd51e)](https://www.codacy.com/gh/dfetch-org/dfetch/dashboard?utm_source=github.com&utm_medium=referral&utm_content=dfetch-org/dfetch&utm_campaign=Badge_Coverage) 6 | [![Documentation Status](https://readthedocs.org/projects/dfetch/badge/?version=latest)](https://dfetch.readthedocs.io/en/latest/?badge=latest) 7 | [![Build](https://github.com/dfetch-org/dfetch/workflows/Test/badge.svg)](https://github.com/dfetch-org/dfetch/actions) 8 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 9 | [![GitHub](https://img.shields.io/github/license/dfetch-org/dfetch)](https://github.com/dfetch-org/dfetch/blob/main/LICENSE) 10 | [![Gitter](https://badges.gitter.im/dfetch-org/community.svg)](https://gitter.im/dfetch-org/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) 11 | [![Libraries.io dependency status for GitHub repo](https://img.shields.io/librariesio/github/dfetch-org/dfetch)](https://libraries.io/github/dfetch-org/dfetch) 12 | ![Maintenance](https://img.shields.io/maintenance/yes/2025) 13 | [![GitHub issues](https://img.shields.io/github/issues/dfetch-org/dfetch)](https://github.com/dfetch-org/dfetch/issues) 14 | ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/dfetch) 15 | [![PyPI](https://img.shields.io/pypi/v/dfetch)](https://pypi.org/project/dfetch/) 16 | [![Contribute with Codespaces](https://img.shields.io/static/v1?label=Codespaces&message=Open&color=blue)](https://codespaces.new/dfetch-org/dfetch) 17 | 18 | 19 | **DFetch can manage dependencies** 20 | 21 | We needed a dependency manager that was flexible enough to retrieve dependencies as plain text 22 | from various sources. `svn externals`, `git submodules` and `git subtrees` solve a similar 23 | problem, but not in a VCS agnostic way or completely user friendly way. 24 | We want self-contained code repositories without any hassle for end-users. 25 | Dfetch must promote upstreaming changes, but allow for local customizations. 26 | The problem is described thoroughly [here](https://embeddedartistry.com/blog/2020/06/22/qa-on-managing-external-dependencies/) and sometimes 27 | is also known as [*vendoring*](https://stackoverflow.com/questions/26217488/what-is-vendoring). 28 | 29 | Other tools that do similar things are ``Zephyr's West``, ``CMake ExternalProject`` and other meta tools. 30 | See [alternatives](https://dfetch.readthedocs.io/en/latest/alternatives.html) for a complete list. 31 | 32 | [**Getting started**](https://dfetch.readthedocs.io/en/latest/getting_started.html) | 33 | [**Manual**](https://dfetch.readthedocs.io/en/latest/manual.html) | 34 | [**Troubleshooting**](https://dfetch.readthedocs.io/en/latest/troubleshooting.html) | 35 | [**Contributing**](https://dfetch.readthedocs.io/en/latest/contributing.html) 36 | 37 | ## Install 38 | 39 | ### Stable 40 | ```bash 41 | pip install dfetch 42 | ``` 43 | 44 | ### latest version 45 | ```bash 46 | pip install https://github.com/dfetch-org/dfetch/archive/main.zip 47 | ``` 48 | -------------------------------------------------------------------------------- /check_quality.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal enabledelayedexpansion 3 | 4 | cd %~dp0 5 | 6 | python create_venv.py --extra_requirements "development" 7 | if not !ERRORLEVEL! == 0 echo "Something went wrong creating the venv." && exit /b !ERRORLEVEL! 8 | 9 | call .\venv\Scripts\activate.bat 10 | pre-commit run 11 | 12 | pause 13 | 14 | exit /b 0 15 | -------------------------------------------------------------------------------- /create_docs.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal enabledelayedexpansion 3 | 4 | cd %~dp0 5 | 6 | python create_venv.py --extra_requirements "docs" 7 | if not !ERRORLEVEL! == 0 echo "Something went wrong creating the venv." && exit /b !ERRORLEVEL! 8 | 9 | call .\venv\Scripts\activate.bat 10 | .\doc\make.bat html 11 | -------------------------------------------------------------------------------- /create_venv.py: -------------------------------------------------------------------------------- 1 | #!python3.12 2 | """Script to setup a venv.""" 3 | 4 | import argparse 5 | import subprocess # nosec 6 | import venv 7 | from typing import Any 8 | 9 | 10 | class MyEnvBuilder(venv.EnvBuilder): 11 | """Create a virtual environment. 12 | 13 | Conditions for use: 14 | + A `requirements.txt` file must exist 15 | By default this is relative to the current run path, but this can be specified programmatically. 16 | """ 17 | 18 | def __init__( 19 | self, 20 | *args: Any, 21 | extra_requirements: str = "", 22 | **kwargs: Any, 23 | ) -> None: # pylint: disable=line-too-long 24 | """:param extra_requirements: Install any additional parts as mentioned in pyproject.toml.""" 25 | super().__init__(*args, **kwargs) 26 | self.extra_requirements = ( 27 | f"[{extra_requirements}]" if extra_requirements else "" 28 | ) 29 | 30 | def post_setup(self, context: Any) -> None: 31 | """Set up proper environment for testing.""" 32 | super().post_setup(context) 33 | 34 | print("Upgrading pip") 35 | self.pip_install(context, "--upgrade", "pip") 36 | print("Installing package and any extra requirements") 37 | self.pip_install(context, "--use-pep517", "-e", f".{self.extra_requirements}") 38 | 39 | @staticmethod 40 | def pip_install(context: Any, *args: Any) -> None: 41 | """Install something using pip. 42 | 43 | We run pip in isolated mode to avoid side effects from 44 | environment vars, the current directory and anything else 45 | intended for the global Python environment 46 | (same as EnvBuilder's _setup_pip) 47 | """ 48 | subprocess.check_call( # nosec 49 | (context.env_exe, "-Im", "pip", "install") + args, 50 | stderr=subprocess.STDOUT, 51 | ) 52 | 53 | 54 | if __name__ == "__main__": 55 | PARSER = argparse.ArgumentParser() 56 | PARSER.add_argument( 57 | "-e", "--extra_requirements", type=str, default="development,test,doc" 58 | ) 59 | ARGS = PARSER.parse_args() 60 | 61 | MyEnvBuilder( 62 | clear=False, 63 | with_pip=True, 64 | extra_requirements=ARGS.extra_requirements, 65 | ).create("venv") 66 | -------------------------------------------------------------------------------- /dfetch.yaml: -------------------------------------------------------------------------------- 1 | manifest: 2 | version: 0.0 3 | 4 | remotes: 5 | - name: github 6 | url-base: https://github.com/ 7 | 8 | projects: 9 | - name: demo-magic 10 | repo-path: paxtonhare/demo-magic.git 11 | dst: doc/generate-casts/demo-magic 12 | src: '*.sh' 13 | 14 | - name: plantuml-c4 15 | repo-path: plantuml-stdlib/C4-PlantUML.git 16 | dst: doc/static/uml/styles/plantuml-c4 17 | src: '*.puml' 18 | -------------------------------------------------------------------------------- /dfetch/__init__.py: -------------------------------------------------------------------------------- 1 | """Dfetch.""" 2 | 3 | __version__ = "0.10.0" 4 | 5 | DEFAULT_MANIFEST_NAME: str = "dfetch.yaml" 6 | -------------------------------------------------------------------------------- /dfetch/__main__.py: -------------------------------------------------------------------------------- 1 | """Find the complete documentation online. 2 | 3 | https://dfetch.rtfd.org 4 | """ 5 | 6 | import argparse 7 | import sys 8 | from typing import Sequence 9 | 10 | import dfetch.commands.check 11 | import dfetch.commands.diff 12 | import dfetch.commands.environment 13 | import dfetch.commands.freeze 14 | import dfetch.commands.import_ 15 | import dfetch.commands.init 16 | import dfetch.commands.report 17 | import dfetch.commands.update 18 | import dfetch.commands.validate 19 | import dfetch.log 20 | import dfetch.util.cmdline 21 | 22 | logger = dfetch.log.setup_root(__name__) 23 | 24 | 25 | class DfetchFatalException(Exception): 26 | """Exception thrown when dfetch did not run successfully.""" 27 | 28 | 29 | def create_parser() -> argparse.ArgumentParser: 30 | """Create the main argument parser.""" 31 | parser = argparse.ArgumentParser( 32 | formatter_class=argparse.RawTextHelpFormatter, epilog=__doc__ 33 | ) 34 | parser.add_argument( 35 | "--verbose", "-v", action="store_true", help="Increase verbosity" 36 | ) 37 | parser.set_defaults(func=_help) 38 | subparsers = parser.add_subparsers(help="commands") 39 | 40 | dfetch.commands.check.Check.create_menu(subparsers) 41 | dfetch.commands.diff.Diff.create_menu(subparsers) 42 | dfetch.commands.environment.Environment.create_menu(subparsers) 43 | dfetch.commands.freeze.Freeze.create_menu(subparsers) 44 | dfetch.commands.import_.Import.create_menu(subparsers) 45 | dfetch.commands.init.Init.create_menu(subparsers) 46 | dfetch.commands.report.Report.create_menu(subparsers) 47 | dfetch.commands.update.Update.create_menu(subparsers) 48 | dfetch.commands.validate.Validate.create_menu(subparsers) 49 | 50 | return parser 51 | 52 | 53 | def _help(args: argparse.Namespace) -> None: 54 | """Show the help.""" 55 | raise RuntimeError("Select a function") 56 | 57 | 58 | def run(argv: Sequence[str]) -> None: 59 | """Start dfetch.""" 60 | logger.print_title() 61 | args = create_parser().parse_args(argv) 62 | 63 | if args.verbose: 64 | dfetch.log.increase_verbosity() 65 | 66 | try: 67 | args.func(args) 68 | except RuntimeError as exc: 69 | for msg in exc.args: 70 | logger.error(msg, stack_info=False) 71 | raise DfetchFatalException from exc 72 | except dfetch.util.cmdline.SubprocessCommandError as exc: 73 | logger.error(exc.message) 74 | raise DfetchFatalException from exc 75 | 76 | 77 | def main() -> None: 78 | """Start dfetch and let it collect arguments from the command-line.""" 79 | try: 80 | run(sys.argv[1:]) 81 | except DfetchFatalException: 82 | sys.exit(1) 83 | 84 | 85 | if __name__ == "__main__": 86 | main() 87 | -------------------------------------------------------------------------------- /dfetch/commands/__init__.py: -------------------------------------------------------------------------------- 1 | """Contains all commandline-related commands.""" 2 | -------------------------------------------------------------------------------- /dfetch/commands/command.py: -------------------------------------------------------------------------------- 1 | """A generic command.""" 2 | 3 | import argparse 4 | import sys 5 | from abc import ABC, abstractmethod 6 | from argparse import ArgumentParser # pylint: disable=unused-import 7 | from typing import TYPE_CHECKING, Type, TypeVar 8 | 9 | if TYPE_CHECKING and sys.version_info >= (3, 10): 10 | from typing import TypeAlias 11 | 12 | SubparserActionType: TypeAlias = ( 13 | "argparse._SubParsersAction[ArgumentParser]" # pyright: ignore[reportPrivateUsage] #pylint: disable=protected-access 14 | ) 15 | else: 16 | SubparserActionType = ( 17 | argparse._SubParsersAction # pyright: ignore[reportPrivateUsage] #pylint: disable=protected-access 18 | ) 19 | 20 | 21 | class Command(ABC): 22 | """An abstract command that dfetch can perform. 23 | 24 | When adding a new command to dfetch this class should be sub-classed. 25 | That subclass should implement: 26 | 27 | - ``create_menu`` which should add an appropriate subparser. 28 | Likely calling parser is enough. 29 | - ``__call__`` which will be called when the user selects the command. 30 | """ 31 | 32 | CHILD_TYPE = TypeVar("CHILD_TYPE", bound="Command") # noqa 33 | 34 | @staticmethod 35 | @abstractmethod 36 | def create_menu(subparsers: SubparserActionType) -> None: 37 | """Add a sub-parser to the given parser. 38 | 39 | Args: 40 | subparsers (argparse._SubParsersAction): subparser that the parser should be added to. 41 | 42 | This method must be implemented by a subclass. It is called when the menu structure is built. 43 | """ 44 | 45 | @abstractmethod 46 | def __call__(self, args: argparse.Namespace) -> None: 47 | """Perform the command. 48 | 49 | Args: 50 | args (argparse.Namespace): arguments as provided by the user. 51 | 52 | Raises: 53 | NotImplementedError: This is an abstract method that should be implemented by a subclass. 54 | """ 55 | 56 | @staticmethod 57 | def parser( 58 | subparsers: SubparserActionType, 59 | command: Type["Command.CHILD_TYPE"], 60 | ) -> "argparse.ArgumentParser": 61 | """Generate the parser. 62 | 63 | The name of the class will be used as command. The class docstring will be split into 64 | the help text, description and epilog. 65 | 66 | Args: 67 | subparsers: The subparser to add the command to. 68 | command: The command class that should be instantiated and called when this command is called. 69 | 70 | Raises: 71 | NotImplementedError: If the child class doesn't have a docstring. 72 | 73 | Returns: 74 | Command: A argparse.ArgumentParser that can be used to add arguments. 75 | """ 76 | if not command.__doc__: 77 | raise NotImplementedError("Must add docstring to class") 78 | help_str, epilog = command.__doc__.split("\n", 1) 79 | 80 | parser = subparsers.add_parser( 81 | command.__name__.lower(), 82 | description=help_str, 83 | help=help_str, 84 | epilog=epilog, 85 | ) 86 | 87 | parser.set_defaults(func=command()) 88 | return parser 89 | -------------------------------------------------------------------------------- /dfetch/commands/common.py: -------------------------------------------------------------------------------- 1 | """Module for common command operations.""" 2 | 3 | import os 4 | from typing import List 5 | 6 | import yaml 7 | 8 | from dfetch.log import get_logger 9 | from dfetch.manifest.manifest import Manifest, get_childmanifests 10 | from dfetch.manifest.project import ProjectEntry 11 | 12 | logger = get_logger(__name__) 13 | 14 | 15 | def check_child_manifests(manifest: Manifest, project: ProjectEntry, path: str) -> None: 16 | """Check for child manifests within a project. 17 | 18 | Args: 19 | manifest (dfetch.manifest.manifest.Manifest): The parent manifest with projects. 20 | project (ProjectEntry): The parent project. 21 | path (str): The path of the parent manifest. 22 | """ 23 | for ( 24 | childmanifest, 25 | childmanifest_path, 26 | ) in get_childmanifests(skip=[path]): 27 | recommendations: List[ProjectEntry] = [] 28 | for childproject in childmanifest.projects: 29 | if childproject.remote_url not in [ 30 | project.remote_url for project in manifest.projects 31 | ]: 32 | recommendations.append(childproject.as_recommendation()) 33 | 34 | if recommendations: 35 | childmanifest_path = os.path.relpath( 36 | childmanifest_path, start=os.path.dirname(path) 37 | ).replace("\\", "/") 38 | _make_recommendation(project, recommendations, childmanifest_path) 39 | 40 | 41 | def _make_recommendation( 42 | project: ProjectEntry, recommendations: List[ProjectEntry], childmanifest_path: str 43 | ) -> None: 44 | """Make recommendations to the user. 45 | 46 | Args: 47 | project (ProjectEntry): The parent project. 48 | recommendations (List[ProjectEntry]): List of recommendations 49 | childmanifest_path (str): Path to the source of recommendations 50 | """ 51 | logger.warning( 52 | "\n".join( 53 | [ 54 | "", 55 | f'"{project.name}" depends on the following project(s) ' 56 | "which are not part of your manifest:", 57 | f"(found in {childmanifest_path})", 58 | ] 59 | ) 60 | ) 61 | 62 | recommendation_json = yaml.dump( 63 | [proj.as_yaml() for proj in recommendations], 64 | indent=4, 65 | sort_keys=False, 66 | ) 67 | logger.warning("") 68 | for line in recommendation_json.splitlines(): 69 | logger.warning(line) 70 | logger.warning("") 71 | -------------------------------------------------------------------------------- /dfetch/commands/environment.py: -------------------------------------------------------------------------------- 1 | """*Dfetch* can generate output about its working environment.""" 2 | 3 | import argparse 4 | import platform 5 | 6 | import dfetch.commands.command 7 | from dfetch.log import get_logger 8 | from dfetch.project import SUPPORTED_PROJECT_TYPES 9 | 10 | logger = get_logger(__name__) 11 | 12 | 13 | class Environment(dfetch.commands.command.Command): 14 | """Get information about the environment dfetch is working in. 15 | 16 | Generate output that can be used by dfetch developers to investigate issues. 17 | """ 18 | 19 | @staticmethod 20 | def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None: 21 | """Add the parser menu for this action.""" 22 | dfetch.commands.command.Command.parser(subparsers, Environment) 23 | 24 | def __call__(self, _: argparse.Namespace) -> None: 25 | """Perform listing the environment.""" 26 | logger.print_info_line("platform", f"{platform.system()} {platform.release()}") 27 | for vcs in SUPPORTED_PROJECT_TYPES: 28 | vcs.list_tool_info() 29 | -------------------------------------------------------------------------------- /dfetch/commands/freeze.py: -------------------------------------------------------------------------------- 1 | """*Dfetch* can freeze the current versions of the projects. 2 | 3 | Say you have the following manifest: 4 | 5 | .. code-block:: yaml 6 | 7 | manifest: 8 | version: 0.0 9 | 10 | projects: 11 | - name: mymodule 12 | url: http://git.mycompany.local/mycompany/mymodule 13 | 14 | 15 | As explained in :ref:`Revision/Branch/Tag` when no version is provided the latest 16 | version of the default branch (e.g. `trunk`, `master`) of ``mymodule`` will 17 | be fetched on a *DFetch* update_. 18 | When your project becomes stable and you want to rely on a specific version 19 | of ``mymodule`` you can run ``dfetch freeze``. 20 | 21 | First *DFetch* will rename your old manifest (appended with ``.backup``). 22 | After that a new manifest is generated with all the projects as in your original 23 | manifest, but each with the specific version as it currently is on disk. 24 | 25 | In our above example this would for instance result in: 26 | 27 | .. code-block:: yaml 28 | 29 | manifest: 30 | version: 0.0 31 | 32 | projects: 33 | - name: mymodule 34 | url: http://git.mycompany.local/mycompany/mymodule 35 | tag: v1.0.0 36 | 37 | .. scenario-include:: ../features/freeze-projects.feature 38 | 39 | """ 40 | 41 | import argparse 42 | import os 43 | import shutil 44 | from typing import List 45 | 46 | import dfetch.commands.command 47 | import dfetch.manifest.project 48 | import dfetch.project 49 | from dfetch import DEFAULT_MANIFEST_NAME 50 | from dfetch.log import get_logger 51 | from dfetch.manifest.manifest import Manifest, get_manifest 52 | from dfetch.manifest.project import ProjectEntry 53 | from dfetch.util.util import catch_runtime_exceptions, in_directory 54 | 55 | logger = get_logger(__name__) 56 | 57 | 58 | class Freeze(dfetch.commands.command.Command): 59 | """Freeze your projects versions in the manifest as they are on disk. 60 | 61 | Generate a new manifest that has all version as they are on disk. 62 | """ 63 | 64 | @staticmethod 65 | def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None: 66 | """Add the parser menu for this action.""" 67 | dfetch.commands.command.Command.parser(subparsers, Freeze) 68 | 69 | def __call__(self, args: argparse.Namespace) -> None: 70 | """Perform the freeze.""" 71 | del args # unused 72 | 73 | manifest, path = get_manifest() 74 | 75 | exceptions: List[str] = [] 76 | projects: List[ProjectEntry] = [] 77 | 78 | with in_directory(os.path.dirname(path)): 79 | for project in manifest.projects: 80 | with catch_runtime_exceptions(exceptions) as exceptions: 81 | on_disk_version = dfetch.project.make(project).on_disk_version() 82 | 83 | if project.version == on_disk_version: 84 | logger.print_info_line( 85 | project.name, 86 | f"Already pinned in manifest on version {project.version}", 87 | ) 88 | elif on_disk_version: 89 | logger.print_info_line( 90 | project.name, f"Freezing on version {on_disk_version}" 91 | ) 92 | project.version = on_disk_version 93 | else: 94 | logger.print_warning_line( 95 | project.name, 96 | "No version on disk, first update with 'dfetch update'", 97 | ) 98 | 99 | projects.append(project) 100 | 101 | manifest = Manifest( 102 | {"version": "0.0", "remotes": manifest.remotes, "projects": projects} 103 | ) 104 | 105 | shutil.move(DEFAULT_MANIFEST_NAME, DEFAULT_MANIFEST_NAME + ".backup") 106 | 107 | manifest.dump(DEFAULT_MANIFEST_NAME) 108 | logger.info(f"Updated manifest ({DEFAULT_MANIFEST_NAME}) in {os.getcwd()}") 109 | -------------------------------------------------------------------------------- /dfetch/commands/init.py: -------------------------------------------------------------------------------- 1 | """*Dfetch* can generate a starting manifest. 2 | 3 | It will be created in the current folder. 4 | """ 5 | 6 | import argparse 7 | import os 8 | import shutil 9 | 10 | import dfetch.commands.command 11 | from dfetch import DEFAULT_MANIFEST_NAME 12 | from dfetch.log import get_logger 13 | from dfetch.resources import TEMPLATE_PATH 14 | 15 | logger = get_logger(__name__) 16 | 17 | 18 | class Init(dfetch.commands.command.Command): 19 | """Initialize a manifest. 20 | 21 | Generate a manifest that can be used as basis for a project. 22 | """ 23 | 24 | @staticmethod 25 | def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None: 26 | """Add the parser menu for this action.""" 27 | dfetch.commands.command.Command.parser(subparsers, Init) 28 | 29 | def __call__(self, args: argparse.Namespace) -> None: 30 | """Perform the init.""" 31 | del args # unused 32 | 33 | if os.path.isfile(DEFAULT_MANIFEST_NAME): 34 | logger.warning(f"{DEFAULT_MANIFEST_NAME} already exists!") 35 | return 36 | 37 | with TEMPLATE_PATH as template_path: 38 | dest = shutil.copyfile(template_path, DEFAULT_MANIFEST_NAME) 39 | 40 | logger.info(f"Created {dest}") 41 | -------------------------------------------------------------------------------- /dfetch/commands/report.py: -------------------------------------------------------------------------------- 1 | """*Dfetch* can generate multiple reports. 2 | 3 | There are several report types that *DFetch* can generate. 4 | """ 5 | 6 | import argparse 7 | import glob 8 | import os 9 | 10 | import infer_license 11 | 12 | import dfetch.commands.command 13 | import dfetch.manifest.manifest 14 | import dfetch.util.util 15 | from dfetch.log import get_logger 16 | from dfetch.manifest.project import ProjectEntry 17 | from dfetch.project.metadata import Metadata 18 | from dfetch.project.vcs import VCS 19 | from dfetch.reporting import REPORTERS, ReportTypes 20 | 21 | logger = get_logger(__name__) 22 | 23 | 24 | class Report(dfetch.commands.command.Command): 25 | """Generate reports containing information about the projects components. 26 | 27 | Report can be stdout, sbom 28 | """ 29 | 30 | @staticmethod 31 | def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None: 32 | """Add the parser menu for this action.""" 33 | parser = dfetch.commands.command.Command.parser(subparsers, Report) 34 | 35 | parser.add_argument( 36 | "-o", 37 | "--outfile", 38 | metavar="", 39 | type=str, 40 | default="report.json", 41 | help="Report filename", 42 | ) 43 | 44 | parser.add_argument( 45 | "projects", 46 | metavar="", 47 | type=str, 48 | nargs="*", 49 | help="Specific project(s) to report", 50 | ) 51 | 52 | parser.add_argument( 53 | "-t", 54 | "--type", 55 | type=ReportTypes, 56 | choices=list(ReportTypes), 57 | default=ReportTypes.STDOUT, 58 | help="Type of report to generate.", 59 | ) 60 | 61 | def __call__(self, args: argparse.Namespace) -> None: 62 | """Generate the report.""" 63 | manifest, path = dfetch.manifest.manifest.get_manifest() 64 | 65 | reporter = REPORTERS[args.type]() 66 | 67 | with dfetch.util.util.in_directory(os.path.dirname(path)): 68 | for project in manifest.selected_projects(args.projects): 69 | determined_license = self._determine_license(project) 70 | version = self._determine_version(project) 71 | reporter.add_project( 72 | project=project, license_name=determined_license, version=version 73 | ) 74 | 75 | if reporter.dump_to_file(args.outfile): 76 | logger.info(f"Generated {reporter.name} report: {args.outfile}") 77 | 78 | @staticmethod 79 | def _determine_license(project: ProjectEntry) -> str: 80 | """Try to determine license of fetched project.""" 81 | if not os.path.exists(project.destination): 82 | logger.print_warning_line( 83 | project.name, "Never fetched, fetch it to get license info." 84 | ) 85 | return "" 86 | 87 | with dfetch.util.util.in_directory(project.destination): 88 | for license_file in filter(VCS.is_license_file, glob.glob("*")): 89 | logger.debug(f"Found license file {license_file} for {project.name}") 90 | guessed_license = infer_license.api.guess_file(license_file) 91 | 92 | if guessed_license: 93 | return str(guessed_license.name) 94 | 95 | logger.print_warning_line( 96 | project.name, f"Could not determine license in {license_file}" 97 | ) 98 | 99 | return "" 100 | 101 | @staticmethod 102 | def _determine_version(project: ProjectEntry) -> str: 103 | """Determine the fetched version.""" 104 | try: 105 | metadata = Metadata.from_file(Metadata.from_project_entry(project).path) 106 | version = metadata.tag or metadata.revision or "" 107 | except FileNotFoundError: 108 | version = project.tag or project.revision or "" 109 | return version 110 | -------------------------------------------------------------------------------- /dfetch/commands/validate.py: -------------------------------------------------------------------------------- 1 | """Note that you can validate your manifest using :ref:`validate`. 2 | 3 | This will parse your :ref:`Manifest` and check if all fields can be parsed. 4 | 5 | .. scenario-include:: ../features/validate-manifest.feature 6 | 7 | """ 8 | 9 | import argparse 10 | import os 11 | 12 | import dfetch.commands.command 13 | from dfetch.log import get_logger 14 | from dfetch.manifest.manifest import find_manifest 15 | from dfetch.manifest.validate import validate 16 | 17 | logger = get_logger(__name__) 18 | 19 | 20 | class Validate(dfetch.commands.command.Command): 21 | """Validate a manifest. 22 | 23 | The Manifest is validated against a schema. See manifest for requirements. 24 | Note that each time either ``update`` or ``check`` is run, the manifest is also validated. 25 | """ 26 | 27 | @staticmethod 28 | def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None: 29 | """Add the parser menu for this action.""" 30 | dfetch.commands.command.Command.parser(subparsers, Validate) 31 | 32 | def __call__(self, args: argparse.Namespace) -> None: 33 | """Perform the validation.""" 34 | del args # unused 35 | 36 | manifest_path = find_manifest() 37 | validate(manifest_path) 38 | manifest_path = os.path.relpath(manifest_path, os.getcwd()) 39 | logger.print_info_line(manifest_path, "valid") 40 | -------------------------------------------------------------------------------- /dfetch/log.py: -------------------------------------------------------------------------------- 1 | """Logging related items.""" 2 | 3 | import logging 4 | from typing import cast 5 | 6 | import coloredlogs 7 | from colorama import Fore 8 | 9 | from dfetch import __version__ 10 | 11 | 12 | class DLogger(logging.Logger): 13 | """Logging class extended with specific log items for dfetch.""" 14 | 15 | def print_info_line(self, name: str, info: str) -> None: 16 | """Print a line of info.""" 17 | self.info(f" {Fore.GREEN}{name:20s}:{Fore.BLUE} {info}") 18 | 19 | def print_warning_line(self, name: str, info: str) -> None: 20 | """Print a line of info.""" 21 | self.info(f" {Fore.GREEN}{name:20s}:{Fore.YELLOW} {info}") 22 | 23 | def print_title(self) -> None: 24 | """Print the DFetch tool title and version.""" 25 | self.info(f"{Fore.BLUE}Dfetch ({__version__})") 26 | 27 | def print_info_field(self, field_name: str, field: str) -> None: 28 | """Print a field with corresponding value.""" 29 | self.print_info_line(field_name, field if field else "") 30 | 31 | 32 | def setup_root(name: str) -> DLogger: 33 | """Create the root logger.""" 34 | logger = get_logger(name) 35 | 36 | msg_format = "%(message)s" 37 | 38 | level_style = { 39 | "critical": {"color": "magenta", "bright": True, "bold": True}, 40 | "debug": {"color": "green", "bright": True, "bold": True}, 41 | "error": {"color": "red", "bright": True, "bold": True}, 42 | "info": {"color": 4, "bright": True, "bold": True}, 43 | "notice": {"color": "magenta", "bright": True, "bold": True}, 44 | "spam": {"color": "green", "faint": True}, 45 | "success": {"color": "green", "bright": True, "bold": True}, 46 | "verbose": {"color": "blue", "bright": True, "bold": True}, 47 | "warning": {"color": "yellow", "bright": True, "bold": True}, 48 | } 49 | 50 | coloredlogs.install(fmt=msg_format, level_styles=level_style, level="INFO") 51 | 52 | return logger 53 | 54 | 55 | def increase_verbosity() -> None: 56 | """Increase the verbosity of the logger.""" 57 | coloredlogs.increase_verbosity() 58 | 59 | 60 | def get_logger(name: str) -> DLogger: 61 | """Get logger for a module.""" 62 | logging.setLoggerClass(DLogger) 63 | return cast(DLogger, logging.getLogger(name)) 64 | -------------------------------------------------------------------------------- /dfetch/manifest/__init__.py: -------------------------------------------------------------------------------- 1 | """Manifest related items.""" 2 | -------------------------------------------------------------------------------- /dfetch/manifest/remote.py: -------------------------------------------------------------------------------- 1 | """Remotes are the external repository where the code should be retrieved from. 2 | 3 | The ``remotes:`` section is not mandatory. 4 | If only one remote is added this is assumed to be the default. 5 | If multiple remotes are listed ``default:`` can be explicitly specified. 6 | If multiple remotes are marked as default, the first marked as default is chosen. 7 | 8 | .. code-block:: yaml 9 | 10 | manifest: 11 | version: 0.0 12 | 13 | remotes: 14 | - name: mycompany-git-modules 15 | url-base: http://git.mycompany.local/mycompany/ 16 | default: true 17 | - name: github 18 | url-base: https://github.com/ 19 | """ 20 | 21 | from typing import Dict, Optional, Union 22 | 23 | from typing_extensions import TypedDict 24 | 25 | _MandatoryRemoteDict = TypedDict("_MandatoryRemoteDict", {"name": str, "url-base": str}) 26 | 27 | 28 | class RemoteDict(_MandatoryRemoteDict, total=False): 29 | """Class representing data types of Remote class construction.""" 30 | 31 | default: Optional[bool] 32 | 33 | 34 | class Remote: 35 | """A single remote entry in the manifest file.""" 36 | 37 | def __init__(self, kwargs: RemoteDict) -> None: 38 | """Create the remote entry.""" 39 | self._name: str = kwargs["name"] 40 | self._url_base: str = kwargs["url-base"] 41 | self._default: bool = bool(kwargs.get("default", False)) 42 | 43 | @classmethod 44 | def from_yaml(cls, yamldata: Union[Dict[str, str], RemoteDict]) -> "Remote": 45 | """Create a remote entry in the manifest from yaml data. 46 | 47 | Returns: 48 | Remote: Entry containing the immutable remote entry 49 | """ 50 | return cls( 51 | { 52 | "name": yamldata["name"], 53 | "url-base": yamldata["url-base"], 54 | "default": bool(yamldata.get("default", False)), 55 | } 56 | ) 57 | 58 | @classmethod 59 | def copy(cls, other: "Remote") -> "Remote": 60 | """Generate a new remote entry in the manifest from another. 61 | 62 | Args: 63 | other (Remote): Other Remote to copy the values from 64 | 65 | Returns: 66 | Remote: Entry containing the immutable remote entry 67 | """ 68 | return cls( 69 | {"name": other.name, "url-base": other.url, "default": other.is_default} 70 | ) 71 | 72 | @property 73 | def name(self) -> str: 74 | """Get the name of the remote.""" 75 | return self._name 76 | 77 | @property 78 | def url(self) -> str: 79 | """Get the url of the remote.""" 80 | return self._url_base 81 | 82 | @property 83 | def is_default(self) -> bool: 84 | """Check if this is a default remote.""" 85 | return self._default 86 | 87 | def __repr__(self) -> str: 88 | """Get a string representation of this remote.""" 89 | return str(self.as_yaml()) 90 | 91 | def as_yaml(self) -> RemoteDict: 92 | """Get this remote as yaml data.""" 93 | yamldata: RemoteDict = {"name": self._name, "url-base": self._url_base} 94 | 95 | if self.is_default: 96 | yamldata["default"] = True 97 | 98 | return yamldata 99 | -------------------------------------------------------------------------------- /dfetch/manifest/validate.py: -------------------------------------------------------------------------------- 1 | """Validate manifests.""" 2 | 3 | import logging 4 | 5 | import pykwalify 6 | from pykwalify.core import Core, SchemaError 7 | from yaml.scanner import ScannerError 8 | 9 | import dfetch.resources 10 | 11 | 12 | def validate(path: str) -> None: 13 | """Validate the given manifest.""" 14 | logging.getLogger(pykwalify.__name__).setLevel(logging.CRITICAL) 15 | 16 | with dfetch.resources.schema_path() as schema_path: 17 | try: 18 | validator = Core(source_file=path, schema_files=[str(schema_path)]) 19 | except ScannerError as err: 20 | raise RuntimeError(f"{schema_path} is not a valid YAML file!") from err 21 | 22 | try: 23 | validator.validate(raise_exception=True) 24 | except SchemaError as err: 25 | raise RuntimeError( 26 | str(err.msg) # pyright: ignore[reportAttributeAccessIssue, reportCallIssue] 27 | ) from err 28 | -------------------------------------------------------------------------------- /dfetch/manifest/version.py: -------------------------------------------------------------------------------- 1 | """Version of a project.""" 2 | 3 | from typing import Any, NamedTuple 4 | 5 | 6 | class Version(NamedTuple): 7 | """Version of a project. 8 | 9 | In DFetch a version consists of a tag, branch or revision. 10 | A tag has precedence over branches/revisions. 11 | """ 12 | 13 | tag: str = "" 14 | branch: str = "" 15 | revision: str = "" 16 | 17 | def __eq__(self, other: Any) -> bool: 18 | """Check if two versions can be considered as equal.""" 19 | if not other: 20 | return False 21 | 22 | if self.tag or other.tag: 23 | return bool(self.tag == other.tag) 24 | 25 | return bool(self.branch == other.branch and self.revision == other.revision) 26 | 27 | def __repr__(self) -> str: 28 | """Get the string representing this version.""" 29 | if self.tag: 30 | return self.tag 31 | 32 | return " - ".join(filter(None, [self.branch.strip(), self.revision])) 33 | -------------------------------------------------------------------------------- /dfetch/project/__init__.py: -------------------------------------------------------------------------------- 1 | """All Project related items.""" 2 | 3 | import dfetch.manifest.project 4 | from dfetch.project.git import GitRepo 5 | from dfetch.project.svn import SvnRepo 6 | from dfetch.project.vcs import VCS 7 | 8 | SUPPORTED_PROJECT_TYPES = [GitRepo, SvnRepo] 9 | 10 | 11 | def make(project_entry: dfetch.manifest.project.ProjectEntry) -> VCS: 12 | """Create a new VCS based on a project from the manifest.""" 13 | for project_type in SUPPORTED_PROJECT_TYPES: 14 | if project_type.NAME == project_entry.vcs: 15 | return project_type(project_entry) 16 | 17 | for project_type in SUPPORTED_PROJECT_TYPES: 18 | project = project_type(project_entry) 19 | 20 | if project.check(): 21 | return project 22 | raise RuntimeError("vcs type unsupported") 23 | -------------------------------------------------------------------------------- /dfetch/project/abstract_check_reporter.py: -------------------------------------------------------------------------------- 1 | """Interface for reporting check results.""" 2 | 3 | from abc import ABC, abstractmethod 4 | 5 | from dfetch.manifest.project import ProjectEntry 6 | from dfetch.manifest.version import Version 7 | 8 | 9 | class AbstractCheckReporter(ABC): 10 | """Reporter for generating report.""" 11 | 12 | @abstractmethod 13 | def __init__(self, manifest_path: str) -> None: 14 | """Create the reporter. 15 | 16 | Args: 17 | manifest_path (str): The path to the manifest. 18 | """ 19 | 20 | @abstractmethod 21 | def unfetched_project( 22 | self, project: ProjectEntry, wanted_version: Version, latest: Version 23 | ) -> None: 24 | """Report an unfetched project. 25 | 26 | Args: 27 | project (ProjectEntry): The unfetched project. 28 | wanted_version (Version): The wanted version. 29 | latest (Version): The latest available version. 30 | """ 31 | 32 | @abstractmethod 33 | def up_to_date_project(self, project: ProjectEntry, latest: Version) -> None: 34 | """Report an up-to-date project. 35 | 36 | Args: 37 | project (ProjectEntry): The up-to-date project 38 | latest (Version): The last version. 39 | """ 40 | 41 | @abstractmethod 42 | def pinned_but_out_of_date_project( 43 | self, project: ProjectEntry, wanted_version: Version, latest: Version 44 | ) -> None: 45 | """Report a pinned but out-of-date project. 46 | 47 | Args: 48 | project (ProjectEntry): Project that is pinned but out-of-date 49 | wanted_version (Version): Version that is wanted by manifest 50 | latest (Version): Available version 51 | """ 52 | 53 | @abstractmethod 54 | def unavailable_project_version( 55 | self, project: ProjectEntry, wanted_version: Version 56 | ) -> None: 57 | """Report a pinned but unavailable project version. 58 | 59 | Args: 60 | project (ProjectEntry): Project that does not have the wanted_version available. 61 | wanted_version (Version): Version that is wanted by manifest 62 | """ 63 | 64 | @abstractmethod 65 | def out_of_date_project( 66 | self, 67 | project: ProjectEntry, 68 | wanted_version: Version, 69 | current: Version, 70 | latest: Version, 71 | ) -> None: 72 | """Report an out-of-date project. 73 | 74 | Args: 75 | project (ProjectEntry): Project that is out-of-date 76 | wanted_version (Version): Version that is wanted by manifest 77 | current (Version): Current version on disk 78 | latest (Version): Available version 79 | """ 80 | 81 | @abstractmethod 82 | def local_changes(self, project: ProjectEntry) -> None: 83 | """Report an project with local changes. 84 | 85 | Args: 86 | project (ProjectEntry): The project with local changes. 87 | """ 88 | 89 | @abstractmethod 90 | def dump_to_file(self) -> None: 91 | """Do nothing.""" 92 | -------------------------------------------------------------------------------- /dfetch/reporting/__init__.py: -------------------------------------------------------------------------------- 1 | """Various reporters for generating reports.""" 2 | 3 | from enum import Enum 4 | from typing import Dict, Type 5 | 6 | from dfetch.reporting.reporter import Reporter 7 | from dfetch.reporting.sbom_reporter import SbomReporter 8 | from dfetch.reporting.stdout_reporter import StdoutReporter 9 | 10 | 11 | class ReportTypes(Enum): 12 | """Enum giving a name to a type of reporter.""" 13 | 14 | SBOM = "sbom" 15 | STDOUT = "list" 16 | 17 | def __str__(self) -> str: 18 | """Get the string.""" 19 | return self.value 20 | 21 | 22 | REPORTERS: Dict[ReportTypes, Type[Reporter]] = { 23 | ReportTypes.STDOUT: StdoutReporter, 24 | ReportTypes.SBOM: SbomReporter, 25 | } 26 | -------------------------------------------------------------------------------- /dfetch/reporting/check/__init__.py: -------------------------------------------------------------------------------- 1 | """Reporters used for checking a project's state.""" 2 | -------------------------------------------------------------------------------- /dfetch/reporting/reporter.py: -------------------------------------------------------------------------------- 1 | """Abstract reporting interface.""" 2 | 3 | from abc import ABC, abstractmethod 4 | 5 | from dfetch.manifest.project import ProjectEntry 6 | 7 | 8 | class Reporter(ABC): 9 | """Reporter for generating report.""" 10 | 11 | name: str = "abstract" 12 | 13 | @abstractmethod 14 | def add_project( 15 | self, project: ProjectEntry, license_name: str, version: str 16 | ) -> None: 17 | """Add a project to the report.""" 18 | 19 | @abstractmethod 20 | def dump_to_file(self, outfile: str) -> bool: 21 | """Do nothing.""" 22 | -------------------------------------------------------------------------------- /dfetch/reporting/stdout_reporter.py: -------------------------------------------------------------------------------- 1 | """*Dfetch* can generate an report on stdout. 2 | 3 | Depending on the state of the projects it will show as much information 4 | from the manifest or the metadata (``.dfetch_data.yaml``). 5 | """ 6 | 7 | from dfetch.log import get_logger 8 | from dfetch.manifest.project import ProjectEntry 9 | from dfetch.project.metadata import Metadata 10 | from dfetch.reporting.reporter import Reporter 11 | 12 | logger = get_logger(__name__) 13 | 14 | 15 | class StdoutReporter(Reporter): 16 | """Reporter for generating report on stdout.""" 17 | 18 | name = "stdout" 19 | 20 | def add_project( 21 | self, project: ProjectEntry, license_name: str, version: str 22 | ) -> None: 23 | """Add a project to the report.""" 24 | del version 25 | logger.print_info_field("project", project.name) 26 | logger.print_info_field(" remote", project.remote) 27 | try: 28 | metadata = Metadata.from_file(Metadata.from_project_entry(project).path) 29 | logger.print_info_field(" remote url", metadata.remote_url) 30 | logger.print_info_field(" branch", metadata.branch) 31 | logger.print_info_field(" tag", metadata.tag) 32 | logger.print_info_field(" last fetch", str(metadata.last_fetch)) 33 | logger.print_info_field(" revision", metadata.revision) 34 | logger.print_info_field(" patch", metadata.patch) 35 | logger.print_info_field(" license", license_name) 36 | 37 | except FileNotFoundError: 38 | logger.print_info_field(" last fetch", "never") 39 | 40 | def dump_to_file(self, outfile: str) -> bool: 41 | """Do nothing.""" 42 | del outfile 43 | return False 44 | -------------------------------------------------------------------------------- /dfetch/resources/__init__.py: -------------------------------------------------------------------------------- 1 | """Resources needed when dfetch is distributed.""" 2 | 3 | try: 4 | import importlib.resources as importlib_resources 5 | except ModuleNotFoundError: 6 | import importlib_resources # type:ignore 7 | 8 | import sys 9 | from pathlib import Path 10 | from typing import ContextManager 11 | 12 | from dfetch import resources # pylint: disable=import-self 13 | 14 | 15 | def _resource_path(filename: str) -> ContextManager[Path]: 16 | """Get the path to the resource.""" 17 | if sys.version_info >= (3, 9): 18 | return importlib_resources.as_file( 19 | importlib_resources.files(resources) / filename 20 | ) 21 | return importlib_resources.path( # pylint: disable=deprecated-method 22 | resources, filename 23 | ) 24 | 25 | 26 | def schema_path() -> ContextManager[Path]: 27 | """Get path to schema.""" 28 | return _resource_path("schema.yaml") 29 | 30 | 31 | TEMPLATE_PATH = _resource_path("template.yaml") 32 | -------------------------------------------------------------------------------- /dfetch/resources/schema.yaml: -------------------------------------------------------------------------------- 1 | # Schema file (schema.yaml) 2 | type: map 3 | mapping: 4 | "manifest": 5 | type: map 6 | required: True 7 | mapping: 8 | "version": { type: number, required: True} 9 | "remotes": 10 | required: False 11 | type: seq 12 | sequence: 13 | - type: map 14 | mapping: 15 | "name": { type: str, required: True, unique: True} 16 | "url-base": { type: str, required: True} 17 | "default": { type: bool } 18 | "projects": 19 | required: True 20 | type: seq 21 | sequence: 22 | - type: map 23 | mapping: 24 | "name": { type: str, required: True, unique: True} 25 | "dst": { type: str, unique: True } 26 | "branch": { type: str } 27 | "tag": { type: str } 28 | "revision": { type: str } 29 | "url": { type: str } 30 | "repo-path": { type: str } 31 | "remote": { type: str } 32 | "patch": { type: str } 33 | "vcs": { type: str, enum: ['git', 'svn'] } 34 | "src": 35 | type: any 36 | "ignore": 37 | required: False 38 | type: seq 39 | sequence: 40 | - type: str 41 | -------------------------------------------------------------------------------- /dfetch/resources/template.yaml: -------------------------------------------------------------------------------- 1 | manifest: 2 | version: 0.0 # DFetch Module syntax version 3 | 4 | remotes: # declare common sources in one place 5 | - name: github 6 | url-base: https://github.com/ 7 | 8 | projects: 9 | - name: cpputest 10 | dst: cpputest/src/ # Destination of this project (relative to this file) 11 | repo-path: cpputest/cpputest.git # Use default github remote 12 | tag: v3.4 # tag 13 | 14 | - name: jsmn # without destination, defaults to project name 15 | repo-path: zserge/jsmn.git # only repo-path is enough 16 | -------------------------------------------------------------------------------- /dfetch/util/__init__.py: -------------------------------------------------------------------------------- 1 | """Non domain specific utilities.""" 2 | -------------------------------------------------------------------------------- /dfetch/util/cmdline.py: -------------------------------------------------------------------------------- 1 | """Module for performing cmd line arguments.""" 2 | 3 | import logging 4 | import os 5 | import subprocess # nosec 6 | from typing import Any, List, Optional, Union # pylint: disable=unused-import 7 | 8 | 9 | class SubprocessCommandError(Exception): 10 | """Error raised when a subprocess fails. 11 | 12 | Whenever a subprocess is executed something can happen. This exception 13 | contains all the results for easier usage later on. 14 | """ 15 | 16 | def __init__( 17 | self, 18 | cmd: Optional[List[str]] = None, 19 | stdout: str = "", 20 | stderr: str = "", 21 | returncode: int = 0, 22 | ): 23 | """Error.""" 24 | cmd_str: str = " ".join(cmd or []) 25 | self._message = f">>>{cmd_str}<<< returned {returncode}:{os.linesep}{stderr}" 26 | self.cmd = cmd_str 27 | self.stderr = stdout 28 | self.stdout = stderr 29 | self.returncode = returncode 30 | super().__init__(self._message) 31 | 32 | @property 33 | def message(self) -> str: 34 | """Return the message of this SubprocessCommandError.""" 35 | return self._message 36 | 37 | 38 | def run_on_cmdline( 39 | logger: logging.Logger, cmd: Union[str, List[str]] 40 | ) -> "subprocess.CompletedProcess[Any]": 41 | """Run a command and log the output, and raise if something goes wrong.""" 42 | logger.debug(f"Running {cmd}") 43 | 44 | if not isinstance(cmd, list): 45 | cmd = cmd.split(" ") 46 | 47 | try: 48 | proc = subprocess.run( # nosec 49 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True 50 | ) 51 | except subprocess.CalledProcessError as exc: 52 | raise SubprocessCommandError( 53 | exc.cmd, 54 | exc.output.decode().strip(), 55 | exc.stderr.decode().strip(), 56 | exc.returncode, 57 | ) from exc 58 | except FileNotFoundError as exc: 59 | cmd = cmd[0] 60 | raise RuntimeError(f"{cmd} not available on system, please install") from exc 61 | 62 | stdout, stderr = proc.stdout, proc.stderr 63 | 64 | _log_output(proc, logger) 65 | 66 | if proc.returncode: 67 | raise SubprocessCommandError( 68 | cmd, stdout.decode(), stderr.decode().strip(), proc.returncode 69 | ) 70 | 71 | return proc 72 | 73 | 74 | def _log_output(proc: subprocess.CompletedProcess, logger: logging.Logger) -> None: # type: ignore 75 | logger.debug(f"Return code: {proc.returncode}") 76 | 77 | _log_output_stream("stdout", proc.stdout, logger) 78 | _log_output_stream("stderr", proc.stderr, logger) 79 | 80 | 81 | def _log_output_stream(name: str, stream: Any, logger: logging.Logger) -> None: 82 | logger.debug(f"{name}:") 83 | try: 84 | for line in stream.decode().split("\n\n"): 85 | logger.debug(line) 86 | except UnicodeDecodeError: 87 | for line in stream.decode(encoding="cp1252").split("\n\n"): 88 | logger.debug(line) 89 | -------------------------------------------------------------------------------- /dfetch/util/versions.py: -------------------------------------------------------------------------------- 1 | """Module for handling version information from strings.""" 2 | 3 | import re 4 | from collections import defaultdict 5 | from typing import Dict, List, Optional, Tuple 6 | 7 | from semver.version import Version 8 | 9 | BASEVERSION = re.compile( 10 | r"""[vV]? 11 | (?P0|[1-9]\d*) 12 | (\. 13 | (?P0|[1-9]\d*) 14 | (\. 15 | (?P0|[1-9]\d*) 16 | )? 17 | )? 18 | """, 19 | re.VERBOSE, 20 | ) 21 | 22 | 23 | def coerce(version: str) -> Tuple[str, Optional[Version], str]: 24 | """Convert an incomplete version string into a semver-compatible Version object. 25 | 26 | * Tries to detect a "basic" version string (``major.minor.patch``). 27 | * If not enough components can be found, missing components are 28 | set to zero to obtain a valid semver version. 29 | 30 | :param str version: the version string to convert 31 | :return: a tuple with a prefix string, a :class:`Version` instance (or ``None`` 32 | if it's not a version) and the rest of the string which doesn't 33 | belong to a basic version. 34 | :rtype: tuple(None, str | :class:`Version` | None, str) 35 | """ 36 | match = None if not version else BASEVERSION.search(version) 37 | if not match: 38 | return ("", None, version) 39 | 40 | ver = { 41 | key: 0 if value is None else int(value) 42 | for key, value in match.groupdict().items() 43 | } 44 | 45 | return ( 46 | match.string[: match.start()], 47 | Version(**ver), 48 | match.string[match.end() :], # noqa:E203 49 | ) 50 | 51 | 52 | def latest_tag_from_list(current_tag: str, available_tags: List[str]) -> str: 53 | """Based on the given tag string and list of tags, get the latest available.""" 54 | parsed_tags = _create_available_version_dict(available_tags) 55 | 56 | prefix, current_version, _ = coerce(current_tag) 57 | 58 | latest_string: str = current_tag 59 | 60 | if current_version: 61 | latest_tag: Version = current_version 62 | for tag in parsed_tags[prefix]: 63 | if tag[0] > latest_tag: 64 | latest_tag, latest_string = tag 65 | 66 | return latest_string 67 | 68 | 69 | def _create_available_version_dict( 70 | available_tags: List[str], 71 | ) -> Dict[str, List[Tuple[Version, str]]]: 72 | """Create a dictionary where each key is a prefix with a list of versions. 73 | 74 | Args: 75 | available_tags (List[str]): A list of available tags. 76 | 77 | Returns: 78 | Dict[str, List[Tuple[Version, str]]]: A dictionary mapping prefixes to lists of versions. 79 | 80 | Example: 81 | >>> available_tags = [ 82 | ... 'release/v1.2.3', 83 | ... 'release/v2.0.0' 84 | ... ] 85 | >>> dict(_create_available_version_dict(available_tags)) 86 | {'release/': [(Version(major=1, minor=2, patch=3, prerelease=None, build=None), 'release/v1.2.3'), 87 | (Version(major=2, minor=0, patch=0, prerelease=None, build=None), 'release/v2.0.0')]} 88 | """ 89 | parsed_tags: Dict[str, List[Tuple[Version, str]]] = defaultdict(list) 90 | for available_tag in available_tags: 91 | prefix, version, _ = coerce(available_tag) 92 | if version: 93 | parsed_tags[prefix] += [(version, available_tag)] 94 | return parsed_tags 95 | 96 | 97 | if __name__ == "__main__": 98 | import doctest 99 | 100 | doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE) 101 | -------------------------------------------------------------------------------- /dfetch/vcs/__init__.py: -------------------------------------------------------------------------------- 1 | """Version control system wrappers in python.""" 2 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = DFetch 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /doc/_ext/scenario_directive.py: -------------------------------------------------------------------------------- 1 | """ 2 | This custom Sphinx directive dynamically includes scenarios from a Gherkin feature file. 3 | 4 | 1. Enable the directive in your Sphinx `conf.py`: 5 | 6 | ```python 7 | extensions = ["your_extension_folder.scenario_directive"] 8 | ``` 9 | 10 | Use it in an .rst file: 11 | 12 | ```rst 13 | .. scenario-include:: path/to/feature_file.feature 14 | :scenario: 15 | Scenario Title 1 16 | Scenario Title 2 17 | ``` 18 | 19 | If `:scenario:` is omitted, all scenarios in the feature file will be included. 20 | The directive automatically detects Scenario: and Scenario Outline: titles. 21 | 22 | """ 23 | 24 | import os 25 | import re 26 | from typing import Tuple 27 | 28 | from docutils import nodes 29 | from docutils.parsers.rst import Directive 30 | 31 | 32 | class ScenarioIncludeDirective(Directive): 33 | """Custom directive to dynamically include scenarios from a Gherkin feature file.""" 34 | 35 | required_arguments = 1 # Only the feature file is required 36 | optional_arguments = 0 37 | final_argument_whitespace = False 38 | option_spec = { 39 | "scenario": str, 40 | } 41 | 42 | def list_of_scenarios(self, feature_file_path: str) -> Tuple[str]: 43 | """Parse the list of scenarios from the feature file""" 44 | env = self.state.document.settings.env 45 | feature_path = os.path.abspath(os.path.join(env.app.srcdir, feature_file_path)) 46 | if not os.path.exists(feature_path): 47 | raise self.error(f"Feature file not found: {feature_path}") 48 | 49 | with open(feature_path, encoding="utf-8") as f: 50 | scenarios = tuple( 51 | m[1] 52 | for m in re.findall( 53 | r"^\s*(Scenario(?: Outline)?):\s*(.+)$", f.read(), re.MULTILINE 54 | ) 55 | ) 56 | return scenarios 57 | 58 | def run(self): 59 | """Generate the same literalinclude block for every scenario.""" 60 | feature_file = self.arguments[0].strip() 61 | 62 | scenarios_available = self.list_of_scenarios(feature_file) 63 | 64 | scenario_titles = [ 65 | title.strip() 66 | for title in self.options.get("scenario", "").splitlines() 67 | if title.strip() 68 | ] or scenarios_available 69 | 70 | container = nodes.section() 71 | 72 | for scenario_title in scenario_titles: 73 | end_before = ( 74 | ":end-before: Scenario:" 75 | if scenario_title != scenarios_available[-1] 76 | else "" 77 | ) 78 | 79 | directive_rst = f""" 80 | .. details:: **Example**: {scenario_title} 81 | 82 | .. literalinclude:: {feature_file} 83 | :language: gherkin 84 | :caption: {feature_file} 85 | :force: 86 | :dedent: 87 | :start-after: Scenario: {scenario_title} 88 | {end_before} 89 | """ 90 | self.state.nested_parse( 91 | self.state_machine.input_lines.__class__(directive_rst.splitlines()), 92 | self.content_offset, 93 | container, 94 | ) 95 | 96 | return container.children 97 | 98 | 99 | def setup(app): 100 | """Setup the directive.""" 101 | app.add_directive("scenario-include", ScenarioIncludeDirective) 102 | -------------------------------------------------------------------------------- /doc/asciicasts/diff.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 197, "height": 38, "timestamp": 1741817055, "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}} 2 | [0.234545, "o", "\u001b[H\u001b[2J\u001b[3J"] 3 | [0.238394, "o", "$ "] 4 | [1.24014, "o", "\u001b["] 5 | [1.420459, "o", "1m"] 6 | [1.510615, "o", "ls"] 7 | [1.600749, "o", " -"] 8 | [1.691082, "o", "l "] 9 | [1.781378, "o", ".\u001b"] 10 | [1.871348, "o", "[0"] 11 | [1.961484, "o", "m"] 12 | [2.962006, "o", "\r\n"] 13 | [2.964684, "o", "total 12\r\n"] 14 | [2.96475, "o", "drwxr-xr-x+ 3 dev dev 4096 Mar 12 22:04 cpputest\r\n-rw-rw-rw- 1 dev dev 733 Mar 12 22:04 dfetch.yaml\r\ndrwxr-xr-x+ 4 dev dev 4096 Mar 12 22:04 jsmn\r\n"] 15 | [2.968562, "o", "$ "] 16 | [3.970109, "o", "\u001b"] 17 | [4.150439, "o", "[1"] 18 | [4.240527, "o", "ml"] 19 | [4.330721, "o", "s "] 20 | [4.420846, "o", "-"] 21 | [4.511213, "o", "l "] 22 | [4.601409, "o", "cp"] 23 | [4.691709, "o", "pu"] 24 | [4.781851, "o", "te"] 25 | [4.872177, "o", "s"] 26 | [5.052491, "o", "t/"] 27 | [5.142729, "o", "sr"] 28 | [5.232728, "o", "c/"] 29 | [5.323074, "o", "RE"] 30 | [5.413105, "o", "A"] 31 | [5.503328, "o", "DM"] 32 | [5.59354, "o", "E."] 33 | [5.683768, "o", "md"] 34 | [5.773725, "o", "\u001b["] 35 | [5.954051, "o", "0"] 36 | [6.044188, "o", "m"] 37 | [7.045011, "o", "\r\n"] 38 | [7.047483, "o", "-rw-rw-rw- 1 dev dev 6777 Mar 12 22:04 cpputest/src/README.md\r\n"] 39 | [7.051371, "o", "$ "] 40 | [8.053625, "o", "\u001b["] 41 | [8.233862, "o", "1m"] 42 | [8.324334, "o", "se"] 43 | [8.41448, "o", "d "] 44 | [8.504891, "o", "-i "] 45 | [8.595088, "o", "'s"] 46 | [8.685142, "o", "/g"] 47 | [8.775441, "o", "it"] 48 | [8.865479, "o", "hu"] 49 | [8.955656, "o", "b/g"] 50 | [9.136049, "o", "it"] 51 | [9.22623, "o", "la"] 52 | [9.316451, "o", "b/"] 53 | [9.40671, "o", "g'"] 54 | [9.496927, "o", " cp"] 55 | [9.587086, "o", "pu"] 56 | [9.677259, "o", "te"] 57 | [9.767281, "o", "st"] 58 | [9.857756, "o", "/s"] 59 | [10.037808, "o", "rc/"] 60 | [10.128184, "o", "RE"] 61 | [10.218345, "o", "AD"] 62 | [10.308558, "o", "ME"] 63 | [10.398535, "o", ".m"] 64 | [10.488909, "o", "d\u001b["] 65 | [10.578945, "o", "0m"] 66 | [11.579457, "o", "\r\n"] 67 | [11.585911, "o", "$ "] 68 | [12.587473, "o", "\u001b["] 69 | [12.76824, "o", "1m"] 70 | [12.858646, "o", "df"] 71 | [12.948835, "o", "et"] 72 | [13.03884, "o", "ch"] 73 | [13.129063, "o", " d"] 74 | [13.219088, "o", "if"] 75 | [13.309255, "o", "f "] 76 | [13.399375, "o", "cp"] 77 | [13.489723, "o", "pu"] 78 | [13.669872, "o", "te"] 79 | [13.759987, "o", "st"] 80 | [13.850113, "o", "\u001b["] 81 | [13.940419, "o", "0m"] 82 | [14.941035, "o", "\r\n"] 83 | [15.191009, "o", "\u001b[1;38;5;4m\u001b[34mDfetch (0.10.0)\u001b[0m\r\n\u001b[0m"] 84 | [15.231142, "o", "\u001b[1;38;5;4m \u001b[32mcpputest :\u001b[34m Generating patch cpputest.patch from 164bb7850e71225143b25915f9f42607d5583ec8 to in /workspaces/dfetch/doc/generate-casts/diff\u001b[0m\r\n\u001b[0m"] 85 | [15.231522, "o", "\u001b[0m"] 86 | [15.265017, "o", "$ "] 87 | [16.266386, "o", "\u001b["] 88 | [16.446689, "o", "1m"] 89 | [16.536883, "o", "ca"] 90 | [16.627202, "o", "t "] 91 | [16.717253, "o", "cpp"] 92 | [16.807542, "o", "ut"] 93 | [16.897752, "o", "es"] 94 | [16.987758, "o", "t."] 95 | [17.078048, "o", "pa"] 96 | [17.168051, "o", "tch"] 97 | [17.348547, "o", "\u001b["] 98 | [17.43883, "o", "0m"] 99 | [18.439514, "o", "\r\n"] 100 | [18.441789, "o", "diff --git a/README.md b/README.md\r\nindex 2655a7b..fc6084e 100644\r\n--- a/README.md\r\n+++ b/README.md\r\n@@ -3,7 +3,7 @@ CppUTest\r\n \r\n CppUTest unit testing and mocking framework for C/C++\r\n \r\n-[More information on the project page](http://cpputest.github.com)\r\n+[More information on the project page](http://cpputest.gitlab.com)\r\n \r\n [![Build Status](https://travis-ci.org/cpputest/cpputest.png?branch=master)](https://travis-ci.org/cpputest/cpputest)\r\n \r\n"] 101 | [21.446309, "o", "$ "] 102 | [21.447681, "o", "\u001b"] 103 | [21.628012, "o", "[1"] 104 | [21.718123, "o", "m\u001b"] 105 | [21.808257, "o", "[0"] 106 | [21.898362, "o", "m"] 107 | [21.898863, "o", "\r\n"] 108 | [21.900874, "o", "/workspaces/dfetch/doc/generate-casts\r\n"] 109 | -------------------------------------------------------------------------------- /doc/asciicasts/environment.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 197, "height": 38, "timestamp": 1741816947, "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}} 2 | [0.009393, "o", "$ "] 3 | [1.010907, "o", "\u001b"] 4 | [1.19121, "o", "[1"] 5 | [1.281376, "o", "md"] 6 | [1.371677, "o", "fe"] 7 | [1.461697, "o", "tc"] 8 | [1.552069, "o", "h "] 9 | [1.642052, "o", "en"] 10 | [1.732376, "o", "vi"] 11 | [1.822366, "o", "ro"] 12 | [1.91248, "o", "nm"] 13 | [2.092733, "o", "e"] 14 | [2.182873, "o", "nt"] 15 | [2.273204, "o", "\u001b["] 16 | [2.3632, "o", "0m"] 17 | [3.363744, "o", "\r\n"] 18 | [3.615978, "o", "\u001b[1;38;5;4m\u001b[34mDfetch (0.10.0)\u001b[0m\r\n\u001b[0m"] 19 | [3.618178, "o", "\u001b[1;38;5;4m \u001b[32mplatform :\u001b[34m Linux 6.8.0-1021-azure\u001b[0m\r\n\u001b[0m"] 20 | [3.619719, "o", "\u001b[1;38;5;4m \u001b[32mgit :\u001b[34m 2.47.1\u001b[0m\r\n\u001b[0m"] 21 | [3.626841, "o", "\u001b[1;38;5;4m \u001b[32msvn :\u001b[34m 1.14.1 (r1886195)\u001b[0m\r\n"] 22 | [3.62707, "o", "\u001b[0m"] 23 | [3.627118, "o", "\u001b[0m"] 24 | [6.661763, "o", "$ "] 25 | [6.663057, "o", "\u001b["] 26 | [6.843632, "o", "1m"] 27 | [6.933625, "o", "\u001b["] 28 | [7.023722, "o", "0m"] 29 | [7.024126, "o", "\r\n"] 30 | -------------------------------------------------------------------------------- /doc/asciicasts/freeze.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 197, "height": 38, "timestamp": 1741817039, "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}} 2 | [0.024788, "o", "\u001b[H\u001b[2J\u001b[3J"] 3 | [0.027435, "o", "$ "] 4 | [1.028907, "o", "\u001b["] 5 | [1.209184, "o", "1m"] 6 | [1.299366, "o", "ca"] 7 | [1.389515, "o", "t "] 8 | [1.479641, "o", "df"] 9 | [1.570016, "o", "et"] 10 | [1.660269, "o", "ch"] 11 | [1.75045, "o", ".y"] 12 | [1.840639, "o", "am"] 13 | [1.930822, "o", "l\u001b"] 14 | [2.111197, "o", "[0"] 15 | [2.201205, "o", "m"] 16 | [3.201925, "o", "\r\n"] 17 | [3.204052, "o", "manifest:\r\n version: 0.0 # DFetch Module syntax version\r\n\r\n remotes: # declare common sources in one place\r\n - name: github\r\n url-base: https://github.com/\r\n\r\n projects:\r\n - name: cpputest\r\n dst: cpputest/src/ # Destination of this project (relative to this file)\r\n repo-path: cpputest/cpputest.git # Use default github remote\r\n tag: v3.4 # tag\r\n\r\n - name: jsmn # without destination, defaults to project name\r\n repo-path: zserge/jsmn.git # only repo-path is enough\r\n"] 18 | [3.207894, "o", "$ "] 19 | [4.209554, "o", "\u001b"] 20 | [4.389824, "o", "[1"] 21 | [4.479975, "o", "md"] 22 | [4.570286, "o", "fe"] 23 | [4.660453, "o", "tc"] 24 | [4.750714, "o", "h "] 25 | [4.840723, "o", "fr"] 26 | [4.933693, "o", "ee"] 27 | [5.022371, "o", "ze"] 28 | [5.112557, "o", "\u001b["] 29 | [5.292758, "o", "0"] 30 | [5.383092, "o", "m"] 31 | [6.384143, "o", "\r\n"] 32 | [6.63437, "o", "\u001b[1;38;5;4m\u001b[34mDfetch (0.10.0)\u001b[0m\r\n\u001b[0m"] 33 | [6.649454, "o", "\u001b[1;38;5;4m \u001b[32mcpputest :\u001b[34m Already pinned in manifest on version v3.4\u001b[0m\r\n\u001b[0m"] 34 | [6.650448, "o", "\u001b[1;38;5;4m \u001b[32mjsmn :\u001b[34m Freezing on version master - 25647e692c7906b96ffd2b05ca54c097948e879c\u001b[0m\r\n\u001b[0m"] 35 | [6.651696, "o", "\u001b[1;38;5;4mUpdated manifest (dfetch.yaml) in /workspaces/dfetch/doc/generate-casts/freeze\u001b[0m\r\n\u001b[0m"] 36 | [6.651888, "o", "\u001b[0m"] 37 | [6.684777, "o", "$ "] 38 | [7.68637, "o", "\u001b["] 39 | [7.86686, "o", "1m"] 40 | [7.957097, "o", "ca"] 41 | [8.047232, "o", "t "] 42 | [8.137217, "o", "df"] 43 | [8.227527, "o", "et"] 44 | [8.31756, "o", "ch"] 45 | [8.407675, "o", ".y"] 46 | [8.497837, "o", "am"] 47 | [8.587957, "o", "l\u001b"] 48 | [8.768151, "o", "[0m"] 49 | [9.769241, "o", "\r\n"] 50 | [9.77142, "o", "manifest:\r\n version: '0.0'\r\n\r\n remotes:\r\n - name: github\r\n url-base: https://github.com/\r\n\r\n projects:\r\n - name: cpputest\r\n dst: cpputest/src/\r\n tag: v3.4\r\n repo-path: cpputest/cpputest.git\r\n\r\n - name: jsmn\r\n revision: 25647e692c7906b96ffd2b05ca54c097948e879c\r\n branch: master\r\n repo-path: zserge/jsmn.git\r\n"] 51 | [9.775225, "o", "$ "] 52 | [10.776831, "o", "\u001b["] 53 | [10.957107, "o", "1m"] 54 | [11.047228, "o", "ls"] 55 | [11.137416, "o", " -"] 56 | [11.227703, "o", "l "] 57 | [11.317911, "o", ".\u001b"] 58 | [11.408129, "o", "[0"] 59 | [11.498338, "o", "m"] 60 | [12.498945, "o", "\r\n"] 61 | [12.501617, "o", "total 16\r\n"] 62 | [12.501744, "o", "drwxr-xr-x+ 3 dev dev 4096 Mar 12 22:03 cpputest\r\n-rw-rw-rw- 1 dev dev 317 Mar 12 22:04 dfetch.yaml\r\n-rw-rw-rw- 1 dev dev 733 Mar 12 22:03 dfetch.yaml.backup\r\ndrwxr-xr-x+ 4 dev dev 4096 Mar 12 22:03 jsmn\r\n"] 63 | [15.506319, "o", "$ "] 64 | [15.507544, "o", "\u001b["] 65 | [15.688161, "o", "1m"] 66 | [15.778058, "o", "\u001b["] 67 | [15.868406, "o", "0m"] 68 | [15.868936, "o", "\r\n"] 69 | [15.87096, "o", "/workspaces/dfetch/doc/generate-casts\r\n"] 70 | -------------------------------------------------------------------------------- /doc/asciicasts/import.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 197, "height": 38, "timestamp": 1741817082, "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}} 2 | [0.009041, "o", "$ "] 3 | [1.015128, "o", "\u001b["] 4 | [1.193684, "o", "1m"] 5 | [1.28419, "o", "ls"] 6 | [1.374424, "o", " -"] 7 | [1.466278, "o", "l\u001b"] 8 | [1.556563, "o", "[0"] 9 | [1.646747, "o", "m"] 10 | [2.647075, "o", "\r\n"] 11 | [2.649701, "o", "total 580\r\n"] 12 | [2.649889, "o", "-rw-rw-rw- 1 dev dev 1137 Mar 12 22:04 CMakeLists.txt\r\n-rw-rw-rw- 1 dev dev 35147 Mar 12 22:04 LICENSE\r\n-rw-rw-rw- 1 dev dev 1796 Mar 12 22:04 README.md\r\n-rw-rw-rw- 1 dev dev 1381 Mar 12 22:04 appveyor.yml\r\n-rwxrwxrwx 1 dev dev 229 Mar 12 22:04 create_doc.sh\r\ndrwxrwxrwx+ 2 dev dev 4096 Mar 12 22:04 data\r\ndrwxrwxrwx+ 4 dev dev 4096 Mar 12 22:04 doc\r\ndrwxrwxrwx+ 4 dev dev 4096 Mar 12 22:04 docs\r\ndrwxrwxrwx+ 2 dev dev 4096 Mar 12 22:04 installer\r\ndrwxrwxrwx+ 4 dev dev 4096 Mar 12 22:04 libraries\r\n-rw-rw-rw- 1 dev dev 505101 Mar 12 22:04 modbusscope_demo.gif\r\ndrwxrwxrwx+ 5 dev dev 4096 Mar 12 22:04 resources\r\ndrwxrwxrwx+ 9 dev dev 4096 Mar 12 22:04 src\r\ndrwxrwxrwx+ 9 dev dev 4096 Mar 12 22:04 tests\r\n"] 13 | [2.653599, "o", "$ "] 14 | [3.655728, "o", "\u001b["] 15 | [3.836105, "o", "1m"] 16 | [3.926206, "o", "ca"] 17 | [4.016572, "o", "t "] 18 | [4.106571, "o", ".gi"] 19 | [4.196696, "o", "tm"] 20 | [4.286882, "o", "od"] 21 | [4.377012, "o", "ul"] 22 | [4.467203, "o", "es"] 23 | [4.557517, "o", "\u001b[0"] 24 | [4.737601, "o", "m"] 25 | [5.738089, "o", "\r\n"] 26 | [5.739795, "o", "[submodule \"tests/googletest\"]\r\n\tpath = tests/googletest\r\n\turl = https://github.com/google/googletest.git\r\n[submodule \"libraries/muparser\"]\r\n\tpath = libraries/muparser\r\n\turl = https://github.com/beltoforion/muparser.git\r\n"] 27 | [5.743254, "o", "$ "] 28 | [6.744771, "o", "\u001b["] 29 | [6.925331, "o", "1m"] 30 | [7.015363, "o", "df"] 31 | [7.105497, "o", "et"] 32 | [7.195583, "o", "ch"] 33 | [7.285978, "o", " i"] 34 | [7.376038, "o", "mp"] 35 | [7.466365, "o", "or"] 36 | [7.556612, "o", "t\u001b"] 37 | [7.646774, "o", "[0"] 38 | [7.826883, "o", "m"] 39 | [8.827699, "o", "\r\n"] 40 | [9.081755, "o", "\u001b[1;38;5;4m\u001b[34mDfetch (0.10.0)\u001b[0m\r\n\u001b[0m"] 41 | [9.692126, "o", "\u001b[1;38;5;4mFound libraries/muparser\u001b[0m\r\n"] 42 | [9.69238, "o", "\u001b[0m\u001b[1;38;5;4mFound tests/googletest\u001b[0m\r\n\u001b[0m"] 43 | [9.694018, "o", "\u001b[1;38;5;4mCreated manifest (dfetch.yaml) in /workspaces/dfetch/doc/generate-casts/ModbusScope\u001b[0m\r\n"] 44 | [9.694154, "o", "\u001b[0m"] 45 | [9.69442, "o", "\u001b[0m"] 46 | [9.729238, "o", "$ "] 47 | [10.730893, "o", "\u001b["] 48 | [10.911196, "o", "1m"] 49 | [11.001305, "o", "ca"] 50 | [11.0915, "o", "t "] 51 | [11.181606, "o", "df"] 52 | [11.271711, "o", "et"] 53 | [11.361844, "o", "ch"] 54 | [11.452017, "o", ".y"] 55 | [11.542146, "o", "am"] 56 | [11.632527, "o", "l\u001b"] 57 | [11.812926, "o", "[0m"] 58 | [12.813538, "o", "\r\n"] 59 | [12.815549, "o", "manifest:\r\n version: '0.0'\r\n\r\n remotes:\r\n - name: github-com-beltoforion\r\n url-base: https://github.com/beltoforion\r\n\r\n - name: github-com-google\r\n url-base: https://github.com/google\r\n\r\n projects:\r\n - name: libraries/muparser\r\n revision: 207d5b77c05c9111ff51ab91082701221220c477\r\n remote: github-com-beltoforion\r\n tag: v2.3.2\r\n repo-path: muparser.git\r\n\r\n - name: tests/googletest\r\n revision: dcc92d0ab6c4ce022162a23566d44f673251eee4\r\n remote: github-com-google\r\n repo-path: googletest.git\r\n"] 60 | [15.8203, "o", "$ "] 61 | [15.821667, "o", "\u001b"] 62 | [16.001996, "o", "[1"] 63 | [16.09215, "o", "m\u001b"] 64 | [16.182264, "o", "[0"] 65 | [16.272665, "o", "m"] 66 | [16.273151, "o", "\r\n"] 67 | -------------------------------------------------------------------------------- /doc/asciicasts/init.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 197, "height": 38, "timestamp": 1741816932, "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}} 2 | [0.007568, "o", "\u001b[H\u001b[2J\u001b[3J"] 3 | [0.010253, "o", "$ "] 4 | [1.012353, "o", "\u001b["] 5 | [1.19269, "o", "1m"] 6 | [1.282837, "o", "ls"] 7 | [1.372931, "o", " -"] 8 | [1.463075, "o", "l\u001b"] 9 | [1.553179, "o", "[0"] 10 | [1.643365, "o", "m"] 11 | [2.643805, "o", "\r\n"] 12 | [2.646214, "o", "total 0\r\n"] 13 | [2.649854, "o", "$ "] 14 | [3.651345, "o", "\u001b["] 15 | [3.831949, "o", "1m"] 16 | [3.921909, "o", "df"] 17 | [4.012108, "o", "et"] 18 | [4.102291, "o", "ch "] 19 | [4.192441, "o", "in"] 20 | [4.282603, "o", "it"] 21 | [4.372755, "o", "\u001b["] 22 | [4.463051, "o", "0m"] 23 | [5.463686, "o", "\r\n"] 24 | [5.713882, "o", "\u001b[1;38;5;4m\u001b[34mDfetch (0.10.0)\u001b[0m\r\n\u001b[0m"] 25 | [5.716546, "o", "\u001b[1;38;5;4mCreated dfetch.yaml\u001b[0m\r\n\u001b[0m"] 26 | [5.716937, "o", "\u001b[0m"] 27 | [5.749856, "o", "$ "] 28 | [6.751644, "o", "\u001b["] 29 | [6.932177, "o", "1m"] 30 | [7.022112, "o", "ls"] 31 | [7.112292, "o", " -"] 32 | [7.202671, "o", "l\u001b"] 33 | [7.292674, "o", "[0"] 34 | [7.383066, "o", "m"] 35 | [8.383864, "o", "\r\n"] 36 | [8.386372, "o", "total 4\r\n"] 37 | [8.386488, "o", "-rw-rw-rw- 1 dev dev 733 Mar 12 22:02 dfetch.yaml\r\n"] 38 | [8.390247, "o", "$ "] 39 | [9.391679, "o", "\u001b["] 40 | [9.571986, "o", "1m"] 41 | [9.662095, "o", "ca"] 42 | [9.752207, "o", "t "] 43 | [9.842323, "o", "df"] 44 | [9.932489, "o", "et"] 45 | [10.022596, "o", "ch"] 46 | [10.112711, "o", ".y"] 47 | [10.202853, "o", "am"] 48 | [10.292946, "o", "l\u001b"] 49 | [10.47324, "o", "[0"] 50 | [10.563349, "o", "m"] 51 | [11.563778, "o", "\r\n"] 52 | [11.565534, "o", "manifest:\r\n version: 0.0 # DFetch Module syntax version\r\n\r\n remotes: # declare common sources in one place\r\n - name: github\r\n url-base: https://github.com/\r\n\r\n projects:\r\n - name: cpputest\r\n dst: cpputest/src/ # Destination of this project (relative to this file)\r\n repo-path: cpputest/cpputest.git # Use default github remote\r\n tag: v3.4 # tag\r\n\r\n - name: jsmn # without destination, defaults to project name\r\n repo-path: zserge/jsmn.git # only repo-path is enough\r\n"] 53 | [14.569641, "o", "$ "] 54 | [14.570979, "o", "\u001b"] 55 | [14.751517, "o", "[1"] 56 | [14.84169, "o", "m\u001b"] 57 | [14.931972, "o", "[0"] 58 | [15.021875, "o", "m"] 59 | [15.022306, "o", "\r\n"] 60 | [15.024279, "o", "/workspaces/dfetch/doc/generate-casts\r\n"] 61 | -------------------------------------------------------------------------------- /doc/asciicasts/report.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 197, "height": 38, "timestamp": 1741817016, "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}} 2 | [0.025568, "o", "\u001b[H\u001b[2J\u001b[3J"] 3 | [0.028447, "o", "$ "] 4 | [1.030314, "o", "\u001b"] 5 | [1.210775, "o", "[1"] 6 | [1.300884, "o", "ml"] 7 | [1.391026, "o", "s "] 8 | [1.481148, "o", "-"] 9 | [1.571301, "o", "l\u001b"] 10 | [1.661471, "o", "[0"] 11 | [1.751848, "o", "m"] 12 | [2.752847, "o", "\r\n"] 13 | [2.755631, "o", "total 12\r\n"] 14 | [2.755691, "o", "drwxr-xr-x+ 3 dev dev 4096 Mar 12 22:03 cpputest\r\n-rw-rw-rw- 1 dev dev 733 Mar 12 22:03 dfetch.yaml\r\ndrwxr-xr-x+ 4 dev dev 4096 Mar 12 22:03 jsmn\r\n"] 15 | [2.759412, "o", "$ "] 16 | [3.760908, "o", "\u001b"] 17 | [3.941209, "o", "[1"] 18 | [4.031314, "o", "md"] 19 | [4.121418, "o", "fe"] 20 | [4.211769, "o", "t"] 21 | [4.301758, "o", "ch"] 22 | [4.392047, "o", " r"] 23 | [4.482039, "o", "ep"] 24 | [4.572205, "o", "or"] 25 | [4.662519, "o", "t"] 26 | [4.842766, "o", "\u001b["] 27 | [4.932907, "o", "0m"] 28 | [5.933712, "o", "\r\n"] 29 | [6.19946, "o", "\u001b[1;38;5;4m\u001b[34mDfetch (0.10.0)\u001b[0m\r\n\u001b[0m"] 30 | [6.231026, "o", "\u001b[1;38;5;4m \u001b[32mproject :\u001b[34m cpputest\u001b[0m\r\n\u001b[0m"] 31 | [6.23116, "o", "\u001b[1;38;5;4m \u001b[32m remote :\u001b[34m github\u001b[0m\r\n\u001b[0m"] 32 | [6.232093, "o", "\u001b[1;38;5;4m \u001b[32m remote url :\u001b[34m https://github.com/cpputest/cpputest.git\u001b[0m\r\n"] 33 | [6.232332, "o", "\u001b[0m\u001b[1;38;5;4m \u001b[32m branch :\u001b[34m master\u001b[0m\r\n\u001b[0m\u001b[1;38;5;4m \u001b[32m tag :\u001b[34m v3.4\u001b[0m\r\n\u001b[0m\u001b[1;38;5;4m \u001b[32m last fetch :\u001b[34m 12/03/2025, 22:03:25\u001b[0m\r\n\u001b[0m"] 34 | [6.232454, "o", "\u001b[1;38;5;4m \u001b[32m revision :\u001b[34m \u001b[0m\r\n\u001b[0m\u001b[1;38;5;4m \u001b[32m patch :\u001b[34m \u001b[0m\r\n\u001b[0m"] 35 | [6.232735, "o", "\u001b[1;38;5;4m \u001b[32m license :\u001b[34m BSD 3-Clause \"New\" or \"Revised\" License\u001b[0m\r\n\u001b[0m"] 36 | [6.234946, "o", "\u001b[1;38;5;4m \u001b[32mproject :\u001b[34m jsmn\u001b[0m\r\n"] 37 | [6.235156, "o", "\u001b[0m\u001b[1;38;5;4m \u001b[32m remote :\u001b[34m github\u001b[0m\r\n\u001b[0m"] 38 | [6.235959, "o", "\u001b[1;38;5;4m \u001b[32m remote url :\u001b[34m https://github.com/zserge/jsmn.git\u001b[0m\r\n"] 39 | [6.236064, "o", "\u001b[0m\u001b[1;38;5;4m \u001b[32m branch :\u001b[34m master\u001b[0m\r\n\u001b[0m"] 40 | [6.236454, "o", "\u001b[1;38;5;4m \u001b[32m tag :\u001b[34m \u001b[0m\r\n\u001b[0m\u001b[1;38;5;4m \u001b[32m last fetch :\u001b[34m 12/03/2025, 22:03:26\u001b[0m\r\n\u001b[0m"] 41 | [6.236502, "o", "\u001b[1;38;5;4m \u001b[32m revision :\u001b[34m 25647e692c7906b96ffd2b05ca54c097948e879c\u001b[0m\r\n\u001b[0m\u001b[1;38;5;4m \u001b[32m patch :\u001b[34m \u001b[0m\r\n\u001b[0m\u001b[1;38;5;4m \u001b[32m license :\u001b[34m MIT License\u001b[0m\r\n\u001b[0m\u001b[0m"] 42 | [9.270847, "o", "/workspaces/dfetch/doc/generate-casts\r\n"] 43 | -------------------------------------------------------------------------------- /doc/asciicasts/validate.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 197, "height": 38, "timestamp": 1741816954, "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}} 2 | [0.296441, "o", "\u001b[H\u001b[2J\u001b[3J"] 3 | [0.299206, "o", "$ "] 4 | [1.301332, "o", "\u001b"] 5 | [1.481876, "o", "[1"] 6 | [1.571869, "o", "md"] 7 | [1.662178, "o", "fe"] 8 | [1.75216, "o", "t"] 9 | [1.842539, "o", "ch"] 10 | [1.932801, "o", " v"] 11 | [2.023, "o", "al"] 12 | [2.113163, "o", "id"] 13 | [2.203311, "o", "a"] 14 | [2.383693, "o", "te"] 15 | [2.473929, "o", "\u001b["] 16 | [2.563865, "o", "0m"] 17 | [3.564316, "o", "\r\n"] 18 | [3.814464, "o", "\u001b[1;38;5;4m\u001b[34mDfetch (0.10.0)\u001b[0m\r\n\u001b[0m"] 19 | [3.826469, "o", "\u001b[1;38;5;4m \u001b[32mdfetch.yaml :\u001b[34m valid\u001b[0m\r\n\u001b[0m"] 20 | [3.826727, "o", "\u001b[0m"] 21 | [6.856959, "o", "/workspaces/dfetch/doc/generate-casts\r\n"] 22 | -------------------------------------------------------------------------------- /doc/changelog.rst: -------------------------------------------------------------------------------- 1 | :tocdepth: 1 2 | 3 | .. _changes: 4 | 5 | ========= 6 | Changelog 7 | ========= 8 | 9 | .. include:: ../CHANGELOG.rst 10 | -------------------------------------------------------------------------------- /doc/generate-casts/README.md: -------------------------------------------------------------------------------- 1 | # Generate casts 2 | 3 | This folder makes it possible to generate asciinema casts. 4 | The solution was completely inspired by https://stackoverflow.com/a/63080929/1149326 5 | It can only run on linux. 6 | 7 | ## Usage 8 | ```console 9 | ./generate-casts.sh 10 | ``` 11 | -------------------------------------------------------------------------------- /doc/generate-casts/basic-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir basic 9 | pushd basic 10 | 11 | dfetch init 12 | clear 13 | 14 | # Run the command 15 | pe "ls -l" 16 | pe "cat dfetch.yaml" 17 | pe "dfetch check" 18 | pe "sed -i 's/v3.4/v4.0/g' dfetch.yaml" 19 | pe "cat dfetch.yaml" 20 | pe "dfetch update" 21 | pe "ls -l" 22 | 23 | PROMPT_TIMEOUT=3 24 | wait 25 | 26 | pei "" 27 | 28 | popd 29 | rm -rf basic 30 | -------------------------------------------------------------------------------- /doc/generate-casts/check-ci-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir check_ci 9 | pushd check_ci 10 | 11 | dfetch init 12 | clear 13 | 14 | # Run the command 15 | pe "cat dfetch.yaml" 16 | pe "dfetch check --jenkins-json jenkins.json --sarif sarif.json" 17 | pe "ls -l ." 18 | pe "cat jenkins.json" 19 | pe "cat sarif.json" 20 | 21 | PROMPT_TIMEOUT=3 22 | wait 23 | 24 | pei "" 25 | 26 | popd 27 | rm -rf check_ci 28 | -------------------------------------------------------------------------------- /doc/generate-casts/check-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir check 9 | pushd check 10 | 11 | dfetch init 12 | clear 13 | 14 | # Run the command 15 | pe "cat dfetch.yaml" 16 | pe "dfetch check" 17 | 18 | PROMPT_TIMEOUT=3 19 | wait 20 | 21 | pei "" 22 | 23 | popd 24 | rm -rf check 25 | -------------------------------------------------------------------------------- /doc/generate-casts/demo-magic/.dfetch_data.yaml: -------------------------------------------------------------------------------- 1 | # This is a generated file by dfetch. Don't edit this, but edit the manifest. 2 | # For more info see https://dfetch.rtfd.io/en/latest/getting_started.html 3 | dfetch: 4 | branch: master 5 | hash: a49df5f5ddba75dca9cb946918277657 6 | last_fetch: 09/02/2025, 20:06:22 7 | patch: '' 8 | remote_url: https://github.com/paxtonhare/demo-magic.git 9 | revision: 8f338e9771502f71a77d6c779aed2dae614fdb0d 10 | tag: '' 11 | -------------------------------------------------------------------------------- /doc/generate-casts/demo-magic/license.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2022 Paxton Hare 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /doc/generate-casts/diff-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir diff 9 | pushd diff 10 | 11 | git init 12 | cp -r ../update/* . 13 | git add . 14 | git commit -m "Initial commit" 15 | clear 16 | 17 | # Run the command 18 | pe "ls -l ." 19 | pe "ls -l cpputest/src/README.md" 20 | pe "sed -i 's/github/gitlab/g' cpputest/src/README.md" 21 | pe "dfetch diff cpputest" 22 | pe "cat cpputest.patch" 23 | 24 | 25 | PROMPT_TIMEOUT=3 26 | wait 27 | 28 | pei "" 29 | 30 | popd 31 | rm -rf diff 32 | -------------------------------------------------------------------------------- /doc/generate-casts/environment-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Remove any existing manifest 8 | clear 9 | 10 | # Run the command 11 | pe "dfetch environment" 12 | 13 | PROMPT_TIMEOUT=3 14 | wait 15 | 16 | pei "" 17 | -------------------------------------------------------------------------------- /doc/generate-casts/freeze-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir freeze 9 | pushd freeze 10 | 11 | cp -r ../update/* . 12 | clear 13 | 14 | # Run the command 15 | pe "cat dfetch.yaml" 16 | pe "dfetch freeze" 17 | pe "cat dfetch.yaml" 18 | pe "ls -l ." 19 | 20 | 21 | PROMPT_TIMEOUT=3 22 | wait 23 | 24 | pei "" 25 | 26 | popd 27 | rm -rf freeze 28 | -------------------------------------------------------------------------------- /doc/generate-casts/generate-casts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Uses relative paths 4 | cd "$(dirname "$0")" 5 | 6 | rm -rf ../asciicasts/* 7 | 8 | asciinema rec --overwrite -c "./basic-demo.sh" ../asciicasts/basic.cast 9 | asciinema rec --overwrite -c "./init-demo.sh" ../asciicasts/init.cast 10 | asciinema rec --overwrite -c "./environment-demo.sh" ../asciicasts/environment.cast 11 | asciinema rec --overwrite -c "./validate-demo.sh" ../asciicasts/validate.cast 12 | asciinema rec --overwrite -c "./check-demo.sh" ../asciicasts/check.cast 13 | asciinema rec --overwrite -c "./check-ci-demo.sh" ../asciicasts/check-ci.cast 14 | asciinema rec --overwrite -c "./update-demo.sh" ../asciicasts/update.cast 15 | 16 | # Depends on artifacts from update 17 | asciinema rec --overwrite -c "./report-demo.sh" ../asciicasts/report.cast 18 | asciinema rec --overwrite -c "./report-sbom-demo.sh" ../asciicasts/sbom.cast 19 | asciinema rec --overwrite -c "./freeze-demo.sh" ../asciicasts/freeze.cast 20 | asciinema rec --overwrite -c "./diff-demo.sh" ../asciicasts/diff.cast 21 | 22 | rm -rf update 23 | 24 | git clone --quiet --depth=1 --branch 3.0.0 https://github.com/jgeudens/ModbusScope.git 2> /dev/null 25 | pushd ModbusScope 26 | git submodule update --quiet --init > /dev/null 27 | asciinema rec --overwrite -c "../import-demo.sh" ../../asciicasts/import.cast 28 | popd 29 | rm -rf ModbusScope 30 | 31 | # Find all files with the .cast extension in the specified directory 32 | files=$(find "../asciicasts" -type f -name '*.cast') 33 | 34 | # Process each file 35 | for file in $files; do 36 | ./strip-setup-from-cast.sh "${file}" 37 | done 38 | -------------------------------------------------------------------------------- /doc/generate-casts/import-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ../demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | clear 8 | 9 | # Run the command 10 | pe "ls -l" 11 | pe "cat .gitmodules" 12 | pe "dfetch import" 13 | pe "cat dfetch.yaml" 14 | 15 | PROMPT_TIMEOUT=3 16 | wait 17 | 18 | pei "" 19 | -------------------------------------------------------------------------------- /doc/generate-casts/init-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir init 9 | pushd init 10 | 11 | clear 12 | 13 | # Run the command 14 | pe "ls -l" 15 | pe "dfetch init" 16 | pe "ls -l" 17 | pe "cat dfetch.yaml" 18 | 19 | PROMPT_TIMEOUT=3 20 | wait 21 | 22 | pei "" 23 | 24 | popd 25 | rm -rf init 26 | -------------------------------------------------------------------------------- /doc/generate-casts/report-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir report 9 | pushd report 10 | 11 | cp -r ../update/* . 12 | clear 13 | 14 | # Run the command 15 | pe "ls -l" 16 | pe "dfetch report" 17 | 18 | PROMPT_TIMEOUT=3 19 | wait 20 | 21 | popd 22 | rm -rf report 23 | -------------------------------------------------------------------------------- /doc/generate-casts/report-sbom-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir report_sbom 9 | pushd report_sbom 10 | 11 | cp -r ../update/* . 12 | clear 13 | 14 | # Run the command 15 | pe "ls -l" 16 | pe "dfetch report -t sbom" 17 | pe "cat report.json" 18 | 19 | 20 | PROMPT_TIMEOUT=3 21 | wait 22 | 23 | popd 24 | rm -rf report_sbom 25 | -------------------------------------------------------------------------------- /doc/generate-casts/strip-setup-from-cast.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Check if a file path was provided as the first argument 4 | if [ -z "$1" ]; then 5 | echo "Error: No file path provided" 6 | exit 1 7 | fi 8 | 9 | # Read the asciicast file into a variable 10 | asciicast=$(<"$1") 11 | 12 | # Find the line containing the escape sequence for clearing the screen 13 | line=$(echo "$asciicast" | grep -n '\\u001b\[H\\u001b\[2J\\u001b\[3J' | cut -d: -f1) 14 | 15 | # Check if a matching line was found 16 | if [ -z "$line" ]; then 17 | echo "Warning: No line containing the escape sequence for clearing the screen was found" 18 | else 19 | echo "Will remove lines 2 --> $line" 20 | # Remove all the lines before the line containing the escape sequence 21 | asciicast=$(echo "$asciicast" | sed -n "2,$((line-1))!p") 22 | 23 | # Write the updated asciicast to the file 24 | echo "$asciicast" > "$1" 25 | 26 | fi 27 | -------------------------------------------------------------------------------- /doc/generate-casts/update-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir update 9 | pushd update 10 | 11 | dfetch init 12 | clear 13 | 14 | # Run the command 15 | pe "ls -l" 16 | pe "cat dfetch.yaml" 17 | pe "dfetch update" 18 | pe "ls -l" 19 | pe "dfetch update" 20 | 21 | PROMPT_TIMEOUT=3 22 | wait 23 | 24 | popd 25 | -------------------------------------------------------------------------------- /doc/generate-casts/validate-demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ./demo-magic/demo-magic.sh 4 | 5 | PROMPT_TIMEOUT=1 6 | 7 | # Copy example manifest 8 | mkdir validate 9 | pushd validate 10 | 11 | dfetch init 12 | clear 13 | 14 | # Run the command 15 | pe "dfetch validate" 16 | 17 | PROMPT_TIMEOUT=3 18 | wait 19 | 20 | popd 21 | rm -rf validate 22 | -------------------------------------------------------------------------------- /doc/images/dfetch_header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/dfetch_header.png -------------------------------------------------------------------------------- /doc/images/dfetch_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/dfetch_logo.png -------------------------------------------------------------------------------- /doc/images/github-actions-result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/github-actions-result.png -------------------------------------------------------------------------------- /doc/images/gitlab-check-pipeline-result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/gitlab-check-pipeline-result.png -------------------------------------------------------------------------------- /doc/images/gitlab-highlighted-manifest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/gitlab-highlighted-manifest.png -------------------------------------------------------------------------------- /doc/images/local-change-github-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/local-change-github-details.png -------------------------------------------------------------------------------- /doc/images/local-change-github.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/local-change-github.png -------------------------------------------------------------------------------- /doc/images/out-of-date-jenkins.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/out-of-date-jenkins.png -------------------------------------------------------------------------------- /doc/images/out-of-date-jenkins2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/doc/images/out-of-date-jenkins2.png -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. Dfetch documentation master file 2 | 3 | .. meta:: 4 | :description: Dfetch is a VCS-agnostic tool that simplifies dependency management by retrieving 5 | source-only dependencies from various repositories, promoting upstream changes and 6 | allowing local customizations. 7 | :keywords: dfetch, dependency management, embedded development, fetch tool, vendoring, multi-repo, dependencies, git, svn, package manager, multi-project, monorepo 8 | :author: Dfetch Contributors 9 | :google-site-verification: yCnoTogJMh7Nm5gxlREDuONIXT4ijHcj972Y5k9p-sU 10 | 11 | .. raw:: html 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | .. image:: images/dfetch_header.png 25 | :width: 100% 26 | :align: center 27 | 28 | .. toctree:: 29 | :maxdepth: 2 30 | 31 | getting_started 32 | manifest 33 | manual 34 | troubleshooting 35 | contributing 36 | changelog 37 | alternatives 38 | legal 39 | internal 40 | 41 | Dfetch - *a source-only no-hassle project-dependency aggregator* 42 | ================================================================ 43 | 44 | What is Dfetch? 45 | --------------- 46 | 47 | We needed a dependency manager that was flexible enough to retrieve dependencies as plain text 48 | from various sources. `svn externals`, `git submodules` and `git subtrees` solve a similar 49 | problem, but not in a vcs agnostic way or completely user friendly way. 50 | We want self-contained code repositories without any hassle for end-users. 51 | Dfetch must promote upstreaming changes, but allow for local customizations. 52 | 53 | Other tools that do similar things are `Zephyr's West`, `CMake ExternalProject` and other meta tools. 54 | See :ref:`alternatives` for a complete list. 55 | 56 | Installation 57 | ------------ 58 | `Dfetch` is a python based cross-platform cli tool. 59 | 60 | Install the latest release with: 61 | 62 | .. code-block:: 63 | 64 | pip install dfetch 65 | 66 | Or install the latest version from the main branch: 67 | 68 | .. code-block:: 69 | 70 | pip install https://github.com/dfetch-org/dfetch/archive/main.zip 71 | 72 | Once installed dfetch output can be seen. 73 | 74 | .. code-block:: 75 | 76 | dfetch --version 77 | 78 | Basic usage 79 | ----------- 80 | 81 | .. asciinema:: asciicasts/basic.cast 82 | -------------------------------------------------------------------------------- /doc/internal.rst: -------------------------------------------------------------------------------- 1 | .. Dfetch documentation internal 2 | 3 | Internal 4 | ======== 5 | *DFetch* is becoming larger everyday. To give it some structure below a description of the internals 6 | 7 | Architecture 8 | ------------ 9 | These diagrams are based on `Simon Brown's C4-model`_. 10 | 11 | .. _`Simon Brown's C4-model` : https://c4model.com/#CoreDiagrams 12 | 13 | C1 - Context 14 | '''''''''''' 15 | .. uml:: /static/uml/c1_dfetch_context.puml 16 | 17 | C2 - Containers 18 | ''''''''''''''' 19 | .. uml:: /static/uml/c2_dfetch_containers.puml 20 | 21 | C3 - Components 22 | ''''''''''''''' 23 | 24 | Commands 25 | ~~~~~~~~ 26 | .. uml:: /static/uml/c3_dfetch_components_commands.puml 27 | 28 | Manifest 29 | ~~~~~~~~ 30 | .. uml:: /static/uml/c3_dfetch_components_manifest.puml 31 | 32 | Project 33 | ~~~~~~~ 34 | .. uml:: /static/uml/c3_dfetch_components_project.puml 35 | -------------------------------------------------------------------------------- /doc/landing-page/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = DFetch 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | touch "$(BUILDDIR)/html/.nojekyll" 22 | -------------------------------------------------------------------------------- /doc/landing-page/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=DFetch 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /doc/landing-page/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | 3 | Sitemap: https://dfetch-org.github.io/sitemap.xml 4 | -------------------------------------------------------------------------------- /doc/landing-page/static/css/custom.css: -------------------------------------------------------------------------------- 1 | @import url("https://fonts.googleapis.com/css?family=Roboto:100,300,300i,400,500,700,900"); 2 | 3 | .sphinxsidebar .caption-text { 4 | font-size: 130%; 5 | } 6 | 7 | .logo { 8 | width: 100%; 9 | } 10 | 11 | .sphinxsidebarwrapper .internal, 12 | .sphinxsidebarwrapper .external { 13 | font-weight: 300; 14 | } 15 | 16 | body { 17 | font-family: Roboto; 18 | } 19 | 20 | h1, 21 | h2, 22 | h3, 23 | h4, 24 | h5, 25 | h6 { 26 | font-family: Roboto !important; 27 | font-weight: 300 !important; 28 | } 29 | 30 | .toctree-l1 { 31 | padding-bottom: 0.5em; 32 | } 33 | 34 | .body p, 35 | .body dd, 36 | .body li { 37 | font-family: Roboto; 38 | font-size: 1em; 39 | font-weight: 300; 40 | line-height: 2; 41 | text-align: justify; 42 | } 43 | 44 | .search { 45 | margin-bottom: 1em; 46 | } 47 | 48 | .sphinxsidebarwrapper ul { 49 | margin-bottom: 1em; 50 | } 51 | 52 | .sphinxsidebar input[type="submit"] { 53 | font-family: "Roboto", serif; 54 | } 55 | 56 | .caption { 57 | margin-top: 1em; 58 | } 59 | 60 | .caption-text { 61 | font-weight: 500; 62 | } 63 | 64 | .sphinxsidebarwrapper .logo-name { 65 | display: none; 66 | } 67 | 68 | .sphinxsidebarwrapper .logo { 69 | margin-bottom: 2em; 70 | } 71 | 72 | img { 73 | height: auto; 74 | max-width: 100%; 75 | } 76 | 77 | div.admonition { 78 | border: none; 79 | border-radius: 0; 80 | box-shadow: 0 2px 2px 0 rgba(0, 0, 0, 0.14), 0 1px 5px 0 rgba(0, 0, 0, 0.12), 81 | 0 3px 1px -2px rgba(0, 0, 0, 0.2); 82 | font-family: Roboto; 83 | padding: 2em; 84 | position: relative; 85 | transition: box-shadow 0.25s; 86 | } 87 | 88 | div.admonition p.admonition-title { 89 | font-family: "Roboto", serif; 90 | font-weight: 300; 91 | margin: 0 0 5px 0; 92 | } 93 | 94 | div.admonition p { 95 | font-weight: 300; 96 | margin-top: 1em; 97 | } 98 | 99 | .admonition a { 100 | font-weight: 300; 101 | } 102 | 103 | strong { 104 | font-weight: 400; 105 | } 106 | 107 | .pre { 108 | padding-left: 0.2em; 109 | padding-right: 0.2em; 110 | } 111 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=DFetch 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /doc/manifest.rst: -------------------------------------------------------------------------------- 1 | .. Dfetch documentation master file 2 | 3 | Manifest 4 | ======== 5 | .. automodule:: dfetch.manifest.manifest 6 | 7 | Remotes 8 | ------- 9 | .. automodule:: dfetch.manifest.remote 10 | 11 | Projects 12 | -------- 13 | .. automodule:: dfetch.manifest.project 14 | -------------------------------------------------------------------------------- /doc/manual.rst: -------------------------------------------------------------------------------- 1 | .. Dfetch documentation master file 2 | 3 | Manual 4 | ====== 5 | 6 | Init 7 | ----- 8 | .. argparse:: 9 | :module: dfetch.__main__ 10 | :func: create_parser 11 | :prog: dfetch 12 | :path: init 13 | 14 | .. asciinema:: asciicasts/init.cast 15 | 16 | .. automodule:: dfetch.commands.init 17 | 18 | Check 19 | ----- 20 | .. argparse:: 21 | :module: dfetch.__main__ 22 | :func: create_parser 23 | :prog: dfetch 24 | :path: check 25 | 26 | .. asciinema:: asciicasts/check.cast 27 | 28 | .. automodule:: dfetch.commands.check 29 | 30 | Reporting 31 | ````````` 32 | .. automodule:: dfetch.reporting.check.reporter 33 | 34 | Jenkins reporter 35 | '''''''''''''''' 36 | .. automodule:: dfetch.reporting.check.jenkins_reporter 37 | 38 | .. asciinema:: asciicasts/check-ci.cast 39 | 40 | Sarif reporter 41 | '''''''''''''' 42 | .. automodule:: dfetch.reporting.check.sarif_reporter 43 | 44 | Code-climate reporter 45 | ''''''''''''''''''''' 46 | .. automodule:: dfetch.reporting.check.code_climate_reporter 47 | 48 | Report 49 | ------ 50 | .. argparse:: 51 | :module: dfetch.__main__ 52 | :func: create_parser 53 | :prog: dfetch 54 | :path: report 55 | 56 | .. asciinema:: asciicasts/report.cast 57 | 58 | .. automodule:: dfetch.commands.report 59 | 60 | List (default) 61 | `````````````` 62 | .. automodule:: dfetch.reporting.stdout_reporter 63 | 64 | Software Bill-of-Materials 65 | `````````````````````````` 66 | .. automodule:: dfetch.reporting.sbom_reporter 67 | 68 | .. asciinema:: asciicasts/sbom.cast 69 | 70 | Update 71 | ------ 72 | .. argparse:: 73 | :module: dfetch.__main__ 74 | :func: create_parser 75 | :prog: dfetch 76 | :path: update 77 | 78 | .. asciinema:: asciicasts/update.cast 79 | 80 | .. automodule:: dfetch.commands.update 81 | 82 | Validate 83 | -------- 84 | .. argparse:: 85 | :module: dfetch.__main__ 86 | :func: create_parser 87 | :prog: dfetch 88 | :path: validate 89 | 90 | .. asciinema:: asciicasts/validate.cast 91 | 92 | .. automodule:: dfetch.commands.validate 93 | 94 | Diff 95 | ----- 96 | .. argparse:: 97 | :module: dfetch.__main__ 98 | :func: create_parser 99 | :prog: dfetch 100 | :path: diff 101 | 102 | .. asciinema:: asciicasts/diff.cast 103 | 104 | .. automodule:: dfetch.commands.diff 105 | 106 | Freeze 107 | ------ 108 | .. argparse:: 109 | :module: dfetch.__main__ 110 | :func: create_parser 111 | :prog: dfetch 112 | :path: freeze 113 | 114 | .. asciinema:: asciicasts/freeze.cast 115 | 116 | .. automodule:: dfetch.commands.freeze 117 | 118 | Environment 119 | ----------- 120 | .. argparse:: 121 | :module: dfetch.__main__ 122 | :func: create_parser 123 | :prog: dfetch 124 | :path: environment 125 | 126 | .. asciinema:: asciicasts/environment.cast 127 | 128 | .. automodule:: dfetch.commands.environment 129 | 130 | 131 | Import 132 | ------ 133 | .. argparse:: 134 | :module: dfetch.__main__ 135 | :func: create_parser 136 | :prog: dfetch 137 | :path: import 138 | 139 | .. asciinema:: asciicasts/import.cast 140 | 141 | .. automodule:: dfetch.commands.import_ 142 | -------------------------------------------------------------------------------- /doc/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | 3 | Disallow: / 4 | 5 | Allow: /en/stable 6 | 7 | Allow: /en/latest 8 | 9 | Sitemap: https://dfetch.readthedocs.io/en/latest/sitemap-custom.xml 10 | -------------------------------------------------------------------------------- /doc/static/css/custom.css: -------------------------------------------------------------------------------- 1 | @import url("https://fonts.googleapis.com/css?family=Roboto:100,300,300i,400,500,700,900"); 2 | 3 | .sphinxsidebar .caption-text { 4 | font-size: 130%; 5 | } 6 | 7 | .logo { 8 | width: 100%; 9 | } 10 | 11 | .sphinxsidebarwrapper .internal, 12 | .sphinxsidebarwrapper .external { 13 | font-weight: 300; 14 | } 15 | 16 | body { 17 | font-family: Roboto; 18 | } 19 | 20 | h1, 21 | h2, 22 | h3, 23 | h4, 24 | h5, 25 | h6 { 26 | font-family: Roboto !important; 27 | font-weight: 300 !important; 28 | } 29 | 30 | .toctree-l1 { 31 | padding-bottom: 0.5em; 32 | } 33 | 34 | .body p, 35 | .body dd, 36 | .body li { 37 | font-family: Roboto; 38 | font-size: 1em; 39 | font-weight: 300; 40 | line-height: 2; 41 | text-align: justify; 42 | } 43 | 44 | .search { 45 | margin-bottom: 1em; 46 | } 47 | 48 | .sphinxsidebarwrapper ul { 49 | margin-bottom: 1em; 50 | } 51 | 52 | .sphinxsidebar input[type="submit"] { 53 | font-family: "Roboto", serif; 54 | } 55 | 56 | .caption { 57 | margin-top: 1em; 58 | } 59 | 60 | .caption-text { 61 | font-weight: 500; 62 | } 63 | 64 | .sphinxsidebarwrapper .logo-name { 65 | display: none; 66 | } 67 | 68 | .sphinxsidebarwrapper .logo { 69 | margin-bottom: 2em; 70 | } 71 | 72 | img { 73 | height: auto; 74 | max-width: 100%; 75 | } 76 | 77 | div.admonition { 78 | border: none; 79 | border-radius: 0; 80 | box-shadow: 0 2px 2px 0 rgba(0, 0, 0, 0.14), 0 1px 5px 0 rgba(0, 0, 0, 0.12), 81 | 0 3px 1px -2px rgba(0, 0, 0, 0.2); 82 | font-family: Roboto; 83 | padding: 2em; 84 | position: relative; 85 | transition: box-shadow 0.25s; 86 | } 87 | 88 | div.admonition p.admonition-title { 89 | font-family: "Roboto", serif; 90 | font-weight: 300; 91 | margin: 0 0 5px 0; 92 | } 93 | 94 | div.admonition p { 95 | font-weight: 300; 96 | margin-top: 1em; 97 | } 98 | 99 | .admonition a { 100 | font-weight: 300; 101 | } 102 | 103 | strong { 104 | font-weight: 400; 105 | } 106 | 107 | .pre { 108 | padding-left: 0.2em; 109 | padding-right: 0.2em; 110 | } 111 | -------------------------------------------------------------------------------- /doc/static/uml/c1_dfetch_context.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Context.puml 4 | 5 | Person(user, "Developer") 6 | 7 | System(DFetch, "Dfetch") 8 | 9 | Rel(user, DFetch, "Uses") 10 | 11 | System_Boundary(Local, "Local") { 12 | System_Ext(git, "Git") 13 | System_Ext(svn, "Svn") 14 | } 15 | 16 | System_Boundary(Remote, "Remote") { 17 | System_Ext(github, "GitHub") 18 | System_Ext(gitlab, "GitLab") 19 | System_Ext(jenkins, "Jenkins") 20 | } 21 | 22 | Rel(DFetch, git, "Uses") 23 | Rel(DFetch, svn, "Uses") 24 | Rel(DFetch, github, "Reports to") 25 | Rel(DFetch, gitlab, "Reports to") 26 | Rel(DFetch, jenkins, "Reports to") 27 | 28 | 29 | @enduml 30 | -------------------------------------------------------------------------------- /doc/static/uml/c2_dfetch_containers.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml 4 | 5 | Person(user, "Developer") 6 | 7 | System_Boundary(DFetch, "Dfetch") { 8 | 9 | Container(contCommands, "Commands", "python", "Single user command to start interacting with dfetch.") 10 | Container(contManifest, "Manifest", "python", "Parsing, editing and finding of manifests.") 11 | Container(contProject, "Project", "python", "Main project that has a manifest.") 12 | Container(contVcs, "Vcs", "python", "Abstraction of various Version Control Systems.") 13 | Container(contReporting, "Reporting", "python", "Output formatters for various reporting formats.") 14 | 15 | Rel(contCommands, contManifest, "Uses") 16 | Rel(contCommands, contReporting, "Uses") 17 | Rel(contCommands, contProject, "Uses") 18 | Rel(contCommands, contVcs, "Uses") 19 | Rel_U(contReporting, contProject, "Implements") 20 | Rel_R(contProject, contManifest, "Has") 21 | Rel_U(contReporting, contManifest, "Uses") 22 | Rel(contProject, contVcs, "Uses") 23 | } 24 | 25 | System_Boundary(Local, "Local") { 26 | System_Ext(git, "Git") 27 | System_Ext(svn, "Svn") 28 | } 29 | 30 | System_Boundary(Remote, "Remote") { 31 | System_Ext(github, "GitHub") 32 | System_Ext(gitlab, "GitLab") 33 | System_Ext(jenkins, "Jenkins") 34 | } 35 | 36 | Rel(contVcs, git, "Uses") 37 | Rel(contVcs, svn, "Uses") 38 | 39 | Rel(contReporting, github, "Reports to") 40 | Rel(contReporting, gitlab, "Reports to") 41 | Rel(contReporting, jenkins, "Reports to") 42 | 43 | 44 | Rel(user, contCommands, "Uses") 45 | 46 | @enduml 47 | -------------------------------------------------------------------------------- /doc/static/uml/c3_dfetch_components_commands.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml 4 | 5 | Person(user, "Developer") 6 | 7 | System_Boundary(DFetch, "Dfetch") { 8 | 9 | Boundary(DFetchCommands, "Commands") { 10 | Component(compCommon, "Common", "python", "Does stuff") 11 | Component(compCommand, "Command", "python", "Does stuff") 12 | 13 | Component(compCheck, "Check", "python", "Does stuff") 14 | Component(compDiff, "Diff", "python", "Does stuff") 15 | Component(compEnv, "Environment", "python", "Does stuff") 16 | Component(compFreeze, "Freeze", "python", "Does stuff") 17 | Component(compImport, "Import", "python", "Does stuff") 18 | Component(compInit, "Init", "python", "Does stuff") 19 | Component(compReport, "Report", "python", "Does stuff") 20 | Component(compUpdate, "Update", "python", "Does stuff") 21 | Component(compValidate, "Validate", "python", "Does stuff") 22 | 23 | Rel_U(compValidate, compCommand, "Extends") 24 | Rel_U(compCheck, compCommand, "Extends") 25 | Rel_U(compDiff, compCommand, "Extends") 26 | Rel_U(compEnv, compCommand, "Extends") 27 | Rel_U(compFreeze, compCommand, "Extends") 28 | Rel_U(compImport, compCommand, "Extends") 29 | Rel_U(compInit, compCommand, "Extends") 30 | Rel_U(compReport, compCommand, "Extends") 31 | Rel_U(compUpdate, compCommand, "Extends") 32 | 33 | Rel_U(compUpdate, compCommon, "Uses") 34 | Rel_U(compCheck, compCommon, "Uses") 35 | } 36 | 37 | Container(contManifest, "Manifest", "python", "Parsing, editing and finding of manifests.") 38 | Container(contProject, "Project", "python", "Main project that has a manifest.") 39 | Container(contVcs, "Vcs", "python", "Abstraction of various Version Control Systems.") 40 | Container(contReporting, "Reporting", "python", "Output formatters for various reporting formats.") 41 | 42 | Rel(compCheck, contManifest, "Uses") 43 | Rel(compCheck, contProject, "Uses") 44 | Rel(compCheck, contReporting, "Uses") 45 | 46 | Rel(compDiff, contManifest, "Uses") 47 | Rel(compDiff, contProject, "Uses") 48 | 49 | Rel(compEnv, contProject, "Uses") 50 | 51 | Rel(compFreeze, contManifest, "Uses") 52 | Rel(compFreeze, contProject, "Uses") 53 | 54 | Rel(compImport, contManifest, "Uses") 55 | Rel(compImport, contProject, "Uses") 56 | Rel(compImport, contVcs, "Uses") 57 | 58 | Rel(compReport, contManifest, "Uses") 59 | Rel(compReport, contProject, "Uses") 60 | Rel(compReport, contReporting, "Uses") 61 | 62 | Rel(compUpdate, contManifest, "Uses") 63 | Rel(compUpdate, contProject, "Uses") 64 | 65 | Rel(compValidate, contManifest, "Uses") 66 | 67 | Rel(contProject, contReporting, "Uses") 68 | Rel_R(contProject, contManifest, "Has") 69 | Rel_U(contReporting, contManifest, "Uses") 70 | Rel(contProject, contVcs, "Uses") 71 | } 72 | 73 | Rel(user, compCommand, "Uses") 74 | 75 | @enduml 76 | -------------------------------------------------------------------------------- /doc/static/uml/c3_dfetch_components_manifest.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml 4 | 5 | Person(user, "Developer") 6 | 7 | System_Boundary(DFetch, "Dfetch") { 8 | 9 | Container(contCommands, "Commands", "python", "Parsing, editing and finding of manifests.") 10 | 11 | Boundary(DFetchManifest, "Manifest") { 12 | Component(compManifest, "Manifest", "python", "Main configuration file describing all projects.") 13 | Component(compProject, "Project", "python", "A single project requirement with optionally specific version") 14 | Component(compRemote, "Remote", "python", "A remote source that contains one or more projects.") 15 | Component(compValidate, "Validate", "python", "Validate a manifest.") 16 | Component(compVersion, "Version", "python", "Check and compare versions.") 17 | 18 | Rel(compManifest, compProject, "Uses") 19 | Rel(compProject, compVersion, "Uses") 20 | Rel_L(compProject, compRemote, "Uses") 21 | Rel(compManifest, compValidate, "Uses") 22 | Rel(compManifest, compRemote, "Uses") 23 | } 24 | 25 | Container(contProject, "Project", "python", "Main project that has a manifest.") 26 | Container(contVcs, "Vcs", "python", "Abstraction of various Version Control Systems.") 27 | Container(contReporting, "Reporting", "python", "Output formatters for various reporting formats.") 28 | 29 | Rel(contCommands, compManifest, "Uses") 30 | Rel(contReporting, contProject, "Extends") 31 | Rel_R(contProject, compManifest, "Has") 32 | Rel(contReporting, compManifest, "Uses") 33 | Rel(contProject, contVcs, "Uses") 34 | } 35 | 36 | Rel(user, contCommands, "Uses") 37 | 38 | @enduml 39 | -------------------------------------------------------------------------------- /doc/static/uml/c3_dfetch_components_project.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml 4 | 5 | Person(user, "Developer") 6 | 7 | System_Boundary(DFetch, "Dfetch") { 8 | 9 | Container(contCommands, "Commands", "python", "Parsing, editing and finding of manifests.") 10 | 11 | Container(contManifest, "Manifest", "python", "Parsing, editing and finding of manifests.") 12 | Boundary(DfetchProject, "Project", "python", "Main project that has a manifest.") { 13 | 14 | Component(compAbstractCheckReporter, "AbstractCheckReporter", "python", "Abstract interface for generating a check report.") 15 | Component(compGit, "Git", "python", "A remote source project based on git.") 16 | Component(compMetadata, "Metadata", "python", "A file containing metadata about a project.") 17 | Component(compSvn, "Svn", "python", "A remote source project based on svn.") 18 | Component(compVcs, "Vcs", "python", "An abstract remote version control system.") 19 | 20 | Rel_U(compGit, compVcs, "Implements") 21 | Rel_U(compSvn, compVcs, "Implements") 22 | Rel(compVcs, compAbstractCheckReporter, "Uses") 23 | Rel_L(compVcs, compMetadata, "Uses") 24 | } 25 | 26 | 27 | Container(contVcs, "Vcs", "python", "Abstraction of various Version Control Systems.") 28 | Container(contReporting, "Reporting", "python", "Output formatters for various reporting formats.") 29 | 30 | Rel(contCommands, compVcs, "Uses") 31 | Rel(contCommands, compGit, "Uses") 32 | Rel(contCommands, compSvn, "Uses") 33 | Rel(contReporting, compAbstractCheckReporter, "Implements") 34 | Rel_R(contReporting, compMetadata, "Uses") 35 | Rel_R(compMetadata, contManifest, "Has") 36 | Rel(compVcs, contManifest, "Has") 37 | Rel(contReporting, contManifest, "Uses") 38 | Rel(compGit, contVcs, "Uses") 39 | Rel(compSvn, contVcs, "Uses") 40 | } 41 | 42 | Rel(user, contCommands, "Uses") 43 | 44 | @enduml 45 | -------------------------------------------------------------------------------- /doc/static/uml/check.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | start 3 | 4 | skinparam monochrome true 5 | skinparam defaultFontName Frutiger 6 | 7 | :Get version on disk; 8 | 9 | if (Tag given?) then (yes) 10 | :Get all tags; 11 | if (Tag exists?) then (no) 12 | stop 13 | endif 14 | :Parse tags; 15 | :Show versions: 16 | - wanted tag 17 | - on disk tag 18 | - available tag; 19 | else (no) 20 | 21 | if (Branch given?) then (no) 22 | if (Revision enough?) then (yes) 23 | if (Does revision exist?) then (no) 24 | stop 25 | else (yes) 26 | :Show versions: 27 | - wanted revision 28 | - on disk revision; 29 | stop 30 | endif 31 | else (no) 32 | :Use default branch; 33 | endif 34 | else (yes) 35 | endif 36 | 37 | :Get latest revision of branch; 38 | :Show versions: 39 | - wanted revision / branch 40 | - on disk revision / branch 41 | - available revision / branch; 42 | endif 43 | 44 | stop 45 | @enduml 46 | -------------------------------------------------------------------------------- /doc/static/uml/styles/plantuml-c4/.dfetch_data.yaml: -------------------------------------------------------------------------------- 1 | # This is a generated file by dfetch. Don't edit this, but edit the manifest. 2 | # For more info see https://dfetch.rtfd.io/en/latest/getting_started.html 3 | dfetch: 4 | branch: master 5 | hash: 017d156a92c1ae227560a716aa093c6a 6 | last_fetch: 09/02/2025, 20:05:33 7 | patch: '' 8 | remote_url: https://github.com/plantuml-stdlib/C4-PlantUML.git 9 | revision: 9e7177f65e5425efd50c10e848ec89e49f6f45be 10 | tag: '' 11 | -------------------------------------------------------------------------------- /doc/static/uml/styles/plantuml-c4/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Ricardo Niepel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /doc/static/uml/update.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | start 3 | 4 | skinparam monochrome true 5 | skinparam defaultFontName Frutiger 6 | 7 | partition "Update required" { 8 | 9 | if (Project +\nMetadata on disk?) then (no) 10 | elseif (Force flag given?) then (yes) 11 | elseif (Version in metadata,\nsame as in manifest?) then (yes) 12 | stop 13 | elseif (Hash on disk same\nas in metadata?) then (yes) 14 | else (no) 15 | stop 16 | endif 17 | 18 | ' if (Hash on disk same\nas in metadata?) then (yes) 19 | ' elseif (Force flag given?) then (yes) 20 | ' else 21 | ' stop 22 | ' endif 23 | } 24 | 25 | partition "Prepare for update" { 26 | :Clear target folder; 27 | } 28 | 29 | partition "Perform Update" { 30 | 31 | if (Tag given?) then (yes) 32 | :Use tag; 33 | elseif (Revision only given\nand enough?) then (yes) 34 | :Use exact revision; 35 | elseif (Branch given?) then (yes) 36 | :Use branch 37 | and optionally 38 | revision; 39 | else 40 | :Use default branch 41 | and optionally 42 | revision; 43 | endif 44 | 45 | :Fetch target: 46 | revision/tag/branch; 47 | 48 | if (Patch given?) then (yes) 49 | :Apply patch; 50 | endif 51 | } 52 | 53 | if (Successful?) then (no) 54 | stop 55 | else (yes) 56 | endif 57 | 58 | partition "Update administration" { 59 | :Hash directory; 60 | :Update Metadata; 61 | } 62 | 63 | stop 64 | @enduml 65 | -------------------------------------------------------------------------------- /doc/troubleshooting.rst: -------------------------------------------------------------------------------- 1 | .. Dfetch documentation master file 2 | 3 | Troubleshooting 4 | =============== 5 | 6 | Although we do our best, *Dfetch* can always do something unexpected. 7 | A great deal of *Dfetch* functionality is dependent on plain-old command line commands. 8 | 9 | First of all, it is important to see what tools the system has. 10 | This can be seen with :ref:`dfetch environment`. 11 | 12 | Each command *Dfetch* performs and its result can be shown with increasing the verbosity 13 | with the `-v` flag. For example, if an :ref:`dfetch import` is giving strange results, re-run it with:: 14 | 15 | dfetch -v import 16 | 17 | Reporting issues 18 | ---------------- 19 | We are glad to help, if you you are stuck, either create an issue_ on github or contact us through gitter_! 20 | 21 | .. _issue: https://github.com/dfetch-org/dfetch/issues 22 | .. _gitter: https://gitter.im/dfetch-org/community 23 | -------------------------------------------------------------------------------- /example/dfetch.yaml: -------------------------------------------------------------------------------- 1 | manifest: 2 | version: 0.0 # DFetch Module syntax 3 | 4 | remotes: # declare common sources in one place 5 | - name: github 6 | url-base: https://github.com/ # Allow git modules 7 | default: true # Set it as default 8 | 9 | - name: sourceforge 10 | url-base: svn://svn.code.sf.net/p/ 11 | 12 | projects: 13 | 14 | - name: cpputest-git-tag 15 | dst: Tests/cpputest-git-tag 16 | url: https://github.com/cpputest/cpputest.git # Use external git directly 17 | tag: v3.4 # revision can also be a tag 18 | 19 | - name: tortoise-svn-branch-rev 20 | dst: Tests/tortoise-svn-branch-rev/ 21 | remote: sourceforge 22 | branch: 1.10.x/src 23 | revision: '28553' 24 | vcs: svn 25 | repo-path: tortoisesvn/code 26 | 27 | - name: tortoise-svn-tag 28 | dst: Tests/tortoise-svn-tag/ 29 | remote: sourceforge 30 | tag: version-1.13.1 31 | src: src 32 | vcs: svn 33 | repo-path: tortoisesvn/code 34 | 35 | - name: cpputest-git-src 36 | dst: Tests/cpputest-git-src 37 | repo-path: cpputest/cpputest.git # Use external git directly 38 | src: src 39 | 40 | - name: cpputest-git-rev-only 41 | dst: Tests/cpputest-git-rev-only 42 | revision: d14505cc9191fcf17ccbd92af1c3409eb3969890 43 | repo-path: cpputest/cpputest.git # Use external git directly 44 | -------------------------------------------------------------------------------- /features/check-specific-projects.feature: -------------------------------------------------------------------------------- 1 | Feature: Checking specific projects 2 | 3 | *DFetch* can check specific projects, this is useful when you have a lot 4 | of projects in your manifest. 5 | 6 | Scenario: Single project is checked 7 | Given the manifest 'dfetch.yaml' 8 | """ 9 | manifest: 10 | version: '0.0' 11 | 12 | remotes: 13 | - name: github-com-dfetch-org 14 | url-base: https://github.com/dfetch-org/test-repo 15 | 16 | projects: 17 | - name: ext/test-repo-rev-only 18 | revision: e1fda19a57b873eb8e6ae37780594cbb77b70f1a 19 | dst: ext/test-repo-rev-only 20 | 21 | - name: ext/test-rev-and-branch 22 | revision: 8df389d0524863b85f484f15a91c5f2c40aefda1 23 | branch: main 24 | dst: ext/test-rev-and-branch 25 | 26 | """ 27 | When I run "dfetch check ext/test-rev-and-branch" 28 | Then the output shows 29 | """ 30 | Dfetch (0.10.0) 31 | ext/test-rev-and-branch: wanted (main - 8df389d0524863b85f484f15a91c5f2c40aefda1), available (main - e1fda19a57b873eb8e6ae37780594cbb77b70f1a) 32 | """ 33 | -------------------------------------------------------------------------------- /features/diff-in-git.feature: -------------------------------------------------------------------------------- 1 | Feature: Diff in git 2 | 3 | If a project contains issues that need to be fixed, the user can work with the *Dfetch'ed* project as 4 | any other piece of code within the project. To upstream the changes back to the original project, *Dfetch* 5 | should allow to generate a patch file. 6 | 7 | Background: 8 | Given a git repository "SomeProject.git" 9 | And a fetched and committed MyProject with the manifest 10 | """ 11 | manifest: 12 | version: 0.0 13 | projects: 14 | - name: SomeProject 15 | url: some-remote-server/SomeProject.git 16 | """ 17 | 18 | Scenario: A patch file is generated 19 | Given "SomeProject/README.md" in MyProject is changed and committed with 20 | """ 21 | An important sentence for the README! 22 | """ 23 | When I run "dfetch diff SomeProject" 24 | Then the patch file 'MyProject/SomeProject.patch' is generated 25 | """ 26 | diff --git a/README.md b/README.md 27 | index 1e65bd6..faa3b21 100644 28 | --- a/README.md 29 | +++ b/README.md 30 | @@ -1 +1,2 @@ 31 | Generated file for SomeProject.git 32 | +An important sentence for the README! 33 | """ 34 | 35 | Scenario: No change is present 36 | When I run "dfetch diff SomeProject" 37 | Then the output shows 38 | """ 39 | Dfetch (0.10.0) 40 | SomeProject : No diffs found since 59efb91396fd369eb113b43382783294dc8ed6d2 41 | """ 42 | 43 | Scenario: Diff is generated on uncommitted changes 44 | Given "SomeProject/README.md" in MyProject is changed with 45 | """ 46 | An important sentence for the README! 47 | """ 48 | When I run "dfetch diff SomeProject" 49 | Then the patch file 'MyProject/SomeProject.patch' is generated 50 | """ 51 | diff --git a/README.md b/README.md 52 | index 1e65bd6..faa3b21 100644 53 | --- a/README.md 54 | +++ b/README.md 55 | @@ -1 +1,2 @@ 56 | Generated file for SomeProject.git 57 | +An important sentence for the README! 58 | """ 59 | -------------------------------------------------------------------------------- /features/diff-in-svn.feature: -------------------------------------------------------------------------------- 1 | Feature: Diff in svn 2 | 3 | If a project contains issues that need to be fixed, the user can work with the *Dfetch'ed* project as 4 | any other piece of code within the project. To upstream the changes back to the original project, *Dfetch* 5 | should allow to generate a patch file. 6 | 7 | Background: 8 | Given a svn-server "SomeProject" 9 | And a fetched and committed MySvnProject with the manifest 10 | """ 11 | manifest: 12 | version: 0.0 13 | projects: 14 | - name: SomeProject 15 | url: some-remote-server/SomeProject 16 | vcs: svn 17 | """ 18 | 19 | Scenario: A patch file is generated 20 | Given "SomeProject/README.md" in MySvnProject is changed, added and committed with 21 | """ 22 | An important sentence for the README! 23 | """ 24 | When I run "dfetch diff SomeProject" in MySvnProject 25 | Then the patch file 'MySvnProject/SomeProject.patch' is generated 26 | """ 27 | Index: SomeProject/README.md 28 | =================================================================== 29 | --- SomeProject/README.md (revision 1) 30 | +++ SomeProject/README.md (working copy) 31 | @@ -1 +1,2 @@ 32 | some content 33 | +An important sentence for the README! 34 | """ 35 | 36 | Scenario: No change is present 37 | When I run "dfetch diff SomeProject" in MySvnProject 38 | Then the output shows 39 | """ 40 | Dfetch (0.10.0) 41 | SomeProject : No diffs found since 1 42 | """ 43 | 44 | Scenario: A patch file is generated on uncommitted changes 45 | Given "SomeProject/README.md" in MySvnProject is changed with 46 | """ 47 | An important sentence for the README! 48 | """ 49 | When I run "dfetch diff SomeProject" in MySvnProject 50 | Then the patch file 'MySvnProject/SomeProject.patch' is generated 51 | """ 52 | Index: SomeProject/README.md 53 | =================================================================== 54 | --- SomeProject/README.md (revision 1) 55 | +++ SomeProject/README.md (working copy) 56 | @@ -1 +1,2 @@ 57 | some content 58 | +An important sentence for the README! 59 | """ 60 | -------------------------------------------------------------------------------- /features/environment.py: -------------------------------------------------------------------------------- 1 | """General hooks for behave tests.""" 2 | 3 | import os 4 | import tempfile 5 | 6 | from behave import fixture, use_fixture 7 | 8 | from dfetch.util.util import safe_rmtree 9 | 10 | 11 | @fixture 12 | def tmpdir(context): 13 | """Create tempdir during test""" 14 | # -- HINT: @behave.fixture is similar to @contextlib.contextmanager 15 | context.orig_cwd = os.getcwd() 16 | context.tmpdir = tempfile.mkdtemp() 17 | os.chdir(context.tmpdir) 18 | context.remotes_dir_path = os.path.abspath( 19 | os.path.join(os.getcwd(), "some-remote-server") 20 | ) 21 | yield context.tmpdir 22 | # -- CLEANUP-FIXTURE PART: 23 | os.chdir(context.orig_cwd) 24 | safe_rmtree(context.tmpdir) 25 | 26 | 27 | def before_scenario(context, _): 28 | """Hook called before scenario is executed.""" 29 | use_fixture(tmpdir, context) 30 | 31 | 32 | def before_all(context): 33 | """Hook called before first test is run.""" 34 | context.config.log_capture = True 35 | context.config.logging_format = "%(message)s" 36 | 37 | context.remotes_dir = "some-remote-server" 38 | -------------------------------------------------------------------------------- /features/fetch-checks-destination.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetch checks destinations 2 | 3 | Destinations marked with the 'dst:' attribute can be misused and lead to 4 | issues in the existing project. For instance path traversal or overwriting 5 | of other fetched projects. *DFetch* should do some sanity checks on the destinations. 6 | 7 | Scenario: No fetch is done directly into manifest folder 8 | Given the manifest 'dfetch.yaml' 9 | """ 10 | manifest: 11 | version: '0.0' 12 | 13 | projects: 14 | - name: ext/test-repo-tag 15 | url: https://github.com/dfetch-org/test-repo 16 | tag: v1 17 | dst: . 18 | 19 | """ 20 | When I run "dfetch update" 21 | Then the output shows 22 | """ 23 | Dfetch (0.10.0) 24 | ext/test-repo-tag : Skipping, path "." is not allowed as destination. 25 | Destination must be in a valid subfolder. "." is not valid! 26 | """ 27 | 28 | Scenario: Path traversal is not allowed 29 | Given the manifest 'dfetch.yaml' 30 | """ 31 | manifest: 32 | version: '0.0' 33 | 34 | projects: 35 | - name: ext/test-repo-tag 36 | url: https://github.com/dfetch-org/test-repo 37 | tag: v1 38 | dst: ../../some-higher-folder 39 | 40 | """ 41 | When I run "dfetch update" 42 | Then the output shows 43 | """ 44 | Dfetch (0.10.0) 45 | ext/test-repo-tag : Skipping, path "../../some-higher-folder" is outside manifest directory tree. 46 | Destination must be in the manifests folder or a subfolder. "../../some-higher-folder" is outside this tree! 47 | """ 48 | -------------------------------------------------------------------------------- /features/fetch-file-pattern-git.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetch file pattern from git repo 2 | 3 | Sometimes all files matching a pattern can be useful. 4 | *DFetch* makes it possible to specify a file pattern from a repository. 5 | 6 | Scenario: A file pattern is fetched from a repo 7 | Given the manifest 'dfetch.yaml' in MyProject 8 | """ 9 | manifest: 10 | version: 0.0 11 | projects: 12 | - name: SomeProjectWithAnInterestingFile 13 | url: some-remote-server/SomeProjectWithAnInterestingFile.git 14 | src: SomeFolder/SomeSubFolder/*.txt 15 | tag: v1 16 | """ 17 | And a git-repository "SomeProjectWithAnInterestingFile.git" with the files 18 | | path | 19 | | SomeFolder/SomeSubFolder/SomeFile.txt | 20 | | SomeFolder/SomeSubFolder/OtherFile.txt | 21 | | SomeFolder/SomeSubFolder/SomeFile.md | 22 | When I run "dfetch update" 23 | Then the output shows 24 | """ 25 | Dfetch (0.10.0) 26 | SomeProjectWithAnInterestingFile: Fetched v1 27 | """ 28 | Then 'MyProject' looks like: 29 | """ 30 | MyProject/ 31 | SomeProjectWithAnInterestingFile/ 32 | .dfetch_data.yaml 33 | OtherFile.txt 34 | SomeFile.txt 35 | dfetch.yaml 36 | """ 37 | -------------------------------------------------------------------------------- /features/fetch-file-pattern-svn.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetch file pattern from svn repo 2 | 3 | Sometimes all files matching a pattern can be useful. 4 | *DFetch* makes it possible to specify a file pattern from a repository. 5 | 6 | Scenario: A file pattern is fetched from a repo 7 | Given the manifest 'dfetch.yaml' in MyProject 8 | """ 9 | manifest: 10 | version: 0.0 11 | projects: 12 | - name: SomeProjectWithAnInterestingFile 13 | url: some-remote-server/SomeProjectWithAnInterestingFile 14 | src: SomeFolder/SomeSubFolder/*.txt 15 | """ 16 | And a svn-server "SomeProjectWithAnInterestingFile" with the files 17 | | path | 18 | | SomeFolder/SomeSubFolder/SomeFile.txt | 19 | | SomeFolder/SomeSubFolder/OtherFile.txt | 20 | | SomeFolder/SomeSubFolder/SomeFile.md | 21 | When I run "dfetch update" 22 | Then the output shows 23 | """ 24 | Dfetch (0.10.0) 25 | SomeProjectWithAnInterestingFile: Fetched trunk - 1 26 | """ 27 | Then 'MyProject' looks like: 28 | """ 29 | MyProject/ 30 | SomeProjectWithAnInterestingFile/ 31 | .dfetch_data.yaml 32 | OtherFile.txt 33 | SomeFile.txt 34 | dfetch.yaml 35 | """ 36 | -------------------------------------------------------------------------------- /features/fetch-git-repo.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetching dependencies from a git repository 2 | 3 | The main functionality of *DFetch* is fetching remote dependencies. 4 | A key VCS that is used in the world is git. *DFetch* makes it possible to 5 | fetch git repositories, using the revision, branch, tag or a combination. 6 | 7 | Scenario: Git projects are specified in the manifest 8 | Given the manifest 'dfetch.yaml' 9 | """ 10 | manifest: 11 | version: '0.0' 12 | 13 | remotes: 14 | - name: github-com-dfetch-org 15 | url-base: https://github.com/dfetch-org/test-repo 16 | 17 | projects: 18 | - name: ext/test-repo-rev-only 19 | revision: e1fda19a57b873eb8e6ae37780594cbb77b70f1a 20 | dst: ext/test-repo-rev-only 21 | 22 | - name: ext/test-rev-and-branch 23 | revision: 8df389d0524863b85f484f15a91c5f2c40aefda1 24 | branch: main 25 | dst: ext/test-rev-and-branch 26 | 27 | - name: ext/test-repo-tag-v1 28 | tag: v1 29 | dst: ext/test-repo-tag-v1 30 | 31 | """ 32 | When I run "dfetch update" 33 | Then the following projects are fetched 34 | | path | 35 | | ext/test-repo-rev-only | 36 | | ext/test-rev-and-branch | 37 | | ext/test-repo-tag-v1 | 38 | 39 | Scenario: Tag is updated in manifest 40 | Given the manifest 'dfetch.yaml' 41 | """ 42 | manifest: 43 | version: '0.0' 44 | 45 | projects: 46 | - name: ext/test-repo-tag 47 | url: https://github.com/dfetch-org/test-repo 48 | tag: v1 49 | 50 | """ 51 | And all projects are updated 52 | When the manifest 'dfetch.yaml' is changed to 53 | """ 54 | manifest: 55 | version: '0.0' 56 | 57 | projects: 58 | - name: ext/test-repo-tag 59 | url: https://github.com/dfetch-org/test-repo 60 | tag: v2.0 61 | 62 | """ 63 | And I run "dfetch update" 64 | Then the output shows 65 | """ 66 | Dfetch (0.10.0) 67 | ext/test-repo-tag : Fetched v2.0 68 | """ 69 | 70 | Scenario: Version check ignored when force flag is given 71 | Given the manifest 'dfetch.yaml' 72 | """ 73 | manifest: 74 | version: '0.0' 75 | 76 | projects: 77 | - name: ext/test-repo-tag 78 | url: https://github.com/dfetch-org/test-repo 79 | tag: v1 80 | 81 | """ 82 | And all projects are updated 83 | When I run "dfetch update --force" 84 | Then the output shows 85 | """ 86 | Dfetch (0.10.0) 87 | ext/test-repo-tag : Fetched v1 88 | """ 89 | -------------------------------------------------------------------------------- /features/fetch-single-file-git.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetch single file from git repo 2 | 3 | Sometimes only one file is enough. *DFetch* makes it possible to specify 4 | only one file from a repository. 5 | 6 | Scenario: A single file is fetched from a repo 7 | Given the manifest 'dfetch.yaml' in MyProject 8 | """ 9 | manifest: 10 | version: 0.0 11 | projects: 12 | - name: SomeProjectWithAnInterestingFile 13 | url: some-remote-server/SomeProjectWithAnInterestingFile.git 14 | src: SomeFolder/SomeSubFolder 15 | tag: v1 16 | """ 17 | And a git-repository "SomeProjectWithAnInterestingFile.git" with the files 18 | | path | 19 | | SomeFolder/SomeSubFolder/SomeFile.txt | 20 | | SomeOtherFolder/SomeOtherFile.txt | 21 | When I run "dfetch update" 22 | Then the output shows 23 | """ 24 | Dfetch (0.10.0) 25 | SomeProjectWithAnInterestingFile: Fetched v1 26 | """ 27 | Then 'MyProject' looks like: 28 | """ 29 | MyProject/ 30 | SomeProjectWithAnInterestingFile/ 31 | .dfetch_data.yaml 32 | SomeFile.txt 33 | dfetch.yaml 34 | """ 35 | 36 | Scenario: A single file is fetched from a repo (dst) 37 | Given the manifest 'dfetch.yaml' in MyProject 38 | """ 39 | manifest: 40 | version: 0.0 41 | projects: 42 | - name: SomeProjectWithAnInterestingFile 43 | url: some-remote-server/SomeProjectWithAnInterestingFile.git 44 | dst: ext 45 | src: SomeFolder/SomeFile.txt 46 | tag: v1 47 | """ 48 | And a git-repository "SomeProjectWithAnInterestingFile.git" with the files 49 | | path | 50 | | SomeFolder/SomeFile.txt | 51 | | SomeOtherFolder/SomeOtherFile.txt | 52 | When I run "dfetch update" 53 | Then the output shows 54 | """ 55 | Dfetch (0.10.0) 56 | SomeProjectWithAnInterestingFile: Fetched v1 57 | """ 58 | Then 'MyProject' looks like: 59 | """ 60 | MyProject/ 61 | dfetch.yaml 62 | ext/ 63 | .dfetch_data.yaml 64 | SomeFile.txt 65 | """ 66 | -------------------------------------------------------------------------------- /features/fetch-single-file-svn.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetch single file from svn repo 2 | 3 | Sometimes only one file is enough. *DFetch* makes it possible to specify 4 | only one file from a repository. 5 | 6 | Scenario: A single file is fetched from svn repo 7 | Given the manifest 'dfetch.yaml' in MyProject 8 | """ 9 | manifest: 10 | version: 0.0 11 | projects: 12 | - name: SomeProjectWithAnInterestingFile 13 | dst: MyOwnFileName.txt 14 | url: some-remote-server/SomeProjectWithAnInterestingFile 15 | src: SomeFolder/SomeFile.txt 16 | """ 17 | And a svn-server "SomeProjectWithAnInterestingFile" with the files 18 | | path | 19 | | SomeFolder/SomeFile.txt | 20 | | SomeOtherFolder/SomeOtherFile.txt | 21 | When I run "dfetch update" 22 | Then the output shows 23 | """ 24 | Dfetch (0.10.0) 25 | SomeProjectWithAnInterestingFile: Fetched trunk - 1 26 | """ 27 | And 'MyProject' looks like: 28 | """ 29 | MyProject/ 30 | .dfetch_data-MyOwnFileName.txt.yaml 31 | MyOwnFileName.txt 32 | dfetch.yaml 33 | """ 34 | -------------------------------------------------------------------------------- /features/fetch-specific-project.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetching specific dependencies 2 | 3 | *DFetch* can update specific projects, this is useful when you have a lot 4 | of projects in your manifest. 5 | 6 | Scenario: Two Specific projects are fetched 7 | Given the manifest 'dfetch.yaml' 8 | """ 9 | manifest: 10 | version: '0.0' 11 | 12 | remotes: 13 | - name: github-com-dfetch-org 14 | url-base: https://github.com/dfetch-org/test-repo 15 | 16 | projects: 17 | - name: ext/test-repo-rev-only 18 | revision: e1fda19a57b873eb8e6ae37780594cbb77b70f1a 19 | dst: ext/test-repo-rev-only 20 | 21 | - name: ext/test-rev-and-branch 22 | revision: 8df389d0524863b85f484f15a91c5f2c40aefda1 23 | branch: main 24 | dst: ext/test-rev-and-branch 25 | 26 | - name: ext/test-repo-tag-v1 27 | tag: v1 28 | dst: ext/test-repo-tag-v1 29 | 30 | """ 31 | When I run "dfetch update ext/test-repo-tag-v1 ext/test-rev-and-branch" 32 | Then the following projects are fetched 33 | | path | 34 | | ext/test-rev-and-branch | 35 | | ext/test-repo-tag-v1 | 36 | -------------------------------------------------------------------------------- /features/fetch-svn-repo.feature: -------------------------------------------------------------------------------- 1 | @remote-svn 2 | Feature: Fetching dependencies from a svn repository 3 | 4 | The main functionality of *DFetch* is fetching remote dependencies. 5 | A key VCS that is used in the world is svn. *DFetch* makes it possible to 6 | fetch svn repositories, using the revision, branch, tag or a combination. 7 | Typically SVN repositories are set-up with a standard layout. 8 | This means there is a 'trunk', 'branches/' and 'tags/'. 9 | Sometimes people don't do this, a user should be able to check 10 | a non-standard SVN repository as well. 11 | 12 | Scenario: SVN projects are specified in the manifest 13 | Given the manifest 'dfetch.yaml' 14 | """ 15 | manifest: 16 | version: '0.0' 17 | 18 | remotes: 19 | - name: cunit 20 | url-base: svn://svn.code.sf.net/p/cunit/code 21 | default: true 22 | 23 | - name: cutter 24 | url-base: svn://svn.code.sf.net/p/cutter/svn/cutter 25 | 26 | projects: 27 | - name: cunit-svn-rev-only 28 | revision: '170' 29 | vcs: svn 30 | dst: ext/cunit-svn-rev-only 31 | 32 | - name: cunit-svn-rev-and-branch 33 | revision: '156' 34 | vcs: svn 35 | branch: mingw64 36 | dst: ext/cunit-svn-rev-and-branch 37 | 38 | - name: cutter-svn-tag 39 | vcs: svn 40 | tag: 1.1.7 41 | src: acmacros 42 | dst: ext/cutter-svn-tag 43 | remote: cutter 44 | 45 | """ 46 | When I run "dfetch update" 47 | Then the following projects are fetched 48 | | path | 49 | | ext/cunit-svn-rev-only | 50 | | ext/cunit-svn-rev-and-branch | 51 | | ext/cutter-svn-tag | 52 | 53 | Scenario: Directory in a non-standard SVN repository can be fetched 54 | Given the manifest 'dfetch.yaml' in MyProject 55 | """ 56 | manifest: 57 | version: 0.0 58 | projects: 59 | - name: SomeProject 60 | url: some-remote-server/SomeProject 61 | src: SomeFolder/ 62 | branch: ' ' 63 | """ 64 | And a non-standard svn-server "SomeProject" with the files 65 | | path | 66 | | SomeFolder/SomeFile.txt | 67 | | SomeOtherFolder/SomeOtherFile.txt | 68 | When I run "dfetch update" 69 | Then 'MyProject' looks like: 70 | """ 71 | MyProject/ 72 | SomeProject/ 73 | .dfetch_data.yaml 74 | SomeFile.txt 75 | dfetch.yaml 76 | """ 77 | -------------------------------------------------------------------------------- /features/fetch-with-ignore-git.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetch with ignore in git 2 | 3 | Sometimes you want to ignore files from a project 4 | These can be specified using the `ignore:` tag 5 | 6 | Background: 7 | Given a git-repository "SomeInterestingProject.git" with the files 8 | | path | 9 | | SomeFolder/SomeSubFolder/SomeFile.txt | 10 | | SomeFolder/SomeSubFolder/OtherFile.txt | 11 | | SomeFolder/SomeSubFolder/SomeFile.md | 12 | | SomeFolder/SomeOtherSubFolder/SomeFile.txt | 13 | | SomeFolder/SomeOtherSubFolder/OtherFile.txt | 14 | 15 | Scenario: A file pattern is fetched from a repo 16 | Given the manifest 'dfetch.yaml' in MyProject 17 | """ 18 | manifest: 19 | version: 0.0 20 | projects: 21 | - name: SomeInterestingProject 22 | url: some-remote-server/SomeInterestingProject.git 23 | src: SomeFolder/SomeSubFolder 24 | ignore: 25 | - OtherFile.txt 26 | tag: v1 27 | """ 28 | When I run "dfetch update" 29 | Then the output shows 30 | """ 31 | Dfetch (0.10.0) 32 | SomeInterestingProject: Fetched v1 33 | """ 34 | Then 'MyProject' looks like: 35 | """ 36 | MyProject/ 37 | SomeInterestingProject/ 38 | .dfetch_data.yaml 39 | SomeFile.md 40 | SomeFile.txt 41 | dfetch.yaml 42 | """ 43 | 44 | Scenario: Combination of directories and a single file can be ignored 45 | Given the manifest 'dfetch.yaml' in MyProject 46 | """ 47 | manifest: 48 | version: 0.0 49 | projects: 50 | - name: SomeInterestingProject 51 | url: some-remote-server/SomeInterestingProject.git 52 | ignore: 53 | - SomeFolder/SomeOtherSubFolder 54 | - SomeFolder/SomeSubFolder/SomeFile.md 55 | tag: v1 56 | """ 57 | When I run "dfetch update" 58 | Then the output shows 59 | """ 60 | Dfetch (0.10.0) 61 | SomeInterestingProject: Fetched v1 62 | """ 63 | Then 'MyProject' looks like: 64 | """ 65 | MyProject/ 66 | SomeInterestingProject/ 67 | .dfetch_data.yaml 68 | SomeFolder/ 69 | SomeSubFolder/ 70 | OtherFile.txt 71 | SomeFile.txt 72 | dfetch.yaml 73 | """ 74 | 75 | Scenario: Ignore overrides the file pattern match in src attribute 76 | Given the manifest 'dfetch.yaml' in MyProject 77 | """ 78 | manifest: 79 | version: 0.0 80 | projects: 81 | - name: SomeInterestingProject 82 | url: some-remote-server/SomeInterestingProject.git 83 | src: SomeFolder/SomeSubFolder/*.txt 84 | ignore: 85 | - /SomeNonExistingPath 86 | - SomeFile.* 87 | tag: v1 88 | """ 89 | When I run "dfetch update" 90 | Then the output shows 91 | """ 92 | Dfetch (0.10.0) 93 | SomeInterestingProject: Fetched v1 94 | """ 95 | Then 'MyProject' looks like: 96 | """ 97 | MyProject/ 98 | SomeInterestingProject/ 99 | .dfetch_data.yaml 100 | OtherFile.txt 101 | dfetch.yaml 102 | """ 103 | -------------------------------------------------------------------------------- /features/fetch-with-ignore-svn.feature: -------------------------------------------------------------------------------- 1 | Feature: Fetch with ignore in svn 2 | 3 | Sometimes you want to ignore files from a project 4 | These can be specified using the `ignore:` tag 5 | 6 | Background: 7 | Given a svn-server "SomeInterestingProject" with the files 8 | | path | 9 | | SomeFolder/SomeSubFolder/SomeFile.txt | 10 | | SomeFolder/SomeSubFolder/OtherFile.txt | 11 | | SomeFolder/SomeSubFolder/SomeFile.md | 12 | | SomeFolder/SomeOtherSubFolder/SomeFile.txt | 13 | | SomeFolder/SomeOtherSubFolder/OtherFile.txt | 14 | 15 | Scenario: A file pattern is fetched from a repo 16 | Given the manifest 'dfetch.yaml' in MyProject 17 | """ 18 | manifest: 19 | version: 0.0 20 | projects: 21 | - name: SomeInterestingProject 22 | url: some-remote-server/SomeInterestingProject 23 | src: SomeFolder/SomeSubFolder 24 | ignore: 25 | - OtherFile.txt 26 | """ 27 | When I run "dfetch update" 28 | Then the output shows 29 | """ 30 | Dfetch (0.10.0) 31 | SomeInterestingProject: Fetched trunk - 1 32 | """ 33 | Then 'MyProject' looks like: 34 | """ 35 | MyProject/ 36 | SomeInterestingProject/ 37 | .dfetch_data.yaml 38 | SomeFile.md 39 | SomeFile.txt 40 | dfetch.yaml 41 | """ 42 | 43 | Scenario: Combination of directories and a single file can be ignored 44 | Given the manifest 'dfetch.yaml' in MyProject 45 | """ 46 | manifest: 47 | version: 0.0 48 | projects: 49 | - name: SomeInterestingProject 50 | url: some-remote-server/SomeInterestingProject 51 | ignore: 52 | - SomeFolder/SomeOtherSubFolder 53 | - SomeFolder/SomeSubFolder/SomeFile.md 54 | """ 55 | When I run "dfetch update" 56 | Then the output shows 57 | """ 58 | Dfetch (0.10.0) 59 | SomeInterestingProject: Fetched trunk - 1 60 | """ 61 | Then 'MyProject' looks like: 62 | """ 63 | MyProject/ 64 | SomeInterestingProject/ 65 | .dfetch_data.yaml 66 | SomeFolder/ 67 | SomeSubFolder/ 68 | OtherFile.txt 69 | SomeFile.txt 70 | dfetch.yaml 71 | """ 72 | 73 | Scenario: Ignore overrides the file pattern match in src attribute 74 | Given the manifest 'dfetch.yaml' in MyProject 75 | """ 76 | manifest: 77 | version: 0.0 78 | projects: 79 | - name: SomeInterestingProject 80 | url: some-remote-server/SomeInterestingProject 81 | src: SomeFolder/SomeSubFolder/*.txt 82 | ignore: 83 | - /SomeNonExistingPath 84 | - SomeFile.* 85 | """ 86 | When I run "dfetch update" 87 | Then the output shows 88 | """ 89 | Dfetch (0.10.0) 90 | SomeInterestingProject: Fetched trunk - 1 91 | """ 92 | Then 'MyProject' looks like: 93 | """ 94 | MyProject/ 95 | SomeInterestingProject/ 96 | .dfetch_data.yaml 97 | OtherFile.txt 98 | dfetch.yaml 99 | """ 100 | -------------------------------------------------------------------------------- /features/freeze-projects.feature: -------------------------------------------------------------------------------- 1 | Feature: Freeze dependencies 2 | 3 | If a user didn't use a revision or branch in his manifest, he can add these automatically with `dfetch freeze`. 4 | During development it shouldn't be a problem to track a branch and always fetch the latest changes, 5 | but when a project becomes more mature, you typically want to freeze the dependencies. 6 | 7 | The same rules apply as for fetching, a tag has precedence over a branch and revision. 8 | 9 | Scenario: Git projects are frozen 10 | Given the manifest 'dfetch.yaml' 11 | """ 12 | manifest: 13 | version: '0.0' 14 | 15 | projects: 16 | - name: ext/test-repo-tag 17 | url: https://github.com/dfetch-org/test-repo 18 | branch: main 19 | 20 | """ 21 | And all projects are updated 22 | When I run "dfetch freeze" 23 | Then the manifest 'dfetch.yaml' is replaced with 24 | """ 25 | manifest: 26 | version: '0.0' 27 | 28 | projects: 29 | - name: ext/test-repo-tag 30 | revision: e1fda19a57b873eb8e6ae37780594cbb77b70f1a 31 | url: https://github.com/dfetch-org/test-repo 32 | branch: main 33 | 34 | """ 35 | 36 | @remote-svn 37 | Scenario: SVN projects are specified in the manifest 38 | Given the manifest 'dfetch.yaml' 39 | """ 40 | manifest: 41 | version: '0.0' 42 | 43 | projects: 44 | - name: cunit-svn 45 | vcs: svn 46 | url: svn://svn.code.sf.net/p/cunit/code 47 | 48 | """ 49 | And all projects are updated 50 | When I run "dfetch freeze" 51 | Then the manifest 'dfetch.yaml' is replaced with 52 | """ 53 | manifest: 54 | version: '0.0' 55 | 56 | projects: 57 | - name: cunit-svn 58 | revision: '170' 59 | url: svn://svn.code.sf.net/p/cunit/code 60 | branch: trunk 61 | vcs: svn 62 | 63 | """ 64 | -------------------------------------------------------------------------------- /features/guard-against-overwriting.feature: -------------------------------------------------------------------------------- 1 | Feature: Guard against overwriting 2 | 3 | Accidentally overwriting local changes could lead to introducing regressions. 4 | To make developers aware of overwriting, store hash after an update and compare 5 | this with hashes just before an update. 6 | 7 | Scenario: No update is done when hash is different 8 | Given MyProject with dependency "SomeProject.git" that must be updated 9 | When "SomeProject/README.md" in MyProject is changed locally 10 | And I run "dfetch update" 11 | Then the output shows 12 | """ 13 | Dfetch (0.10.0) 14 | SomeProject : skipped - local changes after last update (use --force to overwrite) 15 | """ 16 | 17 | Scenario: Force flag overrides local changes check 18 | Given MyProject with dependency "SomeProject.git" that must be updated 19 | When "SomeProject/README.md" in MyProject is changed locally 20 | And I run "dfetch update --force" 21 | Then the output shows 22 | """ 23 | Dfetch (0.10.0) 24 | SomeProject : Fetched v2 25 | """ 26 | -------------------------------------------------------------------------------- /features/handle-invalid-metadata.feature: -------------------------------------------------------------------------------- 1 | Feature: Handle invalid metadata files 2 | 3 | *Dfetch* will keep metadata about the fetched project locally to prevent re-fetching unchanged projects 4 | or replacing locally changed projects. Sometimes the metadata file will become incorrect and this should not lead 5 | to unpredictable behavior in *DFetch*. 6 | For instance, the metadata may be invalid 7 | 8 | Scenario: Invalid metadata is ignored 9 | Given the manifest 'dfetch.yaml' 10 | """ 11 | manifest: 12 | version: '0.0' 13 | 14 | projects: 15 | - name: ext/test-repo-tag 16 | url: https://github.com/dfetch-org/test-repo 17 | tag: v1 18 | 19 | """ 20 | And all projects are updated 21 | And the metadata file ".dfetch_data.yaml" of "ext/test-repo-tag" is corrupt 22 | When I run "dfetch update" 23 | Then the output shows 24 | """ 25 | Dfetch (0.10.0) 26 | ext/test-repo-tag/.dfetch_data.yaml is an invalid metadata file, not checking on disk version! 27 | ext/test-repo-tag/.dfetch_data.yaml is an invalid metadata file, not checking local hash! 28 | ext/test-repo-tag : Fetched v1 29 | """ 30 | -------------------------------------------------------------------------------- /features/import-from-git.feature: -------------------------------------------------------------------------------- 1 | Feature: Importing submodules from an existing git repository 2 | 3 | One alternative to *Dfetch* is git submodules. To make the transition 4 | as easy as possible, a user should be able to generate a manifest that 5 | is filled with the submodules and their pinned versions. 6 | 7 | Scenario: Multiple submodules are imported 8 | Given a git repo with the following submodules 9 | | path | url | revision | 10 | | ext/test-repo1 | https://github.com/dfetch-org/test-repo | e1fda19a57b873eb8e6ae37780594cbb77b70f1a | 11 | | ext/test-repo2 | https://github.com/dfetch-org/test-repo | 8df389d0524863b85f484f15a91c5f2c40aefda1 | 12 | When I run "dfetch import" 13 | Then it should generate the manifest 'dfetch.yaml' 14 | """ 15 | manifest: 16 | version: '0.0' 17 | 18 | remotes: 19 | - name: github-com-dfetch-org 20 | url-base: https://github.com/dfetch-org 21 | 22 | projects: 23 | - name: ext/test-repo1 24 | revision: e1fda19a57b873eb8e6ae37780594cbb77b70f1a 25 | branch: main 26 | repo-path: test-repo 27 | 28 | - name: ext/test-repo2 29 | revision: 8df389d0524863b85f484f15a91c5f2c40aefda1 30 | tag: v1 31 | repo-path: test-repo 32 | 33 | """ 34 | -------------------------------------------------------------------------------- /features/import-from-svn.feature: -------------------------------------------------------------------------------- 1 | Feature: Importing externals from an existing svn repository 2 | 3 | One alternative to *Dfetch* is svn externals. To make the transition 4 | as easy as possible, a user should be able to generate a manifest that 5 | is filled with the externals and their pinned versions. 6 | 7 | Scenario: Multiple externals are imported 8 | Given a svn repo with the following externals 9 | | path | url | revision | 10 | | ext/test-repo1 | https://github.com/dfetch-org/test-repo/trunk | 1 | 11 | | ext/test-repo2 | https://github.com/dfetch-org/test-repo/tags/v2.0 | | 12 | | ext/test-repo3 | https://github.com/dfetch-org/test-repo | | 13 | When I run "dfetch import" 14 | Then it should generate the manifest 'dfetch.yaml' 15 | """ 16 | manifest: 17 | version: '0.0' 18 | 19 | remotes: 20 | - name: github-com-dfetch-org 21 | url-base: https://github.com/dfetch-org 22 | 23 | projects: 24 | - name: ext/test-repo1 25 | revision: '1' 26 | dst: ./ext/test-repo1 27 | repo-path: test-repo 28 | 29 | - name: ext/test-repo2 30 | dst: ./ext/test-repo2 31 | tag: v2.0 32 | repo-path: test-repo 33 | 34 | - name: ext/test-repo3 35 | dst: ./ext/test-repo3 36 | branch: ' ' 37 | repo-path: test-repo 38 | 39 | """ 40 | -------------------------------------------------------------------------------- /features/journey-basic-usage.feature: -------------------------------------------------------------------------------- 1 | Feature: Basic usage journey 2 | 3 | The main user journey is: 4 | - Creating a manifest. 5 | - Fetching all projects in manifest. 6 | - Checking for new updates 7 | - Updating manifest due to changes. 8 | - Fetching new projects. 9 | 10 | Below scenario is described in the getting started and should at least work. 11 | 12 | Scenario: Basic user journey 13 | 14 | Given the manifest 'dfetch.yaml' 15 | """ 16 | manifest: 17 | version: '0.0' 18 | 19 | projects: 20 | - name: ext/test-repo-tag 21 | tag: v1 22 | url: https://github.com/dfetch-org/test-repo 23 | """ 24 | When I run "dfetch update" 25 | Then the following projects are fetched 26 | | path | 27 | | ext/test-repo-tag | 28 | When I run "dfetch check" 29 | Then the output shows 30 | """ 31 | Dfetch (0.10.0) 32 | ext/test-repo-tag : wanted & current (v1), available (v2.0) 33 | """ 34 | When the manifest 'dfetch.yaml' is changed to 35 | """ 36 | manifest: 37 | version: '0.0' 38 | 39 | projects: 40 | - name: ext/test-repo-tag 41 | url: https://github.com/dfetch-org/test-repo 42 | tag: v2.0 43 | """ 44 | And I run "dfetch update" 45 | Then the following projects are fetched 46 | | path | 47 | | ext/test-repo-tag | 48 | -------------------------------------------------------------------------------- /features/list-projects.feature: -------------------------------------------------------------------------------- 1 | Feature: List dependencies 2 | 3 | The report command lists the current state of the projects. It will aggregate the metadata for each project 4 | and list it. This includes metadata from the manifest file and the metadata file (.dfetch_data.yaml) 5 | 6 | Scenario: Git projects are specified in the manifest 7 | Given the manifest 'dfetch.yaml' 8 | """ 9 | manifest: 10 | version: '0.0' 11 | 12 | remotes: 13 | - name: github-com-dfetch-org 14 | url-base: https://github.com/dfetch-org/test-repo 15 | 16 | projects: 17 | - name: ext/test-repo-tag 18 | url: https://github.com/dfetch-org/test-repo 19 | branch: main 20 | 21 | - name: ext/test-rev-and-branch 22 | tag: v1 23 | dst: ext/test-rev-and-branch 24 | 25 | """ 26 | And all projects are updated 27 | When I run "dfetch report" 28 | Then the output shows 29 | """ 30 | Dfetch (0.10.0) 31 | project : ext/test-repo-tag 32 | remote : 33 | remote url : https://github.com/dfetch-org/test-repo 34 | branch : main 35 | tag : 36 | last fetch : 02/07/2021, 20:25:56 37 | revision : e1fda19a57b873eb8e6ae37780594cbb77b70f1a 38 | patch : 39 | license : MIT License 40 | project : ext/test-rev-and-branch 41 | remote : github-com-dfetch-org 42 | remote url : https://github.com/dfetch-org/test-repo 43 | branch : main 44 | tag : v1 45 | last fetch : 02/07/2021, 20:25:56 46 | revision : 47 | patch : 48 | license : MIT License 49 | """ 50 | 51 | @remote-svn 52 | Scenario: SVN projects are specified in the manifest 53 | Given the manifest 'dfetch.yaml' 54 | """ 55 | manifest: 56 | version: '0.0' 57 | 58 | projects: 59 | - name: cutter-svn-tag 60 | url: svn://svn.code.sf.net/p/cutter/svn/cutter 61 | tag: 1.1.7 62 | vcs: svn 63 | src: acmacros 64 | 65 | """ 66 | And all projects are updated 67 | When I run "dfetch report" 68 | Then the output shows 69 | """ 70 | Dfetch (0.10.0) 71 | project : cutter-svn-tag 72 | remote : 73 | remote url : svn://svn.code.sf.net/p/cutter/svn/cutter 74 | branch : 75 | tag : 1.1.7 76 | last fetch : 29/12/2024, 20:09:21 77 | revision : 4007 78 | patch : 79 | license : 80 | """ 81 | 82 | Scenario: Git repo with applied patch 83 | Given MyProject with applied patch 'diff.patch' 84 | When I run "dfetch report" 85 | Then the output shows 86 | """ 87 | Dfetch (0.10.0) 88 | project : ext/test-repo-tag 89 | remote : github-com-dfetch-org 90 | remote url : https://github.com/dfetch-org/test-repo 91 | branch : main 92 | tag : v2.0 93 | last fetch : 02/07/2021, 20:25:56 94 | revision : 95 | patch : diff.patch 96 | license : MIT License 97 | """ 98 | -------------------------------------------------------------------------------- /features/patch-after-fetch-git.feature: -------------------------------------------------------------------------------- 1 | Feature: Patch after fetching from git repo 2 | 3 | Sometimes a patch needs to be applied after fetching. *DFetch* makes it 4 | possible to specify a patch file. 5 | 6 | Scenario: A patch file is applied after fetching 7 | Given the manifest 'dfetch.yaml' 8 | """ 9 | manifest: 10 | version: '0.0' 11 | 12 | remotes: 13 | - name: github-com-dfetch-org 14 | url-base: https://github.com/dfetch-org/test-repo 15 | 16 | projects: 17 | - name: ext/test-repo-tag 18 | tag: v2.0 19 | dst: ext/test-repo-tag 20 | patch: diff.patch 21 | """ 22 | And the patch file 'diff.patch' 23 | """ 24 | diff --git a/README.md b/README.md 25 | index 32d9fad..62248b7 100644 26 | --- a/README.md 27 | +++ b/README.md 28 | @@ -1,2 +1,2 @@ 29 | # Test-repo 30 | -A test repo for testing dfetch. 31 | +A test repo for testing patch. 32 | """ 33 | When I run "dfetch update" 34 | Then the patched 'ext/test-repo-tag/README.md' is 35 | """ 36 | # Test-repo 37 | A test repo for testing patch. 38 | """ 39 | 40 | Scenario: Applying patch file fails 41 | Given the manifest 'dfetch.yaml' 42 | """ 43 | manifest: 44 | version: '0.0' 45 | 46 | remotes: 47 | - name: github-com-dfetch-org 48 | url-base: https://github.com/dfetch-org/test-repo 49 | 50 | projects: 51 | - name: ext/test-repo-tag 52 | tag: v2.0 53 | dst: ext/test-repo-tag 54 | patch: diff.patch 55 | """ 56 | And the patch file 'diff.patch' 57 | """ 58 | diff --git a/README.md b/README1.md 59 | index 32d9fad..62248b7 100644 60 | --- a/README1.md 61 | +++ b/README1.md 62 | @@ -1,2 +1,2 @@ 63 | # Test-repo 64 | -A test repo for testing dfetch. 65 | +A test repo for testing patch. 66 | """ 67 | When I run "dfetch update" 68 | Then the output shows 69 | """ 70 | Dfetch (0.10.0) 71 | ext/test-repo-tag : Fetched v2.0 72 | source/target file does not exist: 73 | --- b'README1.md' 74 | +++ b'README1.md' 75 | Applying patch "diff.patch" failed 76 | """ 77 | -------------------------------------------------------------------------------- /features/patch-after-fetch-svn.feature: -------------------------------------------------------------------------------- 1 | @remote-svn 2 | Feature: Patch after fetching from svn repo 3 | 4 | Sometimes a patch needs to be applied after fetching. *DFetch* makes it 5 | possible to specify a patch file. 6 | 7 | Scenario: A patch file is applied after fetching 8 | Given the manifest 'dfetch.yaml' 9 | """ 10 | manifest: 11 | version: '0.0' 12 | 13 | remotes: 14 | - name: cutter 15 | url-base: svn://svn.code.sf.net/p/cutter/svn/cutter 16 | 17 | projects: 18 | - name: cutter 19 | vcs: svn 20 | tag: 1.1.7 21 | dst: ext/cutter 22 | patch: diff.patch 23 | src: apt 24 | """ 25 | And the patch file 'diff.patch' 26 | """ 27 | Index: build-deb.sh 28 | =================================================================== 29 | --- build-deb.sh (revision 4007) 30 | +++ build-deb.sh (working copy) 31 | @@ -1,1 +1,1 @@ 32 | -#!/bin/sh 33 | +#!/bin/bash 34 | """ 35 | When I run "dfetch update" 36 | Then the first line of 'ext/cutter/build-deb.sh' is changed to 37 | """ 38 | #!/bin/bash 39 | """ 40 | 41 | Scenario: Applying patch file fails 42 | Given the manifest 'dfetch.yaml' 43 | """ 44 | manifest: 45 | version: '0.0' 46 | 47 | remotes: 48 | - name: cutter 49 | url-base: svn://svn.code.sf.net/p/cutter/svn/cutter 50 | 51 | projects: 52 | - name: cutter 53 | vcs: svn 54 | tag: 1.1.7 55 | dst: ext/cutter 56 | patch: diff.patch 57 | src: apt 58 | """ 59 | And the patch file 'diff.patch' 60 | """ 61 | Index: build-deb.sh 62 | =================================================================== 63 | --- build-deb2.sh (revision 4007) 64 | +++ build-deb2.sh (working copy) 65 | @@ -1,1 +1,1 @@ 66 | -#!/bin/sh 67 | +#!/bin/bash 68 | """ 69 | When I run "dfetch update" 70 | Then the output shows 71 | """ 72 | Dfetch (0.10.0) 73 | cutter : Fetched 1.1.7 74 | source/target file does not exist: 75 | --- b'build-deb2.sh' 76 | +++ b'build-deb2.sh' 77 | Applying patch "diff.patch" failed 78 | """ 79 | -------------------------------------------------------------------------------- /features/steps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfetch-org/dfetch/3be35ac2335154b9ee22d3bd324b71f9b5a1b6ec/features/steps/__init__.py -------------------------------------------------------------------------------- /features/steps/manifest_steps.py: -------------------------------------------------------------------------------- 1 | """Steps for features tests.""" 2 | 3 | # pylint: disable=function-redefined, missing-function-docstring, import-error 4 | # pyright: reportRedeclaration=false, reportAttributeAccessIssue=false 5 | 6 | import os 7 | import pathlib 8 | from typing import Optional 9 | 10 | from behave import given, then, when # pylint: disable=no-name-in-module 11 | 12 | from features.steps.generic_steps import check_file, generate_file, remote_server_path 13 | 14 | 15 | def generate_manifest( 16 | context, name="dfetch.yaml", contents: Optional[str] = None, path=None 17 | ): 18 | contents = contents or context.text 19 | manifest = contents.replace( 20 | "url: some-remote-server", f"url: file:///{remote_server_path(context)}" 21 | ) 22 | generate_file(os.path.join(path or os.getcwd(), name), manifest) 23 | 24 | 25 | @given("the manifest '{name}' in {path}") 26 | @given("the manifest '{name}'") 27 | @when("the manifest '{name}' is changed to") 28 | @when("the manifest '{name}' in {path} is changed to") 29 | def step_impl(context, name, path=None): 30 | if path: 31 | pathlib.Path(path).mkdir(parents=True, exist_ok=True) 32 | 33 | generate_manifest(context, name, contents=context.text, path=path) 34 | 35 | 36 | @then("the manifest '{name}' is replaced with") 37 | @then("it should generate the manifest '{name}'") 38 | def step_impl(context, name): 39 | """Check a manifest.""" 40 | check_file(name, context.text) 41 | 42 | 43 | @given("the manifest '{name}' with the projects") 44 | def step_impl(context, name): 45 | projects = "\n".join(f" - name: {row['name']}" for row in context.table) 46 | manifest = f"""manifest: 47 | version: '0.0' 48 | remotes: 49 | - name: github-com-dfetch-org 50 | url-base: https://github.com/dfetch-org/test-repo 51 | 52 | projects: 53 | {projects} 54 | """ 55 | generate_file(os.path.join(os.getcwd(), name), manifest) 56 | -------------------------------------------------------------------------------- /features/steps/svn_steps.py: -------------------------------------------------------------------------------- 1 | """Steps for features tests.""" 2 | 3 | # pylint: disable=function-redefined, missing-function-docstring, import-error 4 | # pyright: reportRedeclaration=false, reportAttributeAccessIssue=false 5 | 6 | import os 7 | import pathlib 8 | import subprocess 9 | 10 | from behave import given # pylint: disable=no-name-in-module 11 | 12 | from dfetch.util.util import in_directory 13 | from features.steps.generic_steps import call_command, extend_file, generate_file 14 | from features.steps.manifest_steps import generate_manifest 15 | 16 | 17 | def create_svn_server_and_repo(context, name="svn-server"): 18 | """Create an local svn server and repo and return the path to the repo.""" 19 | 20 | server_path = os.path.relpath(context.remotes_dir_path) + "/" + name 21 | repo_path = name 22 | 23 | pathlib.Path(server_path).mkdir(parents=True, exist_ok=True) 24 | subprocess.call(["svnadmin", "create", "--fs-type", "fsfs", server_path]) 25 | 26 | current_path = "/".join(os.getcwd().split(os.path.sep) + [server_path]) 27 | subprocess.call(["svn", "checkout", f"file:///{current_path}", repo_path]) 28 | 29 | return repo_path 30 | 31 | 32 | def create_stdlayout(): 33 | pathlib.Path("trunk").mkdir(parents=True, exist_ok=True) 34 | pathlib.Path("branches").mkdir(parents=True, exist_ok=True) 35 | pathlib.Path("tags").mkdir(parents=True, exist_ok=True) 36 | 37 | 38 | def add_and_commit(msg): 39 | subprocess.call(["svn", "add", "--force", "."]) 40 | subprocess.call(["svn", "ci", "-m", f'"{msg}"']) 41 | 42 | 43 | def commit_all(msg): 44 | subprocess.call(["svn", "commit", "--depth", "empty", ".", "-m", f'"{msg}"']) 45 | 46 | 47 | def add_externals(externals): 48 | """Add the given list of dicts as externals.""" 49 | with open("externals", "w", encoding="UTF-8") as external_list: 50 | for external in externals: 51 | revision = f"@{external['revision']}" if external["revision"] else "" 52 | external_list.write(f"{external['url']}{revision} {external['path']}\n") 53 | 54 | subprocess.call(["svn", "propset", "svn:externals", "-F", external_list.name, "."]) 55 | commit_all("Added externals") 56 | subprocess.call(["svn", "update"]) 57 | 58 | 59 | @given("a svn repo with the following externals") 60 | def step_impl(context): 61 | repo_path = create_svn_server_and_repo(context) 62 | os.chdir(repo_path) 63 | add_externals(context.table) 64 | 65 | 66 | @given('a svn-server "{name}"') 67 | @given('a svn-server "{name}" with the files') 68 | def step_impl(context, name): 69 | repo_path = create_svn_server_and_repo(context, name) 70 | 71 | files = context.table or [{"path": "README.md"}] 72 | 73 | with in_directory(repo_path): 74 | create_stdlayout() 75 | with in_directory("trunk"): 76 | for file in files: 77 | generate_file(file["path"], "some content") 78 | add_and_commit("Added files") 79 | 80 | 81 | @given('a non-standard svn-server "{name}" with the files') 82 | def step_impl(context, name): 83 | repo_path = create_svn_server_and_repo(context, name) 84 | 85 | with in_directory(repo_path): 86 | for file in context.table: 87 | generate_file(file["path"], "some content") 88 | add_and_commit("Added files") 89 | 90 | 91 | @given('a non-standard svn-server "{name}"') 92 | def step_impl(context, name): 93 | repo_path = create_svn_server_and_repo(context, name) 94 | 95 | with in_directory(repo_path): 96 | generate_file("SomeFolder/SomeFile.txt", "some content") 97 | add_and_commit("Added files") 98 | 99 | 100 | @given("a fetched and committed MySvnProject with the manifest") 101 | def step_impl(context): 102 | repo_path = create_svn_server_and_repo(context, "MySvnProject") 103 | 104 | with in_directory(repo_path): 105 | generate_manifest(context) 106 | call_command(context, ["update"]) 107 | add_and_commit("Initial commit") 108 | 109 | 110 | @given('"{path}" in {directory} is changed, added and committed with') 111 | def step_impl(context, directory, path): 112 | with in_directory(directory): 113 | extend_file(path, context.text) 114 | add_and_commit("A change") 115 | -------------------------------------------------------------------------------- /features/suggest-project-name.feature: -------------------------------------------------------------------------------- 1 | Feature: Suggest a project name 2 | 3 | Users sometimes mistype project names, this lead to blaming *DFetch* of wrong behavior. 4 | To help the user of *DFetch*, if a user specifies a project name, that can't be found, suggest 5 | the closest. 6 | 7 | Background: 8 | Given the manifest 'dfetch.yaml' with the projects: 9 | | name | 10 | | project with space | 11 | | project-with-l | 12 | | Project-With-Capital | 13 | 14 | Scenario: Name with space 15 | When I run "dfetch check project with space" 16 | Then the output shows 17 | """ 18 | Dfetch (0.10.0) 19 | Not all projects found! "project", "with", "space" 20 | This manifest contains: "project with space", "project-with-l", "Project-With-Capital" 21 | Did you mean: "project with space"? 22 | """ 23 | 24 | Scenario: Name with one token different 25 | When I run "dfetch check project-with-1" 26 | Then the output shows 27 | """ 28 | Dfetch (0.10.0) 29 | Not all projects found! "project-with-1" 30 | This manifest contains: "project with space", "project-with-l", "Project-With-Capital" 31 | Did you mean: "project-with-l"? 32 | """ 33 | 34 | Scenario: Multiple wrong 35 | When I run "dfetch check project-with-1 project-with-space Project-With-Capital" 36 | Then the output shows 37 | """ 38 | Dfetch (0.10.0) 39 | Not all projects found! "project-with-1", "project-with-space" 40 | This manifest contains: "project with space", "project-with-l", "Project-With-Capital" 41 | Did you mean: "project with space" and "project-with-l"? 42 | """ 43 | -------------------------------------------------------------------------------- /features/updated-project-has-dependencies.feature: -------------------------------------------------------------------------------- 1 | Feature: Updated project has dependencies 2 | 3 | When a project has dependencies of its own, it can list them using its own 4 | manifest. *Dfetch* should recommend the user to add these dependencies to 5 | the manifest. 6 | 7 | Scenario: Git projects are specified in the manifest 8 | Given the manifest 'dfetch.yaml' in MyProject 9 | """ 10 | manifest: 11 | version: 0.0 12 | projects: 13 | - name: SomeProjectWithChild 14 | dst: third-party/SomeProjectWithChild 15 | url: some-remote-server/SomeProjectWithChild.git 16 | tag: v1 17 | - name: SomeProjectWithoutChild 18 | dst: third-party/SomeProjectWithoutChild 19 | url: some-remote-server/SomeProjectWithoutChild.git 20 | tag: v1 21 | """ 22 | And a git-repository "SomeProjectWithChild.git" with the manifest: 23 | """ 24 | manifest: 25 | version: 0.0 26 | remotes: 27 | - name: github-com-dfetch-org 28 | url-base: https://github.com/dfetch-org/test-repo 29 | 30 | projects: 31 | - name: SomeOtherProject 32 | dst: SomeOtherProject 33 | url: some-remote-server/SomeOtherProject.git 34 | tag: v1 35 | 36 | - name: ext/test-repo-tag-v1 37 | tag: v1 38 | """ 39 | And a git repository "SomeProjectWithoutChild.git" 40 | When I run "dfetch update" in MyProject 41 | Then the output shows 42 | """ 43 | Dfetch (0.10.0) 44 | SomeProjectWithChild: Fetched v1 45 | 46 | "SomeProjectWithChild" depends on the following project(s) which are not part of your manifest: 47 | (found in third-party/SomeProjectWithChild/dfetch.yaml) 48 | 49 | - name: SomeOtherProject 50 | url: some-remote-server/SomeOtherProject.git 51 | tag: v1 52 | - name: ext/test-repo-tag-v1 53 | url: https://github.com/dfetch-org/test-repo 54 | tag: v1 55 | 56 | SomeProjectWithoutChild: Fetched v1 57 | """ 58 | And 'MyProject' looks like: 59 | """ 60 | MyProject/ 61 | dfetch.yaml 62 | third-party/ 63 | SomeProjectWithChild/ 64 | .dfetch_data.yaml 65 | README.md 66 | dfetch.yaml 67 | SomeProjectWithoutChild/ 68 | .dfetch_data.yaml 69 | README.md 70 | """ 71 | 72 | Scenario: A child-project has an invalid manifest 73 | Given the manifest 'dfetch.yaml' in MyProject 74 | """ 75 | manifest: 76 | version: 0.0 77 | projects: 78 | - name: SomeProject 79 | dst: third-party/SomeProject 80 | url: some-remote-server/SomeProject.git 81 | tag: v1 82 | """ 83 | And a git-repository "SomeProject.git" with the manifest: 84 | """ 85 | very-invalid-manifest 86 | """ 87 | When I run "dfetch update" in MyProject 88 | Then the output shows 89 | """ 90 | Dfetch (0.10.0) 91 | SomeProject : Fetched v1 92 | SomeProject/dfetch.yaml: Schema validation failed: 93 | - Value 'very-invalid-manifest' is not a dict. Value path: ''. 94 | """ 95 | And 'MyProject' looks like: 96 | """ 97 | MyProject/ 98 | dfetch.yaml 99 | third-party/ 100 | SomeProject/ 101 | .dfetch_data.yaml 102 | README.md 103 | dfetch.yaml 104 | """ 105 | -------------------------------------------------------------------------------- /features/validate-manifest.feature: -------------------------------------------------------------------------------- 1 | Feature: Validate a manifest 2 | 3 | *DFetch* can check if the manifest is valid. 4 | 5 | Scenario: A valid manifest is provided 6 | Given the manifest 'dfetch.yaml' 7 | """ 8 | manifest: 9 | version: '0.0' 10 | 11 | remotes: 12 | - name: github-com-dfetch-org 13 | url-base: https://github.com/dfetch-org/test-repo 14 | 15 | projects: 16 | - name: ext/test-repo-rev-only 17 | revision: e1fda19a57b873eb8e6ae37780594cbb77b70f1a 18 | 19 | """ 20 | When I run "dfetch validate" 21 | Then the output shows 22 | """ 23 | Dfetch (0.10.0) 24 | dfetch.yaml : valid 25 | """ 26 | 27 | Scenario: An invalid manifest is provided 28 | Given the manifest 'dfetch.yaml' 29 | """ 30 | manifest-wrong: 31 | version: '0.0' 32 | 33 | remotes: 34 | - name: github-com-dfetch-org 35 | url-base: https://github.com/dfetch-org/test-repo 36 | 37 | projects: 38 | - name: ext/test-repo-rev-only 39 | revision: e1fda19a57b873eb8e6ae37780594cbb77b70f1a 40 | 41 | """ 42 | When I run "dfetch validate" 43 | Then the output shows 44 | """ 45 | Dfetch (0.10.0) 46 | Schema validation failed: 47 | - Cannot find required key 'manifest'. Path: ''. 48 | - Key 'manifest-wrong' was not defined. Path: ''. 49 | """ 50 | -------------------------------------------------------------------------------- /script/release.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Script for simplifying the release.""" 4 | 5 | import glob 6 | import os 7 | import re 8 | from datetime import datetime 9 | 10 | from dfetch import __version__ 11 | 12 | 13 | def replace_pattern_in_files(file_path_pattern, search_pattern, replacement, flags=0): 14 | """ 15 | Searches for a given pattern in all matching files and replaces it with the specified replacement. 16 | 17 | Args: 18 | file_path_pattern (str): The glob pattern for files to search (e.g., "./**/*.feature"). 19 | search_pattern (str): The regex pattern to search for in files. 20 | replacement (str): The replacement string. 21 | flags (int): Optional regex flags (e.g., re.DOTALL for multiline matching). 22 | """ 23 | pattern = re.compile(search_pattern, flags) 24 | 25 | files_changed = [] 26 | 27 | for file_path in glob.glob(file_path_pattern, recursive=True): 28 | with open(file_path, "r", encoding="utf-8") as f: 29 | content = f.read() 30 | 31 | new_content = pattern.sub(replacement, content) 32 | 33 | if content != new_content: 34 | with open(file_path, "w", encoding="utf-8") as f: 35 | f.write(new_content) 36 | files_changed.append(file_path) 37 | print( 38 | f"Replaced '{search_pattern}' with '{replacement}' in {len(files_changed)} files" 39 | ) 40 | 41 | 42 | if __name__ == "__main__": 43 | base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") 44 | 45 | replace_pattern_in_files( 46 | file_path_pattern=f"{base_dir}/CHANGELOG.rst", 47 | search_pattern=r"(Release \d+\.\d+\.\d+) \(unreleased\)", 48 | replacement=r"\1 (released " + datetime.now().strftime("%Y-%m-%d") + ")", 49 | flags=re.DOTALL, 50 | ) 51 | 52 | replace_pattern_in_files( 53 | file_path_pattern=f"{base_dir}/**/*.feature", 54 | search_pattern=r"Dfetch \((\d+)\.(\d+)\.(\d+)\)", 55 | replacement=f"Dfetch ({__version__})", 56 | ) 57 | 58 | # replace_pattern_in_files( 59 | # file_path_pattern=f"{base_dir}/features/report-sbom.feature", 60 | # search_pattern=r'("name":\s*"dfetch",\s*"version":\s*")\d+\.\d+\.\d+(")', 61 | # replacement=r"\1" + __version__ + r"\2", 62 | # flags=re.DOTALL, 63 | # ) 64 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests for this project.""" 2 | -------------------------------------------------------------------------------- /tests/manifest_mock.py: -------------------------------------------------------------------------------- 1 | """Mock for Manifest class.""" 2 | 3 | from unittest.mock import MagicMock, Mock 4 | 5 | from dfetch.manifest.manifest import Manifest 6 | from dfetch.manifest.project import ProjectEntry 7 | 8 | 9 | def mock_manifest(projects): 10 | """Create a manifest mock.""" 11 | 12 | project_mocks = [] 13 | 14 | for project in projects: 15 | mock_project = Mock(spec=ProjectEntry) 16 | mock_project.name = project["name"] 17 | mock_project.destination = "some_dest" 18 | project_mocks += [mock_project] 19 | 20 | mocked_manifest = MagicMock(spec=Manifest, projects=project_mocks) 21 | mocked_manifest.selected_projects.return_value = project_mocks 22 | return mocked_manifest 23 | -------------------------------------------------------------------------------- /tests/run_tests.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal enabledelayedexpansion 3 | 4 | cd %~dp0.. 5 | 6 | python create_venv.py --extra_requirements "test" 7 | if not !ERRORLEVEL! == 0 echo "Something went wrong creating the venv." && exit /b !ERRORLEVEL! 8 | 9 | call .\venv\Scripts\activate.bat 10 | 11 | echo Running tests.... 12 | python -m pytest tests 13 | -------------------------------------------------------------------------------- /tests/test_check.py: -------------------------------------------------------------------------------- 1 | """Test the check command.""" 2 | 3 | # mypy: ignore-errors 4 | # flake8: noqa 5 | 6 | import argparse 7 | from unittest.mock import patch 8 | 9 | import pytest 10 | 11 | from dfetch.commands.check import Check 12 | from tests.manifest_mock import mock_manifest 13 | 14 | DEFAULT_ARGS = argparse.Namespace( 15 | no_recommendations=False, jenkins_json=None, sarif=None, code_climate=None 16 | ) 17 | DEFAULT_ARGS.projects = [] 18 | 19 | 20 | @pytest.mark.parametrize( 21 | "name, projects", 22 | [ 23 | ("empty", []), 24 | ("single_project", [{"name": "my_project"}]), 25 | ("two_projects", [{"name": "first"}, {"name": "second"}]), 26 | ], 27 | ) 28 | def test_check(name, projects): 29 | check = Check() 30 | 31 | with patch("dfetch.manifest.manifest.get_manifest") as mocked_get_manifest: 32 | with patch( 33 | "dfetch.manifest.manifest.get_childmanifests" 34 | ) as mocked_get_childmanifests: 35 | with patch("dfetch.project.make") as mocked_make: 36 | with patch("os.path.exists"): 37 | with patch("dfetch.commands.check.in_directory"): 38 | with patch("dfetch.commands.check.CheckStdoutReporter"): 39 | mocked_get_manifest.return_value = ( 40 | mock_manifest(projects), 41 | "/", 42 | ) 43 | mocked_get_childmanifests.return_value = [] 44 | 45 | check(DEFAULT_ARGS) 46 | 47 | for _ in projects: 48 | mocked_make.return_value.check_for_update.assert_called() 49 | -------------------------------------------------------------------------------- /tests/test_cmdline.py: -------------------------------------------------------------------------------- 1 | """Test the cmdline.""" 2 | 3 | # mypy: ignore-errors 4 | # flake8: noqa 5 | 6 | import os 7 | import subprocess 8 | from subprocess import CalledProcessError, CompletedProcess 9 | from unittest.mock import MagicMock, Mock, patch 10 | 11 | import pytest 12 | 13 | from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline 14 | 15 | LS_CMD = "ls ." 16 | LS_OK_RESULT = CompletedProcess( 17 | returncode=0, stdout="myfile".encode(), stderr="".encode(), args=LS_CMD 18 | ) 19 | LS_NON_ZERO_RESULT = CalledProcessError( 20 | returncode=1, output="myfile".encode(), stderr="".encode(), cmd=LS_CMD 21 | ) 22 | LS_NOK_RESULT = CalledProcessError( 23 | returncode=0, output="myfile".encode(), stderr="".encode(), cmd=LS_CMD 24 | ) 25 | MISSING_CMD_RESULT = FileNotFoundError() 26 | 27 | 28 | @pytest.mark.parametrize( 29 | "name, cmd, cmd_result, expectation", 30 | [ 31 | ("cmd succeeds", LS_CMD, [LS_OK_RESULT], LS_OK_RESULT), 32 | ("cmd non-zero return", LS_CMD, [LS_NON_ZERO_RESULT], SubprocessCommandError), 33 | ("cmd raises", LS_CMD, [LS_NOK_RESULT], SubprocessCommandError), 34 | ("cmd missing", LS_CMD, [MISSING_CMD_RESULT], RuntimeError), 35 | ], 36 | ) 37 | def test_run_on_cmdline(name, cmd, cmd_result, expectation): 38 | with patch("dfetch.util.cmdline.subprocess.run") as subprocess_mock: 39 | subprocess_mock.side_effect = cmd_result 40 | logger_mock = MagicMock() 41 | 42 | if isinstance(expectation, CompletedProcess): 43 | assert expectation == run_on_cmdline(logger_mock, cmd) 44 | else: 45 | with pytest.raises(expectation): 46 | run_on_cmdline(logger_mock, cmd) 47 | -------------------------------------------------------------------------------- /tests/test_import.py: -------------------------------------------------------------------------------- 1 | """Test the import command.""" 2 | 3 | # mypy: ignore-errors 4 | # flake8: noqa 5 | 6 | import argparse 7 | from unittest.mock import patch 8 | 9 | import pytest 10 | 11 | from dfetch.commands.import_ import Import 12 | from dfetch.project.svn import External 13 | from dfetch.vcs.git import Submodule 14 | 15 | DEFAULT_ARGS = argparse.Namespace(non_recursive=False) 16 | 17 | FIRST_SUBMODULE = Submodule( 18 | name="submod1", 19 | sha="1234", 20 | url="http://www.github.com/mod1.git", 21 | toplevel="", 22 | path="somepath1", 23 | branch="master", 24 | tag="", 25 | ) 26 | SECOND_SUBMODULE = Submodule( 27 | name="submod2", 28 | sha="1234", 29 | url="http://www.github.com/mod2.git", 30 | toplevel="", 31 | path="somepath2", 32 | branch="", 33 | tag="v1.2.3", 34 | ) 35 | 36 | 37 | @pytest.mark.parametrize( 38 | "name, submodules", 39 | [ 40 | ("empty", []), 41 | ("single_submodule", [FIRST_SUBMODULE]), 42 | ( 43 | "two_submodules", 44 | [FIRST_SUBMODULE, SECOND_SUBMODULE], 45 | ), 46 | ], 47 | ) 48 | def test_git_import(name, submodules): 49 | import_ = Import() 50 | 51 | with patch("dfetch.commands.import_.GitLocalRepo.submodules") as mocked_submodules: 52 | with patch("dfetch.commands.import_.Manifest") as mocked_manifest: 53 | mocked_submodules.return_value = submodules 54 | 55 | if len(submodules) == 0: 56 | with pytest.raises(RuntimeError): 57 | import_(argparse.Namespace()) 58 | else: 59 | import_(argparse.Namespace()) 60 | 61 | mocked_manifest.assert_called() 62 | 63 | args = mocked_manifest.call_args_list[0][0][0] 64 | 65 | for project_entry in args["projects"]: 66 | assert project_entry.name in [subm.name for subm in submodules] 67 | 68 | # Manifest should have been dumped 69 | mocked_manifest.return_value.dump.assert_called() 70 | 71 | 72 | FIRST_EXTERNAL = External( 73 | name="external1", 74 | revision="1234", 75 | url="http://www.github.com/mod1/", 76 | toplevel="", 77 | path="somepath1", 78 | branch="trunk", 79 | tag="", 80 | src="some/sub/folder", 81 | ) 82 | SECOND_EXTERNAL = External( 83 | name="external2", 84 | revision="1235", 85 | url="http://www.github.com/mod2/", 86 | toplevel="", 87 | path="somepath2", 88 | branch="", 89 | tag="0.0.2", 90 | src="some/sub/folder2/", 91 | ) 92 | 93 | 94 | @pytest.mark.parametrize( 95 | "name, externals", 96 | [ 97 | ("empty", []), 98 | ("single_external", [FIRST_EXTERNAL]), 99 | ( 100 | "two_externals", 101 | [FIRST_EXTERNAL, SECOND_EXTERNAL], 102 | ), 103 | ], 104 | ) 105 | def test_svn_import(name, externals): 106 | import_ = Import() 107 | 108 | with patch("dfetch.commands.import_.SvnRepo.check_path") as check_path: 109 | with patch("dfetch.commands.import_.SvnRepo.externals") as mocked_externals: 110 | with patch("dfetch.commands.import_.Manifest") as mocked_manifest: 111 | with patch("dfetch.commands.import_.GitLocalRepo.is_git") as is_git: 112 | is_git.return_value = False 113 | check_path.return_value = True 114 | mocked_externals.return_value = externals 115 | 116 | if len(externals) == 0: 117 | with pytest.raises(RuntimeError): 118 | import_(argparse.Namespace()) 119 | else: 120 | import_(argparse.Namespace()) 121 | 122 | mocked_manifest.assert_called() 123 | 124 | args = mocked_manifest.call_args_list[0][0][0] 125 | 126 | for project_entry in args["projects"]: 127 | assert project_entry.name in [ext.name for ext in externals] 128 | 129 | # Manifest should have been dumped 130 | mocked_manifest.return_value.dump.assert_called() 131 | -------------------------------------------------------------------------------- /tests/test_project_entry.py: -------------------------------------------------------------------------------- 1 | """Test the Version object command.""" 2 | 3 | # mypy: ignore-errors 4 | # flake8: noqa 5 | 6 | import pytest 7 | 8 | from dfetch.manifest.project import ProjectEntry 9 | 10 | 11 | def test_projectentry_name(): 12 | assert ProjectEntry({"name": "SomeProject"}).name == "SomeProject" 13 | 14 | 15 | def test_projectentry_revision(): 16 | assert ProjectEntry({"name": "SomeProject", "revision": "123"}).revision == "123" 17 | 18 | 19 | def test_projectentry_remote(): 20 | assert ( 21 | ProjectEntry({"name": "SomeProject", "remote": "SomeRemote"}).remote 22 | == "SomeRemote" 23 | ) 24 | 25 | 26 | def test_projectentry_source(): 27 | assert ProjectEntry({"name": "SomeProject", "src": "SomePath"}).source == "SomePath" 28 | 29 | 30 | def test_projectentry_vcs(): 31 | assert ProjectEntry({"name": "SomeProject", "vcs": "git"}).vcs == "git" 32 | 33 | 34 | def test_projectentry_patch(): 35 | assert ( 36 | ProjectEntry({"name": "SomeProject", "patch": "diff.patch"}).patch 37 | == "diff.patch" 38 | ) 39 | 40 | 41 | def test_projectentry_as_yaml(): 42 | assert ProjectEntry({"name": "SomeProject"}).as_yaml() == {"name": "SomeProject"} 43 | 44 | 45 | def test_projectentry_as_str(): 46 | assert ( 47 | str(ProjectEntry({"name": "SomeProject"})) 48 | == "SomeProject latest SomeProject" 49 | ) 50 | -------------------------------------------------------------------------------- /tests/test_report.py: -------------------------------------------------------------------------------- 1 | """Test the report command.""" 2 | 3 | # mypy: ignore-errors 4 | # flake8: noqa 5 | 6 | import argparse 7 | from unittest.mock import patch 8 | 9 | import pytest 10 | 11 | from dfetch.commands.report import Report, ReportTypes 12 | from tests.manifest_mock import mock_manifest 13 | 14 | DEFAULT_ARGS = argparse.Namespace() 15 | DEFAULT_ARGS.projects = [] 16 | DEFAULT_ARGS.type = ReportTypes.STDOUT 17 | DEFAULT_ARGS.outfile = "" 18 | 19 | 20 | @pytest.mark.parametrize( 21 | "name, projects", 22 | [ 23 | ("empty", []), 24 | ("single_project", [{"name": "my_project"}]), 25 | ("two_projects", [{"name": "first"}, {"name": "second"}]), 26 | ], 27 | ) 28 | def test_report(name, projects): 29 | report = Report() 30 | 31 | with patch("dfetch.manifest.manifest.get_manifest") as mocked_get_manifest: 32 | with patch("dfetch.log.DLogger.print_info_line") as mocked_print_info_line: 33 | mocked_get_manifest.return_value = (mock_manifest(projects), "/") 34 | 35 | report(DEFAULT_ARGS) 36 | 37 | if projects: 38 | for project in projects: 39 | mocked_print_info_line.assert_any_call("project", project["name"]) 40 | else: 41 | mocked_print_info_line.assert_not_called() 42 | -------------------------------------------------------------------------------- /tests/test_resources.py: -------------------------------------------------------------------------------- 1 | """Test the resources.""" 2 | 3 | # mypy: ignore-errors 4 | # flake8: noqa 5 | 6 | import os 7 | 8 | import dfetch.resources 9 | 10 | 11 | def test_schema_path() -> None: 12 | """Test that schema path can be used as context manager.""" 13 | 14 | with dfetch.resources.schema_path() as schema_path: 15 | assert os.path.isfile(schema_path) 16 | 17 | 18 | def test_call_schema_path_twice() -> None: 19 | """Had a lot of problems with calling contextmanager twice.""" 20 | 21 | with dfetch.resources.schema_path() as schema_path: 22 | assert os.path.isfile(schema_path) 23 | 24 | with dfetch.resources.schema_path() as schema_path: 25 | assert os.path.isfile(schema_path) 26 | --------------------------------------------------------------------------------