├── tests ├── fixtures │ ├── test-1 │ │ ├── spec │ │ └── autobuild │ │ │ └── defines │ ├── test-2 │ │ ├── spec │ │ ├── 01-sub-1 │ │ │ └── defines │ │ └── 02-sub-2 │ │ │ └── defines │ ├── test-3 │ │ ├── spec │ │ └── autobuild │ │ │ └── defines │ ├── test-5 │ │ ├── spec │ │ └── autobuild │ │ │ └── defines │ ├── test-4 │ │ ├── autobuild │ │ │ └── defines │ │ └── spec │ └── test-6 │ │ ├── autobuild │ │ └── defines │ │ └── spec └── test.py ├── MANIFEST.in ├── acbs ├── __init__.py ├── miniapt_query.pyi ├── const.py ├── query.py ├── crypto.py ├── ab4cfg.py ├── checkpoint.py ├── base.py ├── magic.py ├── resume.py ├── deps.py ├── pm.py ├── bashvar.py ├── find.py ├── parser.py ├── fetch.py ├── main.py └── utils.py ├── src ├── miniapt-query.h └── miniapt-query.cc ├── docs ├── source │ ├── api.rst │ ├── index.rst │ ├── intro.rst │ ├── dx.rst │ ├── install.rst │ ├── appendix.rst │ ├── spec_format.rst │ └── conf.py └── Makefile ├── setup.py ├── .github └── workflows │ └── ci.yml ├── pyproject.toml ├── completions ├── acbs-build ├── _acbs-build └── acbs-build.fish ├── .idea └── TODOs.md ├── bootstrap ├── .gitignore ├── README.md ├── acbs-build └── LICENSE /tests/fixtures/test-1/spec: -------------------------------------------------------------------------------- 1 | VER=1 2 | DUMMYSRC=1 3 | -------------------------------------------------------------------------------- /tests/fixtures/test-2/spec: -------------------------------------------------------------------------------- 1 | VER=1 2 | DUMMYSRC=1 3 | -------------------------------------------------------------------------------- /tests/fixtures/test-3/spec: -------------------------------------------------------------------------------- 1 | VER=1 2 | DUMMYSRC=1 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft tests 2 | graft completions 3 | global-exclude *.py[cod] 4 | -------------------------------------------------------------------------------- /acbs/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '20251101' 2 | __meta_version__ = f'0.1.{__version__}' 3 | -------------------------------------------------------------------------------- /tests/fixtures/test-2/01-sub-1/defines: -------------------------------------------------------------------------------- 1 | PKGNAME='sub-1' 2 | PKGDEP="test-12" 3 | BUILDDEP="test-4" 4 | -------------------------------------------------------------------------------- /tests/fixtures/test-5/spec: -------------------------------------------------------------------------------- 1 | VER=1 2 | SRCS='git::git://github.com/AOSC-Dev/acbs' 3 | CHKSUMS='SKIP' 4 | -------------------------------------------------------------------------------- /tests/fixtures/test-2/02-sub-2/defines: -------------------------------------------------------------------------------- 1 | PKGNAME='sub-2' 2 | PKGDEP="sub-1 test-11" 3 | BUILDDEP="test-7" 4 | -------------------------------------------------------------------------------- /acbs/miniapt_query.pyi: -------------------------------------------------------------------------------- 1 | def apt_init_system() -> bool: ... 2 | def check_if_available(name : str) -> int: ... 3 | -------------------------------------------------------------------------------- /tests/fixtures/test-3/autobuild/defines: -------------------------------------------------------------------------------- 1 | PKGNAME='test-3' 2 | PKGDEP="test-3" 3 | PKGDEP__ARCH="${PKGDEP}" 4 | BUILDDEP="" 5 | BUILDDEP__ARCH="" 6 | -------------------------------------------------------------------------------- /tests/fixtures/test-4/autobuild/defines: -------------------------------------------------------------------------------- 1 | PKGNAME='test-4' 2 | PKGDEP="test-4" 3 | PKGDEP__ARCH="${PKGDEP}" 4 | BUILDDEP="" 5 | BUILDDEP__ARCH="" 6 | -------------------------------------------------------------------------------- /tests/fixtures/test-4/spec: -------------------------------------------------------------------------------- 1 | VER=1 2 | SRCS='git::git://github.com/AOSC-Dev/acbs git::https://github.com/AOSC-Dev/acbs' 3 | CHKSUMS='SKIP SKIP' 4 | -------------------------------------------------------------------------------- /tests/fixtures/test-5/autobuild/defines: -------------------------------------------------------------------------------- 1 | PKGNAME='test-5' 2 | PKGDEP="test-6" 3 | PKGDEP__ARCH="${PKGDEP}" 4 | BUILDDEP="" 5 | BUILDDEP__ARCH="" 6 | -------------------------------------------------------------------------------- /tests/fixtures/test-6/autobuild/defines: -------------------------------------------------------------------------------- 1 | PKGNAME='test-6' 2 | PKGDEP="test-5" 3 | PKGDEP__ARCH="${PKGDEP}" 4 | BUILDDEP="" 5 | BUILDDEP__ARCH="" 6 | -------------------------------------------------------------------------------- /tests/fixtures/test-6/spec: -------------------------------------------------------------------------------- 1 | VER=1 2 | SRCS='git::git://github.com/AOSC-Dev/acbs git::https://github.com/AOSC-Dev/acbs' 3 | CHKSUMS='SKIP SKIP' 4 | -------------------------------------------------------------------------------- /tests/fixtures/test-1/autobuild/defines: -------------------------------------------------------------------------------- 1 | PKGNAME='test-1' 2 | PKGDEP="test-2 test-3" 3 | PKGDEP__ARCH="${PKGDEP} test-17" 4 | BUILDDEP="test-4" 5 | BUILDDEP__ARCH="" 6 | -------------------------------------------------------------------------------- /src/miniapt-query.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int check_available(const char *name); 4 | bool apt_init_system(); 5 | 6 | PYBIND11_MODULE(miniapt_query, m) { 7 | m.doc() = "Query if a package exists in the repository"; 8 | m.def("apt_init_system", &apt_init_system, "Initialize system cache"); 9 | m.def("check_if_available", &check_available, "Check if a package exists in the repository"); 10 | } 11 | -------------------------------------------------------------------------------- /acbs/const.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Some helper constants 3 | ''' 4 | 5 | ANSI_RST = '\033[0m' 6 | ANSI_RED = '\033[91m' 7 | ANSI_BLNK = '\033[5m' 8 | ANSI_CYAN = '\033[36m' 9 | ANSI_LT_CYAN = '\033[96m' 10 | ANSI_GREEN = '\033[32m' 11 | ANSI_YELLOW = '\033[93m' 12 | ANSI_BLUE = '\033[34m' 13 | ANSI_BROWN = '\033[33m' 14 | 15 | # Common paths 16 | CONF_DIR = '/etc/acbs/' 17 | AUTOBUILD_CONF_DIR = '/etc/autobuild/' 18 | DUMP_DIR = '/var/cache/acbs/tarballs/' 19 | TMP_DIR = '/var/cache/acbs/build/' 20 | LOG_DIR = '/var/log/acbs/' 21 | DPKG_DIR = '/var/lib/dpkg/' 22 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | .. ACBS API documentation 2 | 3 | ACBS API Reference 4 | ================== 5 | 6 | ACBS base 7 | --------- 8 | ACBS base contains basic data structures. 9 | 10 | .. autoclass:: acbs.base.ACBSSourceInfo 11 | :members: 12 | .. autoclass:: acbs.base.ACBSPackageInfo 13 | :members: 14 | 15 | ACBS deps 16 | --------- 17 | ACBS deps contains the dependency resolver. 18 | 19 | .. automodule:: acbs.deps 20 | :members: 21 | 22 | ACBS utils 23 | ---------- 24 | ACBS utils contains some helper functions to perform some tasks easier. 25 | 26 | .. automodule:: acbs.utils 27 | :members: 28 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. acbs documentation master file, created by 2 | sphinx-quickstart on Sun Jul 31 18:23:52 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to acbs's documentation! 7 | ================================ 8 | ACBS - The current generation of building toolkit for AOSC OS written in Python. 9 | 10 | Contents: 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | 15 | intro 16 | install 17 | dx 18 | api 19 | spec_format 20 | appendix 21 | 22 | 23 | 24 | Indices and tables 25 | ================== 26 | 27 | * :ref:`genindex` 28 | * :ref:`modindex` 29 | * :ref:`search` 30 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import Extension, setup 2 | 3 | 4 | class get_pybind_include(object): 5 | """Helper class to determine the pybind11 include path 6 | The purpose of this class is to postpone importing pybind11 7 | until it is actually installed, so that the ``get_include()`` 8 | method can be invoked.""" 9 | 10 | def __str__(self): 11 | try: 12 | import pybind11 13 | except ImportError: 14 | return "" 15 | return pybind11.get_include() 16 | 17 | 18 | setup( 19 | ext_modules=[ 20 | Extension( 21 | "acbs.miniapt_query", 22 | sorted(["src/miniapt-query.cc"]), 23 | include_dirs=[str(get_pybind_include())], 24 | extra_link_args=["-lapt-pkg"], 25 | language="c++", 26 | optional=True, 27 | ) 28 | ], 29 | scripts=["acbs-build"] 30 | ) 31 | -------------------------------------------------------------------------------- /docs/source/intro.rst: -------------------------------------------------------------------------------- 1 | .. introduction 2 | 3 | Introduction 4 | ============ 5 | What is ACBS? 6 | ------------- 7 | ACBS (AutoBuild CI Build System) is a re-implementation of the original ABBS (AutoBuild Build Service) in Python. 8 | It's a smart upper structure of the basic Autobuild3 multi-function package builder. 9 | 10 | What happend to ABBS, its ancestor? 11 | ----------------------------------- 12 | ACBS was first created with an intent to extend upon ABBS and improve its sorry reliability and agility. 13 | ACBS supports multi-tree layouts and proper environment handling, providing a more reliable and efficient experience. 14 | Comparing to its ancestor, it improved reliability and added a few more features like an automated dependency 15 | resolver. 16 | 17 | How to install or deploy it? 18 | ---------------------------- 19 | ACBS is only designed to work with Autobuild3 and under Unix-like environment. 20 | The detailed instructions can be found at :doc:`install`. 21 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI checks 2 | 3 | on: 4 | push: 5 | branches: [ staging, production, dx ] 6 | pull_request: 7 | branches: [ staging, production ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Set up Python 3.8 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: "3.8" 20 | cache: 'pip' # caching pip dependencies 21 | - name: Install native dependencies 22 | run: | 23 | sudo apt-get update && sudo apt-get install -y libapt-pkg-dev 24 | sudo snap install --classic astral-uv 25 | - name: Install dependencies 26 | run: | 27 | uv venv 28 | uv pip install '.[dev]' 29 | - name: Type checking using pyright 30 | run: uv run --extra=dev pyright 31 | - name: Lint with Ruff 32 | run: uv run --extra=dev ruff check --config 'output-format="github"' 33 | - name: Run unittests 34 | run: ARCH=amd64 uv run --extra=dev python -m unittest discover ./tests/ 35 | -------------------------------------------------------------------------------- /acbs/query.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Optional, Sequence 3 | 4 | from acbs.const import CONF_DIR, DUMP_DIR, LOG_DIR, TMP_DIR 5 | from acbs.parser import get_tree_by_name 6 | 7 | 8 | def acbs_query(input: str) -> Optional[str]: 9 | input = input.strip() 10 | if not input: 11 | return None 12 | commands = input.split(':') 13 | getter = { 14 | 'tree': acbs_query_tree, 15 | 'path': acbs_query_path 16 | }.get(commands[0]) 17 | if not callable(getter): 18 | return None 19 | return getter(commands) 20 | 21 | 22 | def acbs_query_tree(commands: Sequence[str]) -> Optional[str]: 23 | if len(commands) != 2: 24 | return None 25 | try: 26 | return get_tree_by_name(os.path.join(CONF_DIR, 'forest.conf'), commands[1]) 27 | except Exception: 28 | return None 29 | 30 | 31 | def acbs_query_path(commands: Sequence[str]) -> Optional[str]: 32 | if len(commands) != 2: 33 | return None 34 | return { 35 | 'conf': CONF_DIR, 36 | 'dump': DUMP_DIR, 37 | 'tmp': TMP_DIR, 38 | 'log': LOG_DIR 39 | }.get(commands[1]) 40 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=68.2.0", "pybind11>=2.5.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.setuptools.packages] 6 | find = {} 7 | 8 | [tool.setuptools.package-data] 9 | src = ["*.h"] 10 | 11 | [tool.setuptools.dynamic] 12 | version = { attr = "acbs.__meta_version__" } 13 | 14 | [project] 15 | name = "acbs" 16 | dynamic = ["version"] 17 | authors = [{ name = "liushuyu", email = "liushuyu@aosc.io" }] 18 | description = "AOSC CI Building System" 19 | readme = "README.md" 20 | license = { file = "LICENSE" } 21 | requires-python = ">=3.8" 22 | dependencies = ["pyparsing>=2.4,<4"] 23 | classifiers = [ 24 | "Development Status :: 4 - Beta", 25 | "Programming Language :: Python :: 3", 26 | "Intended Audience :: Developers", 27 | "Operating System :: POSIX :: Linux", 28 | ] 29 | urls = { "Repository" = "https://github.com/AOSC-Dev/acbs" } 30 | 31 | [project.optional-dependencies] 32 | logging = ["pexpect"] 33 | dev = ["pyright", "ruff", "setuptools", "pybind11>=2.5.0", "pexpect"] 34 | 35 | [tool.ruff] 36 | line-length = 127 37 | target-version = "py38" 38 | [tool.ruff.lint.mccabe] 39 | max-complexity = 5 40 | 41 | [tool.pyright] 42 | exclude = ["build", "dist", "venv", ".venv"] 43 | -------------------------------------------------------------------------------- /completions/acbs-build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # $1 name 3 | _acbs_comp_package() { 4 | local tree 5 | tree="$(acbs-build -q 'tree:default' 2>/dev/null)" 6 | if [[ -z "$tree" || ! -d "$tree" ]]; then 7 | return 8 | fi 9 | local PGROUPS="$(find "$tree/groups/" -maxdepth 1 -mindepth 1 -type f -printf 'groups/%f\n')" 10 | COMPREPLY=($(compgen -W "$PGROUPS" -- "${1}")) 11 | if [[ "$1" == *'/'* ]]; then 12 | return 13 | fi 14 | COMPREPLY+=($(find "$tree" -maxdepth 2 -mindepth 2 -type d -not -path "$tree/.git" -name "${1}*" -printf '%f\n')) 15 | } 16 | 17 | _acbs() 18 | { 19 | local cur prev words cword 20 | _init_completion || return 21 | 22 | if [[ $cur == -* ]]; then 23 | COMPREPLY=($(compgen -W '-v --version -d --debug -t --tree -q --query -c --clear -k --skip-deps -g --get -r --resume -w --write -e --reorder -p --print-tasks' -- "$cur")) 24 | elif [[ $prev == "-t" || $prev == "--tree" ]]; then 25 | forest="$(acbs-build -q 'path:conf' 2>/dev/null)/forest.conf" 26 | if [[ "$?" -ne "0" ]]; then 27 | return 28 | fi 29 | COMPREPLY=($(gawk 'match($0,/\[(.*)\]/,m) {print m[1]}' "$forest")) 30 | elif [[ $prev == "-r" || $prev == "--resume" ]]; then 31 | COMPREPLY=($(compgen -f -- "$cur")) 32 | else 33 | _acbs_comp_package "$cur" 34 | fi 35 | } 36 | 37 | complete -F _acbs acbs-build 38 | -------------------------------------------------------------------------------- /acbs/crypto.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from typing import Optional, Tuple 3 | 4 | 5 | def check_hash_hashlib_inner(chksum_type: str, target_file: str) -> Optional[str]: 6 | hash_type = chksum_type.lower() 7 | if hash_type == 'none': 8 | return None 9 | if hash_type not in hashlib.algorithms_available: 10 | raise NotImplementedError( 11 | 'Unsupported hash type %s! Currently supported: %s' % ( 12 | hash_type, ' '.join(sorted(hashlib.algorithms_available)))) 13 | hash_obj = hashlib.new(hash_type) 14 | with open(target_file, 'rb') as f: 15 | for chunk in iter(lambda: f.read(4096), b''): 16 | hash_obj.update(chunk) 17 | target_hash = hash_obj.hexdigest() 18 | return target_hash 19 | 20 | 21 | def hash_url(url: str) -> str: 22 | hash_obj = hashlib.new('sha256') 23 | hash_obj.update(url.encode('utf-8')) 24 | return hash_obj.hexdigest() 25 | 26 | 27 | def check_hash_hashlib(chksum_tuple: Tuple[str, str], target_file: str) -> None: 28 | hash_type, hash_value = chksum_tuple 29 | hash_type = hash_type.lower() 30 | hash_value = hash_value.lower() 31 | target_hash = check_hash_hashlib_inner(hash_type, target_file) 32 | if hash_value != target_hash: 33 | raise RuntimeError('Checksums mismatch of type %s at file %s:\nExpected: %s\nActual: %s' % ( 34 | hash_type, target_file, hash_value, target_hash)) 35 | -------------------------------------------------------------------------------- /.idea/TODOs.md: -------------------------------------------------------------------------------- 1 | ### Documents 2 | - WIP, check back soon! 3 | 4 | ### General Code Logic 5 | - [X] Code cleanup 6 | - [X] ~~Find a better way to handle shell script execution~~ No direct shell script execution now 7 | 8 | ### Multiple source format? [Resolved] 9 | 10 | - Arch `PKGBUILD` like spec: 11 | 12 | This is Bash compatible 13 | 14 | Breaks compatiblity with `abbs` 15 | ```bash 16 | SRCS=('url1', 'url2') 17 | ``` 18 | 19 | - [X] Simple form: 20 | 21 | This is Bash compatible 22 | 23 | Breaks compatiblity with `abbs` 24 | ```bash 25 | SRCS='url1 url2' 26 | ``` 27 | 28 | 29 | ### Check sum format? 30 | 31 | - Arch `PKGBUILD` like spec: 32 | 33 | This is Bash compatible 34 | ```bash 35 | # Matching: Regular Bash expression 36 | md5sum=('d41d8cd98f00b204e9800998ecf8427e') 37 | ``` 38 | 39 | - Simple layout 40 | 41 | This is Bash compatible 42 | ```bash 43 | # Matching: Regular Bash expression 44 | MD5SUM='d41d8cd98f00b204e9800998ecf8427e' 45 | ``` 46 | 47 | - BSD tag like spec: 48 | 49 | This is **not** Bash compatible 50 | ```python 51 | # Matching: RegExp: 52 | match = r'(\w+).*?\((.*?)\).*?\=(.*)' 53 | # Example: 54 | MD5SUM(file)='d41d8cd98f00b204e9800998ecf8427e' 55 | ``` 56 | - [X] Mixed: 57 | 58 | This seems to be Bash compatible 59 | ```bash 60 | # Example: 61 | CHKSUM='md5::d41d8cd98f00b204e9800998ecf8427e' 62 | # Example 2: 63 | CHKSUM='md5(d41d8cd98f00b204e9800998ecf8427e)' 64 | ``` 65 | -------------------------------------------------------------------------------- /acbs/ab4cfg.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from pyparsing import ParseException 4 | 5 | from acbs import bashvar 6 | from acbs.const import AUTOBUILD_CONF_DIR 7 | 8 | from typing import Optional, Dict 9 | 10 | 11 | def _parse_abcfg(abcfg_path: str) -> Dict[str, str]: 12 | with open(abcfg_path) as f: 13 | vars = bashvar.eval_bashvar(f.read(), filename=abcfg_path) 14 | assert isinstance(vars, dict) 15 | return vars 16 | 17 | 18 | def get_arch_override(abcfg_path: str) -> Optional[str]: 19 | vars = _parse_abcfg(abcfg_path) 20 | return vars.get('ARCH') 21 | 22 | 23 | def is_in_stage2_file(abcfg_path: str) -> bool: 24 | vars = _parse_abcfg(abcfg_path) 25 | stage2_val: Optional[str] = vars.get('ABSTAGE2') 26 | return stage2_val == '1' 27 | 28 | 29 | def is_in_stage2_env() -> bool: 30 | return os.environ.get('ABSTAGE2', '') == '1' 31 | 32 | 33 | def is_in_stage2() -> bool: 34 | """ 35 | Return whether the current environment is in a stage2 development phase. 36 | """ 37 | abcfg_path: str = os.path.join(AUTOBUILD_CONF_DIR, 'ab3cfg.sh') 38 | if not os.path.exists(abcfg_path): 39 | abcfg_path = os.path.join(AUTOBUILD_CONF_DIR, 'ab4cfg.sh') 40 | try: 41 | return is_in_stage2_env() or is_in_stage2_file(abcfg_path) 42 | except OSError as e: 43 | raise RuntimeError(f'Unable to read Autobuild config file {abcfg_path}.') from e 44 | except ParseException as e: 45 | raise RuntimeError(f'Error occurred while parsing Autobuild config file {abcfg_path}.') from e 46 | except Exception as e: 47 | raise RuntimeError('Error occurred while checking whether stage 2 mode is enabled.') from e 48 | -------------------------------------------------------------------------------- /bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | export PATH="$HOME/.local/bin:$PATH" 4 | 5 | python3 --version 6 | 7 | echo '[-] Installing required dependencies to local user folder...' 8 | rm -f get-pip.py 9 | wget https://bootstrap.pypa.io/get-pip.py 10 | python3 get-pip.py --user 11 | pip3 install --user pyparsing 12 | echo '[-] Installing acbs (stage 0)...' 13 | python3 setup.py install --user 14 | rm -f get-pip.py 15 | 16 | echo '[-] Testing acbs...' 17 | "$HOME/.local/bin/acbs-build" --version 18 | 19 | if ! command -v autobuild > /dev/null 2>&1; then 20 | echo '[!] Autobuild3 not detected on your system.' 21 | echo '[!] Please manually run the following command:' 22 | # shellcheck disable=SC2016 23 | echo 'export PATH="$HOME:/.local/bin/":$PATH' 24 | exit 0 25 | fi 26 | 27 | if ! command -v git > /dev/null 2>&1; then 28 | echo '[!] GIT not detected on your system.' 29 | exit 1 30 | fi 31 | 32 | echo '[-] Installing acbs (stage 1)...' 33 | TMPDIR="$(mktemp -d)" 34 | cd "$TMPDIR" 35 | git clone --filter=blob:none -b stable https://github.com/AOSC-Dev/aosc-os-abbs 36 | ABBSDIR="$(readlink -f aosc-os-abbs)" 37 | mkdir -p '/etc/acbs/' 38 | mkdir -p '/var/cache/acbs/'{build,tarballs} 39 | mkdir -p '/var/log/acbs/' 40 | [ -f /etc/acbs/forest.conf ] && echo '[-] Backing up forest.conf...' && cp -v /etc/acbs/forest.conf /etc/acbs/forest.conf.bak 41 | cat << EOF > /etc/acbs/forest.conf 42 | [default] 43 | location = ${ABBSDIR} 44 | EOF 45 | "$HOME/.local/bin/acbs-build" acbs 46 | cd .. 47 | 48 | echo '[-] Installing acbs (final)...' 49 | 50 | acbs-build acbs 51 | 52 | echo '[-] Cleaning up...' 53 | rm -rf "$TMPDIR" 54 | acbs-build -c 55 | cp -v /etc/acbs/forest.conf.bak /etc/acbs/forest.conf 56 | -------------------------------------------------------------------------------- /src/miniapt-query.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "miniapt-query.h" 11 | 12 | static pkgPolicy *policy; 13 | static pkgCache *cache; 14 | static bool initialized = false; 15 | 16 | bool apt_init_system() 17 | { 18 | initialized = pkgInitConfig(*_config) && pkgInitSystem(*_config, _system); 19 | return initialized; 20 | } 21 | 22 | int check_available(const char *name) 23 | { 24 | if (!initialized) 25 | return -1; 26 | pkgCacheFile cachefile; 27 | pkgDepCache *depCache = cachefile.GetDepCache(); 28 | if (!depCache) 29 | return -1; 30 | cache = cachefile.GetPkgCache(); 31 | if (!cache) 32 | return -2; 33 | policy = cachefile.GetPolicy(); 34 | if (!policy) 35 | return -3; 36 | if (cachefile->BrokenCount() > 0) 37 | return -4; 38 | 39 | APT::CacheSetHelper helper(true, GlobalError::NOTICE); 40 | const char *list[2] = {name, NULL}; 41 | APT::PackageList pkgset = APT::PackageList::FromCommandLine(cachefile, list, helper); 42 | // returns 0: installed; 1: not installed, available; 2: not installed, not available 43 | for (APT::PackageList::const_iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) 44 | { 45 | if (Pkg->CurrentVer != 0) { 46 | return 0; 47 | } 48 | if (depCache->GetCandidateVersion(Pkg)) { 49 | if (depCache->MarkInstall(Pkg, true, 0, true)) { 50 | return 1; 51 | } 52 | } 53 | } 54 | return 2; 55 | } 56 | -------------------------------------------------------------------------------- /completions/_acbs-build: -------------------------------------------------------------------------------- 1 | #compdef acbs-build 2 | typeset -A opt_args 3 | local context state line 4 | 5 | local -a _acbs_comp_package 6 | function _acbs_comp_package() { 7 | local tree 8 | tree="$(acbs-build -q 'tree:default' 2>/dev/null)" 9 | if [[ -z "$tree" || ! -d "$tree" ]]; then 10 | return 11 | fi 12 | REPLY="$(find "$tree/groups/" -maxdepth 1 -mindepth 1 -type f -printf 'groups/%f\n')" 13 | if [[ "$1" == *'/'* ]]; then 14 | _describe -t commands 'packages' "${(f)REPLY}" -V1 15 | return 16 | fi 17 | REPLY+=" $(find "$tree" -maxdepth 2 -mindepth 2 -type d -not -path "$tree/.git" -printf '%f\n')" 18 | _describe -t commands 'packages' "${(f)REPLY}" -V1 19 | } 20 | 21 | local -a flags 22 | flags=( 23 | '(-v --version)'{-v,--version}'[Show the version and exit]' 24 | '(-d --debug)'{-d,--debug}'[Increase verbosity to ease debugging process]' 25 | '(-t --tree)'{-t,--tree}'[Specify which abbs-tree to use]:tree:' 26 | '(-q --query)'{-q,--query}'[Do a simple ACBS query]' 27 | '(-c --clear)'{-c,--clear}'[Clear build directory]' 28 | '(-k --skip-deps)'{-k,--skip-deps}'[Skip dependency resolution]' 29 | '(-g --get)'{-g,--get}'[Only download source packages without building]' 30 | '(-w --write)'{-w,--write}'[Write spec changes back (Need to be specified with -g)]' 31 | '(-r --resume)'{-r,--resume}'[Resume a previous build attempt]:file:' 32 | '(-e --reorder)'{-e,--reorder}'[Reorder the input build list so that it follows the dependency order]' 33 | '(-p --print-tasks)'{-p,--print-tasks}'[Save the resolved build order to the group folder and exit (dry-run)]' 34 | '(- 1 *)'{-h,--help}'[Show this help]' 35 | '*:: :->subcmd' 36 | ) 37 | 38 | _arguments -s : "$flags[@]" 39 | 40 | if [[ "$state" == "subcmd" ]];then 41 | _acbs_comp_package 42 | fi 43 | -------------------------------------------------------------------------------- /completions/acbs-build.fish: -------------------------------------------------------------------------------- 1 | function __acbs_complete_package 2 | set tree (acbs-build -q 'tree:default' 2>/dev/null) 3 | if test $status != 0 4 | return 5 | end 6 | if test -z $tree -o ! -d $tree 7 | return 8 | end 9 | find "$tree/groups/" -maxdepth 1 -mindepth 1 -type f -printf 'groups/%f\n' 10 | if string match -q -- "*/*" "$current" 11 | return 12 | end 13 | find "$tree" -maxdepth 2 -mindepth 2 -type d -not -path "$tree/.git" -printf '%f\n' 14 | end 15 | 16 | function __acbs_complete_tree 17 | set forest (acbs-build -q 'path:conf' 2>/dev/null)/forest.conf 18 | if test $status != 0 19 | return 20 | end 21 | gawk 'match($0,/\[(.*)\]/,m) {print m[1]}' "$forest" 22 | end 23 | 24 | complete -c acbs-build -f 25 | 26 | complete -c acbs-build -s v -l version -d 'Show the version and exit' 27 | complete -c acbs-build -s d -l debug -d 'Increase verbosity to ease debugging process' 28 | complete -x -c acbs-build -s t -l tree -d 'Specify which abbs-tree to use' -a "(__acbs_complete_tree)" 29 | complete -c acbs-build -s q -l query -d 'Do a simple ACBS query' 30 | complete -c acbs-build -s c -l clear -d 'Clear build directory' 31 | complete -c acbs-build -s k -l skip-deps -d 'Skip dependency resolution' 32 | complete -c acbs-build -s g -l get -d 'Only download source packages without building' 33 | complete -c acbs-build -s e -l reorder -d 'Reorder the input build list so that it follows the dependency order' 34 | complete -c acbs-build -s p -l print-tasks -d 'Save the resolved build order to the group folder and exit (dry-run)' 35 | complete -c acbs-build -n "__fish_contains_opt -s g get" -s w -l write -d 'Write spec changes back' 36 | complete -c acbs-build -s r -l resume -d 'Resume a previous build attempt' -a "(__fish_complete_suffix acbs-ckpt)" 37 | complete -c acbs-build -a "(__acbs_complete_package)" 38 | -------------------------------------------------------------------------------- /acbs/checkpoint.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import io 3 | import os 4 | import pickle 5 | import tarfile 6 | import time 7 | from typing import List 8 | 9 | from acbs.base import ACBSPackageInfo, ACBSShrinkWrap 10 | from acbs.const import DPKG_DIR 11 | 12 | 13 | class Hasher(io.IOBase): 14 | def __init__(self): 15 | self.hash_obj = hashlib.new("sha256") 16 | 17 | def write(self, data: bytes): 18 | self.hash_obj.update(data) 19 | 20 | def hexdigest(self): 21 | return self.hash_obj.hexdigest() 22 | 23 | 24 | def checkpoint_spec(package: ACBSPackageInfo) -> str: 25 | f = Hasher() 26 | with tarfile.open(mode='w|', fileobj=f) as tar: 27 | tar.add(os.path.join(package.script_location, '..')) 28 | return f.hexdigest() 29 | 30 | 31 | def checkpoint_dpkg() -> str: 32 | hasher = hashlib.new("sha256") 33 | with open(os.path.join(DPKG_DIR, 'status'), 'rb') as f: 34 | hasher.update(f.read()) 35 | return hasher.hexdigest() 36 | 37 | 38 | def checkpoint_text(packages: List[ACBSPackageInfo]) -> str: 39 | return '\n'.join([package.name for package in packages]) 40 | 41 | 42 | def checkpoint_to_group(packages: List[ACBSPackageInfo], path: str) -> str: 43 | groups = os.path.join(path, 'groups') 44 | if not os.path.isdir(groups): 45 | os.makedirs(groups) 46 | filename = 'acbs-{}'.format(int(time.time())) 47 | with open(os.path.join(groups, filename), 'wt') as f: 48 | f.write(checkpoint_text(packages)) 49 | return filename 50 | 51 | 52 | def do_shrink_wrap(data: ACBSShrinkWrap, path: str) -> str: 53 | # stamp the spec files 54 | for package in data.packages: 55 | data.sps.append(checkpoint_spec(package)) 56 | data.dpkg_state = checkpoint_dpkg() 57 | filename = os.path.join(path, '{}.acbs-ckpt'.format(int(time.time()))) 58 | with open(filename, 'wb') as f: 59 | pickle.dump(data, f) 60 | return filename 61 | -------------------------------------------------------------------------------- /docs/source/dx.rst: -------------------------------------------------------------------------------- 1 | .. dx 2 | 3 | Director's Cut (DX) Version 4 | =========================== 5 | Background 6 | ---------- 7 | As it was shown in the introduction, ACBS was meant to replace the original ABBS utility with better 8 | reliability and functionalities. However, later it was clear that ACBS itself was riddled with numerous 9 | bugs and issues. 10 | 11 | After years of disrepair, the original ACBS was apparently beyond repair. Although the members of the 12 | AOSC community were getting used to the workarounds and quirks of ACBS, a better solution was still 13 | much desired. 14 | 15 | The original author, liushuyu, decided to face on the challenge. It took him what feels like three months 16 | to complete the rewrite, and the result is then the DX version. 17 | 18 | Version Number 19 | -------------- 20 | The DX version still uses the same version schema as the original. The first release of the DX version is 21 | version ``20200615``. 22 | 23 | Behavioral Changes 24 | ------------------ 25 | Since the DX version is a complete re-implementation, there are some changes to its behaviors: 26 | 27 | 1. **Dependency resolution:** The original version uses a dumb algorithm, which will spawn a new thread to build the dependent package before the package you requested. The new version will use the Tarjan search algorithm to determine what and in which order the dependencies should be built in advance. 28 | #. **Parser:** The original version uses bash itself to parse the build files. This is proven to be unreliable. The new version uses ``bashvar.py`` from ``abbs-meta``. 29 | #. **Source fetching:** The new version now name the source files using their checksum hashes to avoid name collisions. The Git repository is now fetched using the "bare repository" mode, and during the checkout phase, the repository is checked out using "detached" mode. In this case, the ``.git`` folder will not exist in the build directory. 30 | #. **Overall reliability:** The new version now uses type checking to ensure basic stability. No type errors are allowed to pass through under the newly set up CI system. 31 | -------------------------------------------------------------------------------- /acbs/base.py: -------------------------------------------------------------------------------- 1 | import re 2 | from dataclasses import dataclass, field 3 | from typing import Dict, List, Optional, Tuple 4 | 5 | from acbs import __version__ 6 | 7 | 8 | @dataclass 9 | class ACBSSourceInfo: 10 | type: str 11 | url: str 12 | revision: Optional[str] = None 13 | branch: Optional[str] = None 14 | depth: Optional[int] = None 15 | chksum: Tuple[str, str] = ('', '') 16 | source_name: Optional[str] = '' 17 | use_url_name: bool = False 18 | # where the source file/folder is located (on local filesystem) 19 | source_location: Optional[str] = None 20 | enabled: bool = True 21 | # copy the repository to the build directory 22 | copy_repo: bool = False 23 | # this is a tristate: 0 - off; 1 - on (non-recursive); 2 - recursive 24 | submodule: int = 2 25 | 26 | 27 | @dataclass 28 | class ACBSPackageInfo: 29 | name: str 30 | deps: List[str] 31 | location: str 32 | source_uri: List[ACBSSourceInfo] 33 | rel: str = '0' 34 | installables: List[str] = field(default_factory=list) 35 | build_location: str = '' 36 | base_slug: str = '' # group slug (like extra-devel/llvm), if any 37 | group_seq: int = 0 # group sequence number 38 | version: str = '' 39 | epoch: str = '' 40 | subdir: Optional[str] = None 41 | fail_arch: Optional[re.Pattern] = None # fail_arch regex 42 | bin_arch: str = '' 43 | script_location: str = field(init=False) # script location (autobuild directory) 44 | exported: Dict[str, str] = field(default_factory=dict) # extra exported variables from spec 45 | modifiers: str = '' # modifiers to be applied to the source file/folder (only available in autobuild4) 46 | 47 | def __post_init__(self): 48 | self.script_location = self.location 49 | 50 | @staticmethod 51 | def is_in_stage2(modifiers: str) -> bool: 52 | return '+stage2' in modifiers.lower() 53 | 54 | 55 | @dataclass 56 | class ACBSShrinkWrap: 57 | cursor: int 58 | timings: List[Tuple[str, float]] 59 | packages: List[ACBSPackageInfo] 60 | no_deps: bool 61 | # spec states 62 | sps: List[str] = field(default_factory=list) 63 | dpkg_state: str = '' 64 | version: str = __version__ 65 | -------------------------------------------------------------------------------- /acbs/magic.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | """ 4 | Fake magic bindings 5 | """ 6 | import subprocess 7 | 8 | # Flag constants for open and setflags 9 | MAGIC_NONE = NONE = 0 10 | MAGIC_DEBUG = DEBUG = 1 11 | MAGIC_SYMLINK = SYMLINK = 2 12 | MAGIC_COMPRESS = COMPRESS = 4 13 | MAGIC_DEVICES = DEVICES = 8 14 | MAGIC_MIME_TYPE = MIME_TYPE = 16 15 | MAGIC_CONTINUE = CONTINUE = 32 16 | MAGIC_CHECK = CHECK = 64 17 | MAGIC_PRESERVE_ATIME = PRESERVE_ATIME = 128 18 | MAGIC_RAW = RAW = 256 19 | MAGIC_ERROR = ERROR = 512 20 | MAGIC_MIME_ENCODING = MIME_ENCODING = 1024 21 | MAGIC_MIME = MIME = 1040 # MIME_TYPE + MIME_ENCODING 22 | MAGIC_APPLE = APPLE = 2048 23 | 24 | MAGIC_NO_CHECK_COMPRESS = NO_CHECK_COMPRESS = 4096 25 | MAGIC_NO_CHECK_TAR = NO_CHECK_TAR = 8192 26 | MAGIC_NO_CHECK_SOFT = NO_CHECK_SOFT = 16384 27 | MAGIC_NO_CHECK_APPTYPE = NO_CHECK_APPTYPE = 32768 28 | MAGIC_NO_CHECK_ELF = NO_CHECK_ELF = 65536 29 | MAGIC_NO_CHECK_TEXT = NO_CHECK_TEXT = 131072 30 | MAGIC_NO_CHECK_CDF = NO_CHECK_CDF = 262144 31 | MAGIC_NO_CHECK_TOKENS = NO_CHECK_TOKENS = 1048576 32 | MAGIC_NO_CHECK_ENCODING = NO_CHECK_ENCODING = 2097152 33 | 34 | MAGIC_NO_CHECK_BUILTIN = NO_CHECK_BUILTIN = 4173824 35 | 36 | 37 | class fakeMagic(object): 38 | 39 | def __init__(self): 40 | self.flags = 0 41 | self.cmd_args = [] 42 | return 43 | 44 | def magic_open(self, flags=0) -> None: 45 | self.flags = flags 46 | return 47 | 48 | def add_cmds(self) -> None: 49 | self.cmd_args = ['file', '-b'] 50 | if (self.flags & MAGIC_MIME): 51 | self.cmd_args.append('-i') 52 | elif (self.flags & MAGIC_MIME_TYPE): 53 | self.cmd_args.append('--mime-type') 54 | elif (self.flags & MAGIC_SYMLINK): 55 | self.cmd_args.append('-L') 56 | elif (self.flags & MAGIC_COMPRESS): 57 | self.cmd_args.append('-z') 58 | 59 | def load(self) -> None: 60 | pass 61 | 62 | def file(self, *args) -> bytes: 63 | self.add_cmds() 64 | self.cmd_args.append(*args) 65 | return subprocess.check_output(self.cmd_args).strip() 66 | 67 | 68 | mgc = fakeMagic() 69 | 70 | 71 | def open(flags: int): 72 | mgc.magic_open(flags) 73 | return mgc 74 | -------------------------------------------------------------------------------- /docs/source/install.rst: -------------------------------------------------------------------------------- 1 | .. how to install 2 | 3 | Installation 4 | ============ 5 | Get Started 6 | ----------- 7 | ACBS could be deployed in any appropriate directories, and is invoked by calling 8 | ``acbs-build.py`` (you may create a symlink for your convenience). You need to install 9 | all the `mandatory dependencies`_ listed below. You may want to create a configuration 10 | file before using ACBS, although this is not a must, but it is highly recommended though. 11 | 12 | You can use package manager to install it if you are running AOSC OS: 13 | 14 | .. code-block:: bash 15 | 16 | sudo apt install acbs 17 | 18 | Or you can get it from source: https://github.com/AOSC-Dev/acbs/ 19 | If you don't want to clone it using ``git``, you can directly download it: https://github.com/AOSC-Dev/acbs/archive/staging.zip 20 | 21 | ------------ 22 | 23 | ACBS uses an INI-like configuration defining trees to be used, the 24 | configuration file should be stored in ``/etc/acbs/forest.conf``. 25 | 26 | More detailed instructions are listed below. 27 | 28 | Requirements 29 | ------------ 30 | .. _Mandatory dependencies: 31 | 32 | Mandatory dependencies: 33 | 34 | * Python 3 (>= 3.6): Running the program itself. 35 | * GNU File (libmagic): File type detection. 36 | * LibArchive (bsdtar): Archive handling. 37 | * GNU Wget or Aria2: Source downloading. 38 | * Autobuild4_: Package building. 39 | 40 | .. _Optional dependencies: 41 | 42 | Optional dependencies [1]_: 43 | 44 | * libmagic: Python module to detect file type. 45 | * pycryptodome: Python module to verify file checksums. 46 | * pexpect: Python module to simulate PTY sessions and log output to file. 47 | 48 | .. _Autobuild4: https://github.com/AOSC-Dev/autobuild4 49 | 50 | .. [1] Note: By installing optional dependencies, functionalities of ACBS could be enhanced. These dependencies are available on PyPI. 51 | 52 | Initial configurations 53 | ---------------------- 54 | The configuration file located in ``/etc/acbs/forest.conf`` is a INI-like file. 55 | 56 | A bare-minimum configuration example is shown below: 57 | 58 | .. code-block:: ini 59 | 60 | [default] 61 | location = /usr/lib/acbs/repo 62 | 63 | 64 | If you are feeling smart, variable substitutions are also supported: 65 | 66 | .. code-block:: ini 67 | 68 | [vars] 69 | base = /mnt 70 | 71 | [default] 72 | location = ${vars:base}/aosc-os-abbs 73 | 74 | By default, ACBS builds packages from the tree defined in the ``[default]`` block. You can override this 75 | behavior by using ``-t `` parameter. 76 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .venv 111 | env/ 112 | venv/ 113 | ENV/ 114 | env.bak/ 115 | venv.bak/ 116 | 117 | # Spyder project settings 118 | .spyderproject 119 | .spyproject 120 | 121 | # Rope project settings 122 | .ropeproject 123 | 124 | # mkdocs documentation 125 | /site 126 | 127 | # mypy 128 | .mypy_cache/ 129 | .dmypy.json 130 | dmypy.json 131 | 132 | # Pyre type checker 133 | .pyre/ 134 | 135 | # pytype static type analyzer 136 | .pytype/ 137 | 138 | # Cython debug symbols 139 | cython_debug/ 140 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ACBS 2 | ==== 3 | 4 | **ACBS is still under heavy development, but is currently deployed for packaging 5 | for AOSC OS.** 6 | 7 | ACBS (AutoBuild CI Build System) is a re-implementation of the original ABBS 8 | (AutoBuild Build Service) in Python. The re-implementation aims to improve 9 | the horrible reliability and agility of its predecessor, adding with: 10 | 11 | - Multi-tree support (a "forest", so to speak). 12 | - Checksum verification support. 13 | - Cache cleaning and management support. 14 | - Logging support. 15 | - Proper dependency calculation (automatic build sequences, useful for 16 | bootstrapped bases). 17 | 18 | Extra blings are also included: 19 | 20 | - Build timing utilities. 21 | - More detailed error messages. 22 | 23 | Dependencies 24 | ------------ 25 | 26 | Mandatory: 27 | - Python 3 (>= 3.6): Running the program itself. 28 | - GNU File (libmagic): File type detection. 29 | - Util-Linux: File checksum verification. 30 | - LibArchive (bsdtar): Archive handling. 31 | - GNU Wget or Aria2: Source downloading. 32 | - [Autobuild4](https://github.com/AOSC-Dev/autobuild4): Package building. 33 | 34 | Optional: 35 | - libmagic: Python module to detect file type. 36 | - libapt-pkg: Query package information. 37 | - libarchive-c: Python module to handle archives. 38 | - pycrypto: Python module to verify file checksums. 39 | - ptyprocess, pexpect: Build logging. 40 | 41 | Deployment 42 | ---------- 43 | 44 | ACBS could be deployed in any appropriate directories, and is invoked by calling 45 | `acbs-build.py` (you may create a symlink for your convenience). You would need 46 | to create a configuration file before using ACBS. 47 | 48 | ACBS uses an INI-like configuration controlling trees to be used, the 49 | configuration file should be stored in `/etc/acbs/forest.conf`. 50 | 51 | A bare-minimal example is shown below: 52 | 53 | ``` 54 | [default] 55 | location = /usr/lib/acbs/repo 56 | ``` 57 | 58 | If you are feeling smart, variable substitutions are also acceptable: 59 | 60 | ``` 61 | [vars] 62 | base = /mnt 63 | 64 | [default] 65 | location = ${vars:base}/aosc-os-abbs 66 | ``` 67 | 68 | By default, ACBS builds packages from the tree defined in the `[default]` block. 69 | 70 | Usage 71 | ----- 72 | 73 | ``` 74 | usage: acbs-build [-h] [-v] [-d] [-t ACBS_TREE] [-q ACBS_QUERY] [-w] [-c] [-k] [-g] [-r STATE_FILE] [packages [packages ...]] 75 | 76 | ACBS - AOSC CI Build System 77 | Version: 20210227 78 | A small alternative system to port abbs to CI environment to prevent from irregular bash failures 79 | 80 | positional arguments: 81 | packages Packages to be built 82 | 83 | optional arguments: 84 | -h, --help show this help message and exit 85 | -v, --version Show the version and exit 86 | -d, --debug Increase verbosity to ease debugging process 87 | -t ACBS_TREE, --tree ACBS_TREE 88 | Specify which abbs-tree to use 89 | -q ACBS_QUERY, --query ACBS_QUERY 90 | Do a simple ACBS query 91 | -w, --write Write spec changes back (Need to be specified with -g) 92 | -c, --clear Clear build directory 93 | -k, --skip-deps Skip dependency resolution 94 | -g, --get Only download source packages without building 95 | -r STATE_FILE, --resume STATE_FILE 96 | Resume a previous build attempt 97 | ``` 98 | -------------------------------------------------------------------------------- /acbs/resume.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pickle 3 | from typing import Dict, List 4 | 5 | from acbs import __version__ 6 | from acbs.base import ACBSPackageInfo, ACBSShrinkWrap 7 | from acbs.checkpoint import checkpoint_dpkg, checkpoint_spec, checkpoint_to_group 8 | from acbs.const import TMP_DIR 9 | from acbs.find import find_package 10 | from acbs.main import BuildCore 11 | from acbs.pm import check_if_installed 12 | from acbs.utils import make_build_dir, print_build_timings, print_package_names 13 | 14 | 15 | def reassign_build_dir(packages: List[ACBSPackageInfo]): 16 | groups: Dict[str, str] = {} 17 | for package in packages: 18 | if package.base_slug: 19 | directory = groups.get(package.base_slug) 20 | if not directory: 21 | directory = make_build_dir(TMP_DIR) 22 | groups[package.base_slug] = directory 23 | package.build_location = directory 24 | continue 25 | package.build_location = '' 26 | return 27 | 28 | 29 | def check_dpkg_state(state: ACBSShrinkWrap, packages: List[ACBSPackageInfo]) -> bool: 30 | if checkpoint_dpkg() == state.dpkg_state: 31 | return True 32 | logging.warning('DPKG state change detected. Re-checking dependencies...') 33 | for package in packages: 34 | if not check_if_installed(package.name): 35 | return False 36 | return True 37 | 38 | 39 | def do_load_checkpoint(name: str) -> ACBSShrinkWrap: 40 | with open(name, 'rb') as f: 41 | return pickle.load(f) 42 | 43 | 44 | def do_resume_checkpoint(filename: str, args): 45 | def resume_build(): 46 | logging.debug('Queue: {}'.format(resumed_packages)) 47 | logging.info('Packages to be resumed: {}'.format( 48 | print_package_names(resumed_packages, 5))) 49 | build_timings = state.timings.copy() 50 | try: 51 | builder.build_sequential(build_timings, resumed_packages) 52 | except Exception as ex: 53 | # failed again? 54 | logging.exception(ex) 55 | builder.save_checkpoint(build_timings, resumed_packages) 56 | print_build_timings(build_timings, []) 57 | 58 | state = do_load_checkpoint(filename) 59 | builder = BuildCore(args) 60 | stage2 = builder.stage2 61 | logging.info('Resuming from {}'.format(filename)) 62 | if state.version != __version__: 63 | logging.warning( 64 | 'The state was check-pointed with a different version of acbs!') 65 | logging.warning('Undefined behavior might occur!') 66 | if state.no_deps: 67 | leftover = state.packages[state.cursor-1:] 68 | logging.warning('Resuming without dependency resolution.') 69 | logging.info('Resumed. {} packages to go.'.format(len(leftover))) 70 | builder.build_sequential(state.timings, leftover) 71 | return 72 | logging.info('Validating status...') 73 | if len(state.packages) != len(state.sps): 74 | raise ValueError( 75 | 'Inconsistencies detected in the saved state! The file might be corrupted.') 76 | resumed_packages = [] 77 | new_cursor = state.cursor - 1 78 | index = 0 79 | for p, v in zip(state.packages, state.sps): 80 | if checkpoint_spec(p) == v: 81 | resumed_packages.append(p) 82 | index += 1 83 | continue 84 | # the spec files changed 85 | if index < new_cursor: 86 | new_cursor = index 87 | resumed_packages.extend(find_package(p.name, builder.tree_dir, '+stage2' if stage2 else '')) 88 | # index doesn't matter now, since changes have been detected 89 | if not check_dpkg_state(state, resumed_packages[:new_cursor]): 90 | name = checkpoint_to_group( 91 | resumed_packages[new_cursor:], builder.tree_dir) 92 | raise RuntimeError( 93 | 'DPKG state mismatch. Unable to resume.\nACBS has created a new temporary group {} for you to continue.'.format(name)) 94 | resumed_packages = resumed_packages[new_cursor:] 95 | # clear the build directory of the first package 96 | reassign_build_dir(resumed_packages) 97 | if new_cursor != (state.cursor - 1): 98 | logging.warning( 99 | 'Scenario mismatch detected! Dependency resolution will be re-attempted.') 100 | resolved = builder.resolve_deps(resumed_packages, stage2) 101 | logging.info( 102 | 'Dependencies resolved, {} packages in the queue'.format(len(resolved))) 103 | resume_build() 104 | return 105 | resume_build() 106 | return 107 | -------------------------------------------------------------------------------- /acbs/deps.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict, defaultdict, deque 2 | from typing import Deque, Dict, List 3 | 4 | from acbs.find import find_package 5 | from acbs.parser import ACBSPackageInfo, check_buildability 6 | 7 | # package information cache 8 | pool: Dict[str, ACBSPackageInfo] = {} 9 | 10 | 11 | def tarjan_search(packages: 'OrderedDict[str, ACBSPackageInfo]', search_path: str, stage2: bool) -> List[List[ACBSPackageInfo]]: 12 | """This function describes a Tarjan's strongly connected components algorithm. 13 | The resulting list of ACBSPackageInfo are sorted topologically as a byproduct of the algorithm 14 | """ 15 | # Initialize state trackers 16 | lowlink: Dict[str, int] = defaultdict(lambda: -1) 17 | index: Dict[str, int] = defaultdict(lambda: -1) 18 | stackstate: Dict[str, bool] = defaultdict(bool) 19 | stack: Deque[str] = deque() 20 | results: List[List[ACBSPackageInfo]] = [] 21 | packages_list: List[str] = [i for i in packages] 22 | pool.update(packages) 23 | for i in packages_list: 24 | if index[i] == -1: # recurse on each package that is not yet visited 25 | strongly_connected(search_path, packages_list, results, packages, 26 | i, lowlink, index, stackstate, stack, stage2) 27 | return results 28 | 29 | 30 | def prepare_for_reorder(package: ACBSPackageInfo, packages_list: List[str]) -> ACBSPackageInfo: 31 | """This function prepares the package for reordering. 32 | The idea is to move the installable dependencies which are in the build list to the "uninstallable" list. 33 | """ 34 | new_installables = [] 35 | for d in package.installables: 36 | # skip self-dependency 37 | if d == package.name: 38 | new_installables.append(d) 39 | continue 40 | try: 41 | packages_list.index(d) 42 | package.deps.append(d) 43 | except ValueError: 44 | new_installables.append(d) 45 | package.installables = new_installables 46 | return package 47 | 48 | 49 | def strongly_connected(search_path: str, packages_list: List[str], results: list, packages: 'OrderedDict[str, ACBSPackageInfo]', vert: str, lowlink: Dict[str, int], index: Dict[str, int], stackstate: Dict[str, bool], stack: Deque[str], stage2: bool, depth=0): 50 | # update depth indices 51 | index[vert] = depth 52 | lowlink[vert] = depth 53 | depth += 1 54 | stackstate[vert] = True 55 | stack.append(vert) 56 | 57 | # search package begin 58 | print(f'[{len(results) + 1}/{len(pool)}] {vert:30}\r', end='', flush=True) 59 | current_package = packages.get(vert) 60 | if current_package is None: 61 | package = pool.get(vert) or find_package(vert, search_path, '+stage2' if stage2 else '') 62 | if not package: 63 | raise ValueError( 64 | f'Package {vert} not found') 65 | if isinstance(package, list): 66 | for s in package: 67 | if vert == s.name: 68 | current_package = s 69 | pool[s.name] = s 70 | continue 71 | pool[s.name] = s 72 | packages_list.append(s.name) 73 | else: 74 | current_package = package 75 | pool[vert] = current_package 76 | assert current_package is not None 77 | # first check if this dependency is buildable 78 | # when `required_by` argument is present, it will raise an exception when the dependency is unbuildable. 79 | check_buildability( 80 | current_package, stack[-2] if len(stack) > 1 else '') 81 | # search package end 82 | # Look for adjacent packages (dependencies) 83 | for p in current_package.deps: 84 | if index[p] == -1: 85 | # recurse on unvisited packages 86 | strongly_connected(search_path, packages_list, results, packages, 87 | p, lowlink, index, stackstate, stack, stage2, depth) 88 | lowlink[vert] = min(lowlink[p], lowlink[vert]) 89 | # adjacent package is in the stack which means it is part of a loop 90 | elif stackstate[p] is True: 91 | lowlink[vert] = min(lowlink[p], index[vert]) 92 | 93 | w = '' 94 | result = [] 95 | # if this is a root vertex 96 | if lowlink[vert] == index[vert]: 97 | # the current stack contains the vertices that belong to the same loop 98 | # if the stack only contains one vertex, then there is no loop there 99 | while w != vert: 100 | w = stack.pop() 101 | result.append(pool[w]) 102 | stackstate[w] = False 103 | results.append(result) 104 | -------------------------------------------------------------------------------- /acbs-build: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | ''' 3 | ACBS - AOSC CI Build System 4 | A small alternative system to port abbs to CI environment to prevent 5 | from irregular bash failures 6 | ''' 7 | import os 8 | import sys 9 | import shutil 10 | import argparse 11 | import acbs 12 | 13 | from acbs.const import TMP_DIR 14 | from acbs.main import BuildCore 15 | from acbs.query import acbs_query 16 | from acbs.resume import do_resume_checkpoint 17 | 18 | 19 | def main() -> None: 20 | parser = argparse.ArgumentParser(description=help_msg(acbs.__version__ 21 | ), formatter_class=argparse.RawDescriptionHelpFormatter) 22 | parser.add_argument('-v', '--version', 23 | help='Show the version and exit', action="version", version=f'ACBS version {acbs.__version__}') 24 | parser.add_argument( 25 | '-d', '--debug', help='Increase verbosity to ease debugging process', action="store_true") 26 | parser.add_argument( 27 | '-e', '--reorder', help='Reorder the input build list so that it follows the dependency order', action="store_true") 28 | parser.add_argument( 29 | '-p', '--print-tasks', help='Save the resolved build order to the group folder and exit (dry-run)', action="store_true", dest='save_list') 30 | parser.add_argument('-t', '--tree', nargs=1, dest='acbs_tree', 31 | help='Specify which abbs-tree to use') 32 | parser.add_argument('-q', '--query', nargs=1, dest='acbs_query', 33 | help='Do a simple ACBS query') 34 | parser.add_argument('-w', '--write', dest='acbs_write', 35 | help='Write spec changes back (Need to be specified with -g)', 36 | action="store_true") 37 | parser.add_argument('packages', nargs='*', help='Packages to be built') 38 | parser.add_argument('-c', '--clear', help='Clear build directory', 39 | action='store_true', dest='clear_dir') 40 | parser.add_argument('-k', '--skip-deps', help='Skip dependency resolution', 41 | action='store_true', dest='no_deps') 42 | parser.add_argument('-g', '--get', 43 | help='Only download source packages without building', action="store_true") 44 | parser.add_argument('-r', '--resume', nargs=1, dest='state_file', 45 | help='Resume a previous build attempt') 46 | parser.add_argument('-l', '--cache-dir', nargs=1, dest='acbs_dump_dir', 47 | help='Override cache directory') 48 | parser.add_argument('-o', '--log-dir', nargs=1, dest='acbs_log_dir', 49 | help='Override log directory') 50 | parser.add_argument('-b', '--tree-dir', nargs=1, dest='acbs_tree_dir', 51 | help='Override abbs tree directory') 52 | parser.add_argument('-z', '--temp-dir', nargs=1, dest='acbs_temp_dir', 53 | help='Override temp directory') 54 | parser.add_argument('--force-use-apt', help="Only use apt to install dependency", action="store_true", dest="force_use_apt") 55 | parser.add_argument('--generate-package-metadata', help="Generate package metadata", action="store_true", dest="generate_pkg_metadata") 56 | 57 | 58 | args = parser.parse_args() 59 | if args.acbs_temp_dir: 60 | tmp_loc = args.acbs_temp_dir[0] 61 | else: 62 | tmp_loc = TMP_DIR 63 | if args.clear_dir: 64 | clear_tmp(tmp_dir=tmp_loc) 65 | del args.clear_dir 66 | if args.acbs_query: 67 | result = acbs_query(args.acbs_query[0]) 68 | if not result: 69 | sys.exit(1) 70 | print(result) 71 | sys.exit(0) 72 | if args.state_file: 73 | do_resume_checkpoint(args.state_file[0], args) 74 | sys.exit(0) 75 | if args.packages: 76 | # the rest of the arguments are passed to the build process 77 | acbs_instance = BuildCore(args) 78 | acbs_instance.build() 79 | # HACK: Workaround a bug in ArgumentParser 80 | if len(sys.argv) < 2: 81 | parser.print_help() 82 | sys.exit(0) 83 | 84 | 85 | def clear_tmp(tmp_dir: str) -> None: 86 | from time import sleep 87 | 88 | def show_progress(): 89 | try: 90 | print(hide_cursor, end='') 91 | while not complete_status: 92 | for bar in ['-', '\\', '|', '/']: 93 | print('\r[%s/%s] Clearing cache...%s' % 94 | (sub_dirs_deleted, sub_dirs_count, bar), end='') 95 | sys.stdout.flush() 96 | sleep(0.1) 97 | print('\r[OK] Clearing cache... %s\n' % 98 | (show_cursor), end='') 99 | except Exception as ex: 100 | print(show_cursor, end='') 101 | raise ex 102 | import threading 103 | hide_cursor = '\033[?25l' 104 | show_cursor = '\033[?25h' 105 | sub_dirs = os.listdir(tmp_dir) 106 | sub_dirs_count = len(sub_dirs) 107 | if not sub_dirs_count: 108 | print('Build directory clean, no need to clear...') 109 | print(show_cursor, end='') 110 | return 111 | sub_dirs_deleted = 0 112 | complete_status = False 113 | indicator = threading.Thread(target=show_progress) 114 | indicator.start() 115 | for dirs in sub_dirs: 116 | shutil.rmtree(os.path.join(tmp_dir, dirs)) 117 | sub_dirs_deleted += 1 118 | complete_status = True 119 | indicator.join() 120 | return 121 | 122 | 123 | def help_msg(acbs_version: str) -> str: 124 | help_msg = f'''ACBS - AOSC CI Build System\nVersion: {acbs_version}\nA small alternative system to port abbs to CI environment to prevent from irregular bash failures''' 125 | return help_msg 126 | 127 | 128 | if __name__ == '__main__': 129 | main() 130 | -------------------------------------------------------------------------------- /docs/source/appendix.rst: -------------------------------------------------------------------------------- 1 | .. appendix 2 | 3 | Appendix 4 | ======== 5 | Supported VCS 6 | ------------- 7 | 8 | +------------------+------------+-------+--------+--------+----------+-----------+---------------------------------------------------------------+ 9 | | VCS | Supported? | Fetch | Update | Branch | Revision | URL Check | Note | 10 | +==================+============+=======+========+========+==========+===========+===============================================================+ 11 | | Git | Y | Y | Y | Y | Y | Y | Automatically fetch submodules, blobless clone when supported | 12 | +------------------+------------+-------+--------+--------+----------+-----------+---------------------------------------------------------------+ 13 | | Mercurial (hg) | Y | Y | Y | Y | Y | Y | | 14 | +------------------+------------+-------+--------+--------+----------+-----------+---------------------------------------------------------------+ 15 | | Subversion (svn) | Y | Y | Y | Y | Y | N | | 16 | +------------------+------------+-------+--------+--------+----------+-----------+---------------------------------------------------------------+ 17 | | Baazar (bzr) | Y | Y | Y | N | Y | N | Some functionalities are not verified | 18 | +------------------+------------+-------+--------+--------+----------+-----------+---------------------------------------------------------------+ 19 | | Fossil (fossil) | Y | Y | Y | N | Y | N | | 20 | +------------------+------------+-------+--------+--------+----------+-----------+---------------------------------------------------------------+ 21 | | BitKeeper (bk) | N | N | N | N | N | N | Not supported | 22 | +------------------+------------+-------+--------+--------+----------+-----------+---------------------------------------------------------------+ 23 | 24 | Acceptable Prefixes for ``SRCS`` 25 | -------------------------------- 26 | 27 | +--------+-----------------------------------------+---------------------+-----------+ 28 | | Prefix | Source Type | Auto deduction [1]_ | Notes | 29 | +--------+-----------------------------------------+---------------------+-----------+ 30 | | git | Git (VCS) | Yes | | 31 | +--------+-----------------------------------------+---------------------+-----------+ 32 | | hg | Mercurial (VCS) | No | | 33 | +--------+-----------------------------------------+---------------------+-----------+ 34 | | svn | Subversion (VCS) | No | | 35 | +--------+-----------------------------------------+---------------------+-----------+ 36 | | bzr | Baazar (VCS) | No | | 37 | +--------+-----------------------------------------+---------------------+-----------+ 38 | | fossil | Fossil (VCS) | No | | 39 | +--------+-----------------------------------------+---------------------+-----------+ 40 | | tbl | Tarball Archives (Remote Files) | Partial | [2]_ [4]_ | 41 | +--------+-----------------------------------------+---------------------+-----------+ 42 | | file | Opaque Binary Blobs (Remote Files) | No | [3]_ | 43 | +--------+-----------------------------------------+---------------------+-----------+ 44 | | pypi | PyPI Package Source Code (Remote Files) | No | | 45 | +--------+-----------------------------------------+---------------------+-----------+ 46 | 47 | Supported Checksum (Hashing) Algorithm 48 | -------------------------------------- 49 | 50 | +-----------+--------------+------------+ 51 | | Algorithm | Recommended? | Notes | 52 | +-----------+--------------+------------+ 53 | | MD2 | N | | 54 | +-----------+--------------+------------+ 55 | | MD5 | N | | 56 | +-----------+--------------+------------+ 57 | | SHA1 | N | | 58 | +-----------+--------------+------------+ 59 | | SHA256 | Y | | 60 | +-----------+--------------+------------+ 61 | | SHA224 | Y | [5]_ | 62 | +-----------+--------------+------------+ 63 | | SHA384 | Y | | 64 | +-----------+--------------+------------+ 65 | | SHA512 | Y | [6]_ | 66 | +-----------+--------------+------------+ 67 | | BLAKE2B | Y | | 68 | +-----------+--------------+------------+ 69 | | BLAKE2S | Y | | 70 | +-----------+--------------+------------+ 71 | | SHA3_224 | Y | [7]_ | 72 | +-----------+--------------+------------+ 73 | | SHA3_256 | Y | | 74 | +-----------+--------------+------------+ 75 | | SHA3_384 | Y | | 76 | +-----------+--------------+------------+ 77 | | SHA3_512 | Y | [8]_ | 78 | +-----------+--------------+------------+ 79 | 80 | .. [1] Auto deduction means if the prefix is omitted, whether ``acbs`` would try to deduce the missing prefix 81 | .. [2] Will attempt to extract the archive 82 | .. [3] Will leave the files as-is, implies ``SUBDIR='.'`` 83 | .. [4] Only when the extension name contains ``.tar`` or ``.zip`` 84 | .. [5] Although recommended, please consider using ``SHA256`` or better 85 | .. [6] Although recommended, the hash sum is just too long. Currently, ``SHA256`` is sufficient enough. 86 | .. [7] Although recommended, please consider using ``SHA3_256`` or better 87 | .. [8] Although recommended, the hash sum is just too long. Currently, ``SHA3_256`` is sufficient enough. 88 | -------------------------------------------------------------------------------- /docs/source/spec_format.rst: -------------------------------------------------------------------------------- 1 | .. format of spec file 2 | 3 | Specification Files 4 | ===================================================== 5 | defines 6 | ----------- 7 | Defines files are expected to exist in ``:/autobuild/``. ``defines`` files are usually 8 | processed by ``autobuild`` script, however ``acbs`` also use this file to determine 9 | the building order of a given set of packages. 10 | 11 | ``defines`` file MUST contain the following variables: 12 | 13 | * ``PKGNAME`` The name of the package 14 | * ``PKGSEC`` The section/group/"genre" of the package 15 | * ``PKGDES`` The brief description of the package 16 | 17 | This file may also include the following variables: 18 | 19 | * ``PKGDEP`` The mandatory runtime requirements/dependencies 20 | * ``BUILDDEP`` The mandatory compile-time requirements/dependencies 21 | * ``PKGRECOM`` The optional runtime requirements/dependencies (for enhancing UX or add new features) 22 | * ``EPOCH`` The epoch version number of the package 23 | 24 | This file might also include ``autobuild`` specific controlling values. 25 | Consult Autobuild3_ for more information. 26 | 27 | spec 28 | ----------- 29 | Specification (spec) files are expected to exist in ``:/`` (root of the top project folder). 30 | ``defines`` files are solely processed by ``acbs`` to fetch source files and control 31 | ``acbs`` how to transfer controls to ``autobuild``. 32 | 33 | ``spec`` file MUST contain the following variables: 34 | 35 | * ``VER`` The version of the package, it might be not in semantic versioning scheme. 36 | 37 | ``spec`` file SHOULD ONLY contain ONE of the following variables: 38 | 39 | * ``SRCS`` Expected format: ``:::: :: ...`` See footnote [1]_ for details about particular behavior. 40 | * ``DUMMYSRC`` (Bool) If set to 1, indicates this package does not require source files or source files processing cannot be handled well by current version of ``acbs``. 41 | 42 | ``spec`` file may also contain the following variables: 43 | 44 | * ``CHKSUMS`` Expected format: ``:: :: ...`` If set, ``acbs`` will check the checksum of the source files against this value not available if the source is from VCS. [2]_ 45 | * ``SUBDIR`` If set, ``acbs`` will change to specified directory after finishing preparing the source files. (For a list of supported hashing algorithms, see :doc:`appendix`) 46 | * ``SRCTBL`` (String) **[Deprecated]** If set, indicates this package requires "zipped" or archived source files. 47 | * ``SRC`` **[Deprecated]** If set, indicates required source files for this package are in a version controlled repository. (For a list of supported VCS systems, see :doc:`appendix`) 48 | * ``BRCH`` **[Deprecated]** If set, indicates required branch of the repository for the package. 49 | * ``COMMIT`` **[Deprecated]** If set, indicates required commit/revision of the repository for the package. 50 | * ``CHKSUM`` **[Deprecated]** Expected format: ``::`` If set, ``acbs`` will check the checksum of the source file against this value can be omitted if the source is from VCS. 51 | 52 | Details about the ``SRCS`` format: 53 | 54 | * Each source specification in the array accepts one, two or three parameters: 55 | #. One parameter only: ```` 56 | #. Two parameters: ``::`` 57 | #. Three parameters: ``::::`` 58 | 59 | * Currently supported options: 60 | * ``branch``: Name of the branch 61 | * ``commit``: Commit hash 62 | * ``rename``: Rename the source file (including extension name if any) 63 | * ``submodule``: Automatically fetch submodules in the repository. 64 | 65 | * ``true``: Fetch submodules but not recursively (submodules in the submodules are not fetched). 66 | * ``false``: Do not fetch submodules. 67 | * ``recursive``: [Default] Fetch submodules recursively. 68 | 69 | * ``copy-repo``: Automatically copy VCS metadata to the build directory. 70 | 71 | * ``true``: Copy VCS metadata prior to the building process, replaces ``acbs_copy_git``. 72 | * ``false``: [Default] Do not copy VCS metadata. However you can still use ``acbs_copy_git``. 73 | 74 | To specify multiple options, you can join the options with semicolons (``;``) like this: 75 | 76 | .. code-block:: bash 77 | 78 | SRCS="git::rename=lmms-git;commit=94363be::https://github.com/LMMS/lmms" 79 | 80 | The snippet above will make ``acbs`` rename the source directory to ``lmms-git`` and checkout the commit ``94363be``. 81 | 82 | Multiarch Manipulation 83 | ----------- 84 | 85 | In some certain scenarios, packagers may have needs to specify different sources or dependencies for different architectures. Therefore, ACBS provides architecture-specific variables support. Such variables look like below: 86 | 87 | .. code-block:: bash 88 | __= 89 | 90 | Details about architecture-specific variables: 91 | * ```` supports the following values: 92 | * ``SRCS`` and ``CHKSUM`` in ``specs``. Note that different ``SRCS__`` requires different ``CHKSUM__``, otherwise ACBS may refuse to build. 93 | * ``PKGDEP`` and ``BUILDDEP`` in ``autobuild/defines``. 94 | 95 | * ````: By default, ACBS will automatically determine the current architecture of building environment, and replaces ```` with it. Users can manually specify ```` by setting the environment variable ``$CROSS`` or ``$ARCH`` to the desired architecture. 96 | 97 | * If no ``__`` found, ACBS will fallback to use the default value ````. 98 | 99 | .. _Autobuild3: https://wiki.aosc.io/developer/packaging/autobuild3-manual/#the-defines-file 100 | .. [1] Example: 101 | 102 | .. code-block:: bash 103 | 104 | SRCS="git::git://github.com/AOSC-Dev/acbs git::https://github.com/AOSC-Dev/acbs" 105 | 106 | This will make ``acbs`` to download two sets of source files 107 | 108 | .. [2] Example: 109 | 110 | .. code-block:: bash 111 | 112 | CHKSUMS="sha1::a9c55882c935300bec93e209f1ec8a21f75638b7 sha256::4ccdbbd95d4aef058502c8ee07b1abb490f5ef4a4d6ff711440facd0b8eded33" 113 | 114 | This will make ``acbs`` to check two sets of source files 115 | -------------------------------------------------------------------------------- /acbs/pm.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | import subprocess 4 | from typing import Dict, List, Optional 5 | 6 | from acbs.base import ACBSPackageInfo 7 | from acbs.utils import get_arch_name 8 | 9 | installed_cache: Dict[str, bool] = {} 10 | available_cache: Dict[str, bool] = {} 11 | use_native_bindings: bool = True 12 | reorder_mode: bool = False 13 | 14 | try: 15 | from acbs.miniapt_query import apt_init_system 16 | from acbs.miniapt_query import check_if_available as apt_check_if_available 17 | if not apt_init_system(): 18 | raise ImportError('Initialization failure.') 19 | except ImportError: 20 | use_native_bindings = False 21 | apt_init_system = None 22 | apt_check_if_available = None 23 | 24 | 25 | def enable_reorder_mode(enable: Optional[bool]=None) -> bool: 26 | global reorder_mode 27 | if enable is not None: 28 | reorder_mode = enable 29 | return reorder_mode 30 | 31 | 32 | def filter_dependencies(package: ACBSPackageInfo) -> ACBSPackageInfo: 33 | installables = [] 34 | deps = [] 35 | for dep in package.deps: 36 | if check_if_installed(dep): 37 | if reorder_mode: 38 | # HACK: when reordering dependencies, we need to pretend that they needs to be installed 39 | installables.append(dep) 40 | continue 41 | if check_if_available(dep): 42 | installables.append(dep) 43 | continue 44 | deps.append(dep) 45 | package.deps = deps 46 | package.installables = installables 47 | return package 48 | 49 | 50 | def escape_package_name(name: str) -> str: 51 | return re.sub(r'([+*?])', '\\\\\\1', name) 52 | 53 | 54 | def escape_package_name_install(name: str) -> str: 55 | escaped = escape_package_name(name) 56 | if escaped.endswith('+') or escaped.endswith('-'): 57 | return f'{escaped}+' 58 | return escaped 59 | 60 | 61 | def fix_pm_states(escaped: List[str]): 62 | count = 0 63 | while count < 3: 64 | try: 65 | subprocess.call(['dpkg', '--configure', '-a']) 66 | subprocess.check_call(['apt-get', 'install', '-yf']) 67 | if escaped: 68 | command = ['apt-get', 'install', '-y'] 69 | command.extend(escaped) 70 | subprocess.check_call(command, env={'DEBIAN_FRONTEND': 'noninteractive'}) 71 | return 72 | except subprocess.CalledProcessError: 73 | count += 1 74 | continue 75 | raise RuntimeError('Unable to correct package manager states...') 76 | 77 | 78 | def check_if_installed(name: str) -> bool: 79 | logging.debug('Checking if %s is installed' % name) 80 | cached = installed_cache.get(name) 81 | if cached is not None: 82 | return cached 83 | if use_native_bindings: 84 | assert callable(apt_check_if_available) 85 | logging.debug('... using libapt-pkg') 86 | result = apt_check_if_available(name) 87 | if result == 0: 88 | installed_cache[name] = True 89 | return True 90 | elif result == 1: 91 | installed_cache[name] = False 92 | available_cache[name] = True 93 | return False 94 | elif result == 2: 95 | installed_cache[name] = False 96 | available_cache[name] = False 97 | return False 98 | elif result == -4: 99 | fix_pm_states([]) 100 | return check_if_installed(name) 101 | else: 102 | raise RuntimeError(f'libapt-pkg binding returned error: {result}') 103 | try: 104 | subprocess.check_output(['dpkg', '-s', name], stderr=subprocess.STDOUT) 105 | installed_cache[name] = True 106 | return True 107 | except subprocess.CalledProcessError: 108 | installed_cache[name] = False 109 | return False 110 | 111 | 112 | def check_if_available(name: str) -> bool: 113 | logging.debug('Checking if %s is available' % name) 114 | cached = available_cache.get(name) 115 | if cached is not None: 116 | return cached 117 | if use_native_bindings: 118 | assert callable(apt_check_if_available) 119 | logging.debug('... using libapt-pkg') 120 | if apt_check_if_available(name) != 1: 121 | return False 122 | try: 123 | subprocess.check_output( 124 | ['apt-cache', 'show', escape_package_name(name)], stderr=subprocess.STDOUT) 125 | logging.debug('Checking if %s can be installed' % name) 126 | subprocess.check_output( 127 | ['apt-get', 'install', '-s', name], stderr=subprocess.STDOUT, env={'DEBIAN_FRONTEND': 'noninteractive'}) 128 | available_cache[name] = True 129 | return True 130 | except subprocess.CalledProcessError: 131 | available_cache[name] = False 132 | return False 133 | 134 | 135 | def install_from_repo(packages: List[str], force_use_apt=False): 136 | # FIXME: RISC-V build hosts is unreliable when using oma: random lock-ups 137 | # during `oma refresh'. Disabling oma to workaround potential lock-ups. 138 | if get_arch_name() == "riscv64" or force_use_apt: 139 | return install_from_repo_apt(packages) 140 | 141 | return install_from_repo_oma(packages) or install_from_repo_apt(packages) 142 | 143 | 144 | def install_from_repo_apt(packages: List[str]): 145 | logging.debug('Installing %s' % packages) 146 | escaped = [] 147 | for package in packages: 148 | escaped.append(escape_package_name_install(package)) 149 | command = ['apt-get', 'install', '-y', '-o', 'Dpkg::Options::=--force-confnew'] 150 | command.extend(escaped) 151 | try: 152 | subprocess.check_call(command, env={'DEBIAN_FRONTEND': 'noninteractive'}) 153 | except subprocess.CalledProcessError: 154 | logging.warning( 155 | 'Failed to install dependencies, attempting to correct issues...') 156 | fix_pm_states(escaped) 157 | return 158 | 159 | 160 | def install_from_repo_oma(packages: List[str]) -> bool: 161 | logging.debug('Installing %s from oma' % packages) 162 | command = ['oma', 'install', '-y', '--force-confnew', '--no-progress', '--force-unsafe-io', '--no-bell'] 163 | command.extend(packages) 164 | try: 165 | subprocess.check_call(command) 166 | except subprocess.CalledProcessError: 167 | logging.warning( 168 | 'Failed to use oma install dependencies, fallbacking to apt...') 169 | return False 170 | return True 171 | -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import unittest.mock 3 | 4 | import acbs.find 5 | import acbs.parser 6 | import acbs.pm 7 | from acbs.const import TMP_DIR 8 | from acbs.deps import tarjan_search 9 | from acbs.parser import get_deps_graph, parse_url_schema 10 | from acbs.utils import fail_arch_regex, guess_extension_name, make_build_dir 11 | 12 | 13 | def fake_pm(package): 14 | return package 15 | 16 | 17 | def check_scc(resolved): 18 | error = False 19 | for dep in resolved: 20 | if len(dep) > 1 or dep[0].name in dep[0].deps: 21 | # this is a SCC, aka a loop 22 | error = True 23 | elif not error: 24 | continue 25 | return error 26 | 27 | 28 | def find_package_generic(name: str): 29 | acbs.parser.arch = 'none' 30 | acbs.parser.filter_dependencies = fake_pm 31 | make_build_dir_mock = unittest.mock.Mock( 32 | spec=make_build_dir, return_value='/tmp/') 33 | acbs.find.make_build_dir = make_build_dir_mock 34 | return acbs.find.find_package(name, './tests/', modifiers=''), make_build_dir_mock 35 | 36 | 37 | class TestParser(unittest.TestCase): 38 | def test_parse_no_arch(self): 39 | acbs.parser.arch = 'none' 40 | acbs.parser.filter_dependencies = fake_pm 41 | package = acbs.parser.parse_package( 42 | './tests/fixtures/test-1/autobuild', modifiers='') 43 | self.assertEqual(package.deps, ['test-2', 'test-3', 'test-4']) 44 | self.assertEqual(package.version, '1') 45 | self.assertEqual(package.source_uri[0].type, 'none') 46 | 47 | def test_parse_arch(self): 48 | acbs.parser.arch = 'arch' 49 | acbs.parser.filter_dependencies = fake_pm 50 | package = acbs.parser.parse_package( 51 | './tests/fixtures/test-1/autobuild', modifiers='') 52 | self.assertEqual(package.deps, ['test-2', 'test-3', 'test-17']) 53 | self.assertEqual(package.version, '1') 54 | self.assertEqual(package.source_uri[0].type, 'none') 55 | 56 | def test_deps_loop(self): 57 | acbs.parser.arch = 'arch' 58 | acbs.parser.filter_dependencies = fake_pm 59 | package = acbs.parser.parse_package( 60 | './tests/fixtures/test-3/autobuild', modifiers='') 61 | packages = tarjan_search(get_deps_graph([package]), './tests', stage2=False) 62 | error = check_scc(packages) 63 | self.assertEqual(error, True) 64 | 65 | def test_deps_loop_bidirection(self): 66 | acbs.parser.arch = 'arch' 67 | acbs.parser.filter_dependencies = fake_pm 68 | package = acbs.parser.parse_package( 69 | './tests/fixtures/test-5/autobuild', modifiers='') 70 | package2 = acbs.parser.parse_package( 71 | './tests/fixtures/test-6/autobuild', modifiers='') 72 | packages = tarjan_search(get_deps_graph([package, package2]), './tests', stage2=False) 73 | error = check_scc(packages) 74 | self.assertEqual(error, True) 75 | 76 | def test_fail_arch(self): 77 | import re 78 | self.assertEqual(re.compile("^(?!amd64)"), fail_arch_regex("!amd64")) 79 | self.assertEqual(re.compile("^amd64"), fail_arch_regex("amd64")) 80 | self.assertEqual(re.compile("^(amd64|arm64)"), fail_arch_regex("(amd64|arm64)")) 81 | self.assertEqual(re.compile("^(?!amd64|arm64)"), fail_arch_regex("!(amd64|arm64)")) 82 | 83 | def test_parse_url(self): 84 | info = parse_url_schema('tbl::https://example.com', 'sha256::123') 85 | self.assertEqual(info.type, 'tarball') 86 | self.assertEqual(info.url, 'https://example.com') 87 | self.assertEqual(info.chksum, ('sha256', '123')) 88 | info = parse_url_schema('https://example.com/test.tar.gz', 'sha256::123') 89 | self.assertEqual(info.type, 'tarball') 90 | self.assertEqual(info.url, 'https://example.com/test.tar.gz') 91 | self.assertEqual(info.chksum, ('sha256', '123')) 92 | info = parse_url_schema('git://github.com/AOSC-Dev/acbs', 'SKIP') 93 | self.assertEqual(info.type, 'git') 94 | self.assertEqual(info.url, 'git://github.com/AOSC-Dev/acbs') 95 | self.assertEqual(info.chksum, ('none', '')) 96 | info = parse_url_schema('git::commit=abcdef::git://github.com/AOSC-Dev/acbs', 'SKIP') 97 | self.assertEqual(info.type, 'git') 98 | self.assertEqual(info.url, 'git://github.com/AOSC-Dev/acbs') 99 | self.assertEqual(info.revision, 'abcdef') 100 | self.assertEqual(info.chksum, ('none', '')) 101 | info = parse_url_schema('git::commit=a2e5eff::https://github.com/AOSC-Dev/acbs#title', 'SKIP') 102 | self.assertEqual(info.type, 'git') 103 | self.assertEqual(info.url, 'https://github.com/AOSC-Dev/acbs#title') 104 | self.assertEqual(info.revision, 'a2e5eff') 105 | self.assertEqual(info.chksum, ('none', '')) 106 | info = parse_url_schema('tbl::use-url-name=true::https://example.com/test.tar.gz;p=123?test=ok#fragment', 'sha256::123') 107 | self.assertEqual(info.type, 'tarball') 108 | self.assertEqual(info.source_name, 'test.tar.gz') 109 | 110 | def test_parse_new_spec(self): 111 | acbs.parser.arch = 'none' 112 | acbs.parser.filter_dependencies = fake_pm 113 | package = acbs.parser.parse_package( 114 | './tests/fixtures/test-4/autobuild', modifiers='') 115 | self.assertEqual(package.deps, ['test-4']) 116 | self.assertEqual(package.version, '1') 117 | self.assertEqual(package.source_uri[0].type, 'git') 118 | self.assertEqual(package.source_uri[1].type, 'git') 119 | 120 | 121 | class TestSearching(unittest.TestCase): 122 | def test_basic_find(self): 123 | result, _ = find_package_generic('test-1') 124 | self.assertEqual(len(result), 1) 125 | 126 | def test_basic_prefixed_find(self): 127 | result, _ = find_package_generic('fixtures/test-1') 128 | self.assertEqual(len(result), 1) 129 | 130 | def test_subpackage_find(self): 131 | result, make_build_dir_mock = find_package_generic('sub-1') 132 | make_build_dir_mock.assert_called_once_with(TMP_DIR) 133 | self.assertEqual(len(result), 2) 134 | 135 | def test_group_expand(self): 136 | result, make_build_dir_mock = find_package_generic('test-2') 137 | make_build_dir_mock.assert_called_once_with(TMP_DIR) 138 | self.assertEqual(len(result), 2) 139 | 140 | def test_group_prefixed_expand(self): 141 | result, make_build_dir_mock = find_package_generic('fixtures/test-2') 142 | make_build_dir_mock.assert_called_once_with(TMP_DIR) 143 | self.assertEqual(len(result), 2) 144 | 145 | 146 | class TestMisc(unittest.TestCase): 147 | def test_apt_name_escaping(self): 148 | self.assertEqual(acbs.pm.escape_package_name('test++'), 'test\\+\\+') 149 | self.assertEqual(acbs.pm.escape_package_name('test+-'), 'test\\+-') 150 | 151 | def test_apt_install_escaping(self): 152 | self.assertEqual(acbs.pm.escape_package_name_install('test++'), 'test\\+\\++') 153 | self.assertEqual(acbs.pm.escape_package_name_install('test+-'), 'test\\+-+') 154 | 155 | def test_guess_extension_name(self): 156 | self.assertEqual(guess_extension_name('test-1.2.3.tar.gz'), '.tar.gz') 157 | self.assertEqual(guess_extension_name('test-1.2.3.bin'), '.bin') 158 | self.assertEqual(guess_extension_name('test'), '') 159 | 160 | 161 | if __name__ == '__main__': 162 | unittest.main() 163 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 16 | 17 | .PHONY: help 18 | help: 19 | @echo "Please use \`make ' where is one of" 20 | @echo " html to make standalone HTML files" 21 | @echo " dirhtml to make HTML files named index.html in directories" 22 | @echo " singlehtml to make a single large HTML file" 23 | @echo " pickle to make pickle files" 24 | @echo " json to make JSON files" 25 | @echo " htmlhelp to make HTML files and a HTML help project" 26 | @echo " qthelp to make HTML files and a qthelp project" 27 | @echo " applehelp to make an Apple Help Book" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " epub3 to make an epub3" 31 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 32 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 33 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 34 | @echo " text to make text files" 35 | @echo " man to make manual pages" 36 | @echo " texinfo to make Texinfo files" 37 | @echo " info to make Texinfo files and run them through makeinfo" 38 | @echo " gettext to make PO message catalogs" 39 | @echo " changes to make an overview of all changed/added/deprecated items" 40 | @echo " xml to make Docutils-native XML files" 41 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 42 | @echo " linkcheck to check all external links for integrity" 43 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 44 | @echo " coverage to run coverage check of the documentation (if enabled)" 45 | @echo " dummy to check syntax errors of document sources" 46 | 47 | .PHONY: clean 48 | clean: 49 | rm -rf $(BUILDDIR)/* 50 | 51 | .PHONY: html 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | .PHONY: dirhtml 58 | dirhtml: 59 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 60 | @echo 61 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 62 | 63 | .PHONY: singlehtml 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | .PHONY: pickle 70 | pickle: 71 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 72 | @echo 73 | @echo "Build finished; now you can process the pickle files." 74 | 75 | .PHONY: json 76 | json: 77 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 78 | @echo 79 | @echo "Build finished; now you can process the JSON files." 80 | 81 | .PHONY: htmlhelp 82 | htmlhelp: 83 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 84 | @echo 85 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 86 | ".hhp project file in $(BUILDDIR)/htmlhelp." 87 | 88 | .PHONY: qthelp 89 | qthelp: 90 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 91 | @echo 92 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 93 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 94 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/acbs.qhcp" 95 | @echo "To view the help file:" 96 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/acbs.qhc" 97 | 98 | .PHONY: applehelp 99 | applehelp: 100 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 101 | @echo 102 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 103 | @echo "N.B. You won't be able to view it unless you put it in" \ 104 | "~/Library/Documentation/Help or install it in your application" \ 105 | "bundle." 106 | 107 | .PHONY: devhelp 108 | devhelp: 109 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 110 | @echo 111 | @echo "Build finished." 112 | @echo "To view the help file:" 113 | @echo "# mkdir -p $$HOME/.local/share/devhelp/acbs" 114 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/acbs" 115 | @echo "# devhelp" 116 | 117 | .PHONY: epub 118 | epub: 119 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 120 | @echo 121 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 122 | 123 | .PHONY: epub3 124 | epub3: 125 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 126 | @echo 127 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 128 | 129 | .PHONY: latex 130 | latex: 131 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 132 | @echo 133 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 134 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 135 | "(use \`make latexpdf' here to do that automatically)." 136 | 137 | .PHONY: latexpdf 138 | latexpdf: 139 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 140 | @echo "Running LaTeX files through pdflatex..." 141 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 142 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 143 | 144 | .PHONY: latexpdfja 145 | latexpdfja: 146 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 147 | @echo "Running LaTeX files through platex and dvipdfmx..." 148 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 149 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 150 | 151 | .PHONY: text 152 | text: 153 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 154 | @echo 155 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 156 | 157 | .PHONY: man 158 | man: 159 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 160 | @echo 161 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 162 | 163 | .PHONY: texinfo 164 | texinfo: 165 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 166 | @echo 167 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 168 | @echo "Run \`make' in that directory to run these through makeinfo" \ 169 | "(use \`make info' here to do that automatically)." 170 | 171 | .PHONY: info 172 | info: 173 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 174 | @echo "Running Texinfo files through makeinfo..." 175 | make -C $(BUILDDIR)/texinfo info 176 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 177 | 178 | .PHONY: gettext 179 | gettext: 180 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 181 | @echo 182 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 183 | 184 | .PHONY: changes 185 | changes: 186 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 187 | @echo 188 | @echo "The overview file is in $(BUILDDIR)/changes." 189 | 190 | .PHONY: linkcheck 191 | linkcheck: 192 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 193 | @echo 194 | @echo "Link check complete; look for any errors in the above output " \ 195 | "or in $(BUILDDIR)/linkcheck/output.txt." 196 | 197 | .PHONY: doctest 198 | doctest: 199 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 200 | @echo "Testing of doctests in the sources finished, look at the " \ 201 | "results in $(BUILDDIR)/doctest/output.txt." 202 | 203 | .PHONY: coverage 204 | coverage: 205 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 206 | @echo "Testing of coverage in the sources finished, look at the " \ 207 | "results in $(BUILDDIR)/coverage/python.txt." 208 | 209 | .PHONY: xml 210 | xml: 211 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 212 | @echo 213 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 214 | 215 | .PHONY: pseudoxml 216 | pseudoxml: 217 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 218 | @echo 219 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 220 | 221 | .PHONY: dummy 222 | dummy: 223 | $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy 224 | @echo 225 | @echo "Build finished. Dummy builder generates no files." 226 | -------------------------------------------------------------------------------- /acbs/bashvar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import collections 5 | import logging 6 | import re 7 | import subprocess 8 | import tempfile 9 | import warnings 10 | 11 | import pyparsing as pp 12 | 13 | from typing import List, Optional, OrderedDict 14 | 15 | pp.ParserElement.enablePackrat() 16 | 17 | re_variable = re.compile('^\\s*([a-zA-Z_][a-zA-Z0-9_]*)=') 18 | 19 | whitespace = pp.White(ws=' \t').suppress().setName("whitespace") 20 | optwhitespace = pp.Optional(whitespace).setName("optwhitespace") 21 | comment = ('#' + pp.CharsNotIn('\n')).setName("comment") 22 | integer = (pp.Word(pp.nums) | pp.Combine('-' + pp.Word(pp.nums))) 23 | 24 | varname = pp.Word(pp.alphas + '_', pp.alphanums + 25 | '_').setResultsName("varname") 26 | # ${parameter/pattern/string} 27 | substsafe = pp.CharsNotIn('/#%*?[}\'"`\\') 28 | 29 | expansion_param = pp.Group( 30 | pp.Literal('$').setResultsName("expansion") + 31 | # we don't want to parse all the expansions 32 | (( 33 | pp.Literal('{').suppress() + 34 | varname + 35 | pp.Optional( 36 | (pp.Literal(':').setResultsName("exptype") + pp.Group( 37 | pp.Word(pp.nums) | 38 | (whitespace + pp.Combine('-' + pp.Word(pp.nums))) 39 | ).setResultsName("offset") + 40 | pp.Optional(pp.Literal(':') + integer.setResultsName("length"))) 41 | ^ (pp.oneOf('/ // /# /%').setResultsName("exptype") + 42 | pp.Optional(substsafe.setResultsName("pattern") + 43 | pp.Optional(pp.Literal('/') + 44 | pp.Optional(substsafe, '').setResultsName("string"))) 45 | ) 46 | ^ (pp.oneOf("# ## % %%").setResultsName("exptype") + 47 | pp.Optional(substsafe.setResultsName("pattern")) 48 | ) 49 | ) + 50 | pp.Literal('}').suppress() 51 | ) | varname) 52 | ) 53 | 54 | singlequote = pp.Group( 55 | pp.Literal("'").setResultsName("quote") + 56 | pp.Optional(pp.CharsNotIn("'"), '').setResultsName("value") + 57 | pp.Literal("'").suppress() 58 | ).setName("singlequote") 59 | doublequote_escape = ( 60 | (pp.Literal('\\').suppress() + pp.Word('$`"\\', exact=1)) | 61 | pp.Literal('\\\n').suppress() 62 | ) 63 | doublequote = pp.Group( 64 | pp.Literal('"').setResultsName("quote") + 65 | pp.Group(pp.ZeroOrMore( 66 | doublequote_escape | expansion_param | pp.CharsNotIn('$`\\*"') 67 | )).setResultsName("value") + 68 | pp.Literal('"').suppress() 69 | ).setName("doublequote") 70 | 71 | texttoken = ( 72 | singlequote | doublequote | expansion_param | 73 | pp.CharsNotIn('~{}()$\'"`\\*?[] \t\n') 74 | ) 75 | varvalue = pp.Group(pp.ZeroOrMore(texttoken)).setResultsName('varvalue') 76 | varassign = ( 77 | varname + 78 | (pp.Literal('=') | pp.Literal('+=')).setResultsName('operator') + 79 | varvalue 80 | ).setName('varassign').leaveWhitespace() 81 | 82 | line = pp.Group( 83 | pp.lineStart + optwhitespace + 84 | pp.Optional(varassign) + optwhitespace + 85 | pp.Optional(comment).suppress() + 86 | pp.lineEnd.suppress() 87 | ).setName('line').leaveWhitespace() 88 | 89 | bashvarfile = pp.ZeroOrMore(line) 90 | 91 | ParseException = pp.ParseException 92 | 93 | 94 | class VariableWarning(UserWarning): 95 | pass 96 | 97 | 98 | class BashErrorWarning(UserWarning): 99 | pass 100 | 101 | 102 | class ParseError(Exception): 103 | pass 104 | 105 | 106 | def combine_value(tokens, variables): 107 | val = '' 108 | if tokens.get('quote') == '"': 109 | val += combine_value(tokens['value'], variables) 110 | elif tokens.get('quote') == "'": 111 | val += tokens['value'] 112 | elif tokens.get('expansion') == '$': 113 | varname = tokens['varname'] 114 | if varname in variables: 115 | var = variables[varname] 116 | exptype = tokens.get('exptype') 117 | if exptype is None: 118 | pass 119 | elif exptype == ':': 120 | if 'offset' in tokens: 121 | offset = int(tokens['offset'][0].strip()) 122 | if 'length' in tokens: 123 | length = int(tokens['length']) 124 | if length >= 0: 125 | var = var[offset:offset+length] 126 | else: 127 | var = var[offset:length] 128 | else: 129 | var = var[offset:] 130 | elif exptype[0] == '/': 131 | pattern = tokens.get('pattern', '') 132 | newstring = tokens.get('string', '') 133 | if exptype == '/': 134 | var = var.replace(pattern, newstring, 1) 135 | elif exptype == '//': 136 | var = var.replace(pattern, newstring) 137 | elif exptype == '/#': 138 | if var.startswith(pattern): 139 | var = newstring + var[len(pattern):] 140 | elif var.endswith(pattern): # /% 141 | var = var[:-len(pattern)] + newstring 142 | elif exptype[0] == '#': 143 | pattern = tokens.get('pattern', '') 144 | if var.startswith(pattern): 145 | var = var[len(pattern):] 146 | elif exptype[0] == '%': 147 | pattern = tokens.get('pattern', '') 148 | if var.endswith(pattern): 149 | var = var[:-len(pattern)] 150 | val += var 151 | else: 152 | warnings.warn('variable "%s" is undefined' % 153 | varname, VariableWarning) 154 | else: 155 | for tok in tokens: 156 | if isinstance(tok, str): 157 | val += tok 158 | else: 159 | val += combine_value(tok, variables) 160 | return ''.join(val) 161 | 162 | 163 | def eval_bashvar_literal(source: str): 164 | parsed = bashvarfile.parseString(source, parseAll=True) 165 | variables: OrderedDict[str, str] = collections.OrderedDict() 166 | for line in parsed: 167 | if not line: 168 | continue 169 | val = combine_value(line['varvalue'], variables) 170 | if line['operator'] == '=': 171 | variables[line['varname']] = val 172 | elif line['operator'] == '+=': 173 | if line['varname'] in variables: 174 | variables[line['varname']] += val 175 | else: 176 | warnings.warn( 177 | 'variable "%s" is undefined' % line['varname'], VariableWarning) 178 | variables[line['varname']] = val 179 | return variables 180 | 181 | 182 | def uniq(seq): # Dave Kirby 183 | # Order preserving 184 | seen = set() 185 | return [x for x in seq if x not in seen and not seen.add(x)] 186 | 187 | 188 | def eval_bashvar_ext(source: str, filename: Optional[str]=None): 189 | # we don't specify encoding here because the env will do. 190 | var: List[str] = [] 191 | stdin = [] 192 | for ln in source.splitlines(True): 193 | match = re_variable.match(ln) 194 | if match: 195 | var.append(match.group(1)) 196 | stdin.append(ln) 197 | stdin.append('\n') 198 | var = uniq(var) 199 | for v in var: 200 | # workaround variables containing newlines 201 | stdin.append('echo "${%s//$\'\\n\'/\\\\n}"\n' % v) 202 | with tempfile.TemporaryDirectory() as tmpdir: 203 | outs, errs = subprocess.Popen( 204 | ('bash', '-r'), cwd=tmpdir, env={}, 205 | stdin=subprocess.PIPE, stdout=subprocess.PIPE, 206 | stderr=subprocess.PIPE).communicate(''.join(stdin).encode('utf-8')) 207 | if errs: 208 | warnings.warn(errs.decode('utf-8', 'backslashreplace').rstrip(), 209 | BashErrorWarning) 210 | lines = [line.replace('\\n', '\n') for line in outs.decode('utf-8').splitlines()] 211 | if len(var) != len(lines) and not errs: 212 | warnings.warn('bash output not expected', BashErrorWarning) 213 | return collections.OrderedDict(zip(var, lines)) 214 | 215 | 216 | def eval_bashvar(source: str, filename: Optional[str]=None, msg=False): 217 | with warnings.catch_warnings(record=True) as wns: 218 | try: 219 | ret = eval_bashvar_literal(source) 220 | except pp.ParseException: 221 | ret = eval_bashvar_ext(source) 222 | msgs = [] 223 | for w in wns: 224 | if issubclass(w.category, VariableWarning): 225 | logging.debug('%s: %s', filename, w.message) 226 | elif issubclass(w.category, BashErrorWarning): 227 | msgs.append(str(w.message)) 228 | logging.error('%s: %s', filename, w.message) 229 | if msg: 230 | return ret, '\n'.join(msgs) if msgs else None 231 | else: 232 | return ret 233 | 234 | 235 | def read_bashvar(fp, filename=None, msg=False): 236 | return eval_bashvar( 237 | fp.read(), filename or getattr(fp, 'name', None), msg) 238 | -------------------------------------------------------------------------------- /acbs/find.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import Dict, List, Optional 4 | 5 | from acbs.const import TMP_DIR 6 | from acbs.parser import ACBSPackageInfo, ACBSSourceInfo, parse_package, check_buildability 7 | from acbs.utils import make_build_dir 8 | 9 | 10 | def check_package_group(name: str, search_path: str, entry_path: str, modifiers: str, tmp_dir: str = TMP_DIR) -> Optional[List[ACBSPackageInfo]]: 11 | # is this a package group? 12 | if os.path.basename(entry_path) == os.path.basename(name) and os.path.isfile(os.path.join(search_path, entry_path, 'spec')): 13 | stub = ACBSPackageInfo(name, [], '', [ACBSSourceInfo('none', '', '')]) 14 | stub.base_slug = entry_path 15 | return expand_package_group(stub, search_path, modifiers, tmp_dir) 16 | with os.scandir(os.path.join(search_path, entry_path)) as group: 17 | # scan potential package groups 18 | for entry_group in group: 19 | full_search_path = os.path.join( 20 | search_path, entry_path, entry_group.name) 21 | # condition: `defines` inside a folder but not named `autobuild` 22 | if os.path.basename(full_search_path) == 'autobuild' or not os.path.isfile( 23 | os.path.join(full_search_path, 'defines')): 24 | continue 25 | # because the package inside the group will have a different name than the folder name 26 | # we will parse the defines file to decide 27 | result = parse_package(full_search_path, modifiers) 28 | if result and result.name == name: 29 | # name of the package inside the group 30 | package_alias = os.path.basename( 31 | full_search_path) 32 | try: 33 | group_seq = int( 34 | package_alias.split('-')[0]) 35 | except (ValueError, IndexError) as ex: 36 | raise ValueError('Invalid package alias: {alias}'.format( 37 | alias=package_alias)) from ex 38 | group_root = os.path.realpath( 39 | os.path.join(full_search_path, '..')) 40 | group_category = os.path.realpath( 41 | os.path.join(group_root, '..')) 42 | result.base_slug = '{cat}/{root}'.format(cat=os.path.basename( 43 | group_category), root=os.path.basename(group_root)) 44 | result.group_seq = group_seq 45 | group_result = expand_package_group( 46 | result, search_path, modifiers, tmp_dir) 47 | return group_result 48 | return None 49 | 50 | 51 | def filter_unbuildable_packages(packages: List[ACBSPackageInfo], group_name: Optional[str]=None) -> List[ACBSPackageInfo]: 52 | """Filter out packages that are not buildable.""" 53 | filtered_packages = [] 54 | unbuildable = [] 55 | for package in packages: 56 | if check_buildability(package): 57 | filtered_packages.append(package) 58 | else: 59 | unbuildable.append(package.name) 60 | if unbuildable: 61 | logging.warning( 62 | "The following packages %swill be skipped as they are not buildable:\n\t%s", 63 | f"(in {group_name}) " if group_name else "", 64 | " ".join(unbuildable), 65 | ) 66 | return filtered_packages 67 | 68 | 69 | def find_package(name: str, search_path: str, modifiers: str, tmp_dir: str = TMP_DIR) -> List[ACBSPackageInfo]: 70 | if os.path.isfile(os.path.join(search_path, name)): 71 | with open(os.path.join(search_path, name), 'rt') as f: 72 | content = f.read() 73 | packages = content.splitlines() 74 | results = [] 75 | print() 76 | for n, p in enumerate(packages): 77 | print(f'[{n + 1}/{len(packages)}] {name} > {p:15}\r', end='', flush=True) 78 | p = p.strip() 79 | if not p or p.startswith('#'): 80 | continue 81 | found = find_package_inner(p, search_path, modifiers=modifiers, tmp_dir=tmp_dir) 82 | if not found: 83 | raise RuntimeError( 84 | f'Package {p} requested in {name} was not found.') 85 | results.extend(found) 86 | print() 87 | return filter_unbuildable_packages(results, group_name=name) 88 | return find_package_inner(name, search_path, modifiers=modifiers, tmp_dir=tmp_dir) 89 | 90 | 91 | def find_package_inner(name: str, search_path: str, group=False, modifiers: str='', tmp_dir: str = TMP_DIR) -> List[ACBSPackageInfo]: 92 | if os.path.isdir(os.path.join(search_path, name)): 93 | flat_path = os.path.join(search_path, name, 'autobuild') 94 | if os.path.isdir(flat_path): 95 | return [parse_package(os.path.join(search_path, name, 'autobuild'), modifiers)] 96 | # is this a package group? 97 | group_result = check_package_group(name, search_path, name, modifiers, tmp_dir) 98 | if group_result: 99 | return group_result 100 | with os.scandir(search_path) as it: 101 | # scan categories 102 | for entry in it: 103 | if not entry.is_dir(): 104 | continue 105 | with os.scandir(os.path.join(search_path, entry.name)) as inner: 106 | # scan package directories 107 | for entry_inner in inner: 108 | if not entry_inner.is_dir(): 109 | continue 110 | full_search_path = os.path.join( 111 | search_path, entry.name, entry_inner.name, 'autobuild') 112 | if entry_inner.name == name and os.path.isdir(full_search_path): 113 | return [parse_package(full_search_path, modifiers)] 114 | if not group: 115 | continue 116 | # is this a package group? 117 | group_result = check_package_group( 118 | name, search_path, os.path.join(entry.name, entry_inner.name), modifiers, tmp_dir) 119 | if group_result: 120 | return group_result 121 | if group: 122 | return [] 123 | else: 124 | # if cannot find a package without considering it as part of a group 125 | # then re-search with group enabled 126 | return find_package_inner(name, search_path, True, modifiers, tmp_dir) 127 | 128 | 129 | def check_package_groups(packages: List[ACBSPackageInfo]): 130 | """In AOSC OS build rules, the package group need to be built sequentially together. 131 | This function will check if the package inside the group will be built sequentially 132 | """ 133 | groups_seen: Dict[str, int] = {} 134 | for pkg in packages: 135 | base_slug = pkg.base_slug 136 | if not base_slug: 137 | continue 138 | if base_slug in groups_seen: 139 | if groups_seen[base_slug] > pkg.group_seq: 140 | logging.error('Package {} (in {}) has a different sequential order (#{}) after dependency resolution (should be #{})'.format( 141 | pkg.name, base_slug, pkg.group_seq, groups_seen[base_slug] + 1)) 142 | logging.error('This might indicate a dependency cycle between the sub-packages (needs bootstrapping?) ...') 143 | logging.error('... or maybe the sub-package should be named {:02d}-{}'.format(groups_seen[base_slug] + 1, pkg.name)) 144 | logging.error('Please check which situation this package is in and fix it.') 145 | raise ValueError('Specified sub-package order contradicts with the dependency resolution results') 146 | else: 147 | groups_seen[base_slug] = pkg.group_seq 148 | 149 | 150 | def expand_package_group(package: ACBSPackageInfo, search_path: str, modifiers: str, tmp_dir: str = TMP_DIR) -> List[ACBSPackageInfo]: 151 | group_root = os.path.join(search_path, package.base_slug) 152 | original_base = package.base_slug 153 | actionables: List[ACBSPackageInfo] = [] 154 | for entry in os.scandir(group_root): 155 | if not entry.is_dir(): 156 | continue 157 | name = entry.name 158 | splitted = name.split('-', 1) 159 | if len(splitted) != 2: 160 | raise ValueError( 161 | 'Malformed sub-package name: {name}'.format(name=entry.name)) 162 | try: 163 | sequence = int(splitted[0]) 164 | package = parse_package(entry.path, modifiers) 165 | if package: 166 | package.base_slug = original_base 167 | package.group_seq = sequence 168 | actionables.append(package) 169 | except ValueError as ex: 170 | raise ValueError( 171 | 'Malformed sub-package name: {name}'.format(name=entry.name)) from ex 172 | # because the directory order is arbitrary, we need to sort them 173 | actionables = sorted(actionables, key=lambda a: a.group_seq) 174 | # pre-assign build location for sub-packages 175 | location = make_build_dir(tmp_dir) 176 | for a in actionables: 177 | a.build_location = location 178 | return actionables 179 | -------------------------------------------------------------------------------- /acbs/parser.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | import logging 3 | import os 4 | import re 5 | from collections import OrderedDict 6 | from typing import Dict, List, Optional 7 | from urllib.parse import urlparse 8 | 9 | from acbs import bashvar 10 | from acbs.base import ACBSPackageInfo, ACBSSourceInfo 11 | from acbs.pm import filter_dependencies 12 | from acbs.utils import fail_arch_regex, get_arch_name, tarball_pattern 13 | 14 | generate_mode = False 15 | 16 | 17 | def get_defines_file_path(location: str, stage2: bool) -> str: 18 | ''' 19 | Return ${location}/defines or ${location}/defines.stage2 depending on the value of stage2 and whether the .stage2 file exists. 20 | ''' 21 | if stage2 and os.path.exists(os.path.join(location, 'defines.stage2')): 22 | return os.path.join(location, 'defines.stage2') 23 | else: 24 | return os.path.join(location, 'defines') 25 | 26 | 27 | def parse_url_schema(url: str, checksum: str) -> ACBSSourceInfo: 28 | acbs_source_info = ACBSSourceInfo('none', '', '') 29 | url_split = url.split('::', 2) 30 | schema = '' 31 | url_plain = '' 32 | if len(url_split) < 2: 33 | url_plain = url 34 | if re.search(tarball_pattern, url_plain): 35 | schema = 'tarball' 36 | elif url_plain.endswith('.git') or url_plain.startswith('git://'): 37 | schema = 'git' 38 | else: 39 | raise ValueError( 40 | 'Unable to deduce source type for {}.'.format(url_plain)) 41 | elif len(url_split) < 3: 42 | schema = url_split[0].lower() 43 | url_plain = url_split[1] 44 | else: 45 | schema, options, url_plain = url_split 46 | schema = schema.lower() 47 | acbs_source_info = parse_fetch_options(options, acbs_source_info) 48 | acbs_source_info.type = 'tarball' if schema == 'tbl' else schema 49 | chksum_ = checksum.split('::', 1) 50 | if len(chksum_) != 2 and checksum != 'SKIP': 51 | raise ValueError('Malformed checksum: {}'.format(checksum)) 52 | acbs_source_info.chksum = ( 53 | chksum_[0], chksum_[1]) if checksum != 'SKIP' else ('none', '') 54 | acbs_source_info.url = url_plain 55 | if acbs_source_info.use_url_name and acbs_source_info.source_name: 56 | raise ValueError("Option 'use-url-name' can NOT be used with the 'rename' option.") 57 | if acbs_source_info.use_url_name: 58 | parsed = urlparse(url_plain) 59 | acbs_source_info.source_name = os.path.basename(parsed.path) 60 | return acbs_source_info 61 | 62 | 63 | def parse_fetch_options(options: str, acbs_source_info: ACBSSourceInfo): 64 | options_split = options.split(';') 65 | for option in options_split: 66 | k, v = option.split('=') 67 | if k == 'branch': 68 | acbs_source_info.branch = v.strip() 69 | elif k == 'rename': 70 | acbs_source_info.source_name = v.strip() 71 | elif k == 'use-url-name': 72 | acbs_source_info.use_url_name = v.strip() == 'true' 73 | elif k == 'commit': 74 | acbs_source_info.revision = v.strip() 75 | elif k == 'version': 76 | acbs_source_info.revision = v.strip() 77 | elif k == 'copy-repo': 78 | acbs_source_info.copy_repo = v.strip() == 'true' 79 | elif k == 'submodule': 80 | translated = { 81 | 'false': 0, 82 | 'true': 1, 83 | 'recursive': 2, 84 | }.get(v.strip()) 85 | if translated is None: 86 | raise ValueError(f'Invalid submodule directive: {v}') 87 | acbs_source_info.submodule = translated 88 | return acbs_source_info 89 | 90 | 91 | def parse_package_url(var: Dict[str, str], ignore_empty_srcs: bool) -> List[ACBSSourceInfo]: 92 | acbs_source_info: List[ACBSSourceInfo] = [] 93 | sources = var.get('SRCS__{arch}'.format( 94 | arch=arch.upper())) or var.get('SRCS') 95 | checksums = var.get('CHKSUMS__{arch}'.format( 96 | arch=arch.upper())) or var.get('CHKSUMS') 97 | if var.get('DUMMYSRC') in ['y', 'yes', '1']: 98 | acbs_source_info.append(ACBSSourceInfo('none', '', '')) 99 | return acbs_source_info 100 | if sources is None: 101 | if not ignore_empty_srcs: 102 | raise ValueError( 103 | 'Source definition is missing. If that is intended, ' 104 | 'perhaps you want to set DUMMYSRC=1.') 105 | else: 106 | return [] 107 | if checksums is None and not generate_mode: 108 | raise ValueError( 109 | 'Missing checksums. You can use `SKIP` for VCS sources.') 110 | sources_list = sources.strip().split() 111 | checksums_list = checksums.strip().split() if checksums else [ 112 | '::'] * len(sources_list) 113 | if len(sources_list) != len(checksums_list): 114 | raise ValueError( 115 | f'Sources array and checksums array must have the same length (Sources: {len(sources_list)}, Checksums: {len(checksums_list)}).' 116 | ) 117 | for s, c in zip(sources_list, checksums_list): 118 | acbs_source_info.append(parse_url_schema(s, c)) 119 | return acbs_source_info 120 | 121 | 122 | def parse_package(location: str, modifiers: str) -> ACBSPackageInfo: 123 | # Ignore (seemingly) empty srcs on unbuildable archs, if the package 124 | # uses different sources for each (supported) architectures. 125 | ignore_empty_srcs: bool = False 126 | logging.debug('Parsing {}...'.format(location)) 127 | stage2 = ACBSPackageInfo.is_in_stage2(modifiers) 128 | # Call a helper function to check if there's a stage2 defines automatically 129 | defines_location = get_defines_file_path(location, stage2) 130 | spec_location = os.path.join(location, '..', 'spec') 131 | with open(defines_location, 'rt') as f: 132 | var = bashvar.eval_bashvar(f.read(), filename=defines_location) 133 | assert isinstance(var, dict) 134 | with open(spec_location, 'rt') as f: 135 | spec_var = bashvar.eval_bashvar(f.read(), filename=spec_location) 136 | assert isinstance(spec_var, dict) 137 | fail_arch = var.get('FAIL_ARCH') 138 | fail_arch_re: Optional[re.Pattern] = None 139 | if fail_arch: 140 | fail_arch_re = fail_arch_regex(fail_arch) 141 | if fail_arch_re.match(arch): 142 | logging.debug(f'Package {var["PKGNAME"]} is not buildable on current arch: {arch}. ' 143 | 'Any encountered empty SRCS will be ignored.') 144 | # Continue parsing but ignore any source error, since we still 145 | # need the complete tree. 146 | # There are some packages that use different sources for each 147 | # (supported) architectures, but for unbuildable packages the 148 | # source info parser will fail, as there is no SRCS for current 149 | # arch. 150 | ignore_empty_srcs = True 151 | deps_arch: Optional[str] = var.get('PKGDEP__{arch}'.format( 152 | arch=arch.upper())) 153 | # determine whether this is an undefined value or an empty string 154 | deps: str = (var.get('PKGDEP') or '') if deps_arch is None else deps_arch 155 | builddeps_arch: Optional[str] = var.get('BUILDDEP__{arch}'.format( 156 | arch=arch.upper())) 157 | builddeps = var.get( 158 | 'BUILDDEP') if builddeps_arch is None else builddeps_arch 159 | deps += ' ' + (builddeps or '') # add builddep 160 | # architecture specific dependencies 161 | acbs_source_info = parse_package_url(spec_var, ignore_empty_srcs) 162 | if not deps: 163 | result = ACBSPackageInfo( 164 | name=var['PKGNAME'], deps=[], location=location, source_uri=acbs_source_info) 165 | else: 166 | # filter out dependencies that are prefixed with @AB_ (autobuild special placeholders) 167 | deps_iter = filter(lambda d: not d.startswith("@AB_"), deps.split()) 168 | result = ACBSPackageInfo( 169 | name=var['PKGNAME'], deps=list(deps_iter), location=location, source_uri=acbs_source_info) 170 | result.bin_arch = var.get('ABHOST') or arch 171 | release = spec_var.get('REL') or '0' 172 | result.rel = release 173 | version = spec_var.get('VER') 174 | if fail_arch: 175 | result.fail_arch = fail_arch_re 176 | if version: 177 | result.version = version 178 | subdir = spec_var.get('SUBDIR') 179 | if subdir: 180 | result.subdir = subdir 181 | epoch = spec_var.get('EPOCH') 182 | if epoch: 183 | result.epoch = epoch 184 | result.modifiers = modifiers 185 | # collect exported variables (prefixed with `__`) 186 | for k, v in spec_var.items(): 187 | if k.startswith('__'): 188 | result.exported[k] = v 189 | 190 | return filter_dependencies(result) 191 | 192 | 193 | def get_deps_graph(packages: List[ACBSPackageInfo]) -> 'OrderedDict[str, ACBSPackageInfo]': 194 | 'convert flattened list to adjacency list' 195 | result = {} 196 | for i in packages: 197 | result[i.name] = i 198 | return OrderedDict(result) 199 | 200 | 201 | def get_tree_by_name(filename: str, tree_name) -> str: 202 | acbs_config = configparser.ConfigParser( 203 | interpolation=configparser.ExtendedInterpolation()) 204 | with open(filename, 'rt') as conf_file: 205 | try: 206 | acbs_config.read_file(conf_file) 207 | except Exception as ex: 208 | raise Exception('Failed to read configuration file!') from ex 209 | try: 210 | tree_loc_dict = acbs_config[tree_name] 211 | except KeyError as ex: 212 | err_message = 'Tree not found: {}, defined trees: {}'.format(tree_name, 213 | ' '.join(acbs_config.sections())) 214 | raise ValueError(err_message) from ex 215 | try: 216 | tree_loc = tree_loc_dict['location'] 217 | except KeyError as ex: 218 | raise KeyError( 219 | 'Malformed configuration file: missing `location` keyword') from ex 220 | return tree_loc 221 | 222 | 223 | def check_buildability(package: ACBSPackageInfo, required_by: Optional[str]=None) -> bool: 224 | if package.fail_arch and package.fail_arch.match(arch): 225 | if required_by: 226 | raise RuntimeError(f'{package.name} is required by `{required_by}` but is not buildable on `{arch}` (FAIL_ARCH).') 227 | else: 228 | return False 229 | return True 230 | 231 | 232 | arch = os.environ.get('CROSS') or os.environ.get( 233 | 'ARCH') or get_arch_name() or '' 234 | if not arch: 235 | raise ValueError('Unable to determine architecture name') 236 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # acbs documentation build configuration file, created by 5 | # sphinx-quickstart on Sun Jul 31 18:23:52 2016. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | 23 | sys.path.insert(0, os.path.abspath('../..')) 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = [ 35 | 'sphinx.ext.doctest', 36 | 'sphinx.ext.intersphinx', 37 | 'sphinx.ext.todo', 38 | 'sphinx.ext.coverage', 39 | 'sphinx.ext.autodoc' 40 | ] 41 | 42 | # Add any paths that contain templates here, relative to this directory. 43 | templates_path = ['_templates'] 44 | 45 | # The suffix(es) of source filenames. 46 | # You can specify multiple suffix as a list of string: 47 | # 48 | # source_suffix = ['.rst', '.md'] 49 | source_suffix = '.rst' 50 | 51 | # The encoding of source files. 52 | # 53 | # source_encoding = 'utf-8-sig' 54 | 55 | # The master toctree document. 56 | master_doc = 'index' 57 | 58 | # General information about the project. 59 | project = 'acbs' 60 | copyright = '2016-2020, AOSC' 61 | author = 'liushuyu' 62 | 63 | # The version info for the project you're documenting, acts as replacement for 64 | # |version| and |release|, also used in various other places throughout the 65 | # built documents. 66 | # 67 | # The short X.Y version. 68 | version = '0.0.20200707' 69 | # The full version, including alpha/beta/rc tags. 70 | release = '20200707' 71 | 72 | # The language for content autogenerated by Sphinx. Refer to documentation 73 | # for a list of supported languages. 74 | # 75 | # This is also used if you do content translation via gettext catalogs. 76 | # Usually you set "language" from the command line for these cases. 77 | language = None 78 | 79 | # There are two options for replacing |today|: either, you set today to some 80 | # non-false value, then it is used: 81 | # 82 | # today = '' 83 | # 84 | # Else, today_fmt is used as the format for a strftime call. 85 | # 86 | # today_fmt = '%B %d, %Y' 87 | 88 | # List of patterns, relative to source directory, that match files and 89 | # directories to ignore when looking for source files. 90 | # This patterns also effect to html_static_path and html_extra_path 91 | exclude_patterns = [] 92 | 93 | # The reST default role (used for this markup: `text`) to use for all 94 | # documents. 95 | # 96 | # default_role = None 97 | 98 | # If true, '()' will be appended to :func: etc. cross-reference text. 99 | # 100 | # add_function_parentheses = True 101 | 102 | # If true, the current module name will be prepended to all description 103 | # unit titles (such as .. function::). 104 | # 105 | # add_module_names = True 106 | 107 | # If true, sectionauthor and moduleauthor directives will be shown in the 108 | # output. They are ignored by default. 109 | # 110 | # show_authors = False 111 | 112 | # The name of the Pygments (syntax highlighting) style to use. 113 | pygments_style = 'sphinx' 114 | 115 | # A list of ignored prefixes for module index sorting. 116 | # modindex_common_prefix = [] 117 | 118 | # If true, keep warnings as "system message" paragraphs in the built documents. 119 | # keep_warnings = False 120 | 121 | # If true, `todo` and `todoList` produce output, else they produce nothing. 122 | todo_include_todos = True 123 | 124 | 125 | # -- Options for HTML output ---------------------------------------------- 126 | 127 | # The theme to use for HTML and HTML Help pages. See the documentation for 128 | # a list of builtin themes. 129 | # 130 | html_theme = 'alabaster' 131 | 132 | # Theme options are theme-specific and customize the look and feel of a theme 133 | # further. For a list of options available for each theme, see the 134 | # documentation. 135 | # 136 | # html_theme_options = {} 137 | 138 | # Add any paths that contain custom themes here, relative to this directory. 139 | # html_theme_path = [] 140 | 141 | # The name for this set of Sphinx documents. 142 | # " v documentation" by default. 143 | # 144 | # html_title = 'acbs v1' 145 | 146 | # A shorter title for the navigation bar. Default is the same as html_title. 147 | # 148 | # html_short_title = None 149 | 150 | # The name of an image file (relative to this directory) to place at the top 151 | # of the sidebar. 152 | # 153 | # html_logo = None 154 | 155 | # The name of an image file (relative to this directory) to use as a favicon of 156 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 157 | # pixels large. 158 | # 159 | # html_favicon = None 160 | 161 | # Add any paths that contain custom static files (such as style sheets) here, 162 | # relative to this directory. They are copied after the builtin static files, 163 | # so a file named "default.css" will overwrite the builtin "default.css". 164 | html_static_path = ['_static'] 165 | 166 | # Add any extra paths that contain custom files (such as robots.txt or 167 | # .htaccess) here, relative to this directory. These files are copied 168 | # directly to the root of the documentation. 169 | # 170 | # html_extra_path = [] 171 | 172 | # If not None, a 'Last updated on:' timestamp is inserted at every page 173 | # bottom, using the given strftime format. 174 | # The empty string is equivalent to '%b %d, %Y'. 175 | # 176 | # html_last_updated_fmt = None 177 | 178 | # If true, SmartyPants will be used to convert quotes and dashes to 179 | # typographically correct entities. 180 | # 181 | # html_use_smartypants = True 182 | 183 | # Custom sidebar templates, maps document names to template names. 184 | # 185 | # html_sidebars = {} 186 | 187 | # Additional templates that should be rendered to pages, maps page names to 188 | # template names. 189 | # 190 | # html_additional_pages = {} 191 | 192 | # If false, no module index is generated. 193 | # 194 | # html_domain_indices = True 195 | 196 | # If false, no index is generated. 197 | # 198 | # html_use_index = True 199 | 200 | # If true, the index is split into individual pages for each letter. 201 | # 202 | # html_split_index = False 203 | 204 | # If true, links to the reST sources are added to the pages. 205 | # 206 | # html_show_sourcelink = True 207 | 208 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 209 | # 210 | # html_show_sphinx = True 211 | 212 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 213 | # 214 | # html_show_copyright = True 215 | 216 | # If true, an OpenSearch description file will be output, and all pages will 217 | # contain a tag referring to it. The value of this option must be the 218 | # base URL from which the finished HTML is served. 219 | # 220 | # html_use_opensearch = '' 221 | 222 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 223 | # html_file_suffix = None 224 | 225 | # Language to be used for generating the HTML full-text search index. 226 | # Sphinx supports the following languages: 227 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 228 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' 229 | # 230 | # html_search_language = 'en' 231 | 232 | # A dictionary with options for the search language support, empty by default. 233 | # 'ja' uses this config value. 234 | # 'zh' user can custom change `jieba` dictionary path. 235 | # 236 | # html_search_options = {'type': 'default'} 237 | 238 | # The name of a javascript file (relative to the configuration directory) that 239 | # implements a search results scorer. If empty, the default will be used. 240 | # 241 | # html_search_scorer = 'scorer.js' 242 | 243 | # Output file base name for HTML help builder. 244 | htmlhelp_basename = 'acbsdoc' 245 | 246 | # -- Options for LaTeX output --------------------------------------------- 247 | 248 | latex_elements = { 249 | # The paper size ('letterpaper' or 'a4paper'). 250 | # 251 | # 'papersize': 'letterpaper', 252 | 253 | # The font size ('10pt', '11pt' or '12pt'). 254 | # 255 | # 'pointsize': '10pt', 256 | 257 | # Additional stuff for the LaTeX preamble. 258 | # 259 | # 'preamble': '', 260 | 261 | # Latex figure (float) alignment 262 | # 263 | # 'figure_align': 'htbp', 264 | } 265 | 266 | # Grouping the document tree into LaTeX files. List of tuples 267 | # (source start file, target name, title, 268 | # author, documentclass [howto, manual, or own class]). 269 | latex_documents = [ 270 | (master_doc, 'acbs.tex', 'acbs Documentation', 271 | 'liushuyu', 'manual'), 272 | ] 273 | 274 | # The name of an image file (relative to this directory) to place at the top of 275 | # the title page. 276 | # 277 | # latex_logo = None 278 | 279 | # For "manual" documents, if this is true, then toplevel headings are parts, 280 | # not chapters. 281 | # 282 | # latex_use_parts = False 283 | 284 | # If true, show page references after internal links. 285 | # 286 | # latex_show_pagerefs = False 287 | 288 | # If true, show URL addresses after external links. 289 | # 290 | # latex_show_urls = False 291 | 292 | # Documents to append as an appendix to all manuals. 293 | # 294 | # latex_appendices = [] 295 | 296 | # It false, will not define \strong, \code, itleref, \crossref ... but only 297 | # \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added 298 | # packages. 299 | # 300 | # latex_keep_old_macro_names = True 301 | 302 | # If false, no module index is generated. 303 | # 304 | # latex_domain_indices = True 305 | 306 | 307 | # -- Options for manual page output --------------------------------------- 308 | 309 | # One entry per manual page. List of tuples 310 | # (source start file, name, description, authors, manual section). 311 | man_pages = [ 312 | (master_doc, 'acbs', 'acbs Documentation', 313 | [author], 1) 314 | ] 315 | 316 | # If true, show URL addresses after external links. 317 | # 318 | # man_show_urls = False 319 | 320 | 321 | # -- Options for Texinfo output ------------------------------------------- 322 | 323 | # Grouping the document tree into Texinfo files. List of tuples 324 | # (source start file, target name, title, author, 325 | # dir menu entry, description, category) 326 | texinfo_documents = [ 327 | (master_doc, 'acbs', 'acbs Documentation', 328 | author, 'acbs', 'One line description of project.', 329 | 'Miscellaneous'), 330 | ] 331 | 332 | # Documents to append as an appendix to all manuals. 333 | # 334 | # texinfo_appendices = [] 335 | 336 | # If false, no module index is generated. 337 | # 338 | # texinfo_domain_indices = True 339 | 340 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 341 | # 342 | # texinfo_show_urls = 'footnote' 343 | 344 | # If true, do not generate a @detailmenu in the "Top" node's menu. 345 | # 346 | # texinfo_no_detailmenu = False 347 | 348 | 349 | # Example configuration for intersphinx: refer to the Python standard library. 350 | intersphinx_mapping = {'https://docs.python.org/': None} 351 | -------------------------------------------------------------------------------- /acbs/fetch.py: -------------------------------------------------------------------------------- 1 | import http.client 2 | import logging 3 | import os 4 | import shutil 5 | import subprocess 6 | import json 7 | 8 | from typing import Callable, Dict, List, Optional, Tuple 9 | from urllib.parse import urlparse 10 | 11 | from acbs.base import ACBSPackageInfo, ACBSSourceInfo 12 | from acbs.crypto import check_hash_hashlib, hash_url 13 | from acbs.utils import guess_extension_name 14 | 15 | fetcher_signature = Callable[[ACBSSourceInfo, 16 | str, str], Optional[ACBSSourceInfo]] 17 | processor_signature = Callable[[ACBSPackageInfo, int, str], None] 18 | pair_signature = Tuple[fetcher_signature, processor_signature] 19 | generate_mode = False 20 | 21 | 22 | def fetch_source(info: List[ACBSSourceInfo], source_location: str, package_name: str) -> Optional[ACBSSourceInfo]: 23 | logging.info('Fetching required source files...') 24 | count = 0 25 | for i in info: 26 | count += 1 27 | logging.info(f'Fetching source ({count}/{len(info)})...') 28 | # in generate mode, we need to fetch all the sources 29 | if not i.enabled and not generate_mode: 30 | logging.info(f'Source {count} skipped.') 31 | # special handling for PyPI type 32 | url = i.url if i.type != "pypi" else f"pypi://{i.url}/{i.revision}" 33 | url_hash = hash_url(url) 34 | fetch_source_inner(i, source_location, url_hash) 35 | return None 36 | 37 | 38 | def fetch_source_inner(info: ACBSSourceInfo, source_location: str, package_name: str) -> Optional[ACBSSourceInfo]: 39 | type_ = info.type 40 | retry = 0 41 | fetcher: Optional[pair_signature] = handlers.get(type_.upper()) 42 | if not fetcher or not callable(fetcher[0]): 43 | raise NotImplementedError(f'Unsupported source type: {type_}') 44 | while retry < 5: 45 | retry += 1 46 | try: 47 | return fetcher[0](info, source_location, package_name) 48 | except Exception as ex: 49 | logging.exception(ex) 50 | logging.warning(f'Retrying ({retry}/5)...') 51 | continue 52 | raise RuntimeError( 53 | 'Unable to fetch source files, failed 5 times in a row.') 54 | 55 | 56 | def process_source(info: ACBSPackageInfo, source_name: str) -> None: 57 | idx = 0 58 | for source_uri in info.source_uri: 59 | type_ = source_uri.type 60 | fetcher: Optional[pair_signature] = handlers.get(type_.upper()) 61 | if not fetcher or not callable(fetcher[1]): 62 | raise NotImplementedError( 63 | f'Unsupported source type: {type_}') 64 | fetcher[1](info, idx, source_name) 65 | idx += 1 66 | return 67 | 68 | 69 | # Fetchers implementations 70 | def tarball_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]: 71 | if source_location: 72 | filename = hash_url(info.url) 73 | if not info.chksum[1] and not generate_mode: 74 | raise ValueError('No checksum found. Please specify the checksum!') 75 | full_path = os.path.join(source_location, filename) 76 | try: 77 | wget_download(info.url, full_path) 78 | info.source_location = full_path 79 | return info 80 | except Exception: 81 | raise AssertionError('Failed to fetch source with Wget!') 82 | return None 83 | return None 84 | 85 | 86 | def wget_download(url: str, full_path: str): 87 | flag_path = full_path + ".dl" 88 | url_info = urlparse(url) 89 | if os.path.exists(full_path) and not os.path.exists(flag_path): 90 | return 91 | if url_info.hostname == 'sourceforge.net': 92 | if url_info.path.endswith('/download'): 93 | url = url_info._replace(query='failedmirror=cyfuture.dl.sourceforge.net').geturl() 94 | else: 95 | url = url_info._replace(query='failedmirror=cyfuture.dl.sourceforge.net', path=url_info.path+'/download').geturl() 96 | try: 97 | # `touch ${flag_path}`, some servers may not support Range, so this is to ensure 98 | # if the download has finished successfully, we don't overwrite the downloaded file 99 | with open(flag_path, 'wb') as f: 100 | f.write(b'') 101 | subprocess.check_call( 102 | ['wget', '--connect-timeout=20', '-c', url, '-O', full_path]) 103 | os.unlink(flag_path) # delete the flag 104 | return 105 | except Exception: 106 | raise AssertionError('Failed to fetch source with Wget!') 107 | 108 | def tarball_processor_innner(package: ACBSPackageInfo, index: int, source_name: str, decompress=True) -> None: 109 | info = package.source_uri[index] 110 | if not info.source_location: 111 | raise ValueError('Where is the source file?') 112 | logging.info('Computing %s checksum for %s...' % (info.chksum, info.source_location)) 113 | check_hash_hashlib(info.chksum, info.source_location) 114 | 115 | server_filename = os.path.basename(info.url) 116 | extension = guess_extension_name(server_filename) 117 | if len(extension) == 0: 118 | # also guess from downloaded file name 119 | # pypi (maybe other fetcher) use tarball processor, but has no file extension in info.url 120 | extension = guess_extension_name(info.source_location) 121 | 122 | # this name is used in the build directory (will be seen by the build scripts) 123 | # the name will be, e.g. 'acbs-0.1.0.tar.gz' 124 | facade_name = info.source_name or '{name}-{version}{index}{extension}'.format( 125 | name=source_name, version=package.version, extension=extension, 126 | index=('' if index == 0 else ('-%s' % index))) 127 | os.symlink(info.source_location, os.path.join( 128 | package.build_location, facade_name)) 129 | if not decompress: 130 | return 131 | # decompress 132 | logging.info(f'Extracting {facade_name}...') 133 | subprocess.check_call(['bsdtar', '--no-xattrs', '-xf', facade_name], 134 | cwd=package.build_location) 135 | return 136 | 137 | 138 | def tarball_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None: 139 | return tarball_processor_innner(package, index, source_name) 140 | 141 | 142 | def pypi_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]: 143 | # https://warehouse.pypa.io/api-reference/json.html#release 144 | api = f"/pypi/{info.url}/{info.revision}/json" 145 | logging.info("Querying PyPI API endpoint for source URL...") 146 | conn = http.client.HTTPSConnection("pypi.org") 147 | conn.request("GET", api) 148 | response = conn.getresponse() 149 | if response.status != 200: 150 | logging.error(f"Got response {response.status}") 151 | raise RuntimeError("Failed to query PyPI API endpoint") 152 | result = json.load(response) 153 | 154 | actual_url = "" 155 | for r in result["urls"]: 156 | if r["packagetype"] == "sdist": 157 | actual_url = r["url"] 158 | break 159 | if actual_url == "": 160 | raise RuntimeError("Can't find source URL") 161 | logging.info(f"Source URL is {actual_url}") 162 | 163 | ext = guess_extension_name(actual_url) 164 | full_path = os.path.join(source_location, name + ext) 165 | try: 166 | wget_download(actual_url, full_path) 167 | info.source_location = full_path 168 | return info 169 | except Exception: 170 | raise AssertionError('Failed to fetch source with Wget!') 171 | return None 172 | 173 | 174 | def blob_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None: 175 | return tarball_processor_innner(package, index, source_name, False) 176 | 177 | 178 | def git_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]: 179 | full_path = os.path.join(source_location, name) 180 | if not os.path.exists(full_path): 181 | subprocess.check_call(['git', 'clone', '--bare', '--filter=blob:none', info.url, full_path], env={'GIT_TERMINAL_PROMPT': '0'}) 182 | else: 183 | logging.info('Updating repository...') 184 | # --prune: prune remote-tracking branches no longer on remote 185 | # --tags: fetch all tags and associated objects 186 | # --force: force overwrite of local reference 187 | subprocess.check_call( 188 | ['git', 'fetch', 'origin', '+refs/heads/*:refs/heads/*', '--prune', '--tags', '--force'], cwd=full_path, env={'GIT_TERMINAL_PROMPT': '0'}) 189 | info.source_location = full_path 190 | return info 191 | 192 | 193 | def git_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None: 194 | info = package.source_uri[index] 195 | if not info.revision: 196 | raise ValueError( 197 | 'Please specify a specific git commit for this package. (GITCO not defined)') 198 | if not info.source_location: 199 | raise ValueError('Where is the git repository?') 200 | checkout_location = os.path.join(package.build_location, info.source_name or source_name) 201 | os.mkdir(checkout_location) 202 | logging.info(f'Checking out git repository at {info.revision}') 203 | subprocess.check_call( 204 | ['git', '--git-dir', info.source_location, '--work-tree', checkout_location, 205 | 'checkout', '-f', info.revision or '']) 206 | if info.submodule > 0: 207 | logging.info('Fetching submodules (if any)...') 208 | params = [ 209 | 'git', '--git-dir', info.source_location, '--work-tree', checkout_location, 210 | 'submodule', 'update', '--init', '--filter=blob:none' 211 | ] 212 | if info.submodule == 2: 213 | params.append('--recursive') 214 | subprocess.check_call(params, cwd=checkout_location) 215 | if info.copy_repo: 216 | logging.info('Copying git folder...') 217 | shutil.copytree(info.source_location, os.path.join(checkout_location, '.git')) 218 | with open(os.path.join(checkout_location, '.git', 'config'), 'r+') as f: 219 | content = f.read() 220 | content = content.replace('bare = true', 'bare = false') 221 | f.seek(0) 222 | f.write(content) 223 | f.truncate() 224 | return None 225 | with open(os.path.join(package.build_location, '.acbs-script'), 'wt') as f: 226 | f.write( 227 | 'ACBS_SRC=\'%s\';acbs_copy_git(){ abinfo \'Copying git folder...\'; cp -ar "${ACBS_SRC}" .git/; sed -i \'s|bare = true|bare = false|\' \'.git/config\'; }' % (info.source_location)) 228 | return None 229 | 230 | 231 | def svn_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]: 232 | full_path = os.path.join(source_location, name) 233 | if not info.revision: 234 | raise ValueError( 235 | 'Please specify a svn revision for this package. (SVNCO not defined)') 236 | logging.info( 237 | f'Checking out subversion repository at r{info.revision}') 238 | if not os.path.exists(full_path): 239 | subprocess.check_call( 240 | ['svn', 'co', '--force', '-r', info.revision, info.url, full_path]) 241 | else: 242 | subprocess.check_call( 243 | ['svn', 'up', '--force', '-r', info.revision], cwd=full_path) 244 | info.source_location = full_path 245 | return info 246 | 247 | 248 | def svn_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None: 249 | info = package.source_uri[index] 250 | if not info.source_location: 251 | raise ValueError('Where is the subversion repository?') 252 | checkout_location = os.path.join(package.build_location, info.source_name or source_name) 253 | logging.info('Copying subversion repository...') 254 | shutil.copytree(info.source_location, checkout_location) 255 | return 256 | 257 | 258 | def hg_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]: 259 | full_path = os.path.join(source_location, name) 260 | if not os.path.exists(full_path): 261 | subprocess.check_call(['hg', 'clone', '-U', info.url, full_path]) 262 | else: 263 | logging.info('Updating repository...') 264 | subprocess.check_call(['hg', 'pull'], cwd=full_path) 265 | info.source_location = full_path 266 | return info 267 | 268 | 269 | def hg_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None: 270 | info = package.source_uri[index] 271 | if not info.revision: 272 | raise ValueError( 273 | 'Please specify a specific hg commit for this package. (HGCO not defined)') 274 | if not info.source_location: 275 | raise ValueError('Where is the hg repository?') 276 | checkout_location = os.path.join(package.build_location, info.source_name or source_name) 277 | logging.info('Copying hg repository...') 278 | shutil.copytree(info.source_location, checkout_location) 279 | logging.info(f'Checking out hg repository at {info.revision}') 280 | subprocess.check_call( 281 | ['hg', 'update', '-C', '-r', info.revision, '-R', checkout_location]) 282 | if info.copy_repo: 283 | logging.info('Copying hg repository ...') 284 | shutil.copytree(info.source_location, os.path.join(checkout_location, '.hg')) 285 | return None 286 | 287 | 288 | def dummy_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]: 289 | if source_location: 290 | logging.info('Not fetching any source as requested') 291 | return info 292 | return None 293 | 294 | 295 | def dummy_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None: 296 | return None 297 | 298 | 299 | def bzr_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]: 300 | full_path = os.path.join(source_location, name) 301 | if not os.path.exists(full_path): 302 | subprocess.check_call(['bzr', 'branch', '--no-tree', info.url, full_path]) 303 | else: 304 | logging.info('Updating repository...') 305 | subprocess.check_call(['bzr', 'pull'], cwd=full_path) 306 | info.source_location = full_path 307 | return info 308 | 309 | 310 | def bzr_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None: 311 | info = package.source_uri[index] 312 | if not info.revision: 313 | raise ValueError( 314 | 'Please specify a specific bzr revision for this package. (BZRCO not defined)') 315 | if not info.source_location: 316 | raise ValueError('Where is the bzr repository?') 317 | checkout_location = os.path.join(package.build_location, info.source_name or source_name) 318 | logging.info('Copying bzr repository...') 319 | shutil.copytree(info.source_location, checkout_location) 320 | logging.info(f'Checking out bzr repository at {info.revision}') 321 | subprocess.check_call( 322 | ['bzr', 'co', '-r', info.revision], cwd=checkout_location) 323 | return None 324 | 325 | 326 | def fossil_fetch(info: ACBSSourceInfo, source_location: str, name: str) -> Optional[ACBSSourceInfo]: 327 | full_path = os.path.join(source_location, name + '.fossil') 328 | if not os.path.exists(full_path): 329 | subprocess.check_call(['fossil', 'clone', info.url, full_path]) 330 | else: 331 | logging.info('Updating repository...') 332 | subprocess.check_call(['fossil', 'pull', '-R', full_path]) 333 | info.source_location = full_path 334 | return info 335 | 336 | 337 | def fossil_processor(package: ACBSPackageInfo, index: int, source_name: str) -> None: 338 | info = package.source_uri[index] 339 | if not info.revision: 340 | raise ValueError( 341 | 'Please specify a specific fossil commit for this package. (not defined)') 342 | if not info.source_location: 343 | raise ValueError('Where is the fossil repository?') 344 | checkout_location = os.path.join(package.build_location, info.source_name or source_name) 345 | os.mkdir(checkout_location) 346 | logging.info('Opening up the fossil repository...') 347 | subprocess.check_call( 348 | ['fossil', 'open', info.source_location], cwd=checkout_location) 349 | logging.info(f'Checking out fossil repository at {info.revision}') 350 | subprocess.check_call(['fossil', 'update', info.revision], cwd=checkout_location) 351 | return None 352 | 353 | 354 | handlers: Dict[str, pair_signature] = { 355 | 'GIT': (git_fetch, git_processor), 356 | 'SVN': (svn_fetch, svn_processor), 357 | 'BZR': (bzr_fetch, bzr_processor), 358 | 'HG': (hg_fetch, hg_processor), 359 | 'FOSSIL': (fossil_fetch, fossil_processor), 360 | 'TARBALL': (tarball_fetch, tarball_processor), 361 | 'FILE': (tarball_fetch, blob_processor), 362 | 'PYPI': (pypi_fetch, tarball_processor), 363 | 'NONE': (dummy_fetch, dummy_processor), 364 | } 365 | -------------------------------------------------------------------------------- /acbs/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logging.handlers 3 | import os 4 | import sys 5 | import time 6 | import traceback 7 | import fcntl 8 | from pathlib import Path 9 | from typing import List, Tuple 10 | 11 | import acbs.fetch 12 | import acbs.parser 13 | from acbs import __version__ 14 | from acbs.ab4cfg import is_in_stage2 15 | from acbs.base import ACBSPackageInfo 16 | from acbs.checkpoint import ACBSShrinkWrap, checkpoint_to_group, do_shrink_wrap 17 | from acbs.const import AUTOBUILD_CONF_DIR, CONF_DIR, DUMP_DIR, LOG_DIR, TMP_DIR 18 | from acbs.deps import prepare_for_reorder, tarjan_search 19 | from acbs.fetch import fetch_source, process_source 20 | from acbs.find import check_package_groups, find_package 21 | from acbs.parser import check_buildability, get_deps_graph, get_tree_by_name 22 | from acbs.pm import install_from_repo, enable_reorder_mode 23 | from acbs.utils import ( 24 | ACBSLogFormatter, 25 | ACBSLogPlainFormatter, 26 | check_artifact, 27 | full_line_banner, 28 | generate_checksums, 29 | guess_subdir, 30 | has_stamp, 31 | invoke_autobuild, 32 | is_spec_legacy, 33 | make_build_dir, 34 | print_build_timings, 35 | print_package_names, 36 | validate_package_name, 37 | write_checksums, 38 | ) 39 | 40 | 41 | CIEL_LOCK_PATH = '/debs/fresh.lock' 42 | 43 | def ciel_invalidate_cache(): 44 | logging.info('Asking ciel to refresh repository...') 45 | if os.path.exists(CIEL_LOCK_PATH): 46 | with open(CIEL_LOCK_PATH, 'r+') as lock_file: 47 | lock_file_fd = lock_file.fileno() 48 | fcntl.flock(lock_file_fd, fcntl.LOCK_EX) 49 | if lock_file.read(1) != '0': 50 | lock_file.seek(0) 51 | lock_file.truncate(0) 52 | lock_file.write('0') 53 | fcntl.flock(lock_file_fd, fcntl.LOCK_UN) 54 | else: 55 | logging.warning('Ciel did not create lock file, skipping...') 56 | 57 | 58 | def ciel_wait_for_refresh(): 59 | logging.info('Waiting for ciel to refresh repository...') 60 | if os.path.exists(CIEL_LOCK_PATH): 61 | with open(CIEL_LOCK_PATH, 'r') as lock_file: 62 | lock_file_fd = lock_file.fileno() 63 | fcntl.flock(lock_file_fd, fcntl.LOCK_EX) 64 | 65 | success = False 66 | if lock_file.read(1) == '1': 67 | success = True 68 | else: 69 | for _ in range(10): 70 | fcntl.flock(lock_file_fd, fcntl.LOCK_UN) 71 | time.sleep(1) 72 | fcntl.flock(lock_file_fd, fcntl.LOCK_EX) 73 | 74 | lock_file.seek(0) 75 | if lock_file.read(1) == '1': 76 | success = True 77 | break 78 | fcntl.flock(lock_file_fd, fcntl.LOCK_UN) 79 | 80 | if success: 81 | logging.info('Ciel finished refreshing repository...') 82 | else: 83 | logging.warning('Ciel failed to refresh repository in time...') 84 | else: 85 | logging.warning('Ciel did not create lock file, skipping...') 86 | 87 | 88 | class BuildCore(object): 89 | 90 | def __init__(self, args) -> None: 91 | self.debug = args.debug 92 | self.no_deps = args.no_deps 93 | self.dl_only = args.get 94 | self.tree = 'default' 95 | self.build_queue = args.packages 96 | self.generate = args.acbs_write 97 | self.tree_dir = '' 98 | self.package_cursor = 0 99 | self.reorder = args.reorder 100 | self.save_list = args.save_list 101 | self.force_use_apt = args.force_use_apt 102 | self.generate_pkg_metadata = args.generate_pkg_metadata 103 | 104 | # static vars 105 | self.autobuild_conf_dir = AUTOBUILD_CONF_DIR 106 | self.conf_dir = CONF_DIR 107 | if args.acbs_dump_dir is not None: 108 | self.dump_dir = args.acbs_dump_dir[0] 109 | else: 110 | self.dump_dir = DUMP_DIR 111 | if args.acbs_temp_dir is not None: 112 | self.tmp_dir = args.acbs_temp_dir[0] 113 | else: 114 | self.tmp_dir = TMP_DIR 115 | if args.acbs_log_dir is not None: 116 | self.log_dir = args.acbs_log_dir[0] 117 | else: 118 | self.log_dir = LOG_DIR 119 | self.stage2 = is_in_stage2() 120 | if args.acbs_tree: 121 | self.tree = args.acbs_tree[0] 122 | if args.acbs_tree_dir is not None: 123 | self.tree_dir = args.acbs_tree_dir[0] 124 | self.init() 125 | 126 | def init(self) -> None: 127 | sys.excepthook = self.acbs_except_hdr 128 | print(full_line_banner( 129 | f'Welcome to ACBS - {__version__}')) 130 | if self.debug: 131 | log_verbosity = logging.DEBUG 132 | else: 133 | log_verbosity = logging.INFO 134 | try: 135 | for directory in [self.dump_dir, self.tmp_dir, self.conf_dir, 136 | self.log_dir]: 137 | if not os.path.isdir(directory): 138 | os.makedirs(directory) 139 | except Exception: 140 | raise IOError('\033[93mFailed to create work directories\033[0m!') 141 | self.__install_logger(log_verbosity) 142 | if self.tree_dir == '': 143 | # If the user did not specify tree path via -b/--tree-dir, read from config 144 | forest_file = os.path.join(self.conf_dir, 'forest.conf') 145 | if os.path.exists(forest_file): 146 | self.tree_dir = get_tree_by_name(forest_file, self.tree) 147 | if not self.tree_dir: 148 | raise ValueError('Tree not found!') 149 | else: 150 | raise Exception('forest.conf not found') 151 | 152 | def __install_logger(self, str_verbosity=logging.INFO, 153 | file_verbosity=logging.DEBUG): 154 | logger = logging.getLogger() 155 | logger.setLevel(0) # Set to lowest to bypass the initial filter 156 | str_handler = logging.StreamHandler() 157 | str_handler.setLevel(str_verbosity) 158 | if os.environ.get('NO_COLOR'): 159 | str_handler.setFormatter(ACBSLogPlainFormatter()) 160 | else: 161 | str_handler.setFormatter(ACBSLogFormatter()) 162 | logger.addHandler(str_handler) 163 | log_file_handler = logging.handlers.RotatingFileHandler( 164 | os.path.join(self.log_dir, 'acbs-build.log'), mode='a', maxBytes=int(2e5), backupCount=3) 165 | log_file_handler.setLevel(file_verbosity) 166 | log_file_handler.setFormatter(logging.Formatter( 167 | '%(asctime)s:%(levelname)s:%(message)s')) 168 | logger.addHandler(log_file_handler) 169 | 170 | def strip_modifiers(self, p: str) -> Tuple[str, str]: 171 | if ':' in p: 172 | results = p.split(':', 1) 173 | if len(results) == 2: 174 | p, m = results 175 | return p, m 176 | return p, '' 177 | 178 | def build(self) -> None: 179 | packages = [] 180 | build_timings: List[Tuple[str, float]] = [] 181 | acbs.fetch.generate_mode = self.generate 182 | acbs.parser.generate_mode = self.generate 183 | if self.stage2: 184 | logging.info("Life-cycle: currently running in stage2 mode.") 185 | # begin finding and resolving dependencies 186 | logging.info('Searching and resolving dependencies...') 187 | enable_reorder_mode(self.reorder) 188 | for n, i in enumerate(self.build_queue): 189 | i, modifiers = self.strip_modifiers(i) 190 | if not validate_package_name(i): 191 | raise ValueError(f'Invalid package name: `{i}`') 192 | logging.debug(f'Finding {i}...') 193 | print(f'[{n + 1}/{len(self.build_queue)}] {i:30}\r', end='', flush=True) 194 | package = find_package(i, self.tree_dir, modifiers, self.tmp_dir) 195 | if not package: 196 | raise RuntimeError(f'Could not find package {i}') 197 | packages.extend(package) 198 | self.resolve_deps(packages, self.stage2) 199 | if not packages: 200 | logging.info('Nothing to do after dependency resolution') 201 | return 202 | logging.info( 203 | f'Dependencies resolved, {len(packages)} packages in the queue') 204 | logging.debug(f'Queue: {packages}') 205 | logging.info( 206 | f'Packages to be built: {print_package_names(packages, 5)}') 207 | if self.save_list: 208 | filename = checkpoint_to_group(packages, self.tree_dir) 209 | logging.info( 210 | f'ACBS has saved your build queue to groups/{filename}') 211 | return 212 | try: 213 | self.build_sequential(build_timings, packages) 214 | except Exception as ex: 215 | logging.exception(ex) 216 | self.save_checkpoint(build_timings, packages) 217 | print_build_timings(build_timings, []) 218 | 219 | def save_checkpoint(self, build_timings, packages): 220 | logging.info('ACBS is trying to save your build status...') 221 | shrink_wrap = ACBSShrinkWrap( 222 | self.package_cursor, build_timings, packages, self.no_deps) 223 | filename = do_shrink_wrap(shrink_wrap, '/tmp') 224 | logging.info(f'... saved to {filename}') 225 | raise RuntimeError( 226 | f'Build error.\nUse `acbs-build --resume {filename}` to resume after you sorted out the situation.') 227 | 228 | def reorder_deps(self, packages, stage2: bool): 229 | logging.info('Re-ordering packages...') 230 | new_packages = [] 231 | package_names = [p.name for p in packages] 232 | for pkg in packages: 233 | # prepare for re-order if necessary 234 | logging.debug(f'Prepare for re-ordering: {pkg.name}') 235 | new_packages.append(prepare_for_reorder(pkg, package_names)) 236 | graph = get_deps_graph(new_packages) 237 | return tarjan_search(graph, self.tree_dir, stage2) 238 | 239 | def filter_unbuildable(self, packages: List[ACBSPackageInfo]) -> List[ACBSPackageInfo]: 240 | unbuildable = [] 241 | buildable = [] 242 | for p in packages: 243 | if not check_buildability(p): 244 | unbuildable.append(p.name) 245 | else: 246 | buildable.append(p) 247 | if unbuildable: 248 | logging.warning(f'The following packages will be skipped as they are not buildable:\n\t{(" ".join(unbuildable))}') 249 | return buildable 250 | 251 | def resolve_deps(self, packages, stage2: bool): 252 | error = False 253 | if not self.no_deps: 254 | logging.debug('Filtering packages...') 255 | filtered = self.filter_unbuildable(packages) 256 | packages.clear() 257 | packages.extend(filtered) 258 | logging.debug('Converting queue into adjacency graph...') 259 | graph = get_deps_graph(packages) 260 | logging.debug('Running Tarjan search...') 261 | resolved = tarjan_search(graph, self.tree_dir, stage2) 262 | # re-order the packages 263 | if self.reorder: 264 | print() 265 | resolved = self.reorder_deps( 266 | [item for sublist in resolved for item in sublist], self.stage2) 267 | else: 268 | logging.warning('Warning: Dependency resolution disabled!') 269 | resolved = [[package] for package in packages] 270 | # print a newline 271 | print() 272 | packages.clear() # clear package list for the search results 273 | # here we will check if there is any loop in the dependency graph 274 | for dep in resolved: 275 | if len(dep) > 1 or dep[0].name in dep[0].deps: 276 | # this is a SCC, aka a loop 277 | logging.error('Found a loop in the dependency graph: {}'.format( 278 | print_package_names(dep))) 279 | error = True 280 | if self.reorder: 281 | if not self.save_list: 282 | logging.warning( 283 | 'You probably want to add -p option to get a list of ordered packages.') 284 | else: 285 | logging.info( 286 | 'ACBS will still save the build queue. Please keep in mind that the build order inside the loop is not guaranteed.') 287 | error = False 288 | if not error: 289 | packages.extend(dep) 290 | if error: 291 | raise RuntimeError( 292 | 'Dependencies NOT resolved. Couldn\'t continue!') 293 | if not self.reorder: 294 | # TODO: correctly hoist the packages inside the groups 295 | check_package_groups(packages) 296 | return resolved 297 | 298 | def build_sequential(self, build_timings, packages: List[ACBSPackageInfo]): 299 | # build process 300 | for idx, task in enumerate(packages): 301 | self.package_cursor += 1 302 | logging.info( 303 | f'Building {task.name} ({self.package_cursor}/{len(packages)})...') 304 | source_name = task.name 305 | if task.base_slug: 306 | source_name = os.path.basename(task.base_slug) 307 | if not has_stamp(task.build_location) and not self.generate_pkg_metadata: 308 | fetch_source(task.source_uri, self.dump_dir, source_name) 309 | if self.dl_only: 310 | if self.generate: 311 | spec_location = os.path.join( 312 | task.script_location, '..', 'spec') 313 | is_legacy = is_spec_legacy(spec_location) 314 | checksum = generate_checksums(task.source_uri, is_legacy) 315 | write_checksums(spec_location, checksum) 316 | logging.info(f'Updated checksum for {task.name}') 317 | build_timings.append((task.name, -1)) 318 | continue 319 | if not task.build_location: 320 | build_dir = make_build_dir(self.tmp_dir) 321 | task.build_location = build_dir 322 | if not self.generate_pkg_metadata: 323 | process_source(task, source_name) 324 | else: 325 | # First sub-package in a meta-package 326 | if not has_stamp(task.build_location): 327 | if not self.generate_pkg_metadata: 328 | process_source(task, source_name) 329 | Path(os.path.join(task.build_location, '.acbs-stamp')).touch() 330 | build_dir = task.build_location 331 | if task.subdir: 332 | build_dir = os.path.join(build_dir, task.subdir) 333 | else: 334 | subdir = guess_subdir(build_dir) 335 | if not subdir: 336 | raise RuntimeError( 337 | 'Could not determine sub-directory, please specify manually.') 338 | build_dir = os.path.join(build_dir, subdir) 339 | if task.installables and not self.generate_pkg_metadata: 340 | logging.info('Installing dependencies from repository...') 341 | install_from_repo(task.installables, self.force_use_apt) 342 | start = time.monotonic() 343 | task_name = f'{task.name} ({task.bin_arch} @ {task.epoch + ":" if task.epoch else ""}{task.version}-{task.rel})' 344 | try: 345 | scoped_stage2 = ACBSPackageInfo.is_in_stage2(task.modifiers) | self.stage2 346 | invoke_autobuild(task, build_dir, scoped_stage2, self.generate_pkg_metadata) 347 | if not self.generate_pkg_metadata: 348 | check_artifact(task.name, build_dir) 349 | except Exception: 350 | # early printing of build summary before exploding 351 | print_build_timings(build_timings, packages[idx:], time.monotonic() - start) 352 | raise RuntimeError( 353 | f'Build directory of the failed package: {build_dir}') 354 | if not self.generate_pkg_metadata: 355 | build_timings.append((task_name, time.monotonic() - start)) 356 | ciel_invalidate_cache() 357 | ciel_wait_for_refresh() 358 | 359 | def acbs_except_hdr(self, type_, value, tb): 360 | logging.debug('Traceback:\n' + ''.join(traceback.format_tb(tb))) 361 | if self.debug: 362 | sys.__excepthook__(type_, value, tb) 363 | else: 364 | print() 365 | logging.fatal('Oops! \033[93m%s\033[0m: \033[93m%s\033[0m' % ( 366 | str(type_.__name__), str(value))) 367 | -------------------------------------------------------------------------------- /acbs/utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import os 4 | import re 5 | import shutil 6 | import signal 7 | import subprocess 8 | import tempfile 9 | import time 10 | from typing import Dict, List, Optional, Sequence, Tuple, cast 11 | 12 | from acbs import __version__ 13 | from acbs.ab4cfg import get_arch_override 14 | from acbs.base import ACBSPackageInfo, ACBSSourceInfo 15 | from acbs.const import ( 16 | ANSI_BROWN, 17 | ANSI_GREEN, 18 | ANSI_LT_CYAN, 19 | ANSI_RED, 20 | ANSI_RST, 21 | ANSI_YELLOW, 22 | AUTOBUILD_CONF_DIR, 23 | ) 24 | from acbs.crypto import check_hash_hashlib_inner 25 | 26 | build_logging = False 27 | 28 | try: 29 | import pexpect 30 | build_logging = True 31 | except ImportError: 32 | pexpect = None 33 | 34 | chksum_pattern = re.compile(r"CHKSUM(?:S)?=['\"].*?['\"]") 35 | tarball_pattern = re.compile(r'\.(tar\..+|cpio\..+)') 36 | SIGNAMES = dict((k, v) for v, k in reversed(sorted(signal.__dict__.items())) 37 | if v.startswith('SIG') and not v.startswith('SIG_')) 38 | 39 | 40 | def validate_package_name(package_name: str) -> bool: 41 | """ 42 | Validate package name 43 | 44 | :param package_name: name of the package 45 | :returns: True if the package name is valid 46 | """ 47 | if '/' in package_name: 48 | package_name = os.path.basename(package_name) 49 | return re.match(r'^[a-z0-9][a-z0-9\-+\.]*$', package_name) is not None 50 | 51 | 52 | def guess_extension_name_from_contents(filename: str) -> Optional[str]: 53 | from acbs import magic 54 | checker = magic.open(magic.MAGIC_MIME_TYPE) 55 | result = checker.file(filename) 56 | mime_type = result.decode('utf-8').split(';')[0] 57 | return { 58 | "application/zip": "zip", 59 | "application/gzip": "gz", 60 | "application/x-xz": "xz", 61 | "application/vnd.rar": "rar", 62 | "application/vnd.debian.binary-package": "deb", 63 | "application/x-7z-compressed": "7z", 64 | "application/x-xar": "xar", 65 | "application/x-cpio": "cpio", 66 | }.get(mime_type) 67 | 68 | 69 | def guess_extension_name(filename: str) -> str: 70 | """ 71 | Guess extension name based on filename 72 | 73 | :param filename: name of the file 74 | :returns: possible extension name 75 | """ 76 | extension = '' 77 | # determine the extension name to use 78 | re_result = re.search(tarball_pattern, filename) 79 | # handle .tar.* senarios 80 | if re_result: 81 | extension = re_result.group(1) 82 | else: 83 | # normal single extension name 84 | extensions = None 85 | for i in range(len(filename) - 1, -1, -1): 86 | if filename[i] == '.': 87 | extensions = filename[i+1:] 88 | break 89 | # no extension name? 90 | if not extensions: 91 | try: 92 | return guess_extension_name_from_contents(filename) or '' 93 | except Exception: 94 | return '' 95 | else: 96 | # strip out query parameters 97 | extension = extensions.split('?', 1)[0] 98 | if extension: 99 | extension = '.' + extension 100 | return extension 101 | 102 | 103 | def get_arch_name() -> Optional[str]: 104 | """ 105 | Detect architecture of the host machine 106 | 107 | :returns: architecture name 108 | """ 109 | abcfg_path = os.path.join(AUTOBUILD_CONF_DIR, 'ab4cfg.sh') 110 | try: 111 | arch_override = get_arch_override(abcfg_path) 112 | if arch_override: 113 | return arch_override 114 | output = subprocess.check_output(['dpkg', '--print-architecture']) 115 | return output.decode('utf-8').strip() 116 | except Exception: 117 | return None 118 | return None 119 | 120 | 121 | def full_line_banner(msg: str, char='-') -> str: 122 | """ 123 | Print a full line banner with customizable texts 124 | 125 | :param msg: message you want to be printed 126 | :param char: character to use to fill the banner 127 | """ 128 | bars_count = int((shutil.get_terminal_size().columns - len(msg) - 2) / 2) 129 | bars = char*bars_count 130 | return ' '.join((bars, msg, bars)) 131 | 132 | 133 | def print_package_names(packages: List[ACBSPackageInfo], limit: Optional[int] = None) -> str: 134 | """ 135 | Print out the names of packages 136 | 137 | :param packages: list of ACBSPackageInfo objects 138 | :param limit: maximum number of packages to print 139 | :return: a string containing the names of the packages 140 | """ 141 | pkgs = packages 142 | if limit is not None and len(packages) > limit: 143 | pkgs = packages[:limit] 144 | printable_packages = [pkg.name for pkg in pkgs] 145 | more_messages = ' ... and {} more'.format( 146 | len(packages) - limit) if limit and limit < len(packages) else '' 147 | return ', '.join(printable_packages) + more_messages 148 | 149 | 150 | def make_build_dir(path: str) -> str: 151 | return tempfile.mkdtemp(dir=path, prefix='acbs.') 152 | 153 | 154 | def guess_subdir(path: str) -> Optional[str]: 155 | name = None 156 | count = 0 157 | for subdir in os.scandir(path): 158 | if subdir.is_dir(): 159 | name = subdir.name 160 | count += 1 161 | if count > 1: 162 | return None 163 | if count < 1: # probably dummysrc 164 | name = '.' 165 | return name 166 | 167 | 168 | def has_stamp(path: str) -> bool: 169 | return os.path.exists(os.path.join(path, '.acbs-stamp')) 170 | 171 | 172 | def start_build_capture(env: Dict[str, str], build_dir: str): 173 | with tempfile.NamedTemporaryFile(prefix='acbs-build_', suffix='.log', dir=build_dir, delete=False) as f: 174 | logging.info(f'Build log: {f.name}') 175 | header = f'!!ACBS Build Log\n!!Build start: {time.ctime()}\n' 176 | f.write(header.encode()) 177 | assert pexpect 178 | process = pexpect.spawn('autobuild', logfile=f, env=cast(os._Environ, env)) 179 | term_size = shutil.get_terminal_size() 180 | # we need to adjust the pseudo-terminal size to match the actual screen size 181 | process.setwinsize(rows=term_size.lines, 182 | cols=term_size.columns) 183 | process.interact() 184 | # keep killing the process until it finishes 185 | while (not process.isalive()) and (not process.terminated): 186 | process.terminate() 187 | exit_status = process.exitstatus 188 | signal_status = process.signalstatus 189 | if signal_status: 190 | footer = f'\n!!Build killed with {SIGNAMES[signal_status]}' 191 | else: 192 | footer = f'\n!!Build exited with {exit_status}' 193 | f.write(footer.encode()) 194 | if signal_status or exit_status: 195 | raise RuntimeError('autobuild4 did not exit successfully.') 196 | 197 | def start_general_autobuild_metadata(env: Dict[str, str], script_location: str, package_name: str, build_dir: str): 198 | env["AB_WRITE_METADATA"] = "1" 199 | subprocess.check_call(['autobuild', '-p'], env=env) 200 | 201 | path = '' 202 | if script_location.split('/')[-1] == 'autobuild': 203 | path = os.path.join(script_location, '..', '.srcinfo.json') 204 | else: 205 | path = os.path.join(script_location, '..', f'.srcinfo-{package_name}.json') 206 | 207 | shutil.copyfile(os.path.join(build_dir, '.srcinfo.json'), path) 208 | logging.info(f".srcinfo.json saved to: {path}") 209 | 210 | def generate_metadata(task: ACBSPackageInfo) -> str: 211 | tree_commit = 'unknown' 212 | try: 213 | tree_commit = subprocess.check_output( 214 | ['git', '-c', 'safe.directory=/tree', 'describe', '--always', '--dirty'], cwd=task.script_location).decode('utf-8').strip() 215 | except subprocess.CalledProcessError as ex: 216 | logging.warning(f'Could not determine tree commit: {ex}') 217 | return f'X-AOSC-ACBS-Version: {__version__}\nX-AOSC-Commit: {tree_commit}\n' 218 | 219 | 220 | def generate_version_stamp(task: ACBSPackageInfo) -> str: 221 | try: 222 | head_ref = subprocess.check_output( 223 | ['git', '-c', 'safe.directory=/tree', 'symbolic-ref', 'HEAD'], cwd=task.script_location).decode('utf-8').strip() 224 | if head_ref == 'refs/heads/stable': 225 | logging.info('Not using pre-release version stamp') 226 | return '' 227 | 228 | dirty = len(subprocess.check_output( 229 | ['git', '-c', 'safe.directory=/tree', 'status', '--porcelain'], cwd=task.script_location).decode('utf-8').strip()) != 0 230 | timestamp = None 231 | if dirty: 232 | timestamp = int(time.time()) 233 | else: 234 | timestamp = int(subprocess.check_output( 235 | ['git', '-c', 'safe.directory=/tree', 'show', '-s', '--format=%ct', 'HEAD'], cwd=task.script_location).decode('utf-8').strip()) 236 | stamp = ( 237 | datetime.datetime.utcfromtimestamp(timestamp) 238 | .strftime('~pre%Y%m%dT%H%M%SZ') 239 | ) 240 | if dirty: 241 | stamp += '~dirty' 242 | logging.info(f'Using version stamp: {stamp}') 243 | return stamp 244 | except subprocess.CalledProcessError as ex: 245 | logging.warning(f'Could not determine version stamp: {ex}') 246 | return '' 247 | 248 | 249 | def check_artifact(name: str, build_dir: str): 250 | """ 251 | Check if the artifact exists 252 | 253 | :param name: name of the package 254 | :param build_dir: path to the build directory 255 | """ 256 | for f in os.listdir(build_dir): 257 | if f.endswith('.deb') and f.startswith(name): 258 | return 259 | logging.error( 260 | f'{ANSI_RED}Autobuild malfunction! Emergency drop!{ANSI_RST}') 261 | raise RuntimeError( 262 | 'STOP! Autobuild3 malfunction detected! Returned zero status with no artifact.') 263 | 264 | 265 | def invoke_autobuild(task: ACBSPackageInfo, build_dir: str, stage2: bool, generate_pkg_metadata: bool): 266 | dst_dir = os.path.join(build_dir, 'autobuild') 267 | if os.path.exists(dst_dir) and task.group_seq > 1: 268 | shutil.rmtree(dst_dir) 269 | shutil.copytree(task.script_location, dst_dir, symlinks=True) 270 | # Inject variables to defines 271 | acbs_helper = os.path.join(task.build_location, '.acbs-script') 272 | env_dict = os.environ.copy() 273 | env_dict.update({'PKGREL': task.rel, 'PKGVER': task.version, 274 | 'PKGEPOCH': task.epoch or '0', 275 | 'VERSTAMP': generate_version_stamp(task)}) 276 | env_dict.update(task.exported) 277 | if task.modifiers: 278 | env_dict['ABMODIFIERS'] = task.modifiers 279 | defines_file = 'defines' 280 | if stage2 and os.path.exists(os.path.join(build_dir, 'autobuild', 'defines.stage2')): 281 | defines_file = 'defines.stage2' 282 | with open(os.path.join(build_dir, 'autobuild', defines_file), 'at') as f: 283 | f.write('\nPKGREL=\'{}\'\nPKGVER=\'{}\'\nif [ -f \'{}\' ];then source \'{}\' && abinfo "Injected ACBS definitions";fi\n'.format( 284 | task.rel, task.version, acbs_helper, acbs_helper)) 285 | if task.epoch: 286 | f.write(f'PKGEPOCH=\'{task.epoch}\'') 287 | with open(os.path.join(build_dir, 'autobuild', 'extra-dpkg-control'), 'wt') as f: 288 | f.write(generate_metadata(task)) 289 | os.chdir(build_dir) 290 | if build_logging: 291 | if not generate_pkg_metadata: 292 | start_build_capture(env_dict, build_dir) 293 | else: 294 | start_general_autobuild_metadata(env_dict, task.script_location, task.name, build_dir) 295 | return 296 | logging.warning( 297 | 'Build logging not available due to pexpect not installed.') 298 | subprocess.check_call(['autobuild'], env=env_dict) 299 | 300 | 301 | def human_time(full_seconds: float) -> str: 302 | """ 303 | Convert time span (in seconds) to more friendly format 304 | :param full_seconds: Time span in seconds (decimal is acceptable) 305 | """ 306 | if full_seconds < 0: 307 | return 'Download only' 308 | out_str_tmp = '{}'.format( 309 | datetime.timedelta(seconds=full_seconds)) 310 | out_str = out_str_tmp.replace( 311 | ':', f'{ANSI_GREEN}:{ANSI_RST}') 312 | return out_str 313 | 314 | 315 | def format_column(data: Sequence[Tuple[str, ...]]) -> str: 316 | output = '' 317 | col_width = max(len(str(word)) for row in data for word in row) 318 | for row in data: 319 | output = '%s%s\n' % ( 320 | output, ('\t'.join(str(word).ljust(col_width) for word in row))) 321 | return output 322 | 323 | 324 | def format_package_name(package: ACBSPackageInfo) -> str: 325 | return f'{package.name} ({package.bin_arch} @ {package.epoch + ":" if package.epoch else ""}{package.version}-{package.rel})' 326 | 327 | 328 | def print_build_timings(timings: List[Tuple[str, float]], failed_packages: List[ACBSPackageInfo], last_build_time: float=0.0): 329 | """ 330 | Print the build statistics 331 | 332 | :param timings: List of timing data 333 | """ 334 | formatted_timings: List[Tuple[str, str]] = [] 335 | formatted_failed_packages = [format_package_name(pkg) for pkg in failed_packages] 336 | banner = '=' * 40 337 | print(f"\n{banner}") 338 | for timing in timings: 339 | formatted_timings.append((timing[0], human_time(timing[1]))) 340 | print(f" ACBS Build {'Successful' if not failed_packages else 'Failed'}") 341 | print(f"{banner}\n") 342 | if failed_packages: 343 | print("Failed package:") 344 | line_data = (formatted_failed_packages[0], human_time(last_build_time)) 345 | print(format_column([line_data])) 346 | if timings: 347 | print("Package(s) built:") 348 | print(format_column(formatted_timings)) 349 | if len(failed_packages) > 1: 350 | print("Package(s) not built due to previous build failure:") 351 | print('\n'.join(formatted_failed_packages[1:])) 352 | print('') 353 | 354 | 355 | def is_spec_legacy(spec: str) -> bool: 356 | with open(spec, 'rt') as f: 357 | content = f.read() 358 | return content.find('SRCS=') < 0 359 | 360 | 361 | def generate_checksums(info: List[ACBSSourceInfo], legacy=False) -> str: 362 | def calculate_checksum(o: ACBSSourceInfo): 363 | if not o.source_location: 364 | raise ValueError('source_location is None.') 365 | csum = check_hash_hashlib_inner('sha256', o.source_location) 366 | if not csum: 367 | raise ValueError( 368 | f'Unable to calculate checksum for {o.source_location}') 369 | o.chksum = ('sha256', csum) 370 | return o 371 | 372 | if legacy and info[0].type == 'tarball': 373 | info[0] = calculate_checksum(info[0]) 374 | return 'CHKSUM=\"{}\"'.format('::'.join(info[0].chksum)) 375 | output = 'CHKSUMS=\"{}\"' 376 | sums = [] 377 | formatter = ' ' if len(info) < 2 else ' \\\n ' 378 | for i in info: 379 | if i.type in ('tarball', 'file', 'pypi'): 380 | i = calculate_checksum(i) 381 | sums.append('::'.join(i.chksum)) 382 | else: 383 | sums.append('SKIP') 384 | return output.format(formatter.join(sums)) 385 | 386 | 387 | def write_checksums(spec: str, checksums: str): 388 | with open(spec, 'rt') as f: 389 | content = f.read() 390 | if re.search(chksum_pattern, content, re.MULTILINE | re.DOTALL): 391 | content = re.sub(chksum_pattern, checksums, content, 392 | flags=re.MULTILINE | re.DOTALL) 393 | else: 394 | content = content.rstrip() + "\n" + checksums + "\n" 395 | with open(spec, 'wt') as f: 396 | f.write(content) 397 | return 398 | 399 | 400 | def fail_arch_regex(expr: str) -> re.Pattern: 401 | regex = '^' 402 | negated = False 403 | sup_bracket = False 404 | if len(expr) < 3: 405 | raise ValueError('Pattern too short.') 406 | for i, c in enumerate(expr): 407 | if i == 0 and c == '!': 408 | negated = True 409 | if expr[1] != '(': 410 | regex += '(' 411 | sup_bracket = True 412 | continue 413 | if negated: 414 | if c == '(': 415 | regex += '(?!' 416 | continue 417 | elif i == 1 and sup_bracket: 418 | regex += '?!' 419 | regex += c 420 | if sup_bracket: 421 | regex += ')' 422 | return re.compile(regex) 423 | 424 | 425 | class ACBSLogFormatter(logging.Formatter): 426 | """ 427 | ABBS-like format logger formatter class 428 | """ 429 | 430 | def format(self, record): 431 | lvl_map = { 432 | 'WARNING': f'{ANSI_BROWN}WARN{ANSI_RST}', 433 | 'INFO': f'{ANSI_LT_CYAN}INFO{ANSI_RST}', 434 | 'DEBUG': f'{ANSI_GREEN}DEBUG{ANSI_RST}', 435 | 'ERROR': f'{ANSI_RED}ERROR{ANSI_RST}', 436 | 'CRITICAL': f'{ANSI_YELLOW}CRIT{ANSI_RST}' 437 | } 438 | if record.levelno in (logging.WARNING, logging.ERROR, logging.CRITICAL, 439 | logging.INFO, logging.DEBUG): 440 | record.msg = f'[{lvl_map[record.levelname]}]: \033[1m{record.msg}\033[0m' 441 | return super(ACBSLogFormatter, self).format(record) 442 | 443 | 444 | class ACBSLogPlainFormatter(logging.Formatter): 445 | """ 446 | ABBS-like format logger formatter class 447 | ... but with no color codes 448 | """ 449 | 450 | def format(self, record): 451 | lvl_map = { 452 | 'WARNING': 'WARN', 453 | 'INFO': 'INFO', 454 | 'DEBUG': 'DEBUG', 455 | 'ERROR': 'ERROR', 456 | 'CRITICAL': 'CRIT' 457 | } 458 | if record.levelno in (logging.WARNING, logging.ERROR, logging.CRITICAL, 459 | logging.INFO, logging.DEBUG): 460 | record.msg = f'[{lvl_map[record.levelname]}]: {record.msg}' 461 | return super(ACBSLogPlainFormatter, self).format(record) 462 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 2.1, February 1999 3 | 4 | Copyright (C) 1991, 1999 Free Software Foundation, Inc. 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | (This is the first released version of the Lesser GPL. It also counts 10 | as the successor of the GNU Library Public License, version 2, hence 11 | the version number 2.1.) 12 | 13 | Preamble 14 | 15 | The licenses for most software are designed to take away your 16 | freedom to share and change it. By contrast, the GNU General Public 17 | Licenses are intended to guarantee your freedom to share and change 18 | free software--to make sure the software is free for all its users. 19 | 20 | This license, the Lesser General Public License, applies to some 21 | specially designated software packages--typically libraries--of the 22 | Free Software Foundation and other authors who decide to use it. You 23 | can use it too, but we suggest you first think carefully about whether 24 | this license or the ordinary General Public License is the better 25 | strategy to use in any particular case, based on the explanations below. 26 | 27 | When we speak of free software, we are referring to freedom of use, 28 | not price. Our General Public Licenses are designed to make sure that 29 | you have the freedom to distribute copies of free software (and charge 30 | for this service if you wish); that you receive source code or can get 31 | it if you want it; that you can change the software and use pieces of 32 | it in new free programs; and that you are informed that you can do 33 | these things. 34 | 35 | To protect your rights, we need to make restrictions that forbid 36 | distributors to deny you these rights or to ask you to surrender these 37 | rights. These restrictions translate to certain responsibilities for 38 | you if you distribute copies of the library or if you modify it. 39 | 40 | For example, if you distribute copies of the library, whether gratis 41 | or for a fee, you must give the recipients all the rights that we gave 42 | you. You must make sure that they, too, receive or can get the source 43 | code. If you link other code with the library, you must provide 44 | complete object files to the recipients, so that they can relink them 45 | with the library after making changes to the library and recompiling 46 | it. And you must show them these terms so they know their rights. 47 | 48 | We protect your rights with a two-step method: (1) we copyright the 49 | library, and (2) we offer you this license, which gives you legal 50 | permission to copy, distribute and/or modify the library. 51 | 52 | To protect each distributor, we want to make it very clear that 53 | there is no warranty for the free library. Also, if the library is 54 | modified by someone else and passed on, the recipients should know 55 | that what they have is not the original version, so that the original 56 | author's reputation will not be affected by problems that might be 57 | introduced by others. 58 | 59 | Finally, software patents pose a constant threat to the existence of 60 | any free program. We wish to make sure that a company cannot 61 | effectively restrict the users of a free program by obtaining a 62 | restrictive license from a patent holder. Therefore, we insist that 63 | any patent license obtained for a version of the library must be 64 | consistent with the full freedom of use specified in this license. 65 | 66 | Most GNU software, including some libraries, is covered by the 67 | ordinary GNU General Public License. This license, the GNU Lesser 68 | General Public License, applies to certain designated libraries, and 69 | is quite different from the ordinary General Public License. We use 70 | this license for certain libraries in order to permit linking those 71 | libraries into non-free programs. 72 | 73 | When a program is linked with a library, whether statically or using 74 | a shared library, the combination of the two is legally speaking a 75 | combined work, a derivative of the original library. The ordinary 76 | General Public License therefore permits such linking only if the 77 | entire combination fits its criteria of freedom. The Lesser General 78 | Public License permits more lax criteria for linking other code with 79 | the library. 80 | 81 | We call this license the "Lesser" General Public License because it 82 | does Less to protect the user's freedom than the ordinary General 83 | Public License. It also provides other free software developers Less 84 | of an advantage over competing non-free programs. These disadvantages 85 | are the reason we use the ordinary General Public License for many 86 | libraries. However, the Lesser license provides advantages in certain 87 | special circumstances. 88 | 89 | For example, on rare occasions, there may be a special need to 90 | encourage the widest possible use of a certain library, so that it becomes 91 | a de-facto standard. To achieve this, non-free programs must be 92 | allowed to use the library. A more frequent case is that a free 93 | library does the same job as widely used non-free libraries. In this 94 | case, there is little to gain by limiting the free library to free 95 | software only, so we use the Lesser General Public License. 96 | 97 | In other cases, permission to use a particular library in non-free 98 | programs enables a greater number of people to use a large body of 99 | free software. For example, permission to use the GNU C Library in 100 | non-free programs enables many more people to use the whole GNU 101 | operating system, as well as its variant, the GNU/Linux operating 102 | system. 103 | 104 | Although the Lesser General Public License is Less protective of the 105 | users' freedom, it does ensure that the user of a program that is 106 | linked with the Library has the freedom and the wherewithal to run 107 | that program using a modified version of the Library. 108 | 109 | The precise terms and conditions for copying, distribution and 110 | modification follow. Pay close attention to the difference between a 111 | "work based on the library" and a "work that uses the library". The 112 | former contains code derived from the library, whereas the latter must 113 | be combined with the library in order to run. 114 | 115 | GNU LESSER GENERAL PUBLIC LICENSE 116 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 117 | 118 | 0. This License Agreement applies to any software library or other 119 | program which contains a notice placed by the copyright holder or 120 | other authorized party saying it may be distributed under the terms of 121 | this Lesser General Public License (also called "this License"). 122 | Each licensee is addressed as "you". 123 | 124 | A "library" means a collection of software functions and/or data 125 | prepared so as to be conveniently linked with application programs 126 | (which use some of those functions and data) to form executables. 127 | 128 | The "Library", below, refers to any such software library or work 129 | which has been distributed under these terms. A "work based on the 130 | Library" means either the Library or any derivative work under 131 | copyright law: that is to say, a work containing the Library or a 132 | portion of it, either verbatim or with modifications and/or translated 133 | straightforwardly into another language. (Hereinafter, translation is 134 | included without limitation in the term "modification".) 135 | 136 | "Source code" for a work means the preferred form of the work for 137 | making modifications to it. For a library, complete source code means 138 | all the source code for all modules it contains, plus any associated 139 | interface definition files, plus the scripts used to control compilation 140 | and installation of the library. 141 | 142 | Activities other than copying, distribution and modification are not 143 | covered by this License; they are outside its scope. The act of 144 | running a program using the Library is not restricted, and output from 145 | such a program is covered only if its contents constitute a work based 146 | on the Library (independent of the use of the Library in a tool for 147 | writing it). Whether that is true depends on what the Library does 148 | and what the program that uses the Library does. 149 | 150 | 1. You may copy and distribute verbatim copies of the Library's 151 | complete source code as you receive it, in any medium, provided that 152 | you conspicuously and appropriately publish on each copy an 153 | appropriate copyright notice and disclaimer of warranty; keep intact 154 | all the notices that refer to this License and to the absence of any 155 | warranty; and distribute a copy of this License along with the 156 | Library. 157 | 158 | You may charge a fee for the physical act of transferring a copy, 159 | and you may at your option offer warranty protection in exchange for a 160 | fee. 161 | 162 | 2. You may modify your copy or copies of the Library or any portion 163 | of it, thus forming a work based on the Library, and copy and 164 | distribute such modifications or work under the terms of Section 1 165 | above, provided that you also meet all of these conditions: 166 | 167 | a) The modified work must itself be a software library. 168 | 169 | b) You must cause the files modified to carry prominent notices 170 | stating that you changed the files and the date of any change. 171 | 172 | c) You must cause the whole of the work to be licensed at no 173 | charge to all third parties under the terms of this License. 174 | 175 | d) If a facility in the modified Library refers to a function or a 176 | table of data to be supplied by an application program that uses 177 | the facility, other than as an argument passed when the facility 178 | is invoked, then you must make a good faith effort to ensure that, 179 | in the event an application does not supply such function or 180 | table, the facility still operates, and performs whatever part of 181 | its purpose remains meaningful. 182 | 183 | (For example, a function in a library to compute square roots has 184 | a purpose that is entirely well-defined independent of the 185 | application. Therefore, Subsection 2d requires that any 186 | application-supplied function or table used by this function must 187 | be optional: if the application does not supply it, the square 188 | root function must still compute square roots.) 189 | 190 | These requirements apply to the modified work as a whole. If 191 | identifiable sections of that work are not derived from the Library, 192 | and can be reasonably considered independent and separate works in 193 | themselves, then this License, and its terms, do not apply to those 194 | sections when you distribute them as separate works. But when you 195 | distribute the same sections as part of a whole which is a work based 196 | on the Library, the distribution of the whole must be on the terms of 197 | this License, whose permissions for other licensees extend to the 198 | entire whole, and thus to each and every part regardless of who wrote 199 | it. 200 | 201 | Thus, it is not the intent of this section to claim rights or contest 202 | your rights to work written entirely by you; rather, the intent is to 203 | exercise the right to control the distribution of derivative or 204 | collective works based on the Library. 205 | 206 | In addition, mere aggregation of another work not based on the Library 207 | with the Library (or with a work based on the Library) on a volume of 208 | a storage or distribution medium does not bring the other work under 209 | the scope of this License. 210 | 211 | 3. You may opt to apply the terms of the ordinary GNU General Public 212 | License instead of this License to a given copy of the Library. To do 213 | this, you must alter all the notices that refer to this License, so 214 | that they refer to the ordinary GNU General Public License, version 2, 215 | instead of to this License. (If a newer version than version 2 of the 216 | ordinary GNU General Public License has appeared, then you can specify 217 | that version instead if you wish.) Do not make any other change in 218 | these notices. 219 | 220 | Once this change is made in a given copy, it is irreversible for 221 | that copy, so the ordinary GNU General Public License applies to all 222 | subsequent copies and derivative works made from that copy. 223 | 224 | This option is useful when you wish to copy part of the code of 225 | the Library into a program that is not a library. 226 | 227 | 4. You may copy and distribute the Library (or a portion or 228 | derivative of it, under Section 2) in object code or executable form 229 | under the terms of Sections 1 and 2 above provided that you accompany 230 | it with the complete corresponding machine-readable source code, which 231 | must be distributed under the terms of Sections 1 and 2 above on a 232 | medium customarily used for software interchange. 233 | 234 | If distribution of object code is made by offering access to copy 235 | from a designated place, then offering equivalent access to copy the 236 | source code from the same place satisfies the requirement to 237 | distribute the source code, even though third parties are not 238 | compelled to copy the source along with the object code. 239 | 240 | 5. A program that contains no derivative of any portion of the 241 | Library, but is designed to work with the Library by being compiled or 242 | linked with it, is called a "work that uses the Library". Such a 243 | work, in isolation, is not a derivative work of the Library, and 244 | therefore falls outside the scope of this License. 245 | 246 | However, linking a "work that uses the Library" with the Library 247 | creates an executable that is a derivative of the Library (because it 248 | contains portions of the Library), rather than a "work that uses the 249 | library". The executable is therefore covered by this License. 250 | Section 6 states terms for distribution of such executables. 251 | 252 | When a "work that uses the Library" uses material from a header file 253 | that is part of the Library, the object code for the work may be a 254 | derivative work of the Library even though the source code is not. 255 | Whether this is true is especially significant if the work can be 256 | linked without the Library, or if the work is itself a library. The 257 | threshold for this to be true is not precisely defined by law. 258 | 259 | If such an object file uses only numerical parameters, data 260 | structure layouts and accessors, and small macros and small inline 261 | functions (ten lines or less in length), then the use of the object 262 | file is unrestricted, regardless of whether it is legally a derivative 263 | work. (Executables containing this object code plus portions of the 264 | Library will still fall under Section 6.) 265 | 266 | Otherwise, if the work is a derivative of the Library, you may 267 | distribute the object code for the work under the terms of Section 6. 268 | Any executables containing that work also fall under Section 6, 269 | whether or not they are linked directly with the Library itself. 270 | 271 | 6. As an exception to the Sections above, you may also combine or 272 | link a "work that uses the Library" with the Library to produce a 273 | work containing portions of the Library, and distribute that work 274 | under terms of your choice, provided that the terms permit 275 | modification of the work for the customer's own use and reverse 276 | engineering for debugging such modifications. 277 | 278 | You must give prominent notice with each copy of the work that the 279 | Library is used in it and that the Library and its use are covered by 280 | this License. You must supply a copy of this License. If the work 281 | during execution displays copyright notices, you must include the 282 | copyright notice for the Library among them, as well as a reference 283 | directing the user to the copy of this License. Also, you must do one 284 | of these things: 285 | 286 | a) Accompany the work with the complete corresponding 287 | machine-readable source code for the Library including whatever 288 | changes were used in the work (which must be distributed under 289 | Sections 1 and 2 above); and, if the work is an executable linked 290 | with the Library, with the complete machine-readable "work that 291 | uses the Library", as object code and/or source code, so that the 292 | user can modify the Library and then relink to produce a modified 293 | executable containing the modified Library. (It is understood 294 | that the user who changes the contents of definitions files in the 295 | Library will not necessarily be able to recompile the application 296 | to use the modified definitions.) 297 | 298 | b) Use a suitable shared library mechanism for linking with the 299 | Library. A suitable mechanism is one that (1) uses at run time a 300 | copy of the library already present on the user's computer system, 301 | rather than copying library functions into the executable, and (2) 302 | will operate properly with a modified version of the library, if 303 | the user installs one, as long as the modified version is 304 | interface-compatible with the version that the work was made with. 305 | 306 | c) Accompany the work with a written offer, valid for at 307 | least three years, to give the same user the materials 308 | specified in Subsection 6a, above, for a charge no more 309 | than the cost of performing this distribution. 310 | 311 | d) If distribution of the work is made by offering access to copy 312 | from a designated place, offer equivalent access to copy the above 313 | specified materials from the same place. 314 | 315 | e) Verify that the user has already received a copy of these 316 | materials or that you have already sent this user a copy. 317 | 318 | For an executable, the required form of the "work that uses the 319 | Library" must include any data and utility programs needed for 320 | reproducing the executable from it. However, as a special exception, 321 | the materials to be distributed need not include anything that is 322 | normally distributed (in either source or binary form) with the major 323 | components (compiler, kernel, and so on) of the operating system on 324 | which the executable runs, unless that component itself accompanies 325 | the executable. 326 | 327 | It may happen that this requirement contradicts the license 328 | restrictions of other proprietary libraries that do not normally 329 | accompany the operating system. Such a contradiction means you cannot 330 | use both them and the Library together in an executable that you 331 | distribute. 332 | 333 | 7. You may place library facilities that are a work based on the 334 | Library side-by-side in a single library together with other library 335 | facilities not covered by this License, and distribute such a combined 336 | library, provided that the separate distribution of the work based on 337 | the Library and of the other library facilities is otherwise 338 | permitted, and provided that you do these two things: 339 | 340 | a) Accompany the combined library with a copy of the same work 341 | based on the Library, uncombined with any other library 342 | facilities. This must be distributed under the terms of the 343 | Sections above. 344 | 345 | b) Give prominent notice with the combined library of the fact 346 | that part of it is a work based on the Library, and explaining 347 | where to find the accompanying uncombined form of the same work. 348 | 349 | 8. You may not copy, modify, sublicense, link with, or distribute 350 | the Library except as expressly provided under this License. Any 351 | attempt otherwise to copy, modify, sublicense, link with, or 352 | distribute the Library is void, and will automatically terminate your 353 | rights under this License. However, parties who have received copies, 354 | or rights, from you under this License will not have their licenses 355 | terminated so long as such parties remain in full compliance. 356 | 357 | 9. You are not required to accept this License, since you have not 358 | signed it. However, nothing else grants you permission to modify or 359 | distribute the Library or its derivative works. These actions are 360 | prohibited by law if you do not accept this License. Therefore, by 361 | modifying or distributing the Library (or any work based on the 362 | Library), you indicate your acceptance of this License to do so, and 363 | all its terms and conditions for copying, distributing or modifying 364 | the Library or works based on it. 365 | 366 | 10. Each time you redistribute the Library (or any work based on the 367 | Library), the recipient automatically receives a license from the 368 | original licensor to copy, distribute, link with or modify the Library 369 | subject to these terms and conditions. You may not impose any further 370 | restrictions on the recipients' exercise of the rights granted herein. 371 | You are not responsible for enforcing compliance by third parties with 372 | this License. 373 | 374 | 11. If, as a consequence of a court judgment or allegation of patent 375 | infringement or for any other reason (not limited to patent issues), 376 | conditions are imposed on you (whether by court order, agreement or 377 | otherwise) that contradict the conditions of this License, they do not 378 | excuse you from the conditions of this License. If you cannot 379 | distribute so as to satisfy simultaneously your obligations under this 380 | License and any other pertinent obligations, then as a consequence you 381 | may not distribute the Library at all. For example, if a patent 382 | license would not permit royalty-free redistribution of the Library by 383 | all those who receive copies directly or indirectly through you, then 384 | the only way you could satisfy both it and this License would be to 385 | refrain entirely from distribution of the Library. 386 | 387 | If any portion of this section is held invalid or unenforceable under any 388 | particular circumstance, the balance of the section is intended to apply, 389 | and the section as a whole is intended to apply in other circumstances. 390 | 391 | It is not the purpose of this section to induce you to infringe any 392 | patents or other property right claims or to contest validity of any 393 | such claims; this section has the sole purpose of protecting the 394 | integrity of the free software distribution system which is 395 | implemented by public license practices. Many people have made 396 | generous contributions to the wide range of software distributed 397 | through that system in reliance on consistent application of that 398 | system; it is up to the author/donor to decide if he or she is willing 399 | to distribute software through any other system and a licensee cannot 400 | impose that choice. 401 | 402 | This section is intended to make thoroughly clear what is believed to 403 | be a consequence of the rest of this License. 404 | 405 | 12. If the distribution and/or use of the Library is restricted in 406 | certain countries either by patents or by copyrighted interfaces, the 407 | original copyright holder who places the Library under this License may add 408 | an explicit geographical distribution limitation excluding those countries, 409 | so that distribution is permitted only in or among countries not thus 410 | excluded. In such case, this License incorporates the limitation as if 411 | written in the body of this License. 412 | 413 | 13. The Free Software Foundation may publish revised and/or new 414 | versions of the Lesser General Public License from time to time. 415 | Such new versions will be similar in spirit to the present version, 416 | but may differ in detail to address new problems or concerns. 417 | 418 | Each version is given a distinguishing version number. If the Library 419 | specifies a version number of this License which applies to it and 420 | "any later version", you have the option of following the terms and 421 | conditions either of that version or of any later version published by 422 | the Free Software Foundation. If the Library does not specify a 423 | license version number, you may choose any version ever published by 424 | the Free Software Foundation. 425 | 426 | 14. If you wish to incorporate parts of the Library into other free 427 | programs whose distribution conditions are incompatible with these, 428 | write to the author to ask for permission. For software which is 429 | copyrighted by the Free Software Foundation, write to the Free 430 | Software Foundation; we sometimes make exceptions for this. Our 431 | decision will be guided by the two goals of preserving the free status 432 | of all derivatives of our free software and of promoting the sharing 433 | and reuse of software generally. 434 | 435 | NO WARRANTY 436 | 437 | 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO 438 | WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. 439 | EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR 440 | OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY 441 | KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE 442 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 443 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE 444 | LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME 445 | THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 446 | 447 | 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN 448 | WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY 449 | AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU 450 | FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR 451 | CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE 452 | LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING 453 | RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A 454 | FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF 455 | SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH 456 | DAMAGES. 457 | 458 | END OF TERMS AND CONDITIONS 459 | 460 | How to Apply These Terms to Your New Libraries 461 | 462 | If you develop a new library, and you want it to be of the greatest 463 | possible use to the public, we recommend making it free software that 464 | everyone can redistribute and change. You can do so by permitting 465 | redistribution under these terms (or, alternatively, under the terms of the 466 | ordinary General Public License). 467 | 468 | To apply these terms, attach the following notices to the library. It is 469 | safest to attach them to the start of each source file to most effectively 470 | convey the exclusion of warranty; and each file should have at least the 471 | "copyright" line and a pointer to where the full notice is found. 472 | 473 | acbs-demo 474 | Copyright (C) 2016 liushuyu 475 | 476 | This library is free software; you can redistribute it and/or 477 | modify it under the terms of the GNU Lesser General Public 478 | License as published by the Free Software Foundation; either 479 | version 2.1 of the License, or (at your option) any later version. 480 | 481 | This library is distributed in the hope that it will be useful, 482 | but WITHOUT ANY WARRANTY; without even the implied warranty of 483 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 484 | Lesser General Public License for more details. 485 | 486 | You should have received a copy of the GNU Lesser General Public 487 | License along with this library; if not, write to the Free Software 488 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 489 | USA 490 | 491 | Also add information on how to contact you by electronic and paper mail. 492 | 493 | You should also get your employer (if you work as a programmer) or your 494 | school, if any, to sign a "copyright disclaimer" for the library, if 495 | necessary. Here is a sample; alter the names: 496 | 497 | Yoyodyne, Inc., hereby disclaims all copyright interest in the 498 | library `Frob' (a library for tweaking knobs) written by James Random 499 | Hacker. 500 | 501 | {signature of Ty Coon}, 1 April 1990 502 | Ty Coon, President of Vice 503 | 504 | That's all there is to it! 505 | --------------------------------------------------------------------------------