├── nix_bisect ├── __init__.py ├── exceptions.py ├── test_util.py ├── gcroot.py ├── bisect_env.py ├── git_bisect.py ├── derivation.py ├── build_status.py ├── extra_bisect.py ├── bisect_runner.py ├── git.py └── nix.py ├── .gitignore ├── default.nix ├── setup.py ├── flake.lock ├── flake.nix ├── package.nix ├── LICENSE ├── doc └── examples │ ├── digikam.py │ └── system.py ├── CHANGELOG.md ├── README.md └── pylintrc /nix_bisect/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | result* 3 | -------------------------------------------------------------------------------- /nix_bisect/exceptions.py: -------------------------------------------------------------------------------- 1 | class ResourceConstraintException(Exception): 2 | """An operation cannot be executed within a resource limit.""" 3 | 4 | pass 5 | 6 | 7 | class TooManyBuildsException(ResourceConstraintException): 8 | """An operation would need more rebuilds than allowed.""" 9 | 10 | pass 11 | 12 | 13 | class BlacklistedBuildsException(ResourceConstraintException): 14 | """An operation would need to rebuild a blacklisted drv""" 15 | 16 | def __init__(self, drvs): 17 | super().__init__(f"Blacklisted Builds: {drvs}") 18 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { 2 | # `git ls-remote https://github.com/nixos/nixpkgs-channels nixos-unstable` 3 | # Use the flake.lock nixpkgs revision as the default 4 | nixpkgs-rev ? 5 | let 6 | lockFile = builtins.fromJSON (builtins.readFile ./flake.lock); 7 | in 8 | lockFile.nodes.nixpkgs.locked.rev, 9 | pkgsPath ? builtins.fetchTarball { 10 | name = "nixpkgs-${nixpkgs-rev}"; 11 | url = "https://github.com/nixos/nixpkgs/archive/${nixpkgs-rev}.tar.gz"; 12 | }, 13 | pkgs ? import pkgsPath { }, 14 | }: 15 | 16 | pkgs.python3.pkgs.callPackage ./package.nix { } 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Install nix-bisect""" 2 | 3 | from setuptools import setup, find_packages 4 | 5 | setup( 6 | name="nix-bisect", 7 | version="0.4.1", 8 | description="Bisect nix builds", 9 | author="Timo Kaufmann", 10 | packages=find_packages(), 11 | install_requires=["appdirs", "pexpect",], 12 | entry_points={ 13 | "console_scripts": [ 14 | "nix-build-status=nix_bisect.build_status:_main", 15 | "bisect-env=nix_bisect.bisect_env:_main", 16 | "extra-bisect=nix_bisect.extra_bisect:_main", 17 | ] 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixpkgs": { 4 | "locked": { 5 | "lastModified": 1713687659, 6 | "narHash": "sha256-Yd8KuOBpZ0Slau/NxFhMPJI0gBxeax0vq/FD0rqKwuQ=", 7 | "owner": "nixos", 8 | "repo": "nixpkgs", 9 | "rev": "f2d7a289c5a5ece8521dd082b81ac7e4a57c2c5c", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "nixos", 14 | "ref": "nixpkgs-unstable", 15 | "repo": "nixpkgs", 16 | "type": "github" 17 | } 18 | }, 19 | "root": { 20 | "inputs": { 21 | "nixpkgs": "nixpkgs" 22 | } 23 | } 24 | }, 25 | "root": "root", 26 | "version": 7 27 | } 28 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Bisect nix builds. Flake maintained by @n8henrie."; 3 | 4 | inputs.nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; 5 | 6 | outputs = 7 | { self, nixpkgs }: 8 | let 9 | systems = [ 10 | "aarch64-darwin" 11 | "x86_64-linux" 12 | "aarch64-linux" 13 | ]; 14 | eachSystem = f: nixpkgs.lib.genAttrs systems f; 15 | in 16 | { 17 | packages = eachSystem (system: { 18 | default = self.packages.${system}.nix-bisect; 19 | nix-bisect = nixpkgs.legacyPackages.${system}.python3.pkgs.callPackage ./package.nix { }; 20 | }); 21 | 22 | apps = eachSystem (system: self.packages.${system}.default.passthru.apps); 23 | formatter = eachSystem (system: nixpkgs.legacyPackages.${system}.nixfmt-rfc-style); 24 | }; 25 | } 26 | -------------------------------------------------------------------------------- /package.nix: -------------------------------------------------------------------------------- 1 | { 2 | lib, 3 | buildPythonPackage, 4 | appdirs, 5 | numpy, 6 | pexpect, 7 | }: 8 | 9 | let 10 | apps = [ 11 | "bisect-env" 12 | "extra-bisect" 13 | "nix-build-status" 14 | ]; 15 | self = buildPythonPackage rec { 16 | pname = "nix-bisect"; 17 | version = "git"; 18 | src = lib.cleanSource ./.; 19 | 20 | propagatedBuildInputs = [ 21 | appdirs 22 | numpy 23 | pexpect 24 | ]; 25 | 26 | passthru.apps = lib.genAttrs apps (script: { 27 | type = "app"; 28 | program = "${self}/bin/${script}"; 29 | }); 30 | 31 | meta = { 32 | description = "Bisect nix builds"; 33 | homepage = "https://github.com/timokau/nix-bisect"; 34 | license = lib.licenses.mit; 35 | mainProgram = [ "nix-build-status" ]; 36 | }; 37 | }; 38 | in 39 | self 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Timo Kaufmann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /nix_bisect/test_util.py: -------------------------------------------------------------------------------- 1 | """Utilities for testing a build""" 2 | 3 | 4 | from subprocess import run, Popen, PIPE 5 | from .git_bisect import quit_bad, quit_good, quit_skip, abort 6 | 7 | 8 | def exit_code(command): 9 | """Run a shell command and return its exit code""" 10 | result = run(command, shell=True, encoding="utf-8") 11 | return result.returncode 12 | 13 | 14 | def query_user(): 15 | """Query the user for the bisect result and act on it""" 16 | while True: 17 | var = input("Please evaluate the run (good/bad/skip/abort): ") 18 | if var == "good": 19 | quit_good() 20 | if var == "bad": 21 | quit_bad() 22 | if var == "skip": 23 | quit_skip() 24 | if var == "abort": 25 | abort() 26 | 27 | 28 | def script(text, interpreter="sh"): 29 | """Execute a shell script. 30 | 31 | The script is passed to the interpreter via stdin and the return 32 | code of the interpreter is returned.""" 33 | process = Popen(interpreter, stdin=PIPE) 34 | process.communicate(input=text) 35 | process.wait() 36 | return process.returncode 37 | -------------------------------------------------------------------------------- /nix_bisect/gcroot.py: -------------------------------------------------------------------------------- 1 | """Utility functions for dealing with nix gc-roots""" 2 | 3 | from pathlib import Path 4 | import tempfile 5 | import os 6 | 7 | STATE_DIR = Path(os.environ.get("NIX_STATE_DIR", "/nix/var/nix/")) 8 | USER = os.environ.get("USER", "user-unknown") 9 | GCROOT_DIR = Path(STATE_DIR).joinpath("gcroots/per-user").joinpath(USER) 10 | 11 | 12 | def gcroot_path(name): 13 | """Path to a gcroot file with name `name`""" 14 | return GCROOT_DIR.joinpath(name) 15 | 16 | 17 | def tmp_path(name): 18 | """Path to the gcroot indirection in the tmp dir""" 19 | return Path(tempfile.gettempdir()).joinpath(f"nix-bisect-gcroot-{name}") 20 | 21 | 22 | def create_tmp_gcroot(name, target): 23 | """Create an indirect gcroot in tmpdir. 24 | 25 | This has the advantage of automatically being cleaned up in case of a crash. 26 | """ 27 | tmpfile = tmp_path(name) 28 | os.symlink(target, tmpfile) 29 | create_gcroot(name, tmpfile) 30 | 31 | 32 | def create_gcroot(name, target): 33 | """Create a gcroot""" 34 | os.symlink(target, gcroot_path(name)) 35 | 36 | 37 | def delete_gcroot(name): 38 | """Delete a gcroot and its indirect temporary file""" 39 | os.remove(gcroot_path(name)) 40 | 41 | 42 | def delete_tmp_gcroot(name): 43 | """Delete a gcroot and its indirect temporary file""" 44 | os.remove(tmp_path(name)) 45 | delete_gcroot(name) 46 | -------------------------------------------------------------------------------- /doc/examples/digikam.py: -------------------------------------------------------------------------------- 1 | """Bisect a digikam segmentation fault. 2 | 3 | This script was used to bisect a runtime segfault in the digikam 4 | package. It is mostly used to find a valid commit and launch digikam, 5 | the actual testing is then performed manually. 6 | """ 7 | 8 | from nix_bisect import nix, test_util, git_bisect 9 | 10 | 11 | def _main(): 12 | # The digikam attribute changed its name at some point. 13 | try: 14 | digikam = nix.instantiate("digikam") 15 | except nix.InstantiationFailure: 16 | # If this fails to evaluate too, the bisection will abort 17 | # because of the uncaught exception. 18 | digikam = nix.instantiate("kde4.digikam") 19 | 20 | # If a log is present for digikam but the package itself is neither 21 | # in the store nor substitutable, we assume a build failure. This is 22 | # not 100% accurate, but the best we can do. 23 | if nix.log(digikam) is not None and len(nix.build_dry([digikam])[0]) > 0: 24 | print("Cached failure") 25 | git_bisect.quit_skip() 26 | 27 | # Skip on dependency failure. This is mostly done to showcase the 28 | # feature, in this case we don't need to differentiate between 29 | # dependencies and the package itself. 30 | try: 31 | nix.build(nix.dependencies([digikam])) 32 | except nix.BuildFailure: 33 | print("Dependencies failed to build") 34 | git_bisect.quit_skip() 35 | 36 | # Skip on build failure. 37 | try: 38 | build_result = nix.build([digikam]) 39 | except nix.BuildFailure: 40 | print("Digikam failed to build") 41 | git_bisect.quit_skip() 42 | 43 | # Sanity check the package. 44 | if test_util.exit_code(f"{build_result[0]}/bin/digikam -v") != 0: 45 | print("Digikam failed to launch") 46 | git_bisect.quit_skip() 47 | 48 | # Give digikam a clean slate to work with. 49 | test_util.shell( 50 | b""" 51 | echo "cleaning up" 52 | rm -f ~/Pictures/*.db 53 | rm -f ~/.config/digikamrc 54 | rm -rf ~/.local/share/digikam 55 | rm -rf ~/.cache/digikam 56 | """ 57 | ) 58 | 59 | # Now it's time for manual testing. 60 | test_util.exit_code(f"{build_result[0]}/bin/digikam") 61 | test_util.query_user() 62 | 63 | 64 | if __name__ == "__main__": 65 | _main() 66 | -------------------------------------------------------------------------------- /nix_bisect/bisect_env.py: -------------------------------------------------------------------------------- 1 | """Run a command in a temporary environment""" 2 | 3 | import sys 4 | import argparse 5 | import subprocess 6 | from nix_bisect import git_bisect, git 7 | 8 | 9 | class EnvSetupFailedException(Exception): 10 | """Raised when a mandatory action could not be completed""" 11 | 12 | 13 | def run_with_env(function, env_setup): 14 | """Run a function in a certain environment""" 15 | 16 | def pick(rev): 17 | success = git.try_cherry_pick_all(rev) 18 | if not success: 19 | raise EnvSetupFailedException("Cherry-pick failed") 20 | 21 | action_to_function = { 22 | "try_pick": git.try_cherry_pick_all, 23 | "pick": pick, 24 | } 25 | 26 | with git.git_checkpoint(): 27 | for (action, rev) in env_setup: 28 | action_to_function[action](rev) 29 | 30 | return function() 31 | 32 | 33 | def _main(): 34 | # Ordered list of actions to apply 35 | class _AppendShared(argparse.Action): 36 | def __call__(self, parser, namespace, values, option_string=None): 37 | if not "setup_actions" in namespace: 38 | setattr(namespace, "setup_actions", []) 39 | previous = namespace.setup_actions 40 | previous.append((self.dest, values)) 41 | setattr(namespace, "setup_actions", previous) 42 | 43 | parser = argparse.ArgumentParser( 44 | description="Run a program with a certain environment" 45 | ) 46 | parser.add_argument( 47 | "cmd", type=str, help="Command to run", 48 | ) 49 | parser.add_argument( 50 | "args", type=str, nargs=argparse.REMAINDER, 51 | ) 52 | parser.add_argument( 53 | "--try-pick", 54 | action=_AppendShared, 55 | default=[], 56 | help="Cherry pick a commit before building (only if it applies without issues).", 57 | ) 58 | parser.add_argument( 59 | "--pick", 60 | action=_AppendShared, 61 | default=[], 62 | help="Cherry pick a commit before building, abort on failure.", 63 | ) 64 | 65 | try: 66 | args = parser.parse_args() 67 | except SystemExit: 68 | git_bisect.abort() 69 | 70 | setup_actions = args.setup_actions if hasattr(args, "setup_actions") else [] 71 | 72 | def cmd(): 73 | return subprocess.call([args.cmd] + args.args) 74 | 75 | try: 76 | return run_with_env(cmd, setup_actions) 77 | except EnvSetupFailedException: 78 | print("Environment setup failed.") 79 | return 125 80 | 81 | 82 | if __name__ == "__main__": 83 | sys.exit(_main()) 84 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changes for nix-bisect 2 | 3 | ## [0.4.1] - 2020-04-09 4 | 5 | - Gracefully handle `extra-bisect` commands when no good or bad commit is 6 | provided yet. 7 | 8 | ## [0.4.0] - 2020-04-09 9 | 10 | - Support for regular `skip` in the bisect runner. This is in addition to `skip-range`. 11 | - More reliable parsing of `nix build` error output. 12 | - Reuse of old skip-ranges when the cherry-picks that are supposed to unbreak them do not apply. 13 | - Patchsets are now persisted in the git tree, so the bisection can be aborted and resumed. 14 | - Nicer output in the bisect log. 15 | - High-Level Derivation interface. 16 | - Split of the CLI into `nix-build-status`, `extra-bisect` and `bisect-env`. 17 | - Make use of gcroots to prevent repeated builds. 18 | - Some builds can now be blacklisted so that they will always lead to skips. 19 | - Options and argstrings can be passed through to nix. 20 | 21 | This is the first release with outside contributions. Thank you @bhipple and 22 | @lheckemann. 23 | 24 | ## [0.3.0] - 2020-03-24 25 | 26 | - Successful cached builds are no longer unnecessarily fetched. 27 | - We now cache logs of failures, even dependency failures. This enables 28 | reliable skip caching (since `nix log` is not reliable) and transitive build 29 | failure caching (i.e. caching of dependency failures). 30 | - There is a new `nix-bisect` entrypoint which can be called directly instead 31 | of calling the python module. 32 | - The nix file to build an attribute from can now be changed by passing 33 | `--nix-file` or `-f`. 34 | - There is now a new (very experimental) bisect runner that can be enabled with 35 | the `--bisect-runner` flag. When that flag is used, `nix-bisect` should used 36 | standalone instead of as a parameter of `git bisect`. Taking full control of 37 | the bisection opens the door to many possible improvements. 38 | - One such improvement is already implemented: The bisect runner will 39 | treat skips as ranges (i.e. a commit between two skips is assumed to be 40 | skipped as well) and automatically identifies which commit "unbreak" this 41 | range. It then automatically cherry-picks those "unbreak" commits to enable 42 | further bisection. 43 | 44 | ## [0.2.0] - 2020-01-21 45 | 46 | - Added a simple command line interface. 47 | - Various changes to the library 48 | 49 | ## 0.1.0 - 2019-08-13 50 | 51 | - Initial version, library only 52 | 53 | [unreleased]: https://github.com/timokau/nix-bisect/compare/v0.4.1...HEAD 54 | [0.4.1]: https://github.com/timokau/nix-bisect/compare/v0.4.0...v0.4.1 55 | [0.4.0]: https://github.com/timokau/nix-bisect/compare/v0.3.0...v0.4.0 56 | [0.3.0]: https://github.com/timokau/nix-bisect/compare/v0.2.0...v0.3.0 57 | [0.2.0]: https://github.com/timokau/nix-bisect/compare/v0.1.0...v0.2.0 58 | -------------------------------------------------------------------------------- /nix_bisect/git_bisect.py: -------------------------------------------------------------------------------- 1 | """Utilities for git-bisect. 2 | 3 | Importing this file sets up an except-hook as a side-effect that will 4 | cause any uncaught exception to exit with an exit code that aborts the 5 | bisection process. This is usually preferable to indicating a failure 6 | for the current commit, which should be done explicitly. 7 | """ 8 | 9 | import sys 10 | import inspect 11 | 12 | # colors for printing 13 | _ANSI_BLUE = "\033[94m" 14 | _ANSI_GREEN = "\033[92m" 15 | _ANSI_RED = "\033[91m" 16 | _ANSI_RESET = "\033[0m" 17 | _ANSI_BOLD = "\033[1m" 18 | 19 | # make sure uncaught exceptions abort the bisect instead of failing it 20 | def _set_excepthook(): 21 | def _handle_uncaught_exception(exctype, value, trace): 22 | old_hook(exctype, value, trace) 23 | abort() 24 | 25 | sys.excepthook, old_hook = _handle_uncaught_exception, sys.excepthook 26 | 27 | 28 | _set_excepthook() 29 | 30 | 31 | _quit_hooks = [] 32 | 33 | 34 | def register_quit_hook(hook): 35 | _quit_hooks.append(hook) 36 | 37 | 38 | def _call_quit_hooks(result, reason): 39 | for hook in _quit_hooks: 40 | # make it possible for the lazy to pass lambdas without arguments 41 | args = len(inspect.signature(hook).parameters.keys()) 42 | if args == 0: 43 | hook() 44 | elif args == 1: 45 | hook(result) 46 | else: 47 | hook(result, reason) 48 | 49 | 50 | def abort(reason=None): 51 | """Exit with an exit code that aborts a bisect run.""" 52 | _call_quit_hooks("abort", reason) 53 | sys.exit(128) 54 | 55 | 56 | def print_good(): 57 | print(f"{_ANSI_GREEN}bisect: good{_ANSI_RESET}") 58 | 59 | 60 | def print_bad(): 61 | print(f"{_ANSI_RED}bisect: bad{_ANSI_RESET}") 62 | 63 | 64 | def print_skip(reason=None): 65 | skip_reason = "" if reason is None else f" ({reason})" 66 | print(f"{_ANSI_BLUE}bisect: skip{skip_reason}{_ANSI_RESET}") 67 | 68 | 69 | def print_skip_range(reason=None): 70 | skip_reason = "" if reason is None else f" ({reason})" 71 | print(f"{_ANSI_BLUE}bisect: skip-range{skip_reason}{_ANSI_RESET}") 72 | 73 | 74 | def quit_good(reason=None): 75 | """Exit with an exit code that indicates success.""" 76 | _call_quit_hooks("good", reason) 77 | sys.exit(0) 78 | 79 | 80 | def quit_bad(reason=None): 81 | """Exit with an exit code that indicates failure.""" 82 | _call_quit_hooks("bad", reason) 83 | sys.exit(1) 84 | 85 | 86 | def quit_skip(reason=None): 87 | """Exit with an exit code that causes the commit to be skipped.""" 88 | _call_quit_hooks("skip", reason) 89 | sys.exit(125) 90 | 91 | 92 | def quit_skip_range(reason=None): 93 | """Exit with an exit code that causes the commit to be added to a skip range.""" 94 | _call_quit_hooks("skip-range", reason) 95 | sys.exit(129) 96 | -------------------------------------------------------------------------------- /nix_bisect/derivation.py: -------------------------------------------------------------------------------- 1 | """High-Level interface for determining facts about a derivation as efficiently 2 | as possible.""" 3 | 4 | from pathlib import Path 5 | import time 6 | 7 | from nix_bisect import nix, gcroot 8 | 9 | 10 | class Derivation: 11 | """A nix derivation and common operations on it, optimized for bisect""" 12 | 13 | def __init__(self, drv, nix_options=(), max_rebuilds=None, rebuild_blacklist=()): 14 | """Create a new derivation. 15 | 16 | The derivation's methods will throw TooManyBuildsException when the 17 | rebuild limit is exceeded. 18 | """ 19 | self.drv = drv 20 | self.nix_options = nix_options 21 | self.max_rebuilds = max_rebuilds if max_rebuilds is not None else float("inf") 22 | self.rebuild_blacklist = rebuild_blacklist 23 | self._gcroot_name = f"nix-bisect-{Path(drv).name}-{round(time.time() * 1000.0)}" 24 | gcroot.create_tmp_gcroot(self._gcroot_name, drv) 25 | 26 | def __del__(self): 27 | gcroot.delete_tmp_gcroot(self._gcroot_name) 28 | 29 | def immediate_dependencies(self): 30 | """Returns the derivation's immediate dependencies.""" 31 | return nix.references([self.drv]) 32 | 33 | def can_build_deps(self): 34 | """Determines if the derivation's dependencies build would succeed. 35 | 36 | This may or may not actually build or fetch the dependencies. If 37 | possible, cached information is used. 38 | """ 39 | return nix.build_would_succeed( 40 | self.immediate_dependencies(), 41 | nix_options=self.nix_options, 42 | max_rebuilds=self.max_rebuilds - 1, 43 | rebuild_blacklist=self.rebuild_blacklist, 44 | ) 45 | 46 | def sample_dependency_failure(self): 47 | """Returns one dependency failure if it exists. 48 | 49 | This is a cheap-operation of can_build_deps has already been executed. 50 | In contrast, determining all failing dependencies might be much more 51 | expensive as it requires running `nix-build --keep-going`. 52 | """ 53 | # This will use cached failures. 54 | try: 55 | nix.build(self.immediate_dependencies(), nix_options=self.nix_options) 56 | except nix.BuildFailure as bf: 57 | return next(iter(bf.drvs_failed)) 58 | return None 59 | 60 | def can_build(self): 61 | """Determines if the derivation's build would succeed. 62 | 63 | This may or may not actually build or fetch the derivation. If 64 | possible, cached information is used. 65 | """ 66 | return nix.build_would_succeed( 67 | [self.drv], 68 | nix_options=self.nix_options, 69 | max_rebuilds=self.max_rebuilds, 70 | rebuild_blacklist=self.rebuild_blacklist, 71 | ) 72 | 73 | def log_contains(self, line): 74 | """Determines if the derivation's build log contains a line. 75 | 76 | This may or may not actually build or fetch the derivation. If 77 | possible, cached information is used. 78 | """ 79 | return nix.log_contains(self.drv, line) == "yes" 80 | -------------------------------------------------------------------------------- /doc/examples/system.py: -------------------------------------------------------------------------------- 1 | """Bisect a whole nixos + home-manager system""" 2 | 3 | import tempfile 4 | import stat 5 | import time 6 | 7 | from subprocess import Popen, PIPE 8 | from multiprocessing import Process 9 | from pathlib import Path 10 | 11 | from nix_bisect import nix, test_util, git_bisect 12 | 13 | 14 | def _main(): 15 | process = Popen("bash", stdin=PIPE, stdout=PIPE, encoding="UTF-8") 16 | 17 | # Hack to "instantiate" the home-manager system configuration. 18 | with tempfile.TemporaryDirectory() as tmpdirname: 19 | # Monkey-patch "nix-build" by creating a mock nix-build script in a 20 | # temporary directory and then prepending that directory to PATH. 21 | # The goal here is to just print the instantiation result but not 22 | # actually do the building. 23 | nix_build_path = Path(tmpdirname).joinpath("nix-build") 24 | with open(nix_build_path, "w+") as nix_build_mock: 25 | nix_build_mock.write( 26 | """ 27 | #!/usr/bin/env bash 28 | PATH="$old_PATH" nix-instantiate "$@" 29 | """ 30 | ) 31 | nix_build_path.chmod(nix_build_path.stat().st_mode | stat.S_IEXEC) 32 | (stdout, _stderr) = process.communicate( 33 | input=f""" 34 | export old_PATH="$PATH" 35 | export PATH="{tmpdirname}:$PATH" 36 | echo $PWD >&2 37 | home-manager -I nixpkgs=. build 38 | """ 39 | ) 40 | process.wait() 41 | 42 | home = stdout.strip() 43 | print(f"Home: {home}") 44 | # This is what nixos-rebuild instantiates internally 45 | nixos = nix.instantiate("system", nix_file="./nixos") 46 | print(f"Nixos: {nixos}") 47 | 48 | # Build the nixos and home-manager configurations at the same time for 49 | # optimal parallelization. 50 | build_target = [home, nixos] 51 | if len(nix.build_dry(build_target)[0]) > 500: 52 | print("Too many rebuilds, skipping") 53 | git_bisect.quit_skip() 54 | 55 | # Skip on system build failure. 56 | try: 57 | _build_result = nix.build(build_target) 58 | except nix.BuildFailure: 59 | print("System failed to build") 60 | git_bisect.quit_skip() 61 | 62 | # Switch to the previously built system. 63 | if test_util.exit_code("home-manager -I nixpkgs=. switch") != 0: 64 | git_bisect.quit_skip() 65 | if test_util.exit_code("sudo nixos-rebuild switch") != 0: 66 | git_bisect.quit_skip() 67 | 68 | # Test kitty 69 | if test_util.exit_code(f"kitty echo Hello World") != 0: 70 | print("Kitty failed to launch") 71 | git_bisect.quit_bad() 72 | else: 73 | git_bisect.quit_good() 74 | 75 | 76 | class Sudoloop: 77 | """Keeps the sudo password cached""" 78 | 79 | def __init__(self, initialize=True): 80 | self.initialize = initialize 81 | 82 | def sudoloop(): 83 | while True: 84 | # Extend sudo cache by 5 minutes but do not re-query the user 85 | # if the password is not already cached. 86 | test_util.exit_code("sudo -S --validate /dev/null") 87 | time.sleep(4 * 60) 88 | 89 | self.loop_process = Process(target=sudoloop) 90 | 91 | def __enter__(self): 92 | if self.initialize: 93 | # First initialize the cache in a blocking manner. 94 | test_util.exit_code("sudo --validate") 95 | self.loop_process.start() 96 | 97 | def __exit__(self, _type, _value, _traceback): 98 | self.loop_process.terminate() 99 | self.loop_process.join() 100 | 101 | 102 | if __name__ == "__main__": 103 | with Sudoloop(): 104 | _main() 105 | -------------------------------------------------------------------------------- /nix_bisect/build_status.py: -------------------------------------------------------------------------------- 1 | """Determine the status of a nix build as lazily as possible in a 2 | bisect-friendly format""" 3 | 4 | import sys 5 | import argparse 6 | from pathlib import Path 7 | 8 | from nix_bisect import nix, exceptions, git_bisect 9 | from nix_bisect.derivation import Derivation 10 | 11 | 12 | def drvish_to_drv(drvish, nix_file, nix_options, nix_argstr): 13 | """No-op on drv files, otherwise evaluate in the context of nix_file""" 14 | path = Path(drvish) 15 | if path.exists() and path.name.endswith(".drv"): 16 | return str(path) 17 | else: 18 | return nix.instantiate( 19 | drvish, nix_file, nix_options=nix_options, nix_argstr=nix_argstr 20 | ) 21 | 22 | 23 | def build_status( 24 | drvish, 25 | nix_file, 26 | nix_options, 27 | nix_argstr, 28 | failure_line=None, 29 | max_rebuilds=None, 30 | rebuild_blacklist=(), 31 | ): 32 | """Determine the status of `drvish` and return the result as indicated""" 33 | try: 34 | drv = drvish_to_drv( 35 | drvish, nix_file, nix_options=nix_options, nix_argstr=nix_argstr 36 | ) 37 | except nix.InstantiationFailure: 38 | return "instantiation_failure" 39 | print(f"Querying status of {drv}.") 40 | 41 | try: 42 | drv = Derivation( 43 | drv, 44 | nix_options=nix_options, 45 | max_rebuilds=max_rebuilds, 46 | rebuild_blacklist=rebuild_blacklist, 47 | ) 48 | 49 | if not drv.can_build_deps(): 50 | failed = drv.sample_dependency_failure() 51 | print(f"Dependency {failed} failed to build.") 52 | return f"dependency_failure" 53 | 54 | if drv.can_build(): 55 | return "success" 56 | else: 57 | if failure_line is None or drv.log_contains(failure_line): 58 | return "failure" 59 | else: 60 | return "failure_without_line" 61 | except exceptions.ResourceConstraintException as e: 62 | print(e) 63 | return "resource_limit" 64 | 65 | 66 | class _ActionChoices(list): 67 | def __init__(self): 68 | self.named_choices = ["good", "bad", "skip", "skip-range"] 69 | # Add a dummy choice that will only show up in --help but will not 70 | # actually be accepted. 71 | choice_list = self.named_choices + [""] 72 | super().__init__(choice_list) 73 | 74 | # An extension of list that just pretends every integer is a member. Used 75 | # to accept arbitrary return codes as choices (in addition to named 76 | # actions). 77 | def __contains__(self, other): 78 | if self.named_choices.__contains__(other): 79 | return True 80 | try: 81 | _retcode = int(other) 82 | return True 83 | except ValueError: 84 | return False 85 | 86 | 87 | def _main(): 88 | def to_exit_code(action): 89 | try: 90 | return int(action) 91 | except ValueError: 92 | return {"good": 0, "bad": 1, "skip": 125, "skip-range": 129, "abort": 128,}[ 93 | action 94 | ] 95 | 96 | action_choices = _ActionChoices() 97 | 98 | parser = argparse.ArgumentParser( 99 | description="Build a package with nix, suitable for git-bisect." 100 | ) 101 | parser.add_argument( 102 | "drvish", 103 | type=str, 104 | help="Derivation or an attribute/expression that can be resolved to a derivation in the context of the nix file", 105 | ) 106 | parser.add_argument( 107 | "--file", 108 | "-f", 109 | help="Nix file that contains the attribute", 110 | type=str, 111 | default=".", 112 | ) 113 | parser.add_argument( 114 | "--option", 115 | nargs=2, 116 | metavar=("name", "value"), 117 | action="append", 118 | default=[], 119 | help="Set the Nix configuration option `name` to `value`.", 120 | ) 121 | parser.add_argument( 122 | "--argstr", 123 | nargs=2, 124 | metavar=("name", "value"), 125 | action="append", 126 | default=[], 127 | help="Passed on to `nix instantiate`", 128 | ) 129 | parser.add_argument( 130 | "--max-rebuilds", type=int, help="Number of builds to allow.", default=None, 131 | ) 132 | parser.add_argument( 133 | "--failure-line", 134 | help="Line required in the build logs to count as a failure.", 135 | default=None, 136 | ) 137 | parser.add_argument( 138 | "--on-success", 139 | default="good", 140 | choices=action_choices, 141 | help="Bisect action if the expression can be successfully built", 142 | ) 143 | parser.add_argument( 144 | "--on-failure", 145 | default="bad", 146 | choices=action_choices, 147 | help="Bisect action if the expression can be successfully built", 148 | ) 149 | parser.add_argument( 150 | "--on-dependency-failure", 151 | default="skip-range", 152 | choices=action_choices, 153 | help="Bisect action if the expression can be successfully built", 154 | ) 155 | parser.add_argument( 156 | "--on-failure-without-line", 157 | default="skip-range", 158 | choices=action_choices, 159 | help="Bisect action if the expression can be successfully built", 160 | ) 161 | parser.add_argument( 162 | "--on-instantiation-failure", 163 | default="skip-range", 164 | choices=action_choices, 165 | help="Bisect action if the expression cannot be instantiated", 166 | ) 167 | parser.add_argument( 168 | "--on-resource-limit", 169 | default="skip", 170 | choices=action_choices, 171 | help="Bisect action if a resource limit like rebuild count is exceeded", 172 | ) 173 | parser.add_argument( 174 | "--rebuild-blacklist", 175 | action="append", 176 | help="If any derivation matching this regex needs to be rebuilt, the build is skipped", 177 | ) 178 | 179 | try: 180 | args = parser.parse_args() 181 | except SystemExit: 182 | git_bisect.abort() 183 | 184 | status = build_status( 185 | args.drvish, 186 | args.file, 187 | nix_options=args.option, 188 | nix_argstr=args.argstr, 189 | failure_line=args.failure_line, 190 | max_rebuilds=args.max_rebuilds, 191 | rebuild_blacklist=args.rebuild_blacklist 192 | if args.rebuild_blacklist is not None 193 | else (), 194 | ) 195 | action_on_status = { 196 | "success": args.on_success, 197 | "failure": args.on_failure, 198 | "dependency_failure": args.on_dependency_failure, 199 | "failure_without_line": args.on_failure_without_line, 200 | "instantiation_failure": args.on_instantiation_failure, 201 | "resource_limit": args.on_resource_limit, 202 | } 203 | print(f"Build status: {status}") 204 | sys.exit(to_exit_code(action_on_status[status])) 205 | 206 | 207 | if __name__ == "__main__": 208 | sys.exit(_main()) 209 | -------------------------------------------------------------------------------- /nix_bisect/extra_bisect.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import argparse 3 | import subprocess 4 | import shlex 5 | from nix_bisect import bisect_runner, git, git_bisect 6 | 7 | 8 | def _setup_start_parser(parser): 9 | parser.add_argument("bad", nargs="?") 10 | parser.add_argument("good", nargs="*", default=[]) 11 | 12 | def _handle_start(args): 13 | try: 14 | extra_args = [args.bad] if args.bad is not None else [] 15 | extra_args.extend(args.good) 16 | subprocess.check_call(["git", "bisect", "start"] + extra_args) 17 | except subprocess.CalledProcessError: 18 | # `git bisect start` already prints the appropriate error message 19 | return 1 20 | return 0 21 | 22 | parser.set_defaults(func=_handle_start) 23 | 24 | 25 | def _setup_good_parser(parser): 26 | parser.add_argument( 27 | "rev", 28 | type=str, 29 | default="HEAD", 30 | help="Revision that will be marked as good", 31 | nargs="?", 32 | ) 33 | 34 | def _handle_good(args): 35 | print("Good") 36 | bisect_runner.bisect_good(args.rev) 37 | if bisect_runner.has_good_and_bad(): 38 | git.checkout(bisect_runner.BisectRunner().get_next()) 39 | return 0 40 | 41 | parser.set_defaults(func=_handle_good) 42 | 43 | 44 | def _setup_bad_parser(parser): 45 | parser.add_argument( 46 | "rev", 47 | type=str, 48 | default="HEAD", 49 | help="Revision that will be marked as bad", 50 | nargs="?", 51 | ) 52 | 53 | def _handle_bad(args): 54 | bisect_runner.bisect_bad(args.rev) 55 | if bisect_runner.has_good_and_bad(): 56 | git.checkout(bisect_runner.BisectRunner().get_next()) 57 | return 0 58 | 59 | parser.set_defaults(func=_handle_bad) 60 | 61 | 62 | def _setup_skip_parser(parser): 63 | parser.add_argument( 64 | "rev", 65 | type=str, 66 | default="HEAD", 67 | help="Revision that will be marked as belonging to the skip range", 68 | nargs="?", 69 | ) 70 | parser.add_argument( 71 | "--name", 72 | type=str, 73 | default="default", 74 | help="Name of the skip range, purely for display", 75 | ) 76 | 77 | def _handle_skip(args): 78 | bisect_runner.bisect_skip(args.rev) 79 | if bisect_runner.has_good_and_bad(): 80 | git.checkout(bisect_runner.BisectRunner().get_next()) 81 | return 0 82 | 83 | parser.set_defaults(func=_handle_skip) 84 | 85 | 86 | def _setup_skip_range_parser(parser): 87 | parser.add_argument( 88 | "rev", 89 | type=str, 90 | default="HEAD", 91 | help="Revision that will be marked as belonging to the skip range", 92 | nargs="?", 93 | ) 94 | parser.add_argument( 95 | "--name", 96 | type=str, 97 | default="default", 98 | help="Name of the skip range, purely for display", 99 | ) 100 | 101 | def _handle_skip_range(args): 102 | patchset = bisect_runner.read_patchset() 103 | bisect_runner.named_skip(args.name, patchset, args.rev) 104 | if bisect_runner.has_good_and_bad(): 105 | git.checkout(bisect_runner.BisectRunner().get_next()) 106 | return 0 107 | 108 | parser.set_defaults(func=_handle_skip_range) 109 | 110 | 111 | def _setup_env_parser(parser): 112 | parser.add_argument( 113 | "cmd", type=str, help="Command to run", default="bash", nargs="?", 114 | ) 115 | parser.add_argument( 116 | "args", type=str, nargs=argparse.REMAINDER, 117 | ) 118 | 119 | def _handle_env(args): 120 | patchset = bisect_runner.read_patchset() 121 | arg_list = bisect_runner.bisect_env_args(patchset) 122 | arg_list.append(args.cmd) 123 | arg_list.extend(args.args) 124 | return subprocess.call(["bisect-env"] + arg_list) 125 | 126 | parser.set_defaults(func=_handle_env) 127 | 128 | 129 | def _setup_run_parser(parser): 130 | parser.add_argument( 131 | "cmd", type=str, help="Command that controls the bisect", 132 | ) 133 | parser.add_argument( 134 | "args", type=str, nargs=argparse.REMAINDER, 135 | ) 136 | 137 | def _handle_run(args): 138 | if not bisect_runner.has_good_and_bad(): 139 | print("You need to mark at least one good and one bad commit first.") 140 | return 1 141 | 142 | runner = bisect_runner.BisectRunner() 143 | while True: 144 | subprocess_args = ["bisect-env"] 145 | subprocess_args.extend( 146 | bisect_runner.bisect_env_args(bisect_runner.read_patchset()) 147 | ) 148 | subprocess_args.append(args.cmd) 149 | subprocess_args.extend(args.args) 150 | 151 | quoted_cmd = " ".join([shlex.quote(arg) for arg in subprocess_args]) 152 | bisect_runner.bisect_append_log(f"# $ {quoted_cmd}") 153 | print(f"$ {quoted_cmd}") 154 | 155 | return_code = subprocess.call(subprocess_args) 156 | if return_code == 0: 157 | git_bisect.print_good() 158 | bisect_runner.bisect_good("HEAD") 159 | elif return_code == 125: 160 | git_bisect.print_skip() 161 | bisect_runner.bisect_skip("HEAD") 162 | elif return_code == 129: 163 | git_bisect.print_skip_range() 164 | patchset = bisect_runner.read_patchset() 165 | bisect_runner.named_skip("runner-skip", patchset, "HEAD") 166 | elif 1 <= return_code <= 127: 167 | git_bisect.print_bad() 168 | bisect_runner.bisect_bad("HEAD") 169 | else: 170 | break 171 | next_commit = runner.get_next() 172 | if next_commit is None: 173 | break 174 | git.checkout(next_commit) 175 | return 0 176 | 177 | parser.set_defaults(func=_handle_run) 178 | 179 | 180 | def _setup_reset_parser(parser): 181 | parser.add_argument("commit", nargs="?") 182 | 183 | def _handle_reset(args): 184 | try: 185 | extra_args = [args.commit] if args.commit is not None else [] 186 | subprocess.check_call(["git", "bisect", "reset"] + extra_args) 187 | except subprocess.CalledProcessError: 188 | # `git bisect reset` already prints the appropriate error message 189 | return 1 190 | return 0 191 | 192 | parser.set_defaults(func=_handle_reset) 193 | 194 | 195 | def _main(): 196 | parser = argparse.ArgumentParser(description="git-bisect with extra features") 197 | 198 | subparsers = parser.add_subparsers( 199 | title="subcommands", 200 | description="You can use one of the following subcommands:", 201 | help="Each subcommand has its own `--help` page. Also see `man git-bisect` since many of the commands are similar.", 202 | ) 203 | 204 | _setup_good_parser(subparsers.add_parser("good")) 205 | _setup_bad_parser(subparsers.add_parser("bad")) 206 | _setup_skip_parser(subparsers.add_parser("skip")) 207 | _setup_skip_range_parser(subparsers.add_parser("skip-range")) 208 | _setup_env_parser(subparsers.add_parser("env")) 209 | _setup_run_parser(subparsers.add_parser("run")) 210 | _setup_start_parser(subparsers.add_parser("start")) 211 | _setup_reset_parser(subparsers.add_parser("reset")) 212 | 213 | args = parser.parse_args() 214 | if not hasattr(args, "func"): 215 | parser.print_usage() 216 | return 128 217 | return args.func(args) 218 | 219 | 220 | if __name__ == "__main__": 221 | sys.exit(_main()) 222 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nix-bisect -- Bisect Nix Builds 2 | 3 | Thanks to the reproducibility of [nix](https://nixos.org/nix/) and the monorepo approach of [nixpkgs](https://github.com/NixOS/nixpkgs) it is possible to bisect anything from a simple build failure to a regression in your system setup. 4 | 5 | ## Quick Usage Example 6 | 7 | Imagine you just discovered that the `python3.pkgs.rpy2` build is failing on current master (which is assumed to be 0729b8c55e0dfaf302af4c57546871d47a652048): 8 | 9 | ```bash 10 | $ git checkout 0729b8c55e0dfaf302af4c57546871d47a652048 11 | HEAD is now at 0729b8c55e0 Revert Merge #82310: nixos/systemd: apply .link 12 | 13 | $ nix build -f. python3.pkgs.rpy2 14 | builder for '/nix/store/blxlihmb2a4x90x8as9f0hihwag6pa1a-python3.7-rpy2-3.2.6.drv' failed with exit code 1; last 10 log lines: 15 | /nix/store/4lf6ry28hv9ydflwy62blbsca9hqkwq2-python3.7-ipython-7.12.0/lib/python3.7/site-packages/IPython/paths.py:67: UserWarning: IPython parent '/homeless-shelter' is not a writable location, using a temp directory. 16 | " using a temp directory.".format(parent)) 17 | 18 | rpy2/tests/robjects/test_pandas_conversions.py::TestPandasConversions::test_dataframe_int_nan[dtype0] 19 | rpy2/tests/robjects/test_pandas_conversions.py::TestPandasConversions::test_dataframe_int_nan[dtype1] 20 | /build/rpy2-3.2.6/rpy2/robjects/pandas2ri.py:63: UserWarning: Error while trying to convert the column "z". Fall back to string conversion. The error is: int() argument must be a string, a bytes-like object or a number, not 'NAType' 21 | % (name, str(e))) 22 | 23 | -- Docs: https://docs.pytest.org/en/latest/warnings.html 24 | = 4 failed, 674 passed, 12 skipped, 2 xfailed, 1 xpassed, 6 warnings in 30.07s = 25 | [0 built (1 failed), 0.0 MiB DL] 26 | error: build of '/nix/store/blxlihmb2a4x90x8as9f0hihwag6pa1a-python3.7-rpy2-3.2.6.drv' failed 27 | ``` 28 | 29 | as a first reaction, you check the build log to get a hint of what is causing the issue: 30 | 31 | ```bash 32 | nix log /nix/store/blxlihmb2a4x90x8as9f0hihwag6pa1a-python3.7-rpy2-3.2.6.drv 33 | [output elided] 34 | ``` 35 | 36 | you don't immediately recognize the failure. Instead of researching or debugging it, you decide to take advantage of nix and nixpkgs and bisect the failure first. You're fairly confident the build worked a while ago, so you just randomly check a previous commit. You can be generous in the step size here, since `git-biset` has a logarithmic runtime. 37 | 38 | 39 | ``` 40 | git co HEAD~5000 41 | Updating files: 100% (9036/9036), done. 42 | Previous HEAD position was 0729b8c55e0 Revert Merge #82310: nixos/systemd: apply .link 43 | HEAD is now at 43165b29e2e Merge pull request #71894 from timokau/home-manager-2019-10-23 44 | 45 | $ nix build -f. python3.pkgs.rpy2 46 | [152 copied (1362.1 MiB), 377.5 MiB DL] 47 | ``` 48 | 49 | The build succeeded! Now you have a good commit and a bad commit. To make 50 | bisection more robust, the only thing missing is a "failure line", e.g. a line 51 | from the build log to distinguish the failure we're looking for from other 52 | failures that may have come and gone in the meantime. Looking back at the build 53 | log of the failed attempt, the line 54 | 55 | > Incompatible C type sizes. The R array type is 4 bytes while the Python array type is 8 bytes. 56 | 57 | seems pretty distinctive. Now we can go ahead with the bisect. 58 | 59 | ```bash 60 | extra-bisect start 0729b8c55e0dfaf302af4c57546871d47a652048 HEAD 61 | ``` 62 | 63 | Now let `nix-bisect` take care of the actual bisection: 64 | 65 | ```bash 66 | extra-bisect run \ 67 | nix-build-status \ 68 | --max-rebuild 100 \ 69 | --failure-line 'Incompatible C type sizes. The R array type is 4 bytes while the Python array type is 8 bytes.' \ 70 | python3.pkgs.rpy2 71 | ``` 72 | 73 | This takes a while. Fetch a coffee. In fact, fetch the can. On my laptop this 74 | ran for a little over 2 hours. During the run it notices several intermediate 75 | failures which prevent it from deciding whether the commit is good or bad. It 76 | determines which commits fix those intermediate failures and automatically 77 | cherry-picks those commits to continue the bisection 78 | 79 | - fc7e4c926755f47edcda488eaea0c7fb82ff5af9 fix `texlive` 80 | - ff741a5d52550f0bfcb07584c35349f8f9208e0c disable a failing `pandas` test 81 | - eebda1d2f9cdffba3530428b34d97c493cc82677 fix an unrelated `rpy2` failure 82 | 83 | Finally and without any human intervention we get the result: The build was 84 | broken by a recent `pandas` 1.0 update! Some more research reveals that the bug 85 | issue is actually known and already fixed upstream, the upstream repository I 86 | searched first was just outdated. 87 | 88 | As the last step, let's be good open source citizens and upstream our findings: 89 | 90 | - https://github.com/rpy2/rpy2/issues/662 91 | - https://github.com/NixOS/nixpkgs/pull/82773 92 | 93 | ## Explanation and Rationale 94 | 95 | The naive way to bisect a nix build would be 96 | 97 | ```bash 98 | git bisect run nix build -f. attrname 99 | ``` 100 | 101 | This is not perfect though. If you use `nix-bisect` and replace that command with 102 | 103 | ```bash 104 | git bisect run nix-build-status attrname 105 | ``` 106 | 107 | You get the following benefits out of the box: 108 | 109 | - nicer output, with color highlighting on bisect success/failure 110 | - `bisect skip` on dependency failures, `bisect bad` only if the actual attribute fails to build 111 | - the bisect is aborted on ctrl-c, instead of registering as a `bisect bad` 112 | - if there is some unexpected failure, like an instantiation failure, the bisect is aborted instead of registering as a `bisect bad` 113 | 114 | In addition to that out of the box behaviour you can also use it for more 115 | complex use-cases. Consider this example: 116 | 117 | 118 | ``` 119 | git bisect run bisect-env --try-pick e3601e1359ca340b9eda1447436c41f5aa7c5293 nix-build-status --max-rebuilds 500 --failure-line="TypeError:" 'sage.tests.override { files=["src/sage/env.py"]; }' 120 | ``` 121 | 122 | This can be used to track down a failure in the sage build. It should be fairly 123 | self-explanatory. In addition to the benefits mentioned above, it will 124 | 125 | - Try to cherry-pick commit `e3601e1e` into the tree before starting the build. 126 | This is really useful to bisect when there are some already fixed 127 | intermediate failures. This option can be passed multiple times. When the 128 | commit fails to apply (for example because it is already applied on the 129 | current checkout), it is simply not applied. The tree is reset to the exact 130 | state it was before (including unstaged changes) once the build is finished. 131 | 132 | - Skip on any build that would require more than 500 local builds. 133 | 134 | - Register `bisect bad` only when the build fails *and* the specified text 135 | occurs in the build log. If the build fails without the text the current 136 | revision is skipped. 137 | 138 | - Make use of cached builds *and* cached failures (which is only possible with `--failure-line`). 139 | 140 | - Build the *overridden* attribute `sage.tests.override { files=["src/sage/env.py"]; }`. 141 | Plain `nix build` will not allow you to use overrides by default. 142 | 143 | It is very hard, maybe impossible, to build a command-line interface that is 144 | flexible enough to cover any possible use-case for bisecting nix builds. 145 | Because of that, `nix-bisect` is not only a command line tool but primarily a 146 | python library. The CLI is only a convenience for common use-cases. 147 | 148 | If you have a more complex use-case, you can use `nix-bisect` to write a 149 | bisection script in an arguably saner language than bash. You get nice utility 150 | functions and abstractions. 151 | 152 | As an example, 153 | [here](https://github.com/timokau/nix-bisect/blob/712adc0cd3c34bd45c22c03c06d58e83d58da1c3/doc/examples/digikam.py) 154 | is a script I used to debug a digikam segfault. It will build digikam 155 | (transparently dealing with an change of its attrname that happened at some 156 | point), skipping through all build failures. Once a build finally succeeds, it 157 | will prompt me to manually check for a segfault and use my input to decide 158 | whether the current revision is good or bad. 159 | 160 | Keep in mind however that this is very early stages. Barely anything is 161 | documented. I built this to scratch my own itch, and I continue developing it 162 | whenever I need some feature. 163 | 164 | Still, I can already be quite useful for some people. It is not packaged in 165 | nixpkgs, but if you want to try it out simply use `nix-shell` with the 166 | `default.nix` provided in this repository. 167 | -------------------------------------------------------------------------------- /nix_bisect/bisect_runner.py: -------------------------------------------------------------------------------- 1 | """A python reimplementation of git-bisect""" 2 | 3 | import subprocess 4 | from pathlib import Path 5 | import numpy as np 6 | from nix_bisect import git, git_bisect 7 | 8 | 9 | def has_good_and_bad(): 10 | """Determines if the bisect can start""" 11 | return len(get_good_commits()) > 0 and git.rev_parse("refs/bisect/bad") is not None 12 | 13 | 14 | def patchset_identifier(patchset): 15 | """Unique string identifier for a patchset to be used in ref names""" 16 | components = ["patchset"] + patchset 17 | return "/".join(components) 18 | 19 | 20 | def bisect_append_log(msg): 21 | """Adds one line to the bisect log""" 22 | path = Path(git.git_dir()).joinpath("BISECT_LOG") 23 | with open(path, "a") as fp: 24 | fp.write(msg + "\n") 25 | 26 | 27 | def named_skip(name, patchset, commit): 28 | """Mark a commit as belonging to a named skip range. 29 | 30 | In contrast to a regular `git bisect skip`, all commits between two commits 31 | in the range are considered skipped as well. 32 | """ 33 | unique_name = git.rev_parse(commit) 34 | git.update_ref( 35 | f"refs/bisect/break/{patchset_identifier(patchset)}/markers/{name}/{unique_name}", 36 | commit, 37 | ) 38 | bisect_append_log(f"# skip-range: {git.rev_pretty(commit)}") 39 | bisect_append_log(f"extra-bisect skip-range {git.rev_parse(commit)}") 40 | 41 | 42 | def bisect_bad(commit): 43 | """Mark a commit as bad. 44 | 45 | Warning: This may have the side-effect of switching to a different 46 | revision. 47 | 48 | Unfortunately we don't have control about that. In the future we may want 49 | to manage the refs and the bisect-log manually. 50 | """ 51 | git.update_ref(f"refs/bisect/bad", commit) 52 | bisect_append_log(f"# bad: {git.rev_pretty(commit)}") 53 | bisect_append_log(f"git bisect bad {git.rev_parse(commit)}") 54 | 55 | 56 | def bisect_good(commit): 57 | """Mark a commit as good. 58 | 59 | The same disclaimer as for `bisect_bad` applies. 60 | """ 61 | # alternative: `git bisect--helper bisect-write` 62 | rev_parsed = git.rev_parse(commit) 63 | git.update_ref(f"refs/bisect/good-{rev_parsed}", commit) 64 | bisect_append_log(f"# good: {git.rev_pretty(commit)}") 65 | bisect_append_log(f"git bisect good {git.rev_parse(commit)}") 66 | 67 | 68 | def bisect_skip(commit): 69 | """Mark a single commit as skipped. 70 | 71 | This is the traditional `git bisect skip`. The commits skipped with this do 72 | not mark a range, and the algorithm will not attempt to "unbreak" the 73 | commits. 74 | """ 75 | rev_parsed = git.rev_parse(commit) 76 | git.update_ref(f"refs/bisect/skip-{rev_parsed}", commit) 77 | bisect_append_log(f"# skip: {git.rev_pretty(commit)}") 78 | bisect_append_log(f"git bisect skip {git.rev_parse(commit)}") 79 | 80 | 81 | def get_good_commits(): 82 | """Returns all refs that are marked as good.""" 83 | good_refs = [] 84 | for ref in git.get_refs_with_prefix("refs/bisect"): 85 | parts = ref.split("/") 86 | if len(parts) == 3 and parts[2].startswith("good-"): 87 | good_refs.append(ref) 88 | return good_refs 89 | 90 | 91 | def get_skip_range_commits(patchset): 92 | """Returns all refs that are marked with some skip range.""" 93 | return git.get_refs_with_prefix( 94 | f"refs/bisect/break/{patchset_identifier(patchset)}/markers" 95 | ) 96 | 97 | 98 | def within_range(commit, range_markers): 99 | """Whether or not a given commit is enclosed by a pair of range markers""" 100 | reached_by_range = False 101 | for marker in range_markers: 102 | if git.is_ancestor(commit, marker): 103 | reached_by_range = True 104 | break 105 | if not reached_by_range: 106 | return False 107 | 108 | can_reach_range = False 109 | for marker in range_markers: 110 | if git.is_ancestor(marker, commit): 111 | can_reach_range = True 112 | break 113 | return can_reach_range 114 | 115 | 116 | def get_named_skip_refs(name, patchset): 117 | """Returns all commits that are marked with the skip range `name`.""" 118 | return git.get_refs_with_prefix( 119 | f"refs/bisect/break/{patchset_identifier(patchset)}/markers/{name}" 120 | ) 121 | 122 | 123 | def get_skip_ranges(patchset): 124 | """Returns all skip range names""" 125 | return { 126 | ref.split("/")[-2] 127 | for ref in git.get_refs_with_prefix( 128 | f"refs/bisect/break/{patchset_identifier(patchset)}/markers" 129 | ) 130 | } 131 | 132 | 133 | def refs_for_commit(commit): 134 | """Returns all refs that point to a commit.""" 135 | lines = subprocess.check_output(["git", "show-ref"]).decode().splitlines() 136 | result = dict() 137 | for line in lines: 138 | (target, ref) = line.split(" ") 139 | new_set = result.get(target, set()) 140 | new_set.add(ref) 141 | result[target] = new_set 142 | return result.get(commit, []) 143 | 144 | 145 | def skip_ranges_of_commit(commit, patchset): 146 | """Returns all named skip ranges a commit is marked with.""" 147 | skip_ranges = [] 148 | for ref in refs_for_commit(commit): 149 | if ref.startswith(f"refs/bisect/break/{patchset_identifier(patchset)}/markers"): 150 | components = ref.split("/") 151 | skip_ranges.append(components[-2]) 152 | return skip_ranges 153 | 154 | 155 | def clear_refs_with_prefix(prefix): 156 | """Remove all refs that belong to a skip range""" 157 | for ref in git.get_refs_with_prefix(prefix): 158 | git.delete_ref(ref) 159 | 160 | 161 | def read_patchset(): 162 | """Reats the current (i.e. longest) patchset from the refs""" 163 | patchset_refs = git.get_refs_with_prefix("refs/bisect/patchset") 164 | if len(patchset_refs) == 0: 165 | return [] 166 | patchset_identifiers = [ref.split("/")[3:-1] for ref in patchset_refs] 167 | longest_idx = np.argmax([len(ps) for ps in patchset_identifiers]) 168 | patchset = patchset_identifiers[longest_idx] 169 | return patchset 170 | 171 | 172 | def bisect_env_args(patchset): 173 | """Generates arguments for bisect-env to apply the patchset""" 174 | args = [] 175 | for patch in patchset: 176 | args.append(f"--try-pick={patch}") 177 | return args 178 | 179 | 180 | def first_not_skipped(commit_list): 181 | """Returns the first commit of the list that is not marked as skipped""" 182 | for commit in commit_list: 183 | is_skipped = False 184 | for ref in refs_for_commit(commit): 185 | if ref.startswith("refs/bisect/skip-"): 186 | is_skipped = True 187 | if not is_skipped: 188 | return commit 189 | raise Exception("Cannot bisect any further") 190 | 191 | 192 | class BisectRunner: 193 | """Runs a bisection""" 194 | 195 | def get_next(self): 196 | """Computes the next commit to test. 197 | 198 | This takes skip-ranges into account and prioritizes finding the first 199 | commit that unbreaks a skip range. 200 | 201 | May add commits for cherry pick. Returns `False` when the bisect is 202 | finished. 203 | """ 204 | patchset = read_patchset() 205 | considered_good = get_good_commits() + get_skip_range_commits(patchset) 206 | candidates = git.get_bisect_all(considered_good, "refs/bisect/bad") 207 | # It would be better to use a more sophisticated algorithm like 208 | # https://github.com/git/git/commit/ebc9529f0358bdb10192fa27bc75f5d4e452ce90 209 | # This works for now though. 210 | commit = first_not_skipped(candidates) 211 | if git.rev_parse(commit) == git.rev_parse("refs/bisect/bad"): 212 | skip_ranges = [] 213 | good_commits = [git.rev_parse(ref) for ref in get_good_commits()] 214 | for parent in git.parents(commit): 215 | if parent in good_commits: 216 | print(f"First bad found! Here it is: {commit}") 217 | return None 218 | skip_ranges += skip_ranges_of_commit(parent, patchset) 219 | print(f"cherry-pick {commit} to unbreak {skip_ranges}") 220 | patchset.insert(0, commit) 221 | git.update_ref(f"refs/bisect/{patchset_identifier(patchset)}/head", commit) 222 | return self.get_next() 223 | return commit 224 | 225 | def _single_run(self, bisect_fun): 226 | patchset = read_patchset() 227 | with git.git_checkpoint(): 228 | one_patch_succeeded = False 229 | for (i, rev) in enumerate(patchset): 230 | success = git.try_cherry_pick_all(rev) 231 | one_patch_succeeded = success or one_patch_succeeded 232 | if not one_patch_succeeded: 233 | remaining_patchset = patchset[i + 1 :] 234 | for skip_range in get_skip_ranges(remaining_patchset): 235 | if within_range( 236 | "HEAD", get_named_skip_refs(skip_range, remaining_patchset) 237 | ): 238 | print( 239 | f"Commit with remaining patches matches known skip range {skip_range}." 240 | ) 241 | return f"skip {skip_range}" 242 | return bisect_fun() 243 | 244 | def run(self, bisect_fun): 245 | """Finds the first bad commit""" 246 | while True: 247 | next_commit = self.get_next() 248 | if next_commit is None: 249 | return 250 | git.checkout(next_commit) 251 | result = self._single_run(bisect_fun) 252 | if result == "bad": 253 | bisect_bad("HEAD") 254 | git_bisect.print_bad() 255 | elif result == "good": 256 | bisect_good("HEAD") 257 | git_bisect.print_good() 258 | elif result.startswith("skip"): 259 | reason = result[len("skip ") :] 260 | git_bisect.print_skip(reason) 261 | named_skip(reason, read_patchset(), "HEAD") 262 | else: 263 | raise Exception("Unknown bisection result.") 264 | -------------------------------------------------------------------------------- /nix_bisect/git.py: -------------------------------------------------------------------------------- 1 | """Utilities for interacting with git.""" 2 | 3 | import subprocess 4 | from subprocess import run, PIPE 5 | from math import log, floor, ceil 6 | import signal 7 | 8 | 9 | def cur_commit(): 10 | """Returns the rev of the current HEAD.""" 11 | result = run( 12 | ["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE, encoding="utf-8", 13 | ) 14 | result.check_returncode() 15 | return result.stdout.strip() 16 | 17 | 18 | def commits_in_range(rev1, rev2): 19 | """Returns all commits withing a range""" 20 | result = run( 21 | ["git", "log", "--pretty=format:%H", f"{rev1}..{rev2}"], 22 | stdout=PIPE, 23 | stderr=PIPE, 24 | encoding="utf-8", 25 | ) 26 | return result.stdout.splitlines() 27 | 28 | 29 | def bisect_revisions(): 30 | """Returns the amount of still possible first-bad commits. 31 | 32 | This is an approximation.""" 33 | result = run( 34 | ["git", "bisect", "visualize", "--oneline"], 35 | stdout=PIPE, 36 | stderr=PIPE, 37 | encoding="utf-8", 38 | ) 39 | result.check_returncode() 40 | lines = result.stdout.splitlines() 41 | interesting = [line for line in lines if "refs/bisect/skip" not in line] 42 | # the earliest known bad commit will be included in the bisect view 43 | return len(interesting) - 1 44 | 45 | 46 | def bisect_steps_remaining(): 47 | """Estimate of remaining steps, including the current one. 48 | 49 | This is an approximation.""" 50 | # https://github.com/git/git/blob/566a1439f6f56c2171b8853ddbca0ad3f5098770/bisect.c#L1043 51 | return floor(log(bisect_revisions(), 2)) 52 | 53 | 54 | def bisect_status(): 55 | """Reproduce the status line git-bisect prints after each step.""" 56 | return "Bisecting: {} revisions left to test after this (roughly {} steps).".format( 57 | ceil((bisect_revisions() - 1) / 2), bisect_steps_remaining() - 1, 58 | ) 59 | 60 | 61 | class assure_nothing_unstaged: 62 | """Context that temporarily commits staged changes.""" 63 | 64 | def __enter__(self): 65 | self.head_before = cur_commit() 66 | add(".") 67 | commit(f"TMP clean slate") 68 | return None 69 | 70 | def __exit__(self, type, value, traceback): 71 | s = signal.signal(signal.SIGINT, signal.SIG_IGN) 72 | reset(self.head_before) 73 | signal.signal(signal.SIGINT, s) 74 | 75 | 76 | # FIXME create a worktree, periodically re-sync with original 77 | class git_checkpoint: 78 | """Context that remembers the repository's state 79 | 80 | The repositories state (including uncommited changes) is saved when the 81 | context is entered and restored when it is left. 82 | """ 83 | 84 | def __enter__(self): 85 | self.head_before = cur_commit() 86 | 87 | # Create a commit that reflects the current state of the repo. 88 | add(".") 89 | commit(f"TMP clean slate") 90 | self.checkpint_rev = cur_commit() 91 | 92 | # Return to the original commit (soft reset). 93 | reset(self.head_before) 94 | return None 95 | 96 | def __exit__(self, type, value, traceback): 97 | # Don't exit in the middle of cleanup, can happen if someone is 98 | # impatient and hits ctrl-c multiple times. 99 | s = signal.signal(signal.SIGINT, signal.SIG_IGN) 100 | reset(self.checkpint_rev, extra_flags=["--hard"]) 101 | clean(extra_flags=["-f"]) 102 | reset(self.head_before) 103 | signal.signal(signal.SIGINT, s) 104 | 105 | 106 | def parents(rev): 107 | """Returns all parent revisions of a revision""" 108 | return ( 109 | subprocess.check_output(["git", "rev-list", "-n", "1", "--parents", rev]) 110 | .decode() 111 | .strip() 112 | .split(" ")[1:] 113 | ) 114 | 115 | 116 | def try_cherry_pick_all(rev): 117 | """Tries to cherry pick all parents of a (merge) commit.""" 118 | num_par = len(parents(rev)) 119 | any_success = False 120 | for i in range(1, num_par + 1): 121 | any_success = any_success or try_cherry_pick(rev, mainline=i) 122 | return any_success 123 | 124 | 125 | def try_cherry_pick(rev, mainline=1): 126 | rev_name = rev + ("" if mainline == 1 else f"(mainline {mainline})") 127 | with assure_nothing_unstaged(): 128 | result = run( 129 | ["git", "cherry-pick", "--mainline", str(mainline), "-n", rev], 130 | stdout=PIPE, 131 | stderr=PIPE, 132 | encoding="utf-8", 133 | ) 134 | 135 | if result.returncode != 0: 136 | print(f"Cherry-pick of {rev_name} failed") 137 | print(result.stderr.splitlines()[0][len("error: ") - 1 :]) 138 | reset("HEAD", extra_flags=["--hard"]) 139 | return False 140 | 141 | print(f"Cherry-pick of {rev_name} succeeded") 142 | return True 143 | 144 | 145 | def try_revert(rev): 146 | with assure_nothing_unstaged(): 147 | result = run( 148 | ["git", "revert", "-n", rev], stdout=PIPE, stderr=PIPE, encoding="utf-8", 149 | ) 150 | 151 | if result.returncode != 0: 152 | print("Revert failed") 153 | print(result.stderr.splitlines()[0][len("error: ") - 1 :]) 154 | reset("HEAD", extra_flags=["--hard"]) 155 | return False 156 | 157 | print("Revert succeeded") 158 | return True 159 | 160 | 161 | def is_ancestor(ancestor, parent): 162 | """Returns `True` iff `ancestor` is an ancestor of `parent`.""" 163 | try: 164 | subprocess.check_call(["git", "merge-base", "--is-ancestor", ancestor, parent],) 165 | return True 166 | except subprocess.CalledProcessError: 167 | return False 168 | 169 | 170 | def reset(rev, extra_flags=[]): 171 | result = run( 172 | ["git", "reset"] + extra_flags + [rev], 173 | stdout=PIPE, 174 | stderr=PIPE, 175 | encoding="utf-8", 176 | ) 177 | result.check_returncode() 178 | 179 | 180 | def clean(extra_flags=[]): 181 | result = run( 182 | ["git", "clean"] + extra_flags, stdout=PIPE, stderr=PIPE, encoding="utf-8", 183 | ) 184 | result.check_returncode() 185 | 186 | 187 | def add(path): 188 | result = run(["git", "add", path], stdout=PIPE, stderr=PIPE, encoding="utf-8",) 189 | result.check_returncode() 190 | 191 | 192 | def commit(message): 193 | result = run( 194 | ["git", "commit", "--allow-empty", "-m", message], 195 | stdout=PIPE, 196 | stderr=PIPE, 197 | encoding="utf-8", 198 | ) 199 | result.check_returncode() 200 | 201 | 202 | def checkout(commit): 203 | """Runs `git checkout`""" 204 | subprocess.check_call(["git", "checkout", commit]) 205 | 206 | 207 | def get_refs_with_prefix(prefix): 208 | """Returns a list of refs that start with a prefix. 209 | 210 | Internally calls `git for-each-ref`. The prefix has to be complete up to a 211 | `/`, i.e. `some/pre` will find `some/pre/asdf` but not some/prefix. 212 | """ 213 | return ( 214 | subprocess.check_output(["git", "for-each-ref", "--format=%(refname)", prefix],) 215 | .decode() 216 | .splitlines() 217 | ) 218 | 219 | 220 | def rev_list(include, exclude): 221 | """Find all revs that are reachable by `include` but not by `exclude`. 222 | 223 | Runs `git rev-list` internally. 224 | """ 225 | args = include + ["--not"] + [exclude] 226 | return subprocess.check_output(["git", "rev-list"] + args).decode().splitlines() 227 | 228 | 229 | def get_bisect_info(good_commits, bad_commit): 230 | """Returns a dict with info about the current bisect run. 231 | 232 | Internally runs `git rev-list --bisect-vars`. Information includes: 233 | 234 | - bisect_rev: midpoint revision 235 | - bisect_nr: expected number to be tested after bisect_rev 236 | - bisect_good: bisect_nr if good 237 | - bisect_bad: bisect_nr if bad 238 | - bisect_all: commits we are bisecting right now 239 | - biset_step: estimated steps after bisect_rev 240 | """ 241 | args = [bad_commit] + [f"^{commit}" for commit in good_commits] 242 | lines = ( 243 | subprocess.check_output(["git", "rev-list", "--bisect-vars"] + args) 244 | .decode() 245 | .splitlines() 246 | ) 247 | key_values = [line.split("=") for line in lines] 248 | info = dict(key_values) 249 | # this is a quoted string; strip the quotes 250 | info["bisect_rev"] = info["bisect_rev"][1:-1] 251 | for key in ("bisect_nr", "bisect_good", "bisect_bad", "bisect_all", "bisect_steps"): 252 | info[key] = int(info[key]) 253 | return info 254 | 255 | 256 | def get_bisect_all(good_commits, bad_commit): 257 | """Returns a list with potential next commits, sorted by distance 258 | 259 | Internally runs `git rev-list --bisect-all`. 260 | """ 261 | # Could also be combined with --bisect-vars, that may be more efficient. 262 | args = [bad_commit] + [f"^{commit}" for commit in good_commits] 263 | lines = ( 264 | subprocess.check_output(["git", "rev-list", "--bisect-all"] + args) 265 | .decode() 266 | .splitlines() 267 | ) 268 | # first is furthest away, last is equal to bad 269 | commits = [line.split(" ")[0] for line in lines] 270 | return commits 271 | 272 | 273 | def rev_parse(commit_ish, short=False): 274 | """Parses a "commit_ish" to a unique full hash""" 275 | args = ["--short"] if short else [] 276 | return ( 277 | subprocess.check_output(["git", "rev-parse"] + args + [commit_ish]) 278 | .decode() 279 | .strip() 280 | ) 281 | 282 | 283 | def update_ref(ref, value): 284 | """Updates or creates a reference.""" 285 | subprocess.check_call(["git", "update-ref", ref, value]) 286 | 287 | 288 | def delete_ref(ref): 289 | """Deletes a reference.""" 290 | subprocess.check_call(["git", "update-ref", "-d", ref]) 291 | 292 | 293 | def git_dir(): 294 | """Returns the path to the .git directory (works with worktrees)""" 295 | return subprocess.check_output(["git", "rev-parse", "--git-dir"]).decode().strip() 296 | 297 | 298 | def commit_msg(rev): 299 | """Returns the short commit message summary (the first line)""" 300 | return ( 301 | subprocess.check_output(["git", "show", "--pretty=format:%s", "-s", rev]) 302 | .decode() 303 | .strip() 304 | ) 305 | 306 | 307 | def rev_pretty(rev): 308 | """Pretty-print a revision for usage in logs.""" 309 | return f"[{rev_parse(rev, short=True)}] {commit_msg(rev)}" 310 | -------------------------------------------------------------------------------- /nix_bisect/nix.py: -------------------------------------------------------------------------------- 1 | """Wrapper for nix functionality""" 2 | 3 | from subprocess import run, PIPE 4 | import subprocess 5 | from pathlib import Path 6 | from typing import Set 7 | 8 | import struct 9 | import signal 10 | import fcntl 11 | import termios 12 | import json 13 | import re 14 | import sys 15 | 16 | import pexpect 17 | 18 | from appdirs import AppDirs 19 | 20 | from nix_bisect import exceptions 21 | 22 | # Parse the error output of `nix build` 23 | _CANNOT_BUILD_PAT = re.compile(b"cannot build derivation '([^']+)': (.+)") 24 | _BUILD_FAILED_PAT = re.compile(b"build of ('[^']+'(, '[^']+')*) failed") 25 | _BUILDER_FAILED_PAT = re.compile(b"builder for '([^']+)' failed with exit code (\\d+);") 26 | _BUILD_TIMEOUT_PAT = re.compile(b"building of '([^']+)' timed out after.*") 27 | 28 | 29 | def _nix_options_to_flags(nix_options): 30 | option_args = [] 31 | for (name, value) in nix_options: 32 | option_args.append("--option") 33 | option_args.append(name) 34 | option_args.append(value) 35 | return option_args 36 | 37 | 38 | def log(drv): 39 | """Returns the build log of a store path.""" 40 | result = run(["nix", "log", "-f.", drv], stdout=PIPE, stderr=PIPE, encoding="utf-8") 41 | if result.returncode != 0: 42 | return None 43 | return result.stdout 44 | 45 | 46 | def build_dry(drvs, nix_options=()): 47 | """Returns a list of drvs to be built and fetched in order to 48 | realize `drvs`""" 49 | result = run( 50 | ["nix-store", "--realize", "--dry-run"] 51 | + _nix_options_to_flags(nix_options) 52 | + drvs, 53 | stdout=PIPE, 54 | stderr=PIPE, 55 | encoding="utf-8", 56 | ) 57 | result.check_returncode() 58 | lines = result.stderr.splitlines() 59 | to_fetch = [] 60 | to_build = [] 61 | for line in lines: 62 | line = line.strip() 63 | if "will be fetched" in line: 64 | cur = to_fetch 65 | elif "will be built" in line: 66 | cur = to_build 67 | elif line.startswith("/nix/store"): 68 | cur += [line] 69 | elif line.startswith("warning:"): 70 | print(f"dry build: {line}", file=sys.stderr) 71 | elif line != "": 72 | raise RuntimeError(f"dry-run parsing failed, line was:`{line}`") 73 | 74 | return (to_build, to_fetch) 75 | 76 | 77 | class InstantiationFailure(Exception): 78 | """Failure during instantiation.""" 79 | 80 | 81 | def instantiate(attrname, nix_file=".", nix_options=(), nix_argstr=(), expression=True): 82 | """Instantiate an attribute. 83 | 84 | Parameters 85 | ---------- 86 | 87 | attrname: string, 88 | Attribute or expression to instantiate. 89 | 90 | expression: bool 91 | If `True`, arbitrary nix expressions can be evaluated. This 92 | allows for overrides. The nix_file (or the current working 93 | directory by default) will be in scope by default. I.e. the 94 | expression will be implicitly prefixed by 95 | 96 | with (import nix_file {}); 97 | 98 | nix_file: string, 99 | Nix file to instantiate an attribute from. 100 | """ 101 | option_args = _nix_options_to_flags(nix_options) 102 | 103 | if expression: 104 | if nix_file is not None: 105 | # We need to simulate --argstr support since we're calling nixpkgs 106 | # manually to allow for arbitrary nix expressions. 107 | call_args = "" 108 | for (name, val) in nix_argstr: 109 | call_args += f'{name} = "{val}";' 110 | arg = ( 111 | f"with (import {Path(nix_file).absolute()} {{{call_args}}}); {attrname}" 112 | ) 113 | else: 114 | arg = attrname 115 | command = ["nix-instantiate", "-E", arg] + option_args 116 | else: 117 | for name, val in nix_argstr: 118 | option_args.append("--argstr") 119 | option_args.append(name) 120 | option_args.append(val) 121 | command = ["nix-instantiate", nix_file, "-A", arg] + option_args 122 | result = run(command, stdout=PIPE, stderr=PIPE, encoding="utf-8",) 123 | 124 | if result.returncode == 0: 125 | return result.stdout.strip() 126 | 127 | raise InstantiationFailure(result.stderr) 128 | 129 | 130 | def dependencies(drvs, nix_options=()): 131 | """Returns all dependencies of `drvs` that aren't already in the 132 | store.""" 133 | (to_build, to_fetch) = build_dry(drvs, nix_options=nix_options) 134 | to_realize = to_build + to_fetch 135 | for drv in drvs: 136 | try: 137 | to_realize.remove(drv) 138 | except ValueError: 139 | # drv already in store 140 | pass 141 | return to_realize 142 | 143 | 144 | class BuildFailure(Exception): 145 | """A failure during build.""" 146 | 147 | def __init__(self, drvs_failed: Set[str]): 148 | assert len(drvs_failed) > 0, "BuildFailure requires at least one derivation to blame." 149 | super(BuildFailure).__init__() 150 | self.drvs_failed = drvs_failed 151 | 152 | 153 | def _build_uncached(drvs, nix_options=()): 154 | if len(drvs) == 0: 155 | # nothing to do 156 | return "" 157 | 158 | # We need to use pexpect instead of subprocess.Popen here, since `nix 159 | # build` will not produce its regular output when it does not detect a tty. 160 | build_process = pexpect.spawn( 161 | "nix", 162 | ["build", "--no-link"] + _nix_options_to_flags(nix_options) + [d + "^*" if d.endswith(".drv") else d for d in drvs], 163 | logfile=sys.stdout.buffer, 164 | ) 165 | 166 | # adapted from the pexpect docs 167 | def _update_build_winsize(): 168 | s = struct.pack("HHHH", 0, 0, 0, 0) 169 | a = struct.unpack( 170 | "hhhh", fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, s) 171 | ) 172 | if not build_process.closed: 173 | build_process.setwinsize(a[0], a[1]) 174 | 175 | _update_build_winsize() 176 | signal.signal(signal.SIGWINCH, lambda _sig, _data: _update_build_winsize()) 177 | 178 | drvs_failed = set() 179 | try: 180 | while True: 181 | # This will fill the "match" instance attribute. Raises on EOF. We 182 | # can only reliably use this for the final error output, not for 183 | # the streamed output of the actual build (since `nix build` skips 184 | # lines and trims output). Use `nix.log` for that. 185 | build_process.expect( 186 | [ 187 | _CANNOT_BUILD_PAT, 188 | _BUILD_FAILED_PAT, 189 | _BUILD_TIMEOUT_PAT, 190 | _BUILDER_FAILED_PAT, 191 | ], 192 | timeout=None, 193 | ) 194 | 195 | line = build_process.match.group(0) 196 | # Re-match to find out which pattern matched. This doesn't happen very 197 | # often, so the wasted effort isn't too bad. 198 | # Can't wait for https://www.python.org/dev/peps/pep-0572/ 199 | match = _CANNOT_BUILD_PAT.match(line) 200 | if match is not None: 201 | drv = match.group(1).decode() 202 | _reason = match.group(2).decode() 203 | drvs_failed.add(drv) 204 | match = _BUILD_FAILED_PAT.match(line) 205 | if match is not None: 206 | drv_list = match.group(1).decode() 207 | drvs = drv_list.split(", ") 208 | drvs = [drv.strip("'") for drv in drvs] # strip quotes 209 | drvs_failed.update(drvs) 210 | match = _BUILD_TIMEOUT_PAT.match(line) 211 | if match is not None: 212 | drv = match.group(1).decode() 213 | drvs_failed.add(drv) 214 | match = _BUILDER_FAILED_PAT.match(line) 215 | if match is not None: 216 | drv = match.group(1).decode() 217 | _exit_code = match.group(2).decode() 218 | drvs_failed.add(drv) 219 | except pexpect.exceptions.EOF: 220 | pass 221 | 222 | if len(drvs_failed) > 0: 223 | raise BuildFailure(drvs_failed) 224 | 225 | location_process = run( 226 | ["nix-store", "--realize"] + drvs, stdout=PIPE, stderr=PIPE, encoding="utf-8", 227 | ) 228 | location_process.check_returncode() 229 | storepaths = location_process.stdout.split("\n") 230 | return storepaths 231 | 232 | 233 | def log_contains(drv, phrase, write_cache=True): 234 | """Checks if the build log of `drv` contains a phrase 235 | 236 | This may or may not cause a rebuild. Cached logs are only trusted if they 237 | were produced by nix-bisect. May return "yes", "no_fail" or "no_success". 238 | """ 239 | cache_dir = Path(AppDirs("nix-bisect").user_cache_dir) 240 | 241 | # If we already tried this before, we can trust our own cache. 242 | logfile = cache_dir.joinpath("logs").joinpath(Path(drv).name) 243 | if logfile.exists(): 244 | with open(logfile, "r") as f: 245 | log_content = f.read() 246 | # We only save logs of failures. 247 | return "yes" if phrase in log_content else "no_fail" 248 | 249 | # We have to be careful with nix's cache since it might be incomplete. 250 | log_content = log(drv) 251 | if log_content is not None and phrase in log_content: 252 | return "yes" 253 | 254 | # Make sure the cache is populated. 255 | success = True 256 | try: 257 | build([drv], use_cache=False, write_cache=write_cache) 258 | except BuildFailure: 259 | success = False 260 | log_content = log(drv) 261 | 262 | if phrase in log_content: 263 | return "yes" 264 | elif success: 265 | return "no_success" 266 | else: 267 | return "no_fail" 268 | 269 | 270 | def references(drvs): 271 | """Returns all immediate dependencies of `drvs`. 272 | 273 | Runs `nix-store --query --references` internally. 274 | """ 275 | return ( 276 | subprocess.check_output(["nix-store", "--query", "--references"] + drvs) 277 | .decode() 278 | .splitlines() 279 | ) 280 | 281 | 282 | def build_would_succeed( 283 | drvs, 284 | nix_options=(), 285 | max_rebuilds=float("inf"), 286 | rebuild_blacklist=(), 287 | use_cache=True, 288 | write_cache=True, 289 | ): 290 | """Determines build success without actually building if possible""" 291 | rebuilds = build_dry(drvs, nix_options=nix_options)[0] 292 | rebuild_count = len(rebuilds) 293 | if rebuild_count == 0: 294 | return True 295 | 296 | blacklisted = [] 297 | for regex in rebuild_blacklist: 298 | for drv_to_rebuild in rebuilds: 299 | if re.match(regex, drv_to_rebuild) is not None: 300 | blacklisted.append(drv_to_rebuild) 301 | 302 | if len(blacklisted) > 0: 303 | raise exceptions.BlacklistedBuildsException(blacklisted) 304 | elif rebuild_count > max_rebuilds: 305 | raise exceptions.TooManyBuildsException() 306 | 307 | try: 308 | build( 309 | drvs, nix_options=nix_options, use_cache=use_cache, write_cache=write_cache 310 | ) 311 | return True 312 | except BuildFailure: 313 | return False 314 | 315 | 316 | def build(drvs, nix_options=(), use_cache=True, write_cache=True): 317 | """Builds `drvs`, returning a list of store paths""" 318 | cache_dir = Path(AppDirs("nix-bisect").user_cache_dir) 319 | cache_dir.mkdir(parents=True, exist_ok=True) 320 | logs_dir = cache_dir.joinpath("logs") 321 | logs_dir.mkdir(exist_ok=True) 322 | 323 | cache_file = cache_dir.joinpath("build-results.json") 324 | if (use_cache or write_cache) and cache_file.exists(): 325 | with open(cache_file, "r") as cf: 326 | result_cache = json.loads(cf.read()) 327 | else: 328 | result_cache = dict() 329 | 330 | if use_cache: 331 | for drv in drvs: 332 | # innocent till proven guilty 333 | if not result_cache.get(drv, True): 334 | print(f"Cached failure of {drv}.") 335 | raise BuildFailure(set([drv])) 336 | 337 | try: 338 | return _build_uncached(drvs, nix_options) 339 | except BuildFailure as bf: 340 | if write_cache: 341 | for drv in bf.drvs_failed: 342 | # Could save more details here in the future if needed. 343 | result_cache[drv] = False 344 | # If the build finished, we know that we can trust the logs are 345 | # complete if they are available. This is essential for caching 346 | # "skip"s. 347 | failure_log = log(drv) 348 | if failure_log is not None: 349 | with open(logs_dir.joinpath(Path(drv).name), "w") as f: 350 | f.write(failure_log) 351 | 352 | with open(cache_file, "w") as cf: 353 | # Write human-readable json for easy hacking. 354 | cf.write(json.dumps(result_cache, indent=4)) 355 | raise bf 356 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code. 6 | extension-pkg-whitelist=numpy,multiprocessing 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 21 | # number of processors available to use. Anything >1 is currently buggy: 22 | # https://github.com/PyCQA/pylint/issues/374 23 | #jobs=1 24 | 25 | # Control the amount of potential inferred values when inferring a single 26 | # object. This can help the performance when dealing with large functions or 27 | # complex, nested conditions. 28 | limit-inference-results=100 29 | 30 | # List of plugins (as comma separated values of python modules names) to load, 31 | # usually to register additional checkers. 32 | load-plugins= 33 | 34 | # Pickle collected data for later comparisons. 35 | persistent=yes 36 | 37 | # Specify a configuration file. 38 | #rcfile= 39 | 40 | # When enabled, pylint would attempt to guess common misconfiguration and emit 41 | # user-friendly hints instead of false-positive error messages. 42 | suggestion-mode=yes 43 | 44 | # Allow loading of arbitrary C extensions. Extensions are imported into the 45 | # active Python interpreter and may run arbitrary code. 46 | unsafe-load-any-extension=no 47 | 48 | 49 | [MESSAGES CONTROL] 50 | 51 | # Only show warnings with the listed confidence levels. Leave empty to show 52 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. 53 | confidence= 54 | 55 | # Disable the message, report, category or checker with the given id(s). You 56 | # can either give multiple identifiers separated by comma (,) or put this 57 | # option multiple times (only on the command line, not in the configuration 58 | # file where it should appear only once). You can also use "--disable=all" to 59 | # disable everything first and then reenable specific checks. For example, if 60 | # you want to run only the similarities checker, you can use "--disable=all 61 | # --enable=similarities". If you want to run only the classes checker, but have 62 | # no Warning level messages displayed, use "--disable=all --enable=classes 63 | # --disable=W". 64 | 65 | # `if len(seq) != 0` is better than `if seq` 66 | # bad-continuation false-positives, black disagrees 67 | # I disagree with no-else-return, I think a clear "else" is more readable. 68 | # line-too-long is handled by black (differently) 69 | # disagree with too-few-public methods; such classes can still be useful. 70 | disable=len-as-condition,bad-continuation,no-else-return,line-too-long,too-few-public-methods 71 | 72 | # Enable the message, report, category or checker with the given id(s). You can 73 | # either give multiple identifier separated by comma (,) or put this option 74 | # multiple time (only on the command line, not in the configuration file where 75 | # it should appear only once). See also the "--disable" option for examples. 76 | enable=c-extension-no-member 77 | 78 | 79 | [REPORTS] 80 | 81 | # Python expression which should return a note less than 10 (10 is the highest 82 | # note). You have access to the variables errors warning, statement which 83 | # respectively contain the number of errors / warnings messages and the total 84 | # number of statements analyzed. This is used by the global evaluation report 85 | # (RP0004). 86 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 87 | 88 | # Template used to display messages. This is a python new-style format string 89 | # used to format the message information. See doc for all details. 90 | #msg-template= 91 | 92 | # Set the output format. Available formats are text, parseable, colorized, json 93 | # and msvs (visual studio). You can also give a reporter class, e.g. 94 | # mypackage.mymodule.MyReporterClass. 95 | output-format=text 96 | 97 | # Tells whether to display a full report or only the messages. 98 | reports=no 99 | 100 | # Activate the evaluation score. 101 | score=yes 102 | 103 | 104 | [REFACTORING] 105 | 106 | # Maximum number of nested blocks for function / method body 107 | max-nested-blocks=5 108 | 109 | # Complete name of functions that never returns. When checking for 110 | # inconsistent-return-statements if a never returning function is called then 111 | # it will be considered as an explicit return statement and no message will be 112 | # printed. 113 | never-returning-functions=sys.exit 114 | 115 | 116 | [BASIC] 117 | 118 | # Naming style matching correct argument names. 119 | argument-rgx=[a-z_][a-z0-9_]*$ 120 | 121 | # Regular expression matching correct argument names. Overrides argument- 122 | # naming-style. 123 | #argument-rgx= 124 | 125 | # Naming style matching correct attribute names. 126 | attr-rgx=[a-z_][a-z0-9_]*$ 127 | 128 | # Regular expression matching correct attribute names. Overrides attr-naming- 129 | # style. 130 | #attr-rgx= 131 | 132 | # Bad variable names which should always be refused, separated by a comma. 133 | bad-names=foo, 134 | bar, 135 | baz, 136 | toto, 137 | tutu, 138 | tata 139 | 140 | # Naming style matching correct class attribute names. 141 | class-attribute-naming-style=any 142 | 143 | # Regular expression matching correct class attribute names. Overrides class- 144 | # attribute-naming-style. 145 | #class-attribute-rgx= 146 | 147 | # Naming style matching correct class names. 148 | class-naming-style=PascalCase 149 | 150 | # Regular expression matching correct class names. Overrides class-naming- 151 | # style. 152 | #class-rgx= 153 | 154 | # Naming style matching correct constant names. 155 | const-naming-style=UPPER_CASE 156 | 157 | # Regular expression matching correct constant names. Overrides const-naming- 158 | # style. 159 | #const-rgx= 160 | 161 | # Minimum line length for functions/classes that require docstrings, shorter 162 | # ones are exempt. 163 | docstring-min-length=-1 164 | 165 | # Naming style matching correct function names. 166 | function-naming-style=snake_case 167 | 168 | # Regular expression matching correct function names. Overrides function- 169 | # naming-style. 170 | #function-rgx= 171 | 172 | # Good variable names which should always be accepted, separated by a comma. 173 | good-names=Run, 174 | _ 175 | 176 | # Include a hint for the correct naming format with invalid-name. 177 | include-naming-hint=no 178 | 179 | # Naming style matching correct inline iteration names. 180 | inlinevar-naming-style=any 181 | 182 | # Regular expression matching correct inline iteration names. Overrides 183 | # inlinevar-naming-style. 184 | #inlinevar-rgx= 185 | 186 | # Naming style matching correct method names. 187 | method-naming-style=snake_case 188 | 189 | # Regular expression matching correct method names. Overrides method-naming- 190 | # style. 191 | #method-rgx= 192 | 193 | # Naming style matching correct module names. 194 | module-naming-style=snake_case 195 | 196 | # Regular expression matching correct module names. Overrides module-naming- 197 | # style. 198 | #module-rgx= 199 | 200 | # Colon-delimited sets of names that determine each other's naming style when 201 | # the name regexes allow several styles. 202 | name-group= 203 | 204 | # Regular expression which should only match function or class names that do 205 | # not require a docstring. 206 | no-docstring-rgx=^_ 207 | 208 | # List of decorators that produce properties, such as abc.abstractproperty. Add 209 | # to this list to register other decorators that produce valid properties. 210 | # These decorators are taken in consideration only for invalid-name. 211 | property-classes=abc.abstractproperty 212 | 213 | # Naming style matching correct variable names. 214 | variable-rgx=[a-z_][a-z0-9_]*$ 215 | 216 | # Regular expression matching correct variable names. Overrides variable- 217 | # naming-style. 218 | #variable-rgx= 219 | 220 | 221 | [FORMAT] 222 | 223 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 224 | expected-line-ending-format= 225 | 226 | # Regexp for a line that is allowed to be longer than the limit. 227 | ignore-long-lines=^\s*(# )??$ 228 | 229 | # Number of spaces of indent required inside a hanging or continued line. 230 | indent-after-paren=4 231 | 232 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 233 | # tab). 234 | indent-string=' ' 235 | 236 | # Maximum number of characters on a single line. 237 | max-line-length=79 238 | 239 | # Maximum number of lines in a module. 240 | max-module-lines=1000 241 | 242 | # List of optional constructs for which whitespace checking is disabled. `dict- 243 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 244 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 245 | # `empty-line` allows space-only lines. 246 | no-space-check=trailing-comma, 247 | dict-separator 248 | 249 | # Allow the body of a class to be on the same line as the declaration if body 250 | # contains single statement. 251 | single-line-class-stmt=no 252 | 253 | # Allow the body of an if to be on the same line as the test if there is no 254 | # else. 255 | single-line-if-stmt=no 256 | 257 | 258 | [LOGGING] 259 | 260 | # Format style used to check logging format string. `old` means using % 261 | # formatting, while `new` is for `{}` formatting. 262 | logging-format-style=old 263 | 264 | # Logging modules to check that the string format arguments are in logging 265 | # function parameter format. 266 | logging-modules=logging 267 | 268 | 269 | [MISCELLANEOUS] 270 | 271 | # List of note tags to take in consideration, separated by a comma. 272 | notes=FIXME, 273 | XXX, 274 | TODO 275 | 276 | 277 | [SIMILARITIES] 278 | 279 | # Ignore comments when computing similarities. 280 | ignore-comments=yes 281 | 282 | # Ignore docstrings when computing similarities. 283 | ignore-docstrings=yes 284 | 285 | # Ignore imports when computing similarities. 286 | ignore-imports=no 287 | 288 | # Minimum lines number of a similarity. 289 | min-similarity-lines=20 290 | 291 | 292 | [SPELLING] 293 | 294 | # Limits count of emitted suggestions for spelling mistakes. 295 | max-spelling-suggestions=4 296 | 297 | # Spelling dictionary name. Available dictionaries: none. To make it working 298 | # install python-enchant package.. 299 | spelling-dict= 300 | 301 | # List of comma separated words that should not be checked. 302 | spelling-ignore-words= 303 | 304 | # A path to a file that contains private dictionary; one word per line. 305 | spelling-private-dict-file= 306 | 307 | # Tells whether to store unknown words to indicated private dictionary in 308 | # --spelling-private-dict-file option instead of raising a message. 309 | spelling-store-unknown-words=no 310 | 311 | 312 | [TYPECHECK] 313 | 314 | # List of decorators that produce context managers, such as 315 | # contextlib.contextmanager. Add to this list to register other decorators that 316 | # produce valid context managers. 317 | contextmanager-decorators=contextlib.contextmanager 318 | 319 | # List of members which are set dynamically and missed by pylint inference 320 | # system, and so shouldn't trigger E1101 when accessed. Python regular 321 | # expressions are accepted. 322 | generated-members= 323 | 324 | # Tells whether missing members accessed in mixin class should be ignored. A 325 | # mixin class is detected if its name ends with "mixin" (case insensitive). 326 | ignore-mixin-members=yes 327 | 328 | # Tells whether to warn about missing members when the owner of the attribute 329 | # is inferred to be None. 330 | ignore-none=yes 331 | 332 | # This flag controls whether pylint should warn about no-member and similar 333 | # checks whenever an opaque object is returned when inferring. The inference 334 | # can return multiple potential results while evaluating a Python object, but 335 | # some branches might not be evaluated, which results in partial inference. In 336 | # that case, it might be useful to still emit no-member and other checks for 337 | # the rest of the inferred objects. 338 | ignore-on-opaque-inference=yes 339 | 340 | # List of class names for which member attributes should not be checked (useful 341 | # for classes with dynamically set attributes). This supports the use of 342 | # qualified names. 343 | ignored-classes=optparse.Values,thread._local,_thread._local 344 | 345 | # List of module names for which member attributes should not be checked 346 | # (useful for modules/projects where namespaces are manipulated during runtime 347 | # and thus existing member attributes cannot be deduced by static analysis. It 348 | # supports qualified module names, as well as Unix pattern matching. 349 | ignored-modules=multiprocess 350 | 351 | # Show a hint with possible names when a member name was not found. The aspect 352 | # of finding the hint is based on edit distance. 353 | missing-member-hint=yes 354 | 355 | # The minimum edit distance a name should have in order to be considered a 356 | # similar match for a missing member name. 357 | missing-member-hint-distance=1 358 | 359 | # The total number of similar names that should be taken in consideration when 360 | # showing a hint for a missing member. 361 | missing-member-max-choices=1 362 | 363 | 364 | [VARIABLES] 365 | 366 | # List of additional names supposed to be defined in builtins. Remember that 367 | # you should avoid defining new builtins when possible. 368 | additional-builtins= 369 | 370 | # Tells whether unused global variables should be treated as a violation. 371 | allow-global-unused-variables=yes 372 | 373 | # List of strings which can identify a callback function by name. A callback 374 | # name must start or end with one of those strings. 375 | callbacks=cb_, 376 | _cb 377 | 378 | # A regular expression matching the name of dummy variables (i.e. expected to 379 | # not be used). 380 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 381 | 382 | # Argument names that match this expression will be ignored. Default to name 383 | # with leading underscore. 384 | ignored-argument-names=_.*|^ignored_|^unused_ 385 | 386 | # Tells whether we should check for unused import in __init__ files. 387 | init-import=no 388 | 389 | # List of qualified module names which can have objects that can redefine 390 | # builtins. 391 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 392 | 393 | 394 | [CLASSES] 395 | 396 | # List of method names used to declare (i.e. assign) instance attributes. 397 | defining-attr-methods=__init__, 398 | __new__, 399 | setUp 400 | 401 | # List of member names, which should be excluded from the protected access 402 | # warning. 403 | exclude-protected=_asdict, 404 | _fields, 405 | _replace, 406 | _source, 407 | _make 408 | 409 | # List of valid names for the first argument in a class method. 410 | valid-classmethod-first-arg=cls 411 | 412 | # List of valid names for the first argument in a metaclass class method. 413 | valid-metaclass-classmethod-first-arg=cls 414 | 415 | 416 | [DESIGN] 417 | 418 | # Maximum number of arguments for function / method. 419 | max-args=7 420 | 421 | # Maximum number of attributes for a class (see R0902). 422 | max-attributes=7 423 | 424 | # Maximum number of boolean expressions in an if statement. 425 | max-bool-expr=5 426 | 427 | # Maximum number of branch for function / method body. 428 | max-branches=12 429 | 430 | # Maximum number of locals for function / method body. 431 | max-locals=25 432 | 433 | # Maximum number of parents for a class (see R0901). 434 | max-parents=7 435 | 436 | # Maximum number of public methods for a class (see R0904). 437 | max-public-methods=20 438 | 439 | # Maximum number of return / yield for function / method body. 440 | max-returns=6 441 | 442 | # Maximum number of statements in function / method body. 443 | max-statements=50 444 | 445 | # Minimum number of public methods for a class (see R0903). 446 | min-public-methods=2 447 | 448 | 449 | [IMPORTS] 450 | 451 | # Allow wildcard imports from modules that define __all__. 452 | allow-wildcard-with-all=no 453 | 454 | # Analyse import fallback blocks. This can be used to support both Python 2 and 455 | # 3 compatible code, which means that the block might have code that exists 456 | # only in one or another interpreter, leading to false positives when analysed. 457 | analyse-fallback-blocks=no 458 | 459 | # Deprecated modules which should not be used, separated by a comma. 460 | deprecated-modules=optparse,tkinter.tix 461 | 462 | # Create a graph of external dependencies in the given file (report RP0402 must 463 | # not be disabled). 464 | ext-import-graph= 465 | 466 | # Create a graph of every (i.e. internal and external) dependencies in the 467 | # given file (report RP0402 must not be disabled). 468 | import-graph= 469 | 470 | # Create a graph of internal dependencies in the given file (report RP0402 must 471 | # not be disabled). 472 | int-import-graph= 473 | 474 | # Force import order to recognize a module as part of the standard 475 | # compatibility libraries. 476 | known-standard-library= 477 | 478 | # Force import order to recognize a module as part of a third party library. 479 | known-third-party=enchant 480 | 481 | 482 | [EXCEPTIONS] 483 | 484 | # Exceptions that will emit a warning when being caught. Defaults to 485 | # "Exception". 486 | overgeneral-exceptions=Exception 487 | --------------------------------------------------------------------------------