├── .clang-format ├── src ├── strings-portable.hh ├── output-stream-lock.cc ├── worker.hh ├── store.hh ├── meson.build ├── buffered-io.hh ├── output-stream-lock.hh ├── constituents.hh ├── strings-portable.cc ├── eval-args.hh ├── buffered-io.cc ├── drv.hh ├── eval-args.cc ├── constituents.cc ├── drv.cc ├── worker.cc └── nix-eval-jobs.cc ├── .github ├── dependabot.yml └── ISSUE_TEMPLATE │ └── bug_report.md ├── .envrc ├── renovate.json ├── .clang-tidy ├── pyproject.toml ├── .gitignore ├── meson.build ├── dev └── treefmt.nix ├── shell.nix ├── default.nix ├── tests ├── assets │ ├── ci.nix │ └── flake.nix └── test_eval.py ├── CONTRIBUTING.md ├── flake.lock ├── flake.nix ├── README.md └── LICENSE.md /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: llvm 2 | IndentWidth: 4 3 | SortIncludes: false 4 | -------------------------------------------------------------------------------- /src/strings-portable.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | auto get_signal_name(int sig) -> const char *; 4 | auto get_error_name(int err) -> const char *; 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /src/output-stream-lock.cc: -------------------------------------------------------------------------------- 1 | #include "output-stream-lock.hh" 2 | #include 3 | 4 | auto getCoutLock() -> OutputStreamLock & { 5 | static OutputStreamLock coutLock(std::cout); 6 | return coutLock; 7 | } -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | if ! has nix_direnv_version || ! nix_direnv_version 3.0.5; then 2 | source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.5/direnvrc" "sha256-RuwIS+QKFj/T9M2TFXScjBsLR6V3A17YVoEW/Q6AZ1w=" 3 | fi 4 | use flake 5 | -------------------------------------------------------------------------------- /src/worker.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "eval-args.hh" 4 | 5 | class MyArgs; 6 | 7 | namespace nix { 8 | class AutoCloseFD; 9 | class Bindings; 10 | class EvalState; 11 | template class ref; 12 | } // namespace nix 13 | 14 | void worker(MyArgs &args, nix::AutoCloseFD &toParent, 15 | nix::AutoCloseFD &fromParent); 16 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ], 6 | "lockFileMaintenance": { 7 | "enabled": true, 8 | "extends": [ 9 | "schedule:weekly" 10 | ] 11 | }, 12 | "labels": ["dependencies"], 13 | "nix": { 14 | "enabled": true 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/store.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace nix_eval_jobs { 10 | 11 | // Helper to open a store from an optional URL 12 | inline auto openStore(std::optional storeUrl = std::nullopt) 13 | -> nix::ref { 14 | return storeUrl ? nix::openStore(*storeUrl) : nix::openStore(); 15 | } 16 | 17 | } // namespace nix_eval_jobs 18 | -------------------------------------------------------------------------------- /src/meson.build: -------------------------------------------------------------------------------- 1 | src = [ 2 | 'nix-eval-jobs.cc', 3 | 'eval-args.cc', 4 | 'drv.cc', 5 | 'buffered-io.cc', 6 | 'constituents.cc', 7 | 'worker.cc', 8 | 'strings-portable.cc', 9 | 'output-stream-lock.cc' 10 | ] 11 | 12 | executable( 13 | 'nix-eval-jobs', 14 | src, 15 | dependencies: [ 16 | threads_dep, 17 | nlohmann_json_dep, 18 | libcurl_dep, 19 | 20 | nix_store_dep, 21 | nix_fetchers_dep, 22 | nix_expr_dep, 23 | nix_flake_dep, 24 | nix_main_dep, 25 | nix_cmd_dep, 26 | ], 27 | install: true, 28 | ) 29 | -------------------------------------------------------------------------------- /.clang-tidy: -------------------------------------------------------------------------------- 1 | Checks: 2 | - bugprone-* 3 | - concurrency-* 4 | - cppcoreguidelines-* 5 | - google-* 6 | - misc-* 7 | - modernize-* 8 | - performance-* 9 | - portability-* 10 | - readability-* 11 | - -google-readability-todo 12 | 13 | # don't find them too problematic 14 | - -bugprone-easily-swappable-parameters 15 | 16 | # crashes in clang-tidy 19 17 | - -modernize-use-designated-initializers 18 | UseColor: true 19 | CheckOptions: 20 | misc-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic: True 21 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | line-length = 100 3 | target-version = "py312" 4 | [tool.ruff.lint] 5 | select = [ 6 | "E", # pycodestyle errors 7 | "F", # pyflakes 8 | "B", # flake8-bugbear 9 | "I", # isort 10 | "W", # pycodestyle warnings 11 | "C4", # flake8-comprehensions 12 | "UP", # pyupgrade 13 | ] 14 | ignore = [ 15 | "E501", # line too long 16 | ] 17 | 18 | [tool.mypy] 19 | python_version = "3.12" 20 | warn_return_any = true 21 | warn_unused_configs = true 22 | disallow_untyped_defs = true 23 | disallow_incomplete_defs = true 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG] " 5 | labels: bug 6 | assignees: "" 7 | --- 8 | 9 | **Describe the bug** A clear and concise description of what the bug is. 10 | 11 | **To Reproduce** Steps to reproduce the behavior: 12 | 13 | 1. Run '...' 14 | 2. See error 15 | 16 | **Expected behavior** A clear and concise description of what you expected to 17 | happen. 18 | 19 | **Version** nix-eval-jobs version: [e.g. 0.1.6] 20 | 21 | **Additional context** Add any other context about the problem here. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | *.log 4 | 5 | tmp/ 6 | 7 | 8 | # Prerequisites 9 | *.d 10 | 11 | # Compiled Object files 12 | *.slo 13 | *.lo 14 | *.o 15 | *.obj 16 | 17 | # Precompiled Headers 18 | *.gch 19 | *.pch 20 | 21 | # Compiled Dynamic libraries 22 | *.so 23 | *.dylib 24 | *.dll 25 | 26 | # Fortran module files 27 | *.mod 28 | *.smod 29 | 30 | # Compiled Static libraries 31 | *.lai 32 | *.la 33 | *.a 34 | *.lib 35 | 36 | # Executables 37 | *.exe 38 | *.out 39 | *.app 40 | 41 | # build directory 42 | /build 43 | # nix-build 44 | /result 45 | 46 | # Byte-compiled / optimized / DLL files 47 | __pycache__/ 48 | *.py[cod] 49 | *$py.class 50 | 51 | # mypy 52 | .mypy_cache/ 53 | .dmypy.json 54 | dmypy.json 55 | 56 | # nix-direnv 57 | .direnv 58 | -------------------------------------------------------------------------------- /meson.build: -------------------------------------------------------------------------------- 1 | project( 2 | 'nix-eval-jobs', 3 | 'cpp', 4 | version: '0.1.6', 5 | license: 'GPL-3.0', 6 | default_options: [ 7 | 'cpp_std=c++20', 8 | 'warning_level=3' 9 | ] 10 | ) 11 | 12 | threads_dep = dependency('threads', required: true) 13 | nlohmann_json_dep = dependency('nlohmann_json', required: true) 14 | libcurl_dep = dependency('libcurl', required: true) 15 | 16 | nix_store_dep = dependency('nix-store', required: true) 17 | nix_fetchers_dep = dependency('nix-fetchers', required: true) 18 | nix_expr_dep = dependency('nix-expr', required: true) 19 | nix_flake_dep = dependency('nix-flake', required: true) 20 | nix_main_dep = dependency('nix-main', required: true) 21 | nix_cmd_dep = dependency('nix-cmd', required: true) 22 | 23 | subdir('src') 24 | -------------------------------------------------------------------------------- /dev/treefmt.nix: -------------------------------------------------------------------------------- 1 | { pkgs, lib, ... }: 2 | let 3 | supportsDeno = 4 | lib.meta.availableOn pkgs.stdenv.buildPlatform pkgs.deno 5 | && (builtins.tryEval pkgs.deno.outPath).success; 6 | in 7 | { 8 | flakeCheck = pkgs.hostPlatform.system != "riscv64-linux"; 9 | # Used to find the project root 10 | projectRootFile = "flake.lock"; 11 | 12 | programs.deno.enable = supportsDeno; 13 | programs.yamlfmt.enable = true; 14 | 15 | programs.clang-format.enable = true; 16 | programs.clang-format.package = pkgs.llvmPackages_latest.clang-tools; 17 | 18 | programs.deadnix.enable = true; 19 | programs.nixfmt.enable = true; 20 | programs.mypy = { 21 | enable = true; 22 | directories = { 23 | "tests" = { 24 | extraPythonPackages = [ pkgs.python3Packages.pytest ]; 25 | }; 26 | }; 27 | }; 28 | programs.ruff.format = true; 29 | programs.ruff.check = true; 30 | } 31 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { 2 | pkgs ? ( 3 | let 4 | inherit (builtins) fromJSON readFile; 5 | flakeLock = fromJSON (readFile ./flake.lock); 6 | inherit (flakeLock.nodes.nixpkgs) locked; 7 | nixpkgs = 8 | assert locked.type == "github"; 9 | builtins.fetchTarball { 10 | url = "https://github.com/${locked.owner}/${locked.repo}/archive/${locked.rev}.tar.gz"; 11 | sha256 = locked.narHash; 12 | }; 13 | in 14 | import nixpkgs { } 15 | ), 16 | stdenv ? pkgs.stdenv, 17 | lib ? pkgs.lib, 18 | nixComponents, 19 | }: 20 | 21 | let 22 | nix-eval-jobs = pkgs.callPackage ./default.nix { 23 | inherit nixComponents; 24 | }; 25 | in 26 | (pkgs.mkShell.override { inherit stdenv; }) { 27 | inherit (nix-eval-jobs) buildInputs; 28 | nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [ 29 | (pkgs.python3.withPackages (ps: [ ps.pytest ])) 30 | (lib.hiPrio pkgs.llvmPackages.clang-tools) 31 | ]; 32 | 33 | shellHook = lib.optionalString (stdenv.isLinux && nixComponents.nix-everything ? debug) '' 34 | export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nixComponents.nix-everything.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" 35 | ''; 36 | } 37 | -------------------------------------------------------------------------------- /src/buffered-io.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | [[nodiscard]] auto tryWriteLine(int file_descriptor, std::string str) -> int; 9 | 10 | struct FileDeleter { 11 | void operator()(FILE *file) const { 12 | if (file != nullptr) { 13 | std::fclose(file); // NOLINT(cppcoreguidelines-owning-memory) 14 | } 15 | } 16 | }; 17 | 18 | struct MemoryDeleter { 19 | void operator()(void *ptr) const { 20 | // NOLINTBEGIN(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc) 21 | std::free(ptr); 22 | // NOLINTEND(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc) 23 | } 24 | }; 25 | 26 | class LineReader { 27 | public: 28 | LineReader(const LineReader &) = delete; 29 | explicit LineReader(int file_descriptor); 30 | auto operator=(const LineReader &) -> LineReader & = delete; 31 | auto operator=(LineReader &&) -> LineReader & = delete; 32 | ~LineReader() = default; 33 | 34 | LineReader(LineReader &&other) noexcept; 35 | [[nodiscard]] auto readLine() -> std::string_view; 36 | 37 | private: 38 | std::unique_ptr stream = nullptr; 39 | std::unique_ptr buffer = nullptr; 40 | size_t len = 0; 41 | }; 42 | -------------------------------------------------------------------------------- /src/output-stream-lock.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | struct OutputStreamLock { 7 | private: 8 | std::mutex mutex; 9 | std::ostream *stream; 10 | 11 | struct LockedOutputStream { 12 | public: 13 | std::unique_lock lock; 14 | std::ostream *stream; 15 | LockedOutputStream(std::mutex &mutex, std::ostream *stream) 16 | : lock(mutex), stream(stream) {} 17 | LockedOutputStream(const LockedOutputStream &) = delete; 18 | LockedOutputStream(LockedOutputStream &&other) noexcept 19 | : lock(std::move(other.lock)), stream(other.stream) {} 20 | auto operator=(const LockedOutputStream &) 21 | -> LockedOutputStream & = delete; 22 | auto operator=(LockedOutputStream &&) -> LockedOutputStream & = delete; 23 | 24 | template 25 | auto operator<<(const T &value) && -> LockedOutputStream { 26 | // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) 27 | *stream << value; 28 | return std::move(*this); 29 | } 30 | 31 | ~LockedOutputStream() { 32 | if (lock) { 33 | *stream << std::flush; 34 | } 35 | } 36 | }; 37 | 38 | public: 39 | explicit OutputStreamLock(std::ostream &stream) : stream(&stream) {} 40 | 41 | auto lock() -> LockedOutputStream { return {mutex, stream}; } 42 | }; 43 | 44 | auto getCoutLock() -> OutputStreamLock &; -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { 2 | stdenv, 3 | lib, 4 | nixComponents, 5 | pkgs, 6 | }: 7 | 8 | let 9 | revision = "1"; 10 | in 11 | stdenv.mkDerivation { 12 | pname = "nix-eval-jobs"; 13 | version = "${lib.versions.majorMinor nixComponents.nix-cli.version}.${revision}"; 14 | src = lib.fileset.toSource { 15 | fileset = lib.fileset.unions [ 16 | ./.clang-tidy 17 | ./meson.build 18 | ./src/meson.build 19 | (lib.fileset.fileFilter (file: file.hasExt "cc") ./src) 20 | (lib.fileset.fileFilter (file: file.hasExt "hh") ./src) 21 | ]; 22 | root = ./.; 23 | }; 24 | buildInputs = with pkgs; [ 25 | nlohmann_json 26 | curl 27 | nixComponents.nix-store 28 | nixComponents.nix-fetchers 29 | nixComponents.nix-expr 30 | nixComponents.nix-flake 31 | nixComponents.nix-main 32 | nixComponents.nix-cmd 33 | ]; 34 | nativeBuildInputs = 35 | with pkgs; 36 | [ 37 | meson 38 | pkg-config 39 | ninja 40 | # nlohmann_json can be only discovered via cmake files 41 | cmake 42 | ] 43 | ++ lib.optional stdenv.cc.isClang (lib.hiPrio pkgs.llvmPackages.clang-tools); 44 | 45 | passthru = { 46 | inherit nixComponents; 47 | }; 48 | 49 | meta = { 50 | description = "Hydra's builtin hydra-eval-jobs as a standalone"; 51 | homepage = "https://github.com/nix-community/nix-eval-jobs"; 52 | license = lib.licenses.gpl3; 53 | maintainers = with lib.maintainers; [ 54 | adisbladis 55 | mic92 56 | ]; 57 | platforms = lib.platforms.unix; 58 | }; 59 | } 60 | -------------------------------------------------------------------------------- /src/constituents.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | struct DependencyCycle : public std::exception { 20 | std::string a; 21 | std::string b; 22 | std::set remainingAggregates; 23 | 24 | DependencyCycle(std::string nodeA, std::string nodeB, 25 | const std::set &remainingAggregates) 26 | : a(std::move(nodeA)), b(std::move(nodeB)), 27 | remainingAggregates(remainingAggregates) {} 28 | 29 | [[nodiscard]] auto message() const -> std::string { 30 | return nix::fmt("Dependency cycle: %s <-> %s", a, b); 31 | } 32 | }; 33 | 34 | struct AggregateJob { 35 | std::string name; 36 | std::set dependencies; 37 | std::unordered_map brokenJobs; 38 | 39 | auto operator<(const AggregateJob &other) const -> bool { 40 | return name < other.name; 41 | } 42 | }; 43 | 44 | auto resolveNamedConstituents(const std::map &jobs) 45 | -> std::variant, DependencyCycle>; 46 | 47 | void rewriteAggregates(std::map &jobs, 48 | const std::vector &aggregateJobs, 49 | const nix::ref &store, 50 | const nix::Path &gcRootsDir); 51 | -------------------------------------------------------------------------------- /src/strings-portable.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "strings-portable.hh" 3 | 4 | #if defined(__APPLE__) || defined(__FreeBSD__) 5 | // for sys_siglist, sys_errlist, NSIG, sys_nerr 6 | #include //NOLINT(modernize-deprecated-headers) 7 | #include //NOLINT(modernize-deprecated-headers) 8 | #endif 9 | 10 | #ifdef __GLIBC__ 11 | #include //NOLINT(modernize-deprecated-headers) 12 | 13 | // Linux with glibc specific: sigabbrev_np 14 | auto get_signal_name(int sig) -> const char * { 15 | const char *name = sigabbrev_np(sig); 16 | if (name != nullptr) { 17 | return name; 18 | } 19 | return "Unknown signal"; 20 | } 21 | auto get_error_name(int err) -> const char * { 22 | const char *name = strerrorname_np(err); 23 | if (name != nullptr) { 24 | return name; 25 | } 26 | return "Unknown error"; 27 | } 28 | #elif defined(__APPLE__) || defined(__FreeBSD__) 29 | // macOS and FreeBSD have sys_siglist 30 | auto get_signal_name(int sig) -> const char * { 31 | // NOLINTNEXTLINE(misc-include-cleaner) 32 | if (sig >= 0 && sig < NSIG) { 33 | // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index,misc-include-cleaner) 34 | return sys_siglist[sig]; 35 | } 36 | return "Unknown signal"; 37 | } 38 | auto get_error_name(int err) -> const char * { 39 | if (err >= 0 && err < sys_nerr) { 40 | // NOLINTNEXTLINE(misc-include-cleaner) 41 | return sys_errlist[err]; 42 | } 43 | return "Unknown error"; 44 | } 45 | #else 46 | auto get_signal_name(int sig) -> const char * { return strsignal(sig); } 47 | auto get_error_name(int err) -> const char * { return strerror(err); } 48 | #endif 49 | -------------------------------------------------------------------------------- /tests/assets/ci.nix: -------------------------------------------------------------------------------- 1 | { 2 | system ? "x86_64-linux", 3 | }: 4 | 5 | let 6 | dep-a = derivation { 7 | name = "dep-a"; 8 | inherit system; 9 | builder = "/bin/sh"; 10 | args = [ 11 | "-c" 12 | "echo 'bbbbbb' > $out" 13 | ]; 14 | }; 15 | 16 | dep-b = derivation { 17 | name = "dep-b"; 18 | inherit system; 19 | builder = "/bin/sh"; 20 | args = [ 21 | "-c" 22 | "echo 'aaaaaa' > $out" 23 | ]; 24 | }; 25 | in 26 | { 27 | builtJob = derivation { 28 | name = "job1"; 29 | inherit system; 30 | builder = "/bin/sh"; 31 | args = [ 32 | "-c" 33 | "echo 'job1' > $out" 34 | ]; 35 | requiredSystemFeatures = [ "big-parallel" ]; 36 | }; 37 | 38 | dontRecurse = { 39 | # This shouldn't build as `recurseForDerivations = true;` is not set 40 | # recurseForDerivations = true; 41 | 42 | # This should not build 43 | drvB = derivation { 44 | inherit system; 45 | name = "drvA"; 46 | builder = ":"; 47 | }; 48 | }; 49 | 50 | "dotted.attr" = derivation { 51 | name = "dotted"; 52 | inherit system; 53 | builder = "/bin/sh"; 54 | args = [ 55 | "-c" 56 | "echo 'dotted' > $out" 57 | ]; 58 | }; 59 | 60 | package-with-deps = derivation { 61 | name = "package-with-deps"; 62 | inherit system; 63 | builder = "/bin/sh"; 64 | args = [ 65 | "-c" 66 | "echo '${dep-a} ${dep-b}' > $out" 67 | ]; 68 | }; 69 | 70 | recurse = { 71 | # This should build 72 | recurseForDerivations = true; 73 | 74 | # This should not build 75 | drvB = derivation { 76 | inherit system; 77 | name = "drvB"; 78 | builder = ":"; 79 | }; 80 | }; 81 | } 82 | -------------------------------------------------------------------------------- /src/eval-args.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | class MyArgs : virtual public nix::MixEvalArgs, 12 | virtual public nix::MixCommonArgs, 13 | virtual public nix::RootArgs { 14 | public: 15 | static constexpr size_t DEFAULT_MAX_MEMORY_SIZE = 4096; 16 | 17 | virtual ~MyArgs() = default; 18 | std::string releaseExpr; 19 | std::string applyExpr; 20 | std::string selectExpr; 21 | nix::Path gcRootsDir; 22 | bool flake = false; 23 | bool fromArgs = false; 24 | bool meta = false; 25 | bool showTrace = false; 26 | bool impure = false; 27 | bool forceRecurse = false; 28 | bool checkCacheStatus = false; 29 | bool showInputDrvs = false; 30 | bool constituents = false; 31 | bool noInstantiate = false; 32 | size_t nrWorkers = 1; 33 | size_t maxMemorySize = DEFAULT_MAX_MEMORY_SIZE; 34 | 35 | // usually in MixFlakeOptions 36 | nix::flake::LockFlags lockFlags = {.updateLockFile = false, 37 | .writeLockFile = false, 38 | .useRegistries = false, 39 | .allowUnlocked = false, 40 | .referenceLockFilePath = {}, 41 | .outputLockFilePath = {}, 42 | .inputOverrides = {}, 43 | .inputUpdates = {}}; 44 | MyArgs(); 45 | MyArgs(MyArgs &&) = delete; 46 | auto operator=(const MyArgs &) -> MyArgs & = default; 47 | auto operator=(MyArgs &&) -> MyArgs & = delete; 48 | MyArgs(const MyArgs &) = delete; 49 | 50 | void parseArgs(char **argv, int argc); 51 | }; 52 | -------------------------------------------------------------------------------- /src/buffered-io.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | // NOLINTBEGIN(modernize-deprecated-headers) 6 | // misc-include-cleaner wants these headers rather than the C++ version 7 | #include 8 | #include 9 | // NOLINTEND(modernize-deprecated-headers) 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "buffered-io.hh" 18 | #include "strings-portable.hh" 19 | 20 | [[nodiscard]] auto tryWriteLine(int file_descriptor, std::string str) -> int { 21 | str += "\n"; 22 | std::string_view string_view{str}; 23 | while (!string_view.empty()) { 24 | nix::checkInterrupt(); 25 | // NOLINTNEXTLINE(misc-include-cleaner) 26 | const ssize_t res = 27 | write(file_descriptor, string_view.data(), string_view.size()); 28 | if (res == -1 && errno != EINTR) { 29 | return -errno; 30 | } 31 | if (res > 0) { 32 | string_view.remove_prefix(res); 33 | } 34 | } 35 | return 0; 36 | } 37 | 38 | LineReader::LineReader(int file_descriptor) 39 | : stream(fdopen(file_descriptor, "r")) { 40 | if (stream == nullptr) { 41 | throw nix::Error("fdopen(%d) failed: %s", file_descriptor, 42 | get_error_name(errno)); 43 | } 44 | } 45 | 46 | LineReader::LineReader(LineReader &&other) noexcept 47 | : stream(other.stream.release()), buffer(other.buffer.release()), 48 | len(other.len) { 49 | other.stream = nullptr; 50 | other.len = 0; 51 | } 52 | 53 | [[nodiscard]] auto LineReader::readLine() -> std::string_view { 54 | char *buf = buffer.release(); 55 | const ssize_t read = getline(&buf, &len, stream.get()); 56 | buffer.reset(buf); 57 | 58 | if (read == -1) { 59 | return {}; // Return an empty string_view in case of error 60 | } 61 | 62 | nix::checkInterrupt(); 63 | 64 | // Remove trailing newline 65 | char const *line = buffer.get(); 66 | return {line, static_cast(read) - 1}; 67 | } 68 | -------------------------------------------------------------------------------- /src/drv.hh: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | // we need this include or otherwise we cannot instantiate std::optional 6 | #include //NOLINT(misc-include-cleaner) 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "eval-args.hh" 16 | 17 | namespace nix { 18 | class EvalState; 19 | struct PackageInfo; 20 | } // namespace nix 21 | 22 | struct Constituents { 23 | std::vector constituents; 24 | std::vector namedConstituents; 25 | bool globConstituents; 26 | Constituents(std::vector constituents, 27 | std::vector namedConstituents, 28 | bool globConstituents) 29 | : constituents(std::move(constituents)), 30 | namedConstituents(std::move(namedConstituents)), 31 | globConstituents(globConstituents) {}; 32 | }; 33 | 34 | /* The fields of a derivation that are printed in json form */ 35 | struct Drv { 36 | Drv(std::string &attrPath, nix::EvalState &state, 37 | nix::PackageInfo &packageInfo, MyArgs &args, 38 | std::optional constituents); 39 | std::string name; 40 | std::string system; 41 | std::string drvPath; 42 | 43 | std::map> outputs; 44 | 45 | std::optional>> inputDrvs = 46 | std::nullopt; 47 | 48 | std::optional requiredSystemFeatures = std::nullopt; 49 | 50 | // TODO: can we lazily allocate these? 51 | std::vector neededBuilds; 52 | std::vector neededSubstitutes; 53 | std::vector unknownPaths; 54 | 55 | // TODO: we might not need to store this as it can be computed from the 56 | // above 57 | enum class CacheStatus : uint8_t { 58 | Local, 59 | Cached, 60 | NotBuilt, 61 | Unknown 62 | } cacheStatus; 63 | 64 | std::optional meta; 65 | std::optional constituents; 66 | }; 67 | void to_json(nlohmann::json &json, const Drv &drv); 68 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to nix-eval-jobs 2 | 3 | Thank you for considering contributing to nix-eval-jobs! This document provides 4 | guidelines and instructions for contributing to the project. 5 | 6 | ## Development Setup 7 | 8 | 1. Clone the repository: 9 | ```bash 10 | git clone https://github.com/nix-community/nix-eval-jobs.git 11 | cd nix-eval-jobs 12 | ``` 13 | 14 | 2. Set up the development environment: 15 | ```bash 16 | # Using nix 17 | nix develop 18 | # Or using direnv 19 | direnv allow 20 | ``` 21 | 22 | ## Building and Testing 23 | 24 | ### Building 25 | 26 | ```bash 27 | meson setup build 28 | cd build 29 | ninja 30 | ``` 31 | 32 | ### Running Tests 33 | 34 | ```bash 35 | pytest ./tests 36 | ``` 37 | 38 | ### Checking Everything 39 | 40 | To run all builds, tests, and checks: 41 | 42 | ```bash 43 | nix flake check 44 | ``` 45 | 46 | This will: 47 | 48 | - Build the package for all supported platforms 49 | - Run the test suite 50 | - Run all formatters and linters 51 | - Perform static analysis checks 52 | 53 | ## Code Quality Tools 54 | 55 | ### Formatting 56 | 57 | - Clang-format for C++ code 58 | - Ruff for Python code formatting and linting 59 | - Deno and yamlfmt for YAML files 60 | - nixfmt for Nix files 61 | 62 | ### Static Analysis 63 | 64 | - MyPy for Python type checking 65 | - deadnix for Nix code analysis 66 | - clang-tidy for C++ code analysis 67 | ```bash 68 | # Run clang-tidy checks 69 | ninja clang-tidy 70 | # Auto-fix clang-tidy issues where possible 71 | ninja clang-tidy-fix 72 | ``` 73 | 74 | All formatting can be applied using: 75 | 76 | ```bash 77 | nix fmt 78 | ``` 79 | 80 | ## Making Changes 81 | 82 | 1. Create a branch for your changes: 83 | ```bash 84 | git checkout -b your-feature-name 85 | ``` 86 | 87 | 2. Make your changes and commit them with descriptive commit messages: 88 | ```bash 89 | git commit -m "feat: Add new feature X" 90 | ``` 91 | 92 | 3. Push your changes to your fork: 93 | ```bash 94 | git push origin your-feature-name 95 | ``` 96 | 97 | 4. Create a Pull Request against the main repository. 98 | 99 | ## Additional Resources 100 | 101 | - [Project README](README.md) 102 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-parts": { 4 | "inputs": { 5 | "nixpkgs-lib": [ 6 | "nixpkgs" 7 | ] 8 | }, 9 | "locked": { 10 | "lastModified": 1759362264, 11 | "narHash": "sha256-wfG0S7pltlYyZTM+qqlhJ7GMw2fTF4mLKCIVhLii/4M=", 12 | "owner": "hercules-ci", 13 | "repo": "flake-parts", 14 | "rev": "758cf7296bee11f1706a574c77d072b8a7baa881", 15 | "type": "github" 16 | }, 17 | "original": { 18 | "owner": "hercules-ci", 19 | "repo": "flake-parts", 20 | "type": "github" 21 | } 22 | }, 23 | "nix": { 24 | "flake": false, 25 | "locked": { 26 | "lastModified": 1760472641, 27 | "narHash": "sha256-BuKtM7Vr5EcxBXxUENBQPlOBwmNd5mkTRkSmlJi/iQ4=", 28 | "owner": "NixOS", 29 | "repo": "nix", 30 | "rev": "4041bfdb401ad6d1c31a292fab90392254be506a", 31 | "type": "github" 32 | }, 33 | "original": { 34 | "owner": "NixOS", 35 | "repo": "nix", 36 | "type": "github" 37 | } 38 | }, 39 | "nixpkgs": { 40 | "locked": { 41 | "lastModified": 315532800, 42 | "narHash": "sha256-vhAtaRMIQiEghARviANBmSnhGz9Qf2IQJ+nQgsDXnVs=", 43 | "rev": "c12c63cd6c5eb34c7b4c3076c6a99e00fcab86ec", 44 | "type": "tarball", 45 | "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre877036.c12c63cd6c5e/nixexprs.tar.xz" 46 | }, 47 | "original": { 48 | "type": "tarball", 49 | "url": "https://nixos.org/channels/nixpkgs-unstable/nixexprs.tar.xz" 50 | } 51 | }, 52 | "root": { 53 | "inputs": { 54 | "flake-parts": "flake-parts", 55 | "nix": "nix", 56 | "nixpkgs": "nixpkgs", 57 | "treefmt-nix": "treefmt-nix" 58 | } 59 | }, 60 | "treefmt-nix": { 61 | "inputs": { 62 | "nixpkgs": [ 63 | "nixpkgs" 64 | ] 65 | }, 66 | "locked": { 67 | "lastModified": 1760120816, 68 | "narHash": "sha256-gq9rdocpmRZCwLS5vsHozwB6b5nrOBDNc2kkEaTXHfg=", 69 | "owner": "numtide", 70 | "repo": "treefmt-nix", 71 | "rev": "761ae7aff00907b607125b2f57338b74177697ed", 72 | "type": "github" 73 | }, 74 | "original": { 75 | "owner": "numtide", 76 | "repo": "treefmt-nix", 77 | "type": "github" 78 | } 79 | } 80 | }, 81 | "root": "root", 82 | "version": 7 83 | } 84 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Hydra's builtin hydra-eval-jobs as a standalone"; 3 | 4 | inputs.nixpkgs.url = "https://nixos.org/channels/nixpkgs-unstable/nixexprs.tar.xz"; 5 | inputs.nix = { 6 | url = "github:NixOS/nix"; 7 | # We want to control the deps precisely 8 | flake = false; 9 | }; 10 | inputs.flake-parts.url = "github:hercules-ci/flake-parts"; 11 | inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; 12 | inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; 13 | inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; 14 | 15 | outputs = 16 | inputs@{ flake-parts, ... }: 17 | let 18 | inherit (inputs.nixpkgs) lib; 19 | in 20 | flake-parts.lib.mkFlake { inherit inputs; } { 21 | systems = [ 22 | "aarch64-linux" 23 | "riscv64-linux" 24 | "x86_64-linux" 25 | 26 | "aarch64-darwin" 27 | "x86_64-darwin" 28 | ]; 29 | imports = [ inputs.treefmt-nix.flakeModule ]; 30 | 31 | perSystem = 32 | { pkgs, self', ... }: 33 | let 34 | nixDependencies = lib.makeScope pkgs.newScope ( 35 | import (inputs.nix + "/packaging/dependencies.nix") { 36 | inherit pkgs; 37 | inherit (pkgs) stdenv; 38 | inputs = { }; 39 | } 40 | ); 41 | nixComponents = lib.makeScope nixDependencies.newScope ( 42 | import (inputs.nix + "/packaging/components.nix") { 43 | officialRelease = true; 44 | inherit lib pkgs; 45 | src = inputs.nix; 46 | maintainers = [ ]; 47 | } 48 | ); 49 | drvArgs = { 50 | inherit nixComponents; 51 | }; 52 | in 53 | { 54 | treefmt.imports = [ ./dev/treefmt.nix ]; 55 | packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; 56 | packages.clangStdenv-nix-eval-jobs = pkgs.callPackage ./default.nix ( 57 | drvArgs // { stdenv = pkgs.clangStdenv; } 58 | ); 59 | packages.default = self'.packages.nix-eval-jobs; 60 | devShells.default = pkgs.callPackage ./shell.nix drvArgs; 61 | devShells.clang = pkgs.callPackage ./shell.nix (drvArgs // { stdenv = pkgs.clangStdenv; }); 62 | 63 | checks = builtins.removeAttrs self'.packages [ "default" ] // { 64 | shell = self'.devShells.default; 65 | tests = 66 | pkgs.runCommand "nix-eval-jobs-tests" 67 | { 68 | src = lib.fileset.toSource { 69 | fileset = lib.fileset.unions [ 70 | ./tests 71 | ]; 72 | root = ./.; 73 | }; 74 | 75 | buildInputs = [ 76 | self'.packages.nix-eval-jobs 77 | (pkgs.python3.withPackages (ps: [ ps.pytest ])) 78 | ]; 79 | } 80 | '' 81 | # Copy test files 82 | cp -r $src/tests . 83 | 84 | # Set up test environment 85 | export HOME=$TMPDIR 86 | export NIX_STATE_DIR=$TMPDIR/nix-state 87 | export NIX_STORE_DIR=$TMPDIR/nix-store 88 | export NIX_DATA_DIR=$TMPDIR/nix-data 89 | export NIX_LOG_DIR=$TMPDIR/nix-log 90 | export NIX_CONF_DIR=$TMPDIR/nix-conf 91 | 92 | # Use the pre-built nix-eval-jobs binary 93 | export NIX_EVAL_JOBS_BIN=${self'.packages.nix-eval-jobs}/bin/nix-eval-jobs 94 | 95 | # Run the tests 96 | python -m pytest tests/ -v 97 | 98 | # Create output marker 99 | touch $out 100 | ''; 101 | clang-tidy-fix = self'.packages.nix-eval-jobs.overrideAttrs (old: { 102 | nativeBuildInputs = old.nativeBuildInputs ++ [ 103 | pkgs.git 104 | (lib.hiPrio pkgs.llvmPackages.clang-tools) 105 | ]; 106 | buildPhase = '' 107 | export HOME=$TMPDIR 108 | cat > $HOME/.gitconfig < $out" 16 | ]; 17 | }; 18 | in 19 | { 20 | hydraJobs = import ./ci.nix { inherit system; }; 21 | 22 | legacyPackages.x86_64-linux = { 23 | emptyNeeded = rec { 24 | # This is a reproducer for issue #369 where neededBuilds and neededSubstitutes are empty 25 | # when they should contain values 26 | nginx = derivation { 27 | name = "nginx-1.24.0"; 28 | inherit system; 29 | builder = "/bin/sh"; 30 | args = [ 31 | "-c" 32 | "echo 'content' > $out" 33 | ]; 34 | }; 35 | proxyWrapper = derivation { 36 | name = "proxyWrapper"; 37 | system = "aarch64-linux"; 38 | builder = "/bin/sh"; 39 | args = [ 40 | "-c" 41 | "echo '${nginx}' > $out" 42 | ]; 43 | }; 44 | webService = derivation { 45 | name = "webService"; 46 | system = "aarch64-linux"; 47 | builder = "/bin/sh"; 48 | args = [ 49 | "-c" 50 | "echo '${proxyWrapper}' > $out" 51 | ]; 52 | }; 53 | }; 54 | brokenPkgs = { 55 | brokenPackage = throw "this is an evaluation error"; 56 | }; 57 | infiniteRecursionPkgs = { 58 | packageWithInfiniteRecursion = 59 | let 60 | recursion = [ recursion ]; 61 | in 62 | derivation { 63 | inherit system; 64 | name = "drvB"; 65 | recursiveAttr = recursion; 66 | builder = ":"; 67 | }; 68 | }; 69 | success = { 70 | indirect_aggregate = derivation { 71 | name = "indirect_aggregate"; 72 | inherit system; 73 | builder = "/bin/sh"; 74 | args = [ 75 | "-c" 76 | "echo done > $out" 77 | ]; 78 | _hydraAggregate = true; 79 | constituents = [ 80 | "anotherone" 81 | ]; 82 | }; 83 | direct_aggregate = derivation { 84 | name = "direct_aggregate"; 85 | inherit system; 86 | builder = "/bin/sh"; 87 | args = [ 88 | "-c" 89 | "echo done > $out" 90 | ]; 91 | _hydraAggregate = true; 92 | constituents = [ 93 | self.hydraJobs.builtJob 94 | ]; 95 | }; 96 | mixed_aggregate = derivation { 97 | name = "mixed_aggregate"; 98 | inherit system; 99 | builder = "/bin/sh"; 100 | args = [ 101 | "-c" 102 | "echo done > $out" 103 | ]; 104 | _hydraAggregate = true; 105 | constituents = [ 106 | self.hydraJobs.builtJob 107 | "anotherone" 108 | ]; 109 | }; 110 | anotherone = makeTextDrv "constituent" "text"; 111 | }; 112 | failures = { 113 | aggregate = derivation { 114 | name = "aggregate"; 115 | inherit system; 116 | builder = "/bin/sh"; 117 | args = [ 118 | "-c" 119 | "echo done > $out" 120 | ]; 121 | _hydraAggregate = true; 122 | constituents = [ 123 | "doesntexist" 124 | "doesnteval" 125 | ]; 126 | }; 127 | doesnteval = makeTextDrv "constituent" (toString { }); 128 | }; 129 | glob1 = { 130 | constituentA = derivation { 131 | name = "constituentA"; 132 | inherit system; 133 | builder = "/bin/sh"; 134 | args = [ 135 | "-c" 136 | "echo done > $out" 137 | ]; 138 | }; 139 | constituentB = derivation { 140 | name = "constituentB"; 141 | inherit system; 142 | builder = "/bin/sh"; 143 | args = [ 144 | "-c" 145 | "echo done > $out" 146 | ]; 147 | }; 148 | aggregate = derivation { 149 | name = "aggregate"; 150 | inherit system; 151 | builder = "/bin/sh"; 152 | args = [ 153 | "-c" 154 | "echo done > $out" 155 | ]; 156 | _hydraAggregate = true; 157 | _hydraGlobConstituents = true; 158 | constituents = [ "*" ]; 159 | }; 160 | }; 161 | cycle = { 162 | aggregate0 = derivation { 163 | name = "aggregate0"; 164 | inherit system; 165 | builder = "/bin/sh"; 166 | args = [ 167 | "-c" 168 | "echo done > $out" 169 | ]; 170 | _hydraAggregate = true; 171 | _hydraGlobConstituents = true; 172 | constituents = [ "aggregate1" ]; 173 | }; 174 | aggregate1 = derivation { 175 | name = "aggregate1"; 176 | inherit system; 177 | builder = "/bin/sh"; 178 | args = [ 179 | "-c" 180 | "echo done > $out" 181 | ]; 182 | _hydraAggregate = true; 183 | _hydraGlobConstituents = true; 184 | constituents = [ "aggregate0" ]; 185 | }; 186 | }; 187 | glob2 = rec { 188 | packages = { 189 | recurseForDerivations = true; 190 | constituentA = derivation { 191 | name = "constituentA"; 192 | inherit system; 193 | builder = "/bin/sh"; 194 | args = [ 195 | "-c" 196 | "echo done > $out" 197 | ]; 198 | }; 199 | constituentB = derivation { 200 | name = "constituentB"; 201 | inherit system; 202 | builder = "/bin/sh"; 203 | args = [ 204 | "-c" 205 | "echo done > $out" 206 | ]; 207 | }; 208 | }; 209 | aggregate0 = derivation { 210 | name = "aggregate0"; 211 | inherit system; 212 | builder = "/bin/sh"; 213 | args = [ 214 | "-c" 215 | "echo done > $out" 216 | ]; 217 | _hydraAggregate = true; 218 | _hydraGlobConstituents = true; 219 | constituents = [ 220 | "packages.*" 221 | ]; 222 | }; 223 | aggregate1 = derivation { 224 | name = "aggregate1"; 225 | inherit system; 226 | builder = "/bin/sh"; 227 | args = [ 228 | "-c" 229 | "echo done > $out" 230 | ]; 231 | _hydraAggregate = true; 232 | _hydraGlobConstituents = true; 233 | constituents = [ 234 | "tests.*" 235 | ]; 236 | }; 237 | indirect_aggregate0 = derivation { 238 | name = "indirect_aggregate0"; 239 | inherit system; 240 | builder = "/bin/sh"; 241 | args = [ 242 | "-c" 243 | "echo done > $out" 244 | ]; 245 | _hydraAggregate = true; 246 | constituents = [ 247 | "aggregate0" 248 | ]; 249 | }; 250 | mix_aggregate0 = derivation { 251 | name = "mix_aggregate0"; 252 | inherit system; 253 | builder = "/bin/sh"; 254 | args = [ 255 | "-c" 256 | "echo done > $out" 257 | ]; 258 | _hydraAggregate = true; 259 | constituents = [ 260 | "aggregate0" 261 | packages.constituentA 262 | ]; 263 | }; 264 | }; 265 | }; 266 | }; 267 | } 268 | -------------------------------------------------------------------------------- /src/eval-args.cc: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "eval-args.hh" 19 | #include "output-stream-lock.hh" 20 | #include 21 | 22 | MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") { 23 | addFlag({ 24 | .longName = "help", 25 | .aliases = {}, 26 | .shortName = 0, 27 | .description = "show usage information", 28 | .category = "", 29 | .labels = {}, 30 | .handler = {[&]() -> void { 31 | getCoutLock().lock() << "USAGE: nix-eval-jobs [options] expr\n\n"; 32 | for (const auto &[name, flag] : longFlags) { 33 | if (hiddenCategories.contains(flag->category)) { 34 | continue; 35 | } 36 | static constexpr int FLAG_WIDTH = 20; 37 | getCoutLock().lock() 38 | << " --" << std::left << std::setw(FLAG_WIDTH) << name 39 | << " " << flag->description << "\n"; 40 | } 41 | 42 | ::exit(0); // NOLINT(concurrency-mt-unsafe) 43 | }}, 44 | .completer = nullptr, 45 | .experimentalFeature = std::nullopt, 46 | }); 47 | 48 | addFlag({ 49 | .longName = "impure", 50 | .aliases = {}, 51 | .shortName = 0, 52 | .description = "allow impure expressions", 53 | .category = "", 54 | .labels = {}, 55 | .handler = {&impure, true}, 56 | .completer = nullptr, 57 | .experimentalFeature = std::nullopt, 58 | }); 59 | 60 | addFlag({ 61 | .longName = "force-recurse", 62 | .aliases = {}, 63 | .shortName = 0, 64 | .description = "force recursion (don't respect recurseIntoAttrs)", 65 | .category = "", 66 | .labels = {}, 67 | .handler = {&forceRecurse, true}, 68 | .completer = nullptr, 69 | .experimentalFeature = std::nullopt, 70 | }); 71 | 72 | addFlag({ 73 | .longName = "gc-roots-dir", 74 | .aliases = {}, 75 | .shortName = 0, 76 | .description = "garbage collector roots directory", 77 | .category = "", 78 | .labels = {"path"}, 79 | .handler = {&gcRootsDir}, 80 | .completer = nullptr, 81 | .experimentalFeature = std::nullopt, 82 | }); 83 | 84 | addFlag({ 85 | .longName = "workers", 86 | .aliases = {}, 87 | .shortName = 0, 88 | .description = "number of evaluate workers", 89 | .category = "", 90 | .labels = {"workers"}, 91 | .handler = {[this](const std::string &str) -> void { 92 | nrWorkers = std::stoi(str); 93 | }}, 94 | .completer = nullptr, 95 | .experimentalFeature = std::nullopt, 96 | }); 97 | 98 | addFlag({ 99 | .longName = "max-memory-size", 100 | .aliases = {}, 101 | .shortName = 0, 102 | .description = "maximum evaluation memory size in megabyte " 103 | "(4GiB per worker by default)", 104 | .category = "", 105 | .labels = {"size"}, 106 | .handler = {[this](const std::string &str) -> void { 107 | maxMemorySize = std::stoi(str); 108 | }}, 109 | .completer = nullptr, 110 | .experimentalFeature = std::nullopt, 111 | }); 112 | 113 | addFlag({ 114 | .longName = "flake", 115 | .aliases = {}, 116 | .shortName = 0, 117 | .description = "build a flake", 118 | .category = "", 119 | .labels = {}, 120 | .handler = {&flake, true}, 121 | .completer = nullptr, 122 | .experimentalFeature = std::nullopt, 123 | }); 124 | 125 | addFlag({ 126 | .longName = "meta", 127 | .aliases = {}, 128 | .shortName = 0, 129 | .description = "include derivation meta field in output", 130 | .category = "", 131 | .labels = {}, 132 | .handler = {&meta, true}, 133 | .completer = nullptr, 134 | .experimentalFeature = std::nullopt, 135 | }); 136 | 137 | addFlag({ 138 | .longName = "constituents", 139 | .aliases = {}, 140 | .shortName = 0, 141 | .description = 142 | "whether to evaluate constituents for Hydra's aggregate feature", 143 | .category = "", 144 | .labels = {}, 145 | .handler = {&constituents, true}, 146 | .completer = nullptr, 147 | .experimentalFeature = std::nullopt, 148 | }); 149 | 150 | addFlag({ 151 | .longName = "check-cache-status", 152 | .aliases = {}, 153 | .shortName = 0, 154 | .description = "Check if the derivations are present locally or in " 155 | "any configured substituters (i.e. binary cache). The " 156 | "information will be exposed in the `cacheStatus` field " 157 | "of the JSON output.", 158 | .category = "", 159 | .labels = {}, 160 | .handler = {&checkCacheStatus, true}, 161 | .completer = nullptr, 162 | .experimentalFeature = std::nullopt, 163 | }); 164 | 165 | addFlag({ 166 | .longName = "show-input-drvs", 167 | .aliases = {}, 168 | .shortName = 0, 169 | .description = 170 | "Show input derivations in the output for each derivation. " 171 | "This is useful to get direct dependencies of a derivation.", 172 | .category = "", 173 | .labels = {}, 174 | .handler = {&showInputDrvs, true}, 175 | .completer = nullptr, 176 | .experimentalFeature = std::nullopt, 177 | }); 178 | 179 | addFlag({ 180 | .longName = "show-trace", 181 | .aliases = {}, 182 | .shortName = 0, 183 | .description = "print out a stack trace in case of evaluation errors", 184 | .category = "", 185 | .labels = {}, 186 | .handler = {&showTrace, true}, 187 | .completer = nullptr, 188 | .experimentalFeature = std::nullopt, 189 | }); 190 | 191 | addFlag({ 192 | .longName = "no-instantiate", 193 | .aliases = {}, 194 | .shortName = 0, 195 | .description = 196 | "don't instantiate (write) derivations, only evaluate (faster)", 197 | .category = "", 198 | .labels = {}, 199 | .handler = {&noInstantiate, true}, 200 | .completer = nullptr, 201 | .experimentalFeature = std::nullopt, 202 | }); 203 | 204 | addFlag({ 205 | .longName = "expr", 206 | .aliases = {}, 207 | .shortName = 'E', 208 | .description = "treat the argument as a Nix expression", 209 | .category = "", 210 | .labels = {}, 211 | .handler = {&fromArgs, true}, 212 | .completer = nullptr, 213 | .experimentalFeature = std::nullopt, 214 | }); 215 | 216 | addFlag({ 217 | .longName = "apply", 218 | .aliases = {}, 219 | .shortName = 0, 220 | .description = 221 | "Apply provided Nix function to each derivation. " 222 | "The result of this function will be serialized as a JSON value " 223 | "and stored inside `\"extraValue\"` key of the json line output.", 224 | .category = "", 225 | .labels = {"expr"}, 226 | .handler = {&applyExpr}, 227 | .completer = nullptr, 228 | .experimentalFeature = std::nullopt, 229 | }); 230 | 231 | addFlag({ 232 | .longName = "select", 233 | .aliases = {}, 234 | .shortName = 0, 235 | .description = 236 | "Apply provided Nix function to transform the evaluation root. " 237 | "This is applied before any attribute traversal begins. " 238 | "When used with --flake without a fragment, the function receives " 239 | "an attrset with 'outputs' and 'inputs'. " 240 | "When used with a flake fragment, it receives the selected " 241 | "attribute. " 242 | "Examples: " 243 | "--select 'flake: flake.outputs.packages' " 244 | "--select 'flake: flake.inputs.nixpkgs' " 245 | "--select 'outputs: outputs.packages.x86_64-linux'", 246 | .category = "", 247 | .labels = {"expr"}, 248 | .handler = {&selectExpr}, 249 | .completer = nullptr, 250 | .experimentalFeature = std::nullopt, 251 | }); 252 | 253 | // usually in MixFlakeOptions 254 | addFlag({ 255 | .longName = "override-input", 256 | .aliases = {}, 257 | .shortName = 0, 258 | .description = 259 | "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", 260 | .category = category, 261 | .labels = {"input-path", "flake-url"}, 262 | .handler = {[&](const std::string &inputPath, 263 | const std::string &flakeRef) -> void { 264 | // overriden inputs are unlocked 265 | lockFlags.allowUnlocked = true; 266 | lockFlags.inputOverrides.insert_or_assign( 267 | nix::flake::parseInputAttrPath(inputPath), 268 | nix::parseFlakeRef(nix::fetchSettings, flakeRef, 269 | nix::absPath(std::filesystem::path(".")), 270 | true)); 271 | }}, 272 | .completer = nullptr, 273 | .experimentalFeature = std::nullopt, 274 | }); 275 | 276 | addFlag({ 277 | .longName = "reference-lock-file", 278 | .aliases = {}, 279 | .shortName = 0, 280 | .description = "Read the given lock file instead of `flake.lock` " 281 | "within the top-level flake.", 282 | .category = category, 283 | .labels = {"flake-lock-path"}, 284 | .handler = {[&](const std::string &lockFilePath) -> void { 285 | lockFlags.referenceLockFilePath = { 286 | nix::getFSSourceAccessor(), 287 | nix::CanonPath(nix::absPath(lockFilePath))}; 288 | }}, 289 | .completer = completePath, 290 | .experimentalFeature = std::nullopt, 291 | }); 292 | 293 | expectArg("expr", &releaseExpr); 294 | } 295 | 296 | void MyArgs::parseArgs(char **argv, int argc) { 297 | parseCmdline(nix::argvToStrings(argc, argv), false); 298 | } 299 | -------------------------------------------------------------------------------- /src/constituents.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "constituents.hh" 25 | #include "output-stream-lock.hh" 26 | 27 | namespace { 28 | // This is copied from `libutil/topo-sort.hh` in Nix and slightly modified. 29 | // However, I needed a way to use strings as identifiers to sort, but still be 30 | // able to put AggregateJob objects into this function since I'd rather not have 31 | // to transform back and forth between a list of strings and AggregateJobs in 32 | // resolveNamedConstituents. 33 | auto topoSort(const std::set &items) 34 | -> std::vector { 35 | std::vector sorted; 36 | std::set visited; 37 | std::set parents; 38 | 39 | std::map dictIdentToObject; 40 | for (const auto &item : items) { 41 | dictIdentToObject.insert({item.name, item}); 42 | } 43 | 44 | std::function 45 | dfsVisit; 46 | 47 | dfsVisit = [&](const std::string &path, const std::string *parent) -> void { 48 | if (parents.contains(path)) { 49 | dictIdentToObject.erase(path); 50 | dictIdentToObject.erase(*parent); 51 | std::set remaining; 52 | for (auto &[key, value] : dictIdentToObject) { 53 | remaining.insert(key); 54 | } 55 | throw DependencyCycle(path, *parent, remaining); 56 | } 57 | 58 | if (!visited.insert(path).second) { 59 | return; 60 | } 61 | parents.insert(path); 62 | 63 | const std::set references = 64 | dictIdentToObject[path].dependencies; 65 | 66 | for (const auto &ref : references) { 67 | /* Don't traverse into items that don't exist in our starting set. 68 | */ 69 | if (ref != path && dictIdentToObject.contains(ref)) { 70 | dfsVisit(ref, &path); 71 | } 72 | } 73 | 74 | sorted.push_back(dictIdentToObject[path]); 75 | parents.erase(path); 76 | }; 77 | 78 | for (auto &[key, value] : dictIdentToObject) { 79 | dfsVisit(key, nullptr); 80 | } 81 | 82 | return sorted; 83 | } 84 | 85 | auto insertMatchingConstituents( 86 | const std::string &childJobName, const std::string &jobName, 87 | const std::function 88 | &isBroken, 89 | const std::map &jobs, 90 | std::set &results) -> bool { 91 | bool expansionFound = false; 92 | for (const auto &[currentJobName, job] : jobs) { 93 | // Never select the job itself as constituent. Trivial way 94 | // to avoid obvious cycles. 95 | if (currentJobName == jobName) { 96 | continue; 97 | } 98 | auto jobName = currentJobName; 99 | if (fnmatch(childJobName.c_str(), jobName.c_str(), 0) == 0 && 100 | !isBroken(jobName, job)) { 101 | results.insert(jobName); 102 | expansionFound = true; 103 | } 104 | } 105 | 106 | return expansionFound; 107 | } 108 | } // namespace 109 | 110 | namespace { 111 | void addConstituents(nlohmann::json &job, nix::Derivation &drv, 112 | const std::set &dependencies, 113 | const std::map &jobs, 114 | const nix::ref &store) { 115 | for (const auto &childJobName : dependencies) { 116 | auto childDrvPath = store->parseStorePath( 117 | std::string(jobs.find(childJobName)->second["drvPath"])); 118 | auto childDrv = store->readDerivation(childDrvPath); 119 | job["constituents"].push_back(store->printStorePath(childDrvPath)); 120 | drv.inputDrvs.map[childDrvPath].value = { 121 | childDrv.outputs.begin()->first}; 122 | } 123 | } 124 | 125 | auto rewriteDerivation(nlohmann::json &job, nix::Derivation &drv, 126 | const nix::StorePath &drvPath, 127 | const nix::ref &store, 128 | const nix::Path &gcRootsDir) -> bool { 129 | std::string drvName(drvPath.name()); 130 | assert(nix::hasSuffix(drvName, nix::drvExtension)); 131 | drvName.resize(drvName.size() - nix::drvExtension.size()); 132 | 133 | auto hashModulo = hashDerivationModulo(*store, drv, true); 134 | if (hashModulo.kind != nix::DrvHash::Kind::Regular) { 135 | return false; 136 | } 137 | auto hashIter = hashModulo.hashes.find("out"); 138 | if (hashIter == hashModulo.hashes.end()) { 139 | return false; 140 | } 141 | auto outPath = store->makeOutputPath("out", hashIter->second, drvName); 142 | drv.env["out"] = store->printStorePath(outPath); 143 | drv.outputs.insert_or_assign( 144 | "out", nix::DerivationOutput::InputAddressed{.path = outPath}); 145 | 146 | auto newDrvPath = nix::writeDerivation(*store, drv); 147 | auto newDrvPathS = store->printStorePath(newDrvPath); 148 | 149 | if (!gcRootsDir.empty()) { 150 | const nix::Path root = 151 | gcRootsDir + "/" + std::string(nix::baseNameOf(newDrvPathS)); 152 | 153 | if (!nix::pathExists(root)) { 154 | store->addPermRoot(newDrvPath, root); 155 | } 156 | } 157 | 158 | nix::logger->log(nix::lvlDebug, 159 | nix::fmt("rewrote aggregate derivation %s -> %s", 160 | store->printStorePath(drvPath), newDrvPathS)); 161 | 162 | job["drvPath"] = newDrvPathS; 163 | job["outputs"]["out"] = store->printStorePath(outPath); 164 | return true; 165 | } 166 | 167 | void addBrokenJobsError( 168 | nlohmann::json &job, 169 | const std::unordered_map &brokenJobs) { 170 | std::stringstream errorStream; 171 | for (const auto &[jobName, error] : brokenJobs) { 172 | errorStream << jobName << ": " << error << "\n"; 173 | } 174 | job["error"] = errorStream.str(); 175 | } 176 | } // namespace 177 | 178 | auto resolveNamedConstituents(const std::map &jobs) 179 | -> std::variant, DependencyCycle> { 180 | std::set aggregateJobs; 181 | for (auto const &[jobName, job] : jobs) { 182 | auto named = job.find("namedConstituents"); 183 | if (named != job.end() && !named->empty()) { 184 | const bool globConstituents = 185 | job.value("globConstituents", false); 186 | std::unordered_map brokenJobs; 187 | std::set results; 188 | 189 | auto isBroken = [&brokenJobs, 190 | &jobName](const std::string &childJobName, 191 | const nlohmann::json &job) -> bool { 192 | if (job.find("error") != job.end()) { 193 | const std::string error = job["error"]; 194 | nix::logger->log( 195 | nix::lvlError, 196 | nix::fmt( 197 | "aggregate job '%s' references broken job '%s': %s", 198 | jobName, childJobName, error)); 199 | brokenJobs[childJobName] = error; 200 | return true; 201 | } 202 | return false; 203 | }; 204 | 205 | for (const std::string childJobName : *named) { 206 | auto childJobIter = jobs.find(childJobName); 207 | if (childJobIter == jobs.end()) { 208 | if (!globConstituents) { 209 | nix::logger->log( 210 | nix::lvlError, 211 | nix::fmt("aggregate job '%s' references " 212 | "non-existent job '%s'", 213 | jobName, childJobName)); 214 | brokenJobs[childJobName] = "does not exist"; 215 | } else if (!insertMatchingConstituents(childJobName, 216 | jobName, isBroken, 217 | jobs, results)) { 218 | nix::warn("aggregate job '%s' references constituent " 219 | "glob pattern '%s' with no matches", 220 | jobName, childJobName); 221 | brokenJobs[childJobName] = 222 | "constituent glob pattern had no matches"; 223 | } 224 | } else if (!isBroken(childJobName, childJobIter->second)) { 225 | results.insert(childJobName); 226 | } 227 | } 228 | 229 | aggregateJobs.insert(AggregateJob(jobName, results, brokenJobs)); 230 | } 231 | } 232 | 233 | try { 234 | return topoSort(aggregateJobs); 235 | } catch (DependencyCycle &e) { 236 | return e; 237 | } 238 | } 239 | 240 | void rewriteAggregates(std::map &jobs, 241 | const std::vector &aggregateJobs, 242 | const nix::ref &store, 243 | const nix::Path &gcRootsDir) { 244 | for (const auto &aggregateJob : aggregateJobs) { 245 | auto &job = jobs.find(aggregateJob.name)->second; 246 | auto drvPath = store->parseStorePath(std::string(job["drvPath"])); 247 | auto drv = store->readDerivation(drvPath); 248 | 249 | if (aggregateJob.brokenJobs.empty()) { 250 | addConstituents(job, drv, aggregateJob.dependencies, jobs, store); 251 | rewriteDerivation(job, drv, drvPath, store, gcRootsDir); 252 | } 253 | 254 | job.erase("namedConstituents"); 255 | 256 | if (!aggregateJob.brokenJobs.empty()) { 257 | addBrokenJobsError(job, aggregateJob.brokenJobs); 258 | } 259 | 260 | getCoutLock().lock() << job.dump() << "\n"; 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /src/drv.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include // for experimentalFeatureSettings 20 | // required for std::optional 21 | #include //NOLINT(misc-include-cleaner) 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #include "drv.hh" 34 | #include "eval-args.hh" 35 | 36 | namespace { 37 | 38 | auto queryOutputs(nix::PackageInfo &packageInfo, nix::EvalState &state, 39 | const std::string &attrPath) 40 | -> std::map> { 41 | std::map> outputs; 42 | 43 | try { 44 | nix::PackageInfo::Outputs outputsQueried; 45 | 46 | // CA derivations do not have static output paths, so we have to 47 | // fallback if we encounter an error 48 | try { 49 | outputsQueried = packageInfo.queryOutputs(true); 50 | } catch (const nix::Error &e) { 51 | // Handle CA derivation errors 52 | if (!nix::experimentalFeatureSettings.isEnabled( 53 | nix::Xp::CaDerivations)) { 54 | throw; 55 | } 56 | outputsQueried = packageInfo.queryOutputs(false); 57 | } 58 | for (auto &[outputName, optOutputPath] : outputsQueried) { 59 | if (optOutputPath) { 60 | outputs[outputName] = 61 | state.store->printStorePath(*optOutputPath); 62 | } else { 63 | outputs[outputName] = std::nullopt; 64 | } 65 | } 66 | } catch (const std::exception &e) { 67 | state 68 | .error( 69 | "derivation '%s' does not have valid outputs: %s", attrPath, 70 | e.what()) 71 | .debugThrow(); 72 | } 73 | 74 | return outputs; 75 | } 76 | 77 | auto queryMeta(nix::PackageInfo &packageInfo, nix::EvalState &state) 78 | -> std::optional { 79 | nlohmann::json meta_; 80 | for (const auto &metaName : packageInfo.queryMetaNames()) { 81 | nix::NixStringContext context; 82 | std::stringstream stream; 83 | 84 | auto *metaValue = packageInfo.queryMeta(metaName); 85 | // Skip non-serialisable types 86 | if (metaValue == nullptr) { 87 | continue; 88 | } 89 | 90 | nix::printValueAsJSON(state, true, *metaValue, nix::noPos, stream, 91 | context); 92 | 93 | meta_[metaName] = nlohmann::json::parse(stream.str()); 94 | } 95 | return meta_; 96 | } 97 | 98 | auto queryInputDrvs(const nix::Derivation &drv, nix::Store &store) 99 | -> std::map> { 100 | std::map> drvs; 101 | for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) { 102 | std::set inputDrvOutputs; 103 | for (const auto &outputName : inputNode.value) { 104 | inputDrvOutputs.insert(outputName); 105 | } 106 | drvs[store.printStorePath(inputDrvPath)] = inputDrvOutputs; 107 | } 108 | return drvs; 109 | } 110 | 111 | auto queryCacheStatus( 112 | nix::Store &store, 113 | std::map> &outputs, 114 | std::vector &neededBuilds, 115 | std::vector &neededSubstitutes, 116 | std::vector &unknownPaths, const nix::Derivation &drv) 117 | -> Drv::CacheStatus { 118 | 119 | std::vector paths; 120 | // Add output paths 121 | for (auto const &[key, val] : outputs) { 122 | if (val) { 123 | paths.push_back(followLinksToStorePathWithOutputs(store, *val)); 124 | } 125 | } 126 | 127 | // Add input derivation paths 128 | for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) { 129 | paths.push_back( 130 | nix::StorePathWithOutputs(inputDrvPath, inputNode.value)); 131 | } 132 | 133 | auto missing = store.queryMissing(toDerivedPaths(paths)); 134 | 135 | if (!missing.willBuild.empty()) { 136 | // TODO: can we expose the topological sort order as a graph? 137 | auto sorted = store.topoSortPaths(missing.willBuild); 138 | std::ranges::reverse(sorted.begin(), sorted.end()); 139 | for (auto &path : sorted) { 140 | neededBuilds.push_back(store.printStorePath(path)); 141 | } 142 | } 143 | if (!missing.willSubstitute.empty()) { 144 | std::vector willSubstituteSorted = {}; 145 | std::ranges::for_each(missing.willSubstitute.begin(), 146 | missing.willSubstitute.end(), 147 | [&](const nix::StorePath &path) -> void { 148 | willSubstituteSorted.push_back(&path); 149 | }); 150 | std::ranges::sort( 151 | willSubstituteSorted.begin(), willSubstituteSorted.end(), 152 | [](const nix::StorePath *lhs, const nix::StorePath *rhs) -> bool { 153 | if (lhs->name() == rhs->name()) { 154 | return lhs->to_string() < rhs->to_string(); 155 | } 156 | return lhs->name() < rhs->name(); 157 | }); 158 | for (const auto *path : willSubstituteSorted) { 159 | neededSubstitutes.push_back(store.printStorePath(*path)); 160 | } 161 | } 162 | 163 | if (!missing.unknown.empty()) { 164 | for (const auto &path : missing.unknown) { 165 | unknownPaths.push_back(store.printStorePath(path)); 166 | } 167 | } 168 | 169 | if (missing.willBuild.empty() && missing.unknown.empty()) { 170 | if (missing.willSubstitute.empty()) { 171 | // cacheStatus is Local if: 172 | // - there's nothing to build 173 | // - there's nothing to substitute 174 | return Drv::CacheStatus::Local; 175 | } 176 | // cacheStatus is Cached if: 177 | // - there's nothing to build 178 | // - there are paths to substitute 179 | return Drv::CacheStatus::Cached; 180 | } 181 | return Drv::CacheStatus::NotBuilt; 182 | }; 183 | 184 | } // namespace 185 | 186 | /* The fields of a derivation that are printed in json form */ 187 | Drv::Drv(std::string &attrPath, nix::EvalState &state, 188 | nix::PackageInfo &packageInfo, MyArgs &args, 189 | std::optional constituents) 190 | : constituents(std::move(constituents)) { 191 | 192 | auto store = state.store; 193 | 194 | name = packageInfo.queryName(); 195 | 196 | // Query outputs using helper function 197 | outputs = queryOutputs(packageInfo, state, attrPath); 198 | drvPath = store->printStorePath(packageInfo.requireDrvPath()); 199 | 200 | // Check if we can read derivations (requires LocalFSStore and not in 201 | // read-only mode) 202 | auto localStore = store.dynamic_pointer_cast(); 203 | const bool canReadDerivation = localStore && !nix::settings.readOnlyMode; 204 | 205 | if (canReadDerivation) { 206 | // We can read the derivation directly for precise information 207 | auto drv = localStore->readDerivation(packageInfo.requireDrvPath()); 208 | 209 | // Use the more precise system from the derivation 210 | system = drv.platform; 211 | 212 | if (args.checkCacheStatus) { 213 | // TODO: is this a bottleneck, where we should batch these queries? 214 | cacheStatus = 215 | queryCacheStatus(*store, outputs, neededBuilds, 216 | neededSubstitutes, unknownPaths, drv); 217 | } else { 218 | cacheStatus = Drv::CacheStatus::Unknown; 219 | } 220 | 221 | if (args.showInputDrvs) { 222 | inputDrvs = queryInputDrvs(drv, *store); 223 | } 224 | 225 | auto drvOptions = nix::DerivationOptions::fromStructuredAttrs( 226 | drv.env, drv.structuredAttrs); 227 | requiredSystemFeatures = 228 | std::optional(drvOptions.getRequiredSystemFeatures(drv)); 229 | } else { 230 | // Fall back to basic info from PackageInfo 231 | // This happens when: 232 | // - In read-only/no-instantiate mode 233 | // - Store is not a LocalFSStore (e.g., remote store) 234 | system = packageInfo.querySystem(); 235 | cacheStatus = Drv::CacheStatus::Unknown; 236 | // Can't get input derivations without reading the .drv file 237 | } 238 | 239 | // Handle metadata (works in both modes) 240 | if (args.meta) { 241 | meta = queryMeta(packageInfo, state); 242 | } 243 | } 244 | 245 | void to_json(nlohmann::json &json, const Drv &drv) { 246 | json = nlohmann::json{{"name", drv.name}, 247 | {"system", drv.system}, 248 | {"drvPath", drv.drvPath}, 249 | {"outputs", drv.outputs}}; 250 | 251 | if (drv.meta.has_value()) { 252 | json["meta"] = drv.meta.value(); 253 | } 254 | if (drv.inputDrvs) { 255 | json["inputDrvs"] = drv.inputDrvs.value(); 256 | } 257 | 258 | if (drv.requiredSystemFeatures) { 259 | json["requiredSystemFeatures"] = drv.requiredSystemFeatures.value(); 260 | } 261 | 262 | if (auto constituents = drv.constituents) { 263 | json["constituents"] = constituents->constituents; 264 | json["namedConstituents"] = constituents->namedConstituents; 265 | json["globConstituents"] = constituents->globConstituents; 266 | } 267 | 268 | if (drv.cacheStatus != Drv::CacheStatus::Unknown) { 269 | // Deprecated field 270 | json["isCached"] = drv.cacheStatus == Drv::CacheStatus::Cached || 271 | drv.cacheStatus == Drv::CacheStatus::Local; 272 | 273 | switch (drv.cacheStatus) { 274 | case Drv::CacheStatus::Cached: 275 | json["cacheStatus"] = "cached"; 276 | break; 277 | case Drv::CacheStatus::Local: 278 | json["cacheStatus"] = "local"; 279 | break; 280 | default: 281 | json["cacheStatus"] = "notBuilt"; 282 | break; 283 | } 284 | json["neededBuilds"] = drv.neededBuilds; 285 | json["neededSubstitutes"] = drv.neededSubstitutes; 286 | // TODO: is it useful to include "unknown" paths at all? 287 | // json["unknown"] = drv.unknownPaths; 288 | } 289 | } 290 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nix-eval-jobs 2 | 3 | This project evaluates nix attribute sets in parallel with streamable json 4 | output. This is useful for time and memory intensive evaluations such as NixOS 5 | machines, i.e. in a CI context. The evaluation is done with a controllable 6 | number of threads that are restarted when their memory consumption exceeds a 7 | certain threshold. 8 | 9 | To facilitate integration, nix-eval-jobs creates garbage collection roots for 10 | each evaluated derivation (drv file, not the build) within the provided 11 | attribute. This prevents race conditions between the nix garbage collection 12 | service and user-started nix builds processes. 13 | 14 | ## Why using nix-eval-jobs? 15 | 16 | - Faster evaluation by using threads 17 | - Memory used for evaluation is reclaimed after nix-eval-jobs finish, so that 18 | the build can use it. 19 | - Evaluation of jobs can fail individually 20 | 21 | ## Example 22 | 23 | In the following example we evaluate the hydraJobs attribute of the 24 | [patchelf](https://github.com/NixOS/patchelf) flake: 25 | 26 | ```console 27 | $ nix-eval-jobs --gc-roots-dir gcroot --flake 'github:NixOS/patchelf#hydraJobs' 28 | {"attr":"coverage","attrPath":["coverage"],"drvPath":"/nix/store/fmbqzaq8mim1423879lhn9whs6imx5w4-patchelf-coverage-0.18.0.drv","inputDrvs":{"/nix/store/23632hx2c98lbbjld279dx0w08lxn6kp-hook.drv":["out"],"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/ap9g09fxbicj836zm88d56dn3ff4clxl-stdenv-linux.drv":["out"],"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv":["out"],"/nix/store/vslywm6kbazi37q1vbq8y7bi884yc6yx-lcov-1.16.drv":["out"],"/nix/store/y964yq4vz1gsn7azd44vyg65gnr4gpvi-hook.drv":["out"]},"name":"patchelf-coverage-0.18.0","outputs":{"out":"/nix/store/gfni9sbhhwhxxfqziq1fs3n82bvw962l-patchelf-coverage-0.18.0"},"system":"x86_64-linux"} 29 | {"attr":"patchelf-win32","attrPath":["patchelf-win32"],"drvPath":"/nix/store/s38l0fg5ja6j8qpws7slw2ws0c6v0qcf-patchelf-i686-w64-mingw32-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/b2p151ilwqpd47fbmzz50a5cmj12ixbf-hook.drv":["out"],"/nix/store/fbnhh18m4jh6cwa92am2sv3aqzjnzpdd-stdenv-linux.drv":["out"]},"name":"patchelf-i686-w64-mingw32-0.18.0","outputs":{"out":"/nix/store/w8r4h1xk71fryb99df8aszp83kfhw3bc-patchelf-i686-w64-mingw32-0.18.0"},"system":"x86_64-linux"} 30 | {"attr":"patchelf-win64","attrPath":["patchelf-win64"],"drvPath":"/nix/store/wxpym6d3dxr1w9syhinp7f058gwxfmd3-patchelf-x86_64-w64-mingw32-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/71lv5lsr1y59bv1b91jc9gg0n85kf1sq-stdenv-linux.drv":["out"],"/nix/store/b2p151ilwqpd47fbmzz50a5cmj12ixbf-hook.drv":["out"]},"name":"patchelf-x86_64-w64-mingw32-0.18.0","outputs":{"out":"/nix/store/fkq5428l2xsb84yj0cc6q1lkvsrga7sv-patchelf-x86_64-w64-mingw32-0.18.0"},"system":"x86_64-linux"} 31 | {"attr":"release","attrPath":["release"],"drvPath":"/nix/store/3xpwg8f623dpkh6cblv2fzcq5n99xl0j-patchelf-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/9rmihrl9ys0sap6827xyns0y73vqafjx-patchelf-0.18.0.drv":["out"],"/nix/store/am2zqx3pyc1i14f888jna785h0f841sg-patchelf-0.18.0.drv":["out"],"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv":["out"],"/nix/store/csjiccxbwpfv55m8kqs2xwrkkha14dnq-patchelf-0.18.0.drv":["out"],"/nix/store/jsrnpxdx5vmpnakd9bkb3sk3lgh0k8hm-patchelf-0.18.0.drv":["out"],"/nix/store/k8a51ax83554c67g98xf3y751vjgjs7m-patchelf-0.18.0.drv":["out"],"/nix/store/wq3ncl207isqqkqmsa5ql4fg19jbrhxg-stdenv-linux.drv":["out"]},"name":"patchelf-0.18.0","outputs":{"out":"/nix/store/d0mzprvv3vhasj23r1a6qn8qip0srbc4-patchelf-0.18.0"},"system":"x86_64-linux"} 32 | {"attr":"tarball","attrPath":["tarball"],"drvPath":"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/9d754glmsvpjm5kxvgsjslvgv356kbmn-libtool-2.4.7.drv":["out"],"/nix/store/ap9g09fxbicj836zm88d56dn3ff4clxl-stdenv-linux.drv":["out"],"/nix/store/f1ksgsyplvb0sli4pls6k6vsfvmv519d-autoconf-2.71.drv":["out"],"/nix/store/jf58lcnch1bmpbi2188c59w5zr1cqrx2-automake-1.16.5.drv":["out"]},"name":"patchelf-tarball-0.18.0","outputs":{"out":"/nix/store/72pz5awc7gpwdqxrdsy8j0bvg2n7z78q-patchelf-tarball-0.18.0"},"system":"x86_64-linux"} 33 | ``` 34 | 35 | The output here is newline-seperated json according to https://jsonlines.org. 36 | 37 | The code is derived from [hydra's](https://github.com/nixos/hydra) eval-jobs 38 | executable. 39 | 40 | ## Further options 41 | 42 | ```console 43 | USAGE: nix-eval-jobs [options] expr 44 | 45 | --apply Apply provided Nix function to each derivation. The result of this function will be serialized as a JSON value and stored inside `"extraValue"` key of the json line output. 46 | --arg Pass the value *expr* as the argument *name* to Nix functions. 47 | --arg-from-file Pass the contents of file *path* as the argument *name* to Nix functions. 48 | --arg-from-stdin Pass the contents of stdin as the argument *name* to Nix functions. 49 | --argstr Pass the string *string* as the argument *name* to Nix functions. 50 | --check-cache-status Check if the derivations are present locally or in any configured substituters (i.e. binary cache). The information will be exposed in the `cacheStatus` field of the JSON output. 51 | --constituents whether to evaluate constituents for Hydra's aggregate feature 52 | --debug Set the logging verbosity level to 'debug'. 53 | --eval-store 54 | The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) 55 | to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. 56 | 57 | --expr treat the argument as a Nix expression 58 | --flake build a flake 59 | --force-recurse force recursion (don't respect recurseIntoAttrs) 60 | --gc-roots-dir garbage collector roots directory 61 | --help show usage information 62 | --impure allow impure expressions 63 | --include 64 | Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) 65 | 66 | This option may be given multiple times. 67 | 68 | Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). 69 | 70 | --log-format Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`. 71 | --max-memory-size maximum evaluation memory size in megabyte (4GiB per worker by default) 72 | --meta include derivation meta field in output 73 | --no-instantiate don't instantiate (write) derivations, only evaluate (faster) 74 | --option Set the Nix configuration setting *name* to *value* (overriding `nix.conf`). 75 | --override-flake Override the flake registries, redirecting *original-ref* to *resolved-ref*. 76 | --override-input Override a specific flake input (e.g. `dwarffs/nixpkgs`). 77 | --quiet Decrease the logging verbosity level. 78 | --reference-lock-file Read the given lock file instead of `flake.lock` within the top-level flake. 79 | --repair During evaluation, rewrite missing or corrupted files in the Nix store. During building, rebuild missing or corrupted store paths. 80 | --select Apply provided Nix function to transform the evaluation root. This is applied before any attribute traversal begins. When used with --flake without a fragment, the function receives an attrset with 'outputs' and 'inputs'. When used with a flake fragment, it receives the selected attribute. Examples: --select 'flake: flake.outputs.packages' --select 'flake: flake.inputs.nixpkgs' --select 'outputs: outputs.packages.x86_64-linux' 81 | --show-input-drvs Show input derivations in the output for each derivation. This is useful to get direct dependencies of a derivation. 82 | --show-trace print out a stack trace in case of evaluation errors 83 | --verbose Increase the logging verbosity level. 84 | --workers number of evaluate workers 85 | ``` 86 | 87 | ## Potential use-cases for the tool 88 | 89 | **Faster evaluator in deployment tools.** When evaluating NixOS machines, 90 | evaluation can take several minutes when run on a single core. This limits 91 | scalability for large deployments with deployment tools such as 92 | [NixOps](https://github.com/NixOS/nixops). 93 | 94 | **Faster evaluator in CIs.** In addition to evaluation speed for CIs, it is also 95 | useful if evaluation of individual jobs in CIs can fail, as opposed to failing 96 | the entire jobset. For CIs that allow dynamic build steps to be created, one can 97 | also take advantage of the fact that nix-eval-jobs outputs the derivation path 98 | separately. This allows separate logs and success status per job instead of a 99 | single large log file. In the 100 | [wiki](https://github.com/nix-community/nix-eval-jobs/wiki#ci-example-configurations) 101 | we collect example ci configuration for various CIs. 102 | 103 | ## Projects using nix-eval-jobs 104 | 105 | - [Hydra](https://github.com/NixOS/hydra) - The Nix-based continuous build 106 | system 107 | - [nix-fast-build](https://github.com/Mic92/nix-fast-build) - Combine the power 108 | of nix-eval-jobs with nix-output-monitor to speed-up your evaluation and 109 | building process 110 | - [buildbot-nix](https://github.com/Mic92/buildbot-nix) - A nixos module to make 111 | buildbot a proper Nix-CI 112 | - [colmena](https://github.com/zhaofengli/colmena) - A simple, stateless NixOS 113 | deployment tool 114 | - [robotnix](https://github.com/danielfullmer/robotnix) - Build Android (AOSP) 115 | using Nix, used in their 116 | [CI](https://github.com/danielfullmer/robotnix/blob/38b80700ee4265c306dcfdcce45056e32ab2973f/.github/workflows/instantiate.yml#L18) 117 | 118 | ## FAQ 119 | 120 | ### How can I check if my package already have been uploaded in the binary cache? 121 | 122 | If you provide the `--check-cache-status`, the json will contain a 123 | `"cacheStatus"` key in its json, with the following values: 124 | 125 | | Value | Meaning | 126 | | -------- | ------------------------------------------------------- | 127 | | local | Package is present locally | 128 | | cached | Package is present in the binary cache, but not locally | 129 | | notBuilt | Package needs to be built. | 130 | 131 | ### How can I evaluate nixpkgs? 132 | 133 | If you want to evaluate nixpkgs in the same way 134 | [hydra](https://hydra.nixos.org/) does it, use this snippet: 135 | 136 | ```console 137 | $ nix-eval-jobs --force-recurse pkgs/top-level/release.nix 138 | ``` 139 | 140 | ### nix-eval-jobs consumes too much memory / is too slow 141 | 142 | By default, nix-eval-jobs spawns as many worker processes as there are hardware 143 | threads in the system and limits the memory usage for each worker to 4GB. 144 | 145 | However, keep in mind that each worker process may need to re-evaluate shared 146 | dependencies of the attributes, which can introduce some overhead for each 147 | evaluation or cause workers to exceed their memory limit. If you encounter these 148 | situations, you can tune the following options: 149 | 150 | `--workers`: This option allows you to set the number of evaluation workers that 151 | nix-eval-jobs should spawn. You can increase or decrease this number to optimize 152 | the evaluation speed and memory usage. For example, if you have a system with 153 | many CPU cores but limited memory, you may want to reduce the number of workers 154 | to avoid exceeding the memory limit. 155 | 156 | `--max-memory-size`: This option allows you to adjust the memory limit for each 157 | worker process. By default, it's set to 4GiB, but you can increase or decrease 158 | this value as needed. For example, if you have a system with a lot of memory and 159 | want to speed up the evaluation, you may want to increase the memory limit to 160 | allow workers to cache more data in memory before getting restarted by 161 | nix-eval-jobs. Note that this is not a hard limit and memory usage may rise 162 | above the limit momentarily before the worker process exits. 163 | 164 | Overall, tuning these options can help you optimize the performance and memory 165 | usage of nix-eval-jobs to better fit your system and evaluation needs. 166 | -------------------------------------------------------------------------------- /src/worker.cc: -------------------------------------------------------------------------------- 1 | // doesn't exist on macOS 2 | // IWYU pragma: no_include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | // NOLINTBEGIN(modernize-deprecated-headers) 17 | // misc-include-cleaner wants this header rather than the C++ version 18 | #include 19 | // NOLINTEND(modernize-deprecated-headers) 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | 48 | #include "worker.hh" 49 | #include "drv.hh" 50 | #include "buffered-io.hh" 51 | #include "eval-args.hh" 52 | #include "store.hh" 53 | 54 | namespace nix { 55 | struct Expr; 56 | } // namespace nix 57 | 58 | namespace { 59 | auto releaseExprTopLevelValue(nix::EvalState &state, nix::Bindings &autoArgs, 60 | MyArgs &args) -> nix::Value * { 61 | nix::Value vTop; 62 | 63 | if (args.fromArgs) { 64 | nix::Expr *expr = 65 | state.parseExprFromString(args.releaseExpr, state.rootPath(".")); 66 | state.eval(expr, vTop); 67 | } else { 68 | state.evalFile(lookupFileArg(state, args.releaseExpr), vTop); 69 | } 70 | 71 | auto *vRoot = state.allocValue(); 72 | 73 | state.autoCallFunction(autoArgs, vTop, *vRoot); 74 | 75 | return vRoot; 76 | } 77 | 78 | auto evaluateFlake(const nix::ref &state, 79 | const std::string &releaseExpr, 80 | const nix::flake::LockFlags &lockFlags) -> nix::Value * { 81 | auto [flakeRef, fragment, outputSpec] = 82 | nix::parseFlakeRefWithFragmentAndExtendedOutputsSpec( 83 | nix::fetchSettings, releaseExpr, 84 | nix::absPath(std::filesystem::path("."))); 85 | 86 | nix::InstallableFlake flake{{}, state, std::move(flakeRef), 87 | fragment, outputSpec, {}, 88 | {}, lockFlags}; 89 | 90 | // If no fragment specified, use callFlake to get the full flake structure 91 | // (just like :lf in the REPL) 92 | if (fragment.empty()) { 93 | auto *value = state->allocValue(); 94 | nix::flake::callFlake(*state, *flake.getLockedFlake(), *value); 95 | return value; 96 | } 97 | // Fragment specified, use normal evaluation 98 | return flake.toValue(*state).first; 99 | } 100 | 101 | auto attrPathJoin(nlohmann::json input) -> std::string { 102 | return std::accumulate( 103 | input.begin(), input.end(), std::string(), 104 | [](const std::string &acc, std::string str) -> std::basic_string { 105 | // Escape token if containing dots 106 | if (str.find('.') != std::string::npos) { 107 | str = "\"" + str + "\""; 108 | } 109 | return acc.empty() ? str : acc + "." + str; 110 | }); 111 | } 112 | 113 | auto extractConstituents(nix::EvalState &state, nix::Value *value, 114 | const MyArgs &args) -> std::optional { 115 | if (!args.constituents) { 116 | return std::nullopt; 117 | } 118 | 119 | std::vector constituents; 120 | std::vector namedConstituents; 121 | bool globConstituents = false; 122 | 123 | const auto *aggregateAttr = 124 | value->attrs()->get(state.symbols.create("_hydraAggregate")); 125 | 126 | if (aggregateAttr != nullptr && 127 | state.forceBool(*aggregateAttr->value, aggregateAttr->pos, 128 | "while evaluating the `_hydraAggregate` attribute")) { 129 | 130 | const auto *constituentsAttr = 131 | value->attrs()->get(state.symbols.create("constituents")); 132 | 133 | if (constituentsAttr == nullptr) { 134 | state 135 | .error( 136 | "derivation must have a 'constituents' attribute") 137 | .debugThrow(); 138 | } 139 | 140 | // Extract constituent paths from context 141 | nix::NixStringContext context; 142 | state.coerceToString( 143 | constituentsAttr->pos, *constituentsAttr->value, context, 144 | "while evaluating the `constituents` attribute", true, false); 145 | 146 | for (const auto &ctx : context) { 147 | std::visit( 148 | nix::overloaded{ 149 | [&](const nix::NixStringContextElem::Built &built) -> void { 150 | constituents.push_back( 151 | built.drvPath->to_string(*state.store)); 152 | }, 153 | [&](const nix::NixStringContextElem::Opaque &opaque 154 | [[maybe_unused]]) -> void {}, 155 | [&](const nix::NixStringContextElem::DrvDeep &drvDeep 156 | [[maybe_unused]]) -> void {}, 157 | }, 158 | ctx.raw); 159 | } 160 | 161 | // Extract named constituents 162 | state.forceList(*constituentsAttr->value, constituentsAttr->pos, 163 | "while evaluating the `constituents` attribute"); 164 | auto constituentsList = constituentsAttr->value->listView(); 165 | 166 | for (const auto &val : constituentsList) { 167 | state.forceValue(*val, nix::noPos); 168 | if (val->type() == nix::nString) { 169 | namedConstituents.emplace_back(val->c_str()); 170 | } 171 | } 172 | 173 | // Check for glob constituents 174 | const auto *glob = 175 | value->attrs()->get(state.symbols.create("_hydraGlobConstituents")); 176 | globConstituents = 177 | glob != nullptr && 178 | state.forceBool( 179 | *glob->value, glob->pos, 180 | "while evaluating the `_hydraGlobConstituents` attribute"); 181 | } 182 | 183 | return Constituents(constituents, namedConstituents, globConstituents); 184 | } 185 | 186 | auto applyExprToValue(nix::EvalState &state, nix::Value *value, 187 | const std::string &applyExpr) -> nlohmann::json { 188 | if (applyExpr.empty()) { 189 | return nlohmann::json{}; 190 | } 191 | 192 | auto *expr = state.parseExprFromString(applyExpr, state.rootPath(".")); 193 | 194 | nix::Value vApply; 195 | nix::Value vRes; 196 | 197 | state.eval(expr, vApply); 198 | state.callFunction(vApply, *value, vRes, nix::noPos); 199 | state.forceAttrs(vRes, nix::noPos, "apply needs to evaluate to an attrset"); 200 | 201 | nix::NixStringContext context; 202 | std::stringstream stream; 203 | nix::printValueAsJSON(state, true, vRes, nix::noPos, stream, context); 204 | 205 | return nlohmann::json::parse(stream.str()); 206 | } 207 | 208 | auto registerGCRoot(nix::EvalState &state, const Drv &drv, const MyArgs &args) 209 | -> void { 210 | if (args.gcRootsDir.empty() || nix::settings.readOnlyMode || 211 | drv.drvPath.empty()) { 212 | return; 213 | } 214 | 215 | const nix::Path root = 216 | args.gcRootsDir + "/" + std::string(nix::baseNameOf(drv.drvPath)); 217 | 218 | if (!nix::pathExists(root)) { 219 | auto localStore = state.store.dynamic_pointer_cast(); 220 | if (localStore) { 221 | auto storePath = localStore->parseStorePath(drv.drvPath); 222 | localStore->addPermRoot(storePath, root); 223 | } 224 | // If not a local store, we can't create GC roots 225 | } 226 | } 227 | 228 | auto collectAttrsForRecursion(nix::EvalState &state, nix::Value *value, 229 | const nlohmann::json &path, const MyArgs &args) 230 | -> nlohmann::json { 231 | auto attrs = nlohmann::json::array(); 232 | bool recurse = 233 | args.forceRecurse || 234 | path.empty(); // Don't require recurseForDerivations for top-level 235 | 236 | for (auto &attr : value->attrs()->lexicographicOrder(state.symbols)) { 237 | const std::string_view &name = state.symbols[attr->name]; 238 | attrs.push_back(name); 239 | 240 | if (!args.forceRecurse && name == "recurseForDerivations") { 241 | const auto *attrv = 242 | value->attrs()->get(nix::EvalState::s.recurseForDerivations); 243 | recurse = state.forceBool(*attrv->value, attrv->pos, 244 | "while evaluating recurseForDerivations"); 245 | } 246 | } 247 | 248 | return recurse ? attrs : nlohmann::json::array(); 249 | } 250 | 251 | auto processDerivation(nix::EvalState &state, nix::Value *value, 252 | std::string &attrPathS, const nlohmann::json &path, 253 | MyArgs &args, nlohmann::json &reply) -> void { 254 | auto packageInfo = nix::getDerivation(state, *value, false); 255 | if (!packageInfo) { 256 | auto attrs = collectAttrsForRecursion(state, value, path, args); 257 | reply["attrs"] = attrs; 258 | return; 259 | } 260 | 261 | // Extract constituents if enabled 262 | auto maybeConstituents = extractConstituents(state, value, args); 263 | 264 | // Apply expression if provided 265 | if (!args.applyExpr.empty()) { 266 | reply["extraValue"] = applyExprToValue(state, value, args.applyExpr); 267 | } 268 | 269 | // Create derivation info 270 | auto drv = Drv(attrPathS, state, *packageInfo, args, maybeConstituents); 271 | reply.update(drv); 272 | 273 | // Register GC root 274 | registerGCRoot(state, drv, args); 275 | } 276 | 277 | auto initializeRootValue(const nix::ref &state, 278 | nix::Bindings &autoArgs, MyArgs &args) 279 | -> nix::Value * { 280 | nix::Value *vEvaluated = 281 | args.flake ? evaluateFlake(state, args.releaseExpr, args.lockFlags) 282 | : releaseExprTopLevelValue(*state, autoArgs, args); 283 | 284 | if (args.selectExpr.empty()) { 285 | return vEvaluated; 286 | } 287 | 288 | // Apply the provided select function 289 | auto *selectExpr = 290 | state->parseExprFromString(args.selectExpr, state->rootPath(".")); 291 | 292 | nix::Value vSelect; 293 | state->eval(selectExpr, vSelect); 294 | 295 | nix::Value *vSelected = state->allocValue(); 296 | state->callFunction(vSelect, *vEvaluated, *vSelected, nix::noPos); 297 | state->forceAttrs( 298 | *vSelected, nix::noPos, 299 | "'--select' must evaluate to an attrset (the traversal root)"); 300 | 301 | return vSelected; 302 | } 303 | 304 | auto shouldRestart(const MyArgs &args) -> bool { 305 | struct rusage resourceUsage = {}; // NOLINT(misc-include-cleaner) 306 | getrusage(RUSAGE_SELF, &resourceUsage); 307 | const size_t maxrss = 308 | resourceUsage 309 | .ru_maxrss; // NOLINT(cppcoreguidelines-pro-type-union-access) 310 | static constexpr size_t KB_TO_BYTES = 1024; 311 | return maxrss > args.maxMemorySize * KB_TO_BYTES; 312 | } 313 | 314 | auto processJobRequest(nix::EvalState &state, LineReader &fromReader, 315 | nix::AutoCloseFD &toParent, nix::Bindings &autoArgs, 316 | nix::Value *vRoot, MyArgs &args) -> bool { 317 | /* Wait for the collector to send us a job name. */ 318 | if (tryWriteLine(toParent.get(), "next") < 0) { 319 | return false; // main process died 320 | } 321 | 322 | auto line = fromReader.readLine(); 323 | if (line == "exit") { 324 | return false; 325 | } 326 | 327 | if (!nix::hasPrefix(line, "do ")) { 328 | std::cerr << "worker error: received invalid command '" << line 329 | << "'\n"; 330 | abort(); 331 | } 332 | 333 | auto path = nlohmann::json::parse(line.substr(3)); 334 | auto attrPathS = attrPathJoin(path); 335 | 336 | /* Evaluate it and send info back to the collector. */ 337 | nlohmann::json reply = 338 | nlohmann::json{{"attr", attrPathS}, {"attrPath", path}}; 339 | 340 | try { 341 | auto *vTmp = 342 | nix::findAlongAttrPath(state, attrPathS, autoArgs, *vRoot).first; 343 | 344 | auto *value = state.allocValue(); 345 | state.autoCallFunction(autoArgs, *vTmp, *value); 346 | 347 | if (value->type() == nix::nAttrs) { 348 | processDerivation(state, value, attrPathS, path, args, reply); 349 | } else { 350 | // We ignore everything that cannot be built 351 | reply["attrs"] = nlohmann::json::array(); 352 | } 353 | } catch (nix::EvalError &e) { 354 | const auto &err = e.info(); 355 | std::ostringstream oss; 356 | nix::showErrorInfo(oss, err, nix::loggerSettings.showTrace.get()); 357 | auto msg = oss.str(); 358 | 359 | // Transmit the error in JSON output 360 | reply["error"] = nix::filterANSIEscapes(msg, true); 361 | // Print to STDERR for Hydra UI 362 | std::cerr << msg << "\n"; 363 | } catch (const std::exception &e) { 364 | // FIXME: for some reason the catch block above doesn't trigger on macOS 365 | // (?) 366 | const auto *msg = e.what(); 367 | reply["error"] = nix::filterANSIEscapes(msg, true); 368 | std::cerr << msg << '\n'; 369 | } 370 | 371 | if (tryWriteLine(toParent.get(), reply.dump()) < 0) { 372 | return false; // main process died 373 | } 374 | 375 | /* Check if we should restart due to memory usage */ 376 | return !shouldRestart(args); 377 | } 378 | 379 | } // namespace 380 | 381 | void worker( 382 | MyArgs &args, 383 | nix::AutoCloseFD &toParent, // NOLINT(bugprone-easily-swappable-parameters) 384 | nix::AutoCloseFD &fromParent) { 385 | 386 | auto evalStore = nix_eval_jobs::openStore(args.evalStoreUrl); 387 | auto state = nix::make_ref( 388 | args.lookupPath, evalStore, nix::fetchSettings, nix::evalSettings); 389 | nix::Bindings &autoArgs = *args.getAutoArgs(*state); 390 | 391 | nix::Value *vRoot = initializeRootValue(state, autoArgs, args); 392 | 393 | LineReader fromReader(fromParent.release()); 394 | 395 | while (processJobRequest(*state, fromReader, toParent, autoArgs, vRoot, 396 | args)) { 397 | // Continue processing jobs until we need to exit 398 | } 399 | 400 | if (tryWriteLine(toParent.get(), "restart") < 0) { 401 | return; // main process died 402 | }; 403 | } 404 | -------------------------------------------------------------------------------- /tests/test_eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import os 5 | import subprocess 6 | from pathlib import Path 7 | from tempfile import TemporaryDirectory 8 | from typing import Any 9 | 10 | TEST_ROOT = Path(__file__).parent.resolve() 11 | PROJECT_ROOT = TEST_ROOT.parent 12 | # Allow overriding the binary path with environment variable 13 | BIN = Path( 14 | os.environ.get("NIX_EVAL_JOBS_BIN", str(PROJECT_ROOT.joinpath("build", "src", "nix-eval-jobs"))) 15 | ) 16 | # Common flags for all test invocations 17 | COMMON_FLAGS = ["--extra-experimental-features", "nix-command flakes"] 18 | 19 | 20 | def check_gc_root(gcRootDir: str, drvPath: str) -> None: 21 | """ 22 | Make sure the expected GC root exists in the given dir 23 | """ 24 | link_name = os.path.basename(drvPath) 25 | symlink_path = os.path.join(gcRootDir, link_name) 26 | assert os.path.islink(symlink_path) and drvPath == os.readlink(symlink_path) 27 | 28 | 29 | def common_test(extra_args: list[str]) -> list[dict[str, Any]]: 30 | with TemporaryDirectory() as tempdir: 31 | cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta", *COMMON_FLAGS, *extra_args] 32 | res = subprocess.run( 33 | cmd, 34 | cwd=TEST_ROOT.joinpath("assets"), 35 | text=True, 36 | check=True, 37 | stdout=subprocess.PIPE, 38 | ) 39 | 40 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 41 | assert len(results) == 4 42 | 43 | built_job = results[0] 44 | assert built_job["attr"] == "builtJob" 45 | assert built_job["name"] == "job1" 46 | assert built_job["outputs"]["out"].endswith("-job1") 47 | assert built_job["drvPath"].endswith(".drv") 48 | # No meta field in bare derivations 49 | 50 | dotted_job = results[1] 51 | assert dotted_job["attr"] == '"dotted.attr"' 52 | assert dotted_job["attrPath"] == ["dotted.attr"] 53 | 54 | package_with_deps = results[2] 55 | assert package_with_deps["attr"] == "package-with-deps" 56 | assert package_with_deps["name"] == "package-with-deps" 57 | 58 | recurse_drv = results[3] 59 | assert recurse_drv["attr"] == "recurse.drvB" 60 | assert recurse_drv["name"] == "drvB" 61 | 62 | assert len(list(Path(tempdir).iterdir())) == 4 63 | return results 64 | 65 | 66 | def test_flake() -> None: 67 | results = common_test(["--flake", ".#hydraJobs"]) 68 | for result in results: 69 | assert "isCached" not in result # legacy 70 | assert "cacheStatus" not in result 71 | assert "neededBuilds" not in result 72 | assert "neededSubstitutes" not in result 73 | assert "requiredSystemFeatures" in result 74 | 75 | 76 | def test_query_cache_status() -> None: 77 | results = common_test(["--flake", ".#hydraJobs", "--check-cache-status"]) 78 | # FIXME in the nix sandbox we cannot query binary caches 79 | # this would need some local one 80 | for result in results: 81 | assert "isCached" in result # legacy 82 | assert "cacheStatus" in result 83 | assert "neededBuilds" in result 84 | assert "neededSubstitutes" in result 85 | 86 | 87 | def test_expression() -> None: 88 | results = common_test(["ci.nix"]) 89 | for result in results: 90 | assert "isCached" not in result # legacy 91 | assert "cacheStatus" not in result 92 | assert "requiredSystemFeatures" in result 93 | if result["attr"] == "builtJob": 94 | assert isinstance(result["requiredSystemFeatures"], list) 95 | assert "big-parallel" in result["requiredSystemFeatures"] 96 | 97 | with open(TEST_ROOT.joinpath("assets/ci.nix")) as ci_nix: 98 | common_test(["-E", ci_nix.read()]) 99 | 100 | 101 | def test_input_drvs() -> None: 102 | results = common_test(["ci.nix", "--show-input-drvs"]) 103 | for result in results: 104 | assert "inputDrvs" in result 105 | 106 | 107 | def test_eval_error() -> None: 108 | with TemporaryDirectory() as tempdir: 109 | cmd = [ 110 | str(BIN), 111 | "--gc-roots-dir", 112 | tempdir, 113 | "--meta", 114 | "--workers", 115 | "1", 116 | *COMMON_FLAGS, 117 | "--flake", 118 | ".#legacyPackages.x86_64-linux.brokenPkgs", 119 | ] 120 | res = subprocess.run( 121 | cmd, 122 | cwd=TEST_ROOT.joinpath("assets"), 123 | text=True, 124 | stdout=subprocess.PIPE, 125 | ) 126 | print(res.stdout) 127 | attrs = json.loads(res.stdout) 128 | assert attrs["attr"] == "brokenPackage" 129 | assert "this is an evaluation error" in attrs["error"] 130 | 131 | 132 | def test_no_gcroot_dir() -> None: 133 | cmd = [ 134 | str(BIN), 135 | "--meta", 136 | "--workers", 137 | "1", 138 | *COMMON_FLAGS, 139 | "--flake", 140 | ".#legacyPackages.x86_64-linux.brokenPkgs", 141 | ] 142 | res = subprocess.run( 143 | cmd, 144 | cwd=TEST_ROOT.joinpath("assets"), 145 | text=True, 146 | stdout=subprocess.PIPE, 147 | ) 148 | print(res.stdout) 149 | attrs = json.loads(res.stdout) 150 | assert attrs["attr"] == "brokenPackage" 151 | assert "this is an evaluation error" in attrs["error"] 152 | 153 | 154 | def test_constituents() -> None: 155 | with TemporaryDirectory() as tempdir: 156 | cmd = [ 157 | str(BIN), 158 | "--gc-roots-dir", 159 | tempdir, 160 | "--meta", 161 | "--workers", 162 | "1", 163 | *COMMON_FLAGS, 164 | "--flake", 165 | ".#legacyPackages.x86_64-linux.success", 166 | "--constituents", 167 | ] 168 | res = subprocess.run( 169 | cmd, 170 | cwd=TEST_ROOT.joinpath("assets"), 171 | text=True, 172 | stdout=subprocess.PIPE, 173 | ) 174 | print(res.stdout) 175 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 176 | assert len(results) == 4 177 | child = results[0] 178 | assert child["attr"] == "anotherone" 179 | direct = results[1] 180 | assert direct["attr"] == "direct_aggregate" 181 | indirect = results[2] 182 | assert indirect["attr"] == "indirect_aggregate" 183 | mixed = results[3] 184 | assert mixed["attr"] == "mixed_aggregate" 185 | 186 | def absent_or_empty(f: str, d: dict) -> bool: 187 | return f not in d or len(d[f]) == 0 188 | 189 | assert absent_or_empty("namedConstituents", direct) 190 | assert absent_or_empty("namedConstituents", indirect) 191 | assert absent_or_empty("namedConstituents", mixed) 192 | 193 | assert direct["constituents"][0].endswith("-job1.drv") 194 | 195 | assert indirect["constituents"][0] == child["drvPath"] 196 | 197 | assert mixed["constituents"][0].endswith("-job1.drv") 198 | assert mixed["constituents"][1] == child["drvPath"] 199 | 200 | assert "error" not in direct 201 | assert "error" not in indirect 202 | assert "error" not in mixed 203 | 204 | check_gc_root(tempdir, direct["drvPath"]) 205 | check_gc_root(tempdir, indirect["drvPath"]) 206 | check_gc_root(tempdir, mixed["drvPath"]) 207 | 208 | 209 | def test_constituents_all() -> None: 210 | with TemporaryDirectory() as tempdir: 211 | cmd = [ 212 | str(BIN), 213 | "--gc-roots-dir", 214 | tempdir, 215 | "--meta", 216 | "--workers", 217 | "1", 218 | *COMMON_FLAGS, 219 | "--flake", 220 | ".#legacyPackages.x86_64-linux.glob1", 221 | "--constituents", 222 | ] 223 | res = subprocess.run( 224 | cmd, 225 | cwd=TEST_ROOT.joinpath("assets"), 226 | text=True, 227 | stdout=subprocess.PIPE, 228 | ) 229 | print(res.stdout) 230 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 231 | assert len(results) == 3 232 | assert [x["name"] for x in results] == [ 233 | "constituentA", 234 | "constituentB", 235 | "aggregate", 236 | ] 237 | aggregate = results[2] 238 | assert len(aggregate["constituents"]) == 2 239 | assert aggregate["constituents"][0].endswith("constituentA.drv") 240 | assert aggregate["constituents"][1].endswith("constituentB.drv") 241 | 242 | 243 | def test_constituents_glob_misc() -> None: 244 | with TemporaryDirectory() as tempdir: 245 | cmd = [ 246 | str(BIN), 247 | "--gc-roots-dir", 248 | tempdir, 249 | "--meta", 250 | "--workers", 251 | "1", 252 | *COMMON_FLAGS, 253 | "--flake", 254 | ".#legacyPackages.x86_64-linux.glob2", 255 | "--constituents", 256 | ] 257 | res = subprocess.run( 258 | cmd, 259 | cwd=TEST_ROOT.joinpath("assets"), 260 | text=True, 261 | stdout=subprocess.PIPE, 262 | ) 263 | print(res.stdout) 264 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 265 | assert len(results) == 6 266 | assert [x["name"] for x in results] == [ 267 | "constituentA", 268 | "constituentB", 269 | "aggregate0", 270 | "aggregate1", 271 | "indirect_aggregate0", 272 | "mix_aggregate0", 273 | ] 274 | aggregate = results[2] 275 | assert len(aggregate["constituents"]) == 2 276 | assert aggregate["constituents"][0].endswith("constituentA.drv") 277 | assert aggregate["constituents"][1].endswith("constituentB.drv") 278 | aggregate = results[4] 279 | assert len(aggregate["constituents"]) == 1 280 | assert aggregate["constituents"][0].endswith("aggregate0.drv") 281 | failed = results[3] 282 | assert "constituents" in failed 283 | assert failed["error"] == "tests.*: constituent glob pattern had no matches\n" 284 | 285 | assert results[4]["constituents"][0] == results[2]["drvPath"] 286 | assert results[5]["constituents"][0] == results[0]["drvPath"] 287 | assert results[5]["constituents"][1] == results[2]["drvPath"] 288 | 289 | 290 | def test_constituents_cycle() -> None: 291 | with TemporaryDirectory() as tempdir: 292 | cmd = [ 293 | str(BIN), 294 | "--gc-roots-dir", 295 | tempdir, 296 | "--meta", 297 | "--workers", 298 | "1", 299 | *COMMON_FLAGS, 300 | "--flake", 301 | ".#legacyPackages.x86_64-linux.cycle", 302 | "--constituents", 303 | ] 304 | res = subprocess.run( 305 | cmd, 306 | cwd=TEST_ROOT.joinpath("assets"), 307 | text=True, 308 | stdout=subprocess.PIPE, 309 | ) 310 | print(res.stdout) 311 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 312 | assert len(results) == 2 313 | assert [x["name"] for x in results] == ["aggregate0", "aggregate1"] 314 | for i in results: 315 | assert i["error"] == "Dependency cycle: aggregate0 <-> aggregate1" 316 | 317 | 318 | def test_constituents_error() -> None: 319 | with TemporaryDirectory() as tempdir: 320 | cmd = [ 321 | str(BIN), 322 | "--gc-roots-dir", 323 | tempdir, 324 | "--meta", 325 | "--workers", 326 | "1", 327 | *COMMON_FLAGS, 328 | "--flake", 329 | ".#legacyPackages.x86_64-linux.failures", 330 | "--constituents", 331 | ] 332 | res = subprocess.run( 333 | cmd, 334 | cwd=TEST_ROOT.joinpath("assets"), 335 | text=True, 336 | stdout=subprocess.PIPE, 337 | ) 338 | print(res.stdout) 339 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 340 | assert len(results) == 2 341 | child = results[0] 342 | assert child["attr"] == "doesnteval" 343 | assert "error" in child 344 | aggregate = results[1] 345 | assert aggregate["attr"] == "aggregate" 346 | assert "namedConstituents" not in aggregate 347 | assert "doesntexist: does not exist\n" in aggregate["error"] 348 | assert "constituents" in aggregate 349 | 350 | 351 | def test_empty_needed() -> None: 352 | """Test for issue #369 where neededBuilds and neededSubstitutes are empty when they shouldn't be""" 353 | with TemporaryDirectory() as tempdir: 354 | cmd = [ 355 | str(BIN), 356 | "--gc-roots-dir", 357 | tempdir, 358 | "--meta", 359 | "--check-cache-status", 360 | "--workers", 361 | "1", 362 | *COMMON_FLAGS, 363 | "--flake", 364 | ".#legacyPackages.x86_64-linux.emptyNeeded", 365 | ] 366 | res = subprocess.run( 367 | cmd, 368 | cwd=TEST_ROOT.joinpath("assets"), 369 | text=True, 370 | check=True, 371 | stdout=subprocess.PIPE, 372 | ) 373 | print(res.stdout) 374 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 375 | 376 | # Should be 3 results - nginx, proxyWrapper, and webService 377 | assert len(results) == 3 378 | 379 | # Find the results for each attr 380 | web_service_result = next(r for r in results if r["attr"] == "webService") 381 | proxy_wrapper_result = next(r for r in results if r["attr"] == "proxyWrapper") 382 | nginx_result = next(r for r in results if r["attr"] == "nginx") 383 | 384 | # webService should have proxyWrapper.drv in its neededBuilds 385 | assert len(web_service_result["neededBuilds"]) > 0 386 | assert any( 387 | proxy_wrapper_result["drvPath"] in drv for drv in web_service_result["neededBuilds"] 388 | ) 389 | 390 | # proxyWrapper should have nginx in its neededBuilds (since nginx is a derivation dependency) 391 | assert len(proxy_wrapper_result["neededBuilds"]) > 0 392 | assert any(nginx_result["drvPath"] in drv for drv in proxy_wrapper_result["neededBuilds"]) 393 | 394 | 395 | def test_apply() -> None: 396 | with TemporaryDirectory() as tempdir: 397 | applyExpr = """drv: { 398 | the-name = drv.name; 399 | version = drv.version or null; 400 | }""" 401 | 402 | cmd = [ 403 | str(BIN), 404 | "--gc-roots-dir", 405 | tempdir, 406 | "--workers", 407 | "1", 408 | "--apply", 409 | applyExpr, 410 | *COMMON_FLAGS, 411 | "--flake", 412 | ".#hydraJobs", 413 | ] 414 | res = subprocess.run( 415 | cmd, 416 | cwd=TEST_ROOT.joinpath("assets"), 417 | text=True, 418 | check=True, 419 | stdout=subprocess.PIPE, 420 | ) 421 | 422 | print(res.stdout) 423 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 424 | 425 | assert len(results) == 4 # sanity check that we assert against all jobs 426 | 427 | # Check that nix-eval-jobs applied the expression correctly 428 | # and extracted 'version' as 'version' and 'name' as 'the-name' 429 | assert results[0]["extraValue"]["the-name"] == "job1" 430 | assert results[0]["extraValue"]["version"] is None 431 | assert results[1]["extraValue"]["the-name"] == "dotted" 432 | assert results[1]["extraValue"]["version"] is None 433 | assert results[2]["extraValue"]["the-name"] == "package-with-deps" 434 | assert results[2]["extraValue"]["version"] is None 435 | assert results[3]["extraValue"]["the-name"] == "drvB" 436 | assert results[3]["extraValue"]["version"] is None 437 | 438 | 439 | def test_select_flake() -> None: 440 | """Test the --select option to filter flake outputs before evaluation""" 441 | with TemporaryDirectory() as tempdir: 442 | # Test 1: Select specific attributes from hydraJobs 443 | cmd = [ 444 | str(BIN), 445 | "--gc-roots-dir", 446 | tempdir, 447 | "--meta", 448 | *COMMON_FLAGS, 449 | "--flake", 450 | ".#hydraJobs", 451 | "--select", 452 | "outputs: { inherit (outputs) builtJob recurse; }", 453 | ] 454 | res = subprocess.run( 455 | cmd, 456 | cwd=TEST_ROOT.joinpath("assets"), 457 | text=True, 458 | check=True, 459 | stdout=subprocess.PIPE, 460 | ) 461 | 462 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 463 | # Should only have the two selected jobs 464 | assert len(results) == 2 465 | attrs = {r["attr"] for r in results} 466 | assert attrs == {"builtJob", "recurse.drvB"} 467 | 468 | # Test 2: Select from the whole flake (outputs and inputs) 469 | # When using --flake . we get a structure with 'outputs' and 'inputs' 470 | cmd = [ 471 | str(BIN), 472 | "--gc-roots-dir", 473 | tempdir, 474 | "--meta", 475 | "--workers", 476 | "1", 477 | *COMMON_FLAGS, 478 | "--flake", 479 | ".", 480 | "--select", 481 | "flake: flake.outputs.hydraJobs", # Select just hydraJobs from outputs 482 | ] 483 | res = subprocess.run( 484 | cmd, 485 | cwd=TEST_ROOT.joinpath("assets"), 486 | text=True, 487 | check=True, 488 | stdout=subprocess.PIPE, 489 | ) 490 | 491 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 492 | # Should get the 4 hydraJobs 493 | assert len(results) == 4 494 | attrs = {r["attr"] for r in results} 495 | assert "builtJob" in attrs 496 | assert '"dotted.attr"' in attrs 497 | 498 | 499 | def test_recursion_error() -> None: 500 | with TemporaryDirectory() as tempdir: 501 | cmd = [ 502 | str(BIN), 503 | "--gc-roots-dir", 504 | tempdir, 505 | "--meta", 506 | "--workers", 507 | "1", 508 | *COMMON_FLAGS, 509 | "--flake", 510 | ".#legacyPackages.x86_64-linux.infiniteRecursionPkgs", 511 | ] 512 | res = subprocess.run( 513 | cmd, 514 | cwd=TEST_ROOT.joinpath("assets"), 515 | text=True, 516 | stderr=subprocess.PIPE, 517 | ) 518 | assert res.returncode == 1 519 | print(res.stderr) 520 | assert "packageWithInfiniteRecursion" in res.stderr 521 | expected_errors = [ 522 | "error: BUG: while reading result for attrPath 'packageWithInfiniteRecursion', worker pipe got closed but evaluation worker still running?", 523 | "possible infinite recursion", 524 | "error: while reading result for attrPath 'packageWithInfiniteRecursion', evaluation worker got killed by signal 6", 525 | ] 526 | assert any(err in res.stderr for err in expected_errors) 527 | 528 | 529 | def test_no_instantiate_mode() -> None: 530 | """Test that --no-instantiate flag works correctly""" 531 | with TemporaryDirectory() as tempdir: 532 | cmd = [ 533 | str(BIN), 534 | "--gc-roots-dir", 535 | tempdir, 536 | "--meta", 537 | "--no-instantiate", 538 | *COMMON_FLAGS, 539 | "--flake", 540 | ".#hydraJobs", 541 | ] 542 | res = subprocess.run( 543 | cmd, 544 | cwd=TEST_ROOT.joinpath("assets"), 545 | text=True, 546 | check=True, 547 | stdout=subprocess.PIPE, 548 | ) 549 | 550 | results = [json.loads(r) for r in res.stdout.split("\n") if r] 551 | assert len(results) == 4 552 | 553 | # Check that all results have the expected structure 554 | for result in results: 555 | # In no-instantiate mode, drvPath should still be present (from the attr) 556 | assert "drvPath" in result 557 | assert result["drvPath"].endswith(".drv") 558 | 559 | # System should still be present (from querySystem fallback) 560 | assert "system" in result 561 | assert result["system"] != "" 562 | 563 | # Name should still be present 564 | assert "name" in result 565 | 566 | # Outputs should still be present and populated 567 | assert "outputs" in result 568 | assert isinstance(result["outputs"], dict) 569 | # Most derivations should have at least an "out" output 570 | if result["attr"] != "recurse.drvB": # This one might be special 571 | assert len(result["outputs"]) > 0 572 | assert "out" in result["outputs"] 573 | assert result["outputs"]["out"] != "" 574 | 575 | # Cache status should not be present (it's Unknown and not included) 576 | assert "cacheStatus" not in result 577 | assert "neededBuilds" not in result 578 | assert "neededSubstitutes" not in result 579 | 580 | # Input drvs should not be present (requires reading derivation from store) 581 | assert "inputDrvs" not in result 582 | 583 | # Required system features should not be present (requires reading derivation from store) 584 | assert "requiredSystemFeatures" not in result 585 | 586 | # Verify specific outputs for known derivations 587 | built_job = next(r for r in results if r["attr"] == "builtJob") 588 | assert built_job["outputs"]["out"].endswith("-job1") 589 | 590 | # No GC roots should be created in no-instantiate mode 591 | assert len(list(Path(tempdir).iterdir())) == 0 592 | -------------------------------------------------------------------------------- /src/nix-eval-jobs.cc: -------------------------------------------------------------------------------- 1 | // NOLINTBEGIN(modernize-deprecated-headers) 2 | // misc-include-cleaner wants these header rather than the C++ versions 3 | #include 4 | #include 5 | #include 6 | // NOLINTEND(modernize-deprecated-headers) 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include // NOLINT(misc-header-include-cycle) 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include // NOLINT(misc-header-include-cycle) 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | 54 | #include "eval-args.hh" 55 | #include "buffered-io.hh" 56 | #include "worker.hh" 57 | #include "strings-portable.hh" 58 | #include "output-stream-lock.hh" 59 | #include "constituents.hh" 60 | #include "store.hh" 61 | 62 | namespace { 63 | MyArgs myArgs; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) 64 | 65 | using Processor = std::function; 67 | 68 | void handleConstituents(std::map &jobs, 69 | const MyArgs &args) { 70 | 71 | auto store = nix_eval_jobs::openStore(args.evalStoreUrl); 72 | auto localStore = store.dynamic_pointer_cast(); 73 | 74 | if (!localStore) { 75 | nix::warn("constituents feature requires a local store, skipping " 76 | "aggregate rewriting"); 77 | return; 78 | } 79 | 80 | auto localStoreRef = nix::ref(localStore); 81 | 82 | std::visit( 83 | nix::overloaded{ 84 | [&](const std::vector &namedConstituents) -> void { 85 | rewriteAggregates(jobs, namedConstituents, localStoreRef, 86 | args.gcRootsDir); 87 | }, 88 | [&](const DependencyCycle &cycle) -> void { 89 | nix::logger->log(nix::lvlError, 90 | nix::fmt("Found dependency cycle " 91 | "between jobs '%s' and '%s'", 92 | cycle.a, cycle.b)); 93 | jobs[cycle.a]["error"] = cycle.message(); 94 | jobs[cycle.b]["error"] = cycle.message(); 95 | 96 | getCoutLock().lock() << jobs[cycle.a].dump() << "\n" 97 | << jobs[cycle.b].dump() << "\n"; 98 | 99 | for (const auto &jobName : cycle.remainingAggregates) { 100 | jobs[jobName]["error"] = 101 | "Skipping aggregate because of a dependency " 102 | "cycle"; 103 | getCoutLock().lock() << jobs[jobName].dump() << "\n"; 104 | } 105 | }, 106 | }, 107 | resolveNamedConstituents(jobs)); 108 | } 109 | 110 | /* Auto-cleanup of fork's process and fds. */ 111 | struct Proc { 112 | nix::AutoCloseFD to, from; 113 | nix::Pid pid; 114 | 115 | Proc(const Proc &) = delete; 116 | Proc(Proc &&) = delete; 117 | auto operator=(const Proc &) -> Proc & = delete; 118 | auto operator=(Proc &&) -> Proc & = delete; 119 | 120 | explicit Proc(const Processor &proc) { 121 | nix::Pipe toPipe; 122 | nix::Pipe fromPipe; 123 | toPipe.create(); 124 | fromPipe.create(); 125 | auto childPid = startProcess( 126 | [&, 127 | toFd{std::make_shared( 128 | std::move(fromPipe.writeSide))}, 129 | fromFd{std::make_shared( 130 | std::move(toPipe.readSide))}]() -> void { 131 | nix::logger->log( 132 | nix::lvlDebug, 133 | nix::fmt("created worker process %d", getpid())); 134 | try { 135 | proc(myArgs, *toFd, *fromFd); 136 | } catch (nix::Error &e) { 137 | nlohmann::json err; 138 | const auto &msg = e.msg(); 139 | err["error"] = nix::filterANSIEscapes(msg, true); 140 | nix::logger->log(nix::lvlError, msg); 141 | if (tryWriteLine(toFd->get(), err.dump()) < 0) { 142 | return; // main process died 143 | }; 144 | // Don't forget to print it into the STDERR log, this is 145 | // what's shown in the Hydra UI. 146 | if (tryWriteLine(toFd->get(), "restart") < 0) { 147 | return; // main process died 148 | } 149 | } 150 | }, 151 | nix::ProcessOptions{.allowVfork = false}); 152 | 153 | to = std::move(toPipe.writeSide); 154 | from = std::move(fromPipe.readSide); 155 | pid = childPid; 156 | } 157 | 158 | ~Proc() = default; 159 | }; 160 | 161 | // We'd highly prefer using std::thread here; but this won't let us configure 162 | // the stack size. macOS uses 512KiB size stacks for non-main threads, and musl 163 | // defaults to 128k. While Nix configures a 64MiB size for the main thread, this 164 | // doesn't propagate to the threads we launch here. It turns out, running the 165 | // evaluator under an anemic stack of 0.5MiB has it overflow way too quickly. 166 | // Hence, we have our own custom Thread struct. 167 | // NOLINTBEGIN(misc-include-cleaner) 168 | // False positive: pthread.h is included but clang-tidy doesn't recognize it 169 | struct Thread { 170 | pthread_t thread = {}; 171 | 172 | Thread(const Thread &) = delete; 173 | Thread(Thread &&) noexcept = default; 174 | ~Thread() = default; 175 | auto operator=(const Thread &) -> Thread & = delete; 176 | auto operator=(Thread &&) -> Thread & = delete; 177 | 178 | explicit Thread(std::function func) { 179 | pthread_attr_t attr = {}; 180 | 181 | auto funcPtr = 182 | std::make_unique>(std::move(func)); 183 | 184 | int status = pthread_attr_init(&attr); 185 | if (status != 0) { 186 | throw nix::SysError(status, "calling pthread_attr_init"); 187 | } 188 | 189 | struct AttrGuard { 190 | pthread_attr_t &attr; 191 | explicit AttrGuard(pthread_attr_t &attribute) : attr(attribute) {} 192 | AttrGuard(const AttrGuard &) = delete; 193 | auto operator=(const AttrGuard &) -> AttrGuard & = delete; 194 | AttrGuard(AttrGuard &&) = delete; 195 | auto operator=(AttrGuard &&) -> AttrGuard & = delete; 196 | ~AttrGuard() { (void)pthread_attr_destroy(&attr); } 197 | }; 198 | const AttrGuard attrGuard(attr); 199 | 200 | static constexpr size_t STACK_SIZE_MB = 64; 201 | static constexpr size_t KB_SIZE = 1024; 202 | status = pthread_attr_setstacksize( 203 | &attr, static_cast(STACK_SIZE_MB) * KB_SIZE * KB_SIZE); 204 | if (status != 0) { 205 | throw nix::SysError(status, "calling pthread_attr_setstacksize"); 206 | } 207 | status = pthread_create(&thread, &attr, Thread::init, funcPtr.get()); 208 | if (status != 0) { 209 | throw nix::SysError(status, "calling pthread_launch"); 210 | } 211 | [[maybe_unused]] auto *res = 212 | funcPtr.release(); // will be deleted in init() 213 | } 214 | 215 | void join() const { 216 | const int status = pthread_join(thread, nullptr); 217 | if (status != 0) { 218 | throw nix::SysError(status, "calling pthread_join"); 219 | } 220 | } 221 | 222 | private: 223 | static auto init(void *ptr) -> void * { 224 | std::unique_ptr> func; 225 | func.reset(static_cast *>(ptr)); 226 | 227 | (*func)(); 228 | return nullptr; 229 | } 230 | }; 231 | // NOLINTEND(misc-include-cleaner) 232 | 233 | struct State { 234 | std::set todo = 235 | nlohmann::json::array({nlohmann::json::array()}); 236 | std::set active; 237 | std::map jobs; 238 | std::exception_ptr exc; 239 | }; 240 | 241 | void handleBrokenWorkerPipe(Proc &proc, std::string_view msg) { 242 | // we already took the process status from Proc, no 243 | // need to wait for it again to avoid error messages 244 | // NOLINTNEXTLINE(misc-include-cleaner) 245 | const pid_t pid = proc.pid.release(); 246 | while (true) { 247 | int status = 0; 248 | const int result = waitpid(pid, &status, WNOHANG); 249 | if (result == 0) { 250 | kill(pid, SIGKILL); 251 | throw nix::Error( 252 | "BUG: while %s, worker pipe got closed but evaluation " 253 | "worker still running?", 254 | msg); 255 | } 256 | 257 | if (result == -1) { 258 | kill(pid, SIGKILL); 259 | throw nix::Error( 260 | "BUG: while %s, waitpid for evaluation worker failed: %s", msg, 261 | get_error_name(errno)); 262 | } 263 | if (WIFEXITED(status)) { 264 | if (WEXITSTATUS(status) == 1) { 265 | throw nix::Error( 266 | "while %s, evaluation worker exited with exit code 1, " 267 | "(possible infinite recursion)", 268 | msg); 269 | } 270 | throw nix::Error("while %s, evaluation worker exited with %d", msg, 271 | WEXITSTATUS(status)); 272 | } 273 | 274 | if (WIFSIGNALED(status)) { 275 | switch (WTERMSIG(status)) { 276 | case SIGKILL: 277 | throw nix::Error( 278 | "while %s, evaluation worker got killed by SIGKILL, " 279 | "maybe " 280 | "memory limit reached?", 281 | msg); 282 | break; 283 | #ifdef __APPLE__ 284 | case SIGBUS: 285 | throw nix::Error( 286 | "while %s, evaluation worker got killed by SIGBUS, " 287 | "(possible infinite recursion)", 288 | msg); 289 | break; 290 | #else 291 | case SIGSEGV: 292 | throw nix::Error( 293 | "while %s, evaluation worker got killed by SIGSEGV, " 294 | "(possible infinite recursion)", 295 | msg); 296 | #endif 297 | default: 298 | throw nix::Error("while %s, evaluation worker got killed by " 299 | "signal %d (%s)", 300 | msg, WTERMSIG(status), 301 | get_signal_name(WTERMSIG(status))); 302 | } 303 | } // else ignore WIFSTOPPED and WIFCONTINUED 304 | } 305 | } 306 | 307 | auto joinAttrPath(const nlohmann::json &attrPath) -> std::string { 308 | std::string joined; 309 | for (const auto &element : attrPath) { 310 | if (!joined.empty()) { 311 | joined += '.'; 312 | } 313 | joined += element.get(); 314 | } 315 | return joined; 316 | } 317 | 318 | namespace { 319 | auto checkWorkerStatus(LineReader *fromReader, Proc *proc) -> std::string_view { 320 | auto line = fromReader->readLine(); 321 | if (line.empty()) { 322 | handleBrokenWorkerPipe(*proc, "checking worker process"); 323 | } 324 | if (line != "next" && line != "restart") { 325 | try { 326 | auto json = nlohmann::json::parse(line); 327 | throw nix::Error("worker error: %s", std::string(json["error"])); 328 | } catch (const nlohmann::json::exception &e) { 329 | throw nix::Error( 330 | "Received invalid JSON from worker: %s\n json: '%s'", e.what(), 331 | line); 332 | } 333 | } 334 | return line; 335 | } 336 | 337 | auto getNextJob(nix::Sync &state_, std::condition_variable &wakeup, 338 | Proc *proc) -> std::optional { 339 | nlohmann::json attrPath; 340 | while (true) { 341 | nix::checkInterrupt(); 342 | auto state(state_.lock()); 343 | if ((state->todo.empty() && state->active.empty()) || state->exc) { 344 | if (tryWriteLine(proc->to.get(), "exit") < 0) { 345 | handleBrokenWorkerPipe(*proc, "sending exit"); 346 | } 347 | return std::nullopt; 348 | } 349 | if (!state->todo.empty()) { 350 | attrPath = *state->todo.begin(); 351 | state->todo.erase(state->todo.begin()); 352 | state->active.insert(attrPath); 353 | return attrPath; 354 | } 355 | state.wait(wakeup); 356 | } 357 | } 358 | 359 | auto processWorkerResponse(LineReader *fromReader, 360 | const nlohmann::json &attrPath, Proc *proc, 361 | nix::Sync &state_) 362 | -> std::vector { 363 | // Read response from worker 364 | auto respString = fromReader->readLine(); 365 | if (respString.empty()) { 366 | auto msg = 367 | "reading result for attrPath '" + joinAttrPath(attrPath) + "'"; 368 | handleBrokenWorkerPipe(*proc, msg); 369 | } 370 | 371 | // Parse JSON response 372 | nlohmann::json response; 373 | try { 374 | response = nlohmann::json::parse(respString); 375 | } catch (const nlohmann::json::exception &e) { 376 | throw nix::Error("Received invalid JSON from worker: %s\n json: '%s'", 377 | e.what(), respString); 378 | } 379 | 380 | // Process the response 381 | std::vector newAttrs; 382 | if (response.find("attrs") != response.end()) { 383 | for (const auto &attr : response["attrs"]) { 384 | nlohmann::json newAttr = nlohmann::json(response["attrPath"]); 385 | newAttr.emplace_back(attr); 386 | newAttrs.push_back(newAttr); 387 | } 388 | } else { 389 | { 390 | auto state(state_.lock()); 391 | state->jobs.insert_or_assign(response["attr"], response); 392 | } 393 | auto named = response.find("namedConstituents"); 394 | if (named == response.end() || named->empty()) { 395 | getCoutLock().lock() << respString << "\n"; 396 | } 397 | } 398 | 399 | return newAttrs; 400 | } 401 | 402 | void updateJobQueue(nix::Sync &state_, std::condition_variable &wakeup, 403 | const nlohmann::json &attrPath, 404 | const std::vector &newAttrs) { 405 | auto state(state_.lock()); 406 | state->active.erase(attrPath); 407 | for (const auto &newAttr : newAttrs) { 408 | state->todo.insert(newAttr); 409 | } 410 | wakeup.notify_all(); 411 | } 412 | } // namespace 413 | 414 | void collector(nix::Sync &state_, std::condition_variable &wakeup) { 415 | try { 416 | std::optional> proc_; 417 | std::optional> fromReader_; 418 | 419 | while (true) { 420 | // Initialize worker if needed 421 | if (!proc_.has_value()) { 422 | proc_ = std::make_unique(worker); 423 | } 424 | if (!fromReader_.has_value()) { 425 | fromReader_ = 426 | std::make_unique(proc_.value()->from.release()); 427 | } 428 | 429 | auto line = checkWorkerStatus(fromReader_.value().get(), 430 | proc_.value().get()); 431 | if (line == "restart") { 432 | // Reset worker 433 | proc_ = std::nullopt; 434 | fromReader_ = std::nullopt; 435 | continue; 436 | } 437 | 438 | auto maybeAttrPath = 439 | getNextJob(state_, wakeup, proc_.value().get()); 440 | if (!maybeAttrPath.has_value()) { 441 | return; 442 | } 443 | const auto &attrPath = maybeAttrPath.value(); 444 | 445 | if (tryWriteLine(proc_.value()->to.get(), "do " + attrPath.dump()) < 446 | 0) { 447 | auto msg = "sending attrPath '" + joinAttrPath(attrPath) + "'"; 448 | handleBrokenWorkerPipe(*proc_.value(), msg); 449 | } 450 | 451 | auto newAttrs = 452 | processWorkerResponse(fromReader_.value().get(), attrPath, 453 | proc_.value().get(), state_); 454 | 455 | updateJobQueue(state_, wakeup, attrPath, newAttrs); 456 | } 457 | } catch (...) { 458 | auto state(state_.lock()); 459 | state->exc = std::current_exception(); 460 | wakeup.notify_all(); 461 | } 462 | } 463 | 464 | void validateIncompatibleFlags(const MyArgs &args) { 465 | if (!args.noInstantiate) { 466 | return; 467 | } 468 | 469 | const std::vector> flagChecks = { 470 | {args.showInputDrvs, "--show-input-drvs"}, 471 | {args.checkCacheStatus, "--check-cache-status"}, 472 | {args.constituents, "--constituents"}}; 473 | 474 | std::string incompatibleFlags; 475 | for (const auto &[isSet, flagName] : flagChecks) { 476 | if (isSet) { 477 | incompatibleFlags += 478 | (incompatibleFlags.empty() ? "" : ", ") + std::string(flagName); 479 | } 480 | } 481 | 482 | if (!incompatibleFlags.empty()) { 483 | throw nix::UsageError( 484 | nix::fmt("--no-instantiate is incompatible with: %s. " 485 | "These features require instantiated derivations.", 486 | incompatibleFlags)); 487 | } 488 | } 489 | } // namespace 490 | 491 | auto main(int argc, char **argv) -> int { 492 | /* We are doing the garbage collection by killing forks */ 493 | setenv("GC_DONT_GC", "1", 1); // NOLINT(concurrency-mt-unsafe) 494 | 495 | /* Because of an objc quirk[1], calling curl_global_init for the first time 496 | after fork() will always result in a crash. 497 | Up until now the solution has been to set 498 | OBJC_DISABLE_INITIALIZE_FORK_SAFETY for every nix process to ignore that 499 | error. Instead of working around that error we address it at the core - 500 | by calling curl_global_init here, which should mean curl will already 501 | have been initialized by the time we try to do so in a forked process. 502 | 503 | [1] 504 | https://github.com/apple-oss-distributions/objc4/blob/01edf1705fbc3ff78a423cd21e03dfc21eb4d780/runtime/objc-initialize.mm#L614-L636 505 | */ 506 | curl_global_init(CURL_GLOBAL_ALL); 507 | 508 | auto args = std::span(argv, argc); 509 | 510 | return nix::handleExceptions(args[0], [&]() -> void { 511 | nix::initNix(); 512 | nix::initGC(); 513 | nix::flakeSettings.configureEvalSettings(nix::evalSettings); 514 | 515 | myArgs.parseArgs(argv, argc); 516 | 517 | validateIncompatibleFlags(myArgs); 518 | 519 | /* FIXME: The build hook in conjunction with import-from-derivation is 520 | * causing "unexpected EOF" during eval */ 521 | nix::settings.builders = ""; 522 | 523 | /* Set no-instantiate mode if requested (makes evaluation faster) */ 524 | if (myArgs.noInstantiate) { 525 | nix::settings.readOnlyMode = true; 526 | } 527 | 528 | /* When building a flake, use pure evaluation (no access to 529 | 'getEnv', 'currentSystem' etc. */ 530 | if (myArgs.impure) { 531 | nix::evalSettings.pureEval = false; 532 | } else if (myArgs.flake) { 533 | nix::evalSettings.pureEval = true; 534 | } 535 | 536 | if (myArgs.releaseExpr.empty()) { 537 | throw nix::UsageError("no expression specified"); 538 | } 539 | 540 | if (!myArgs.gcRootsDir.empty()) { 541 | myArgs.gcRootsDir = std::filesystem::absolute(myArgs.gcRootsDir); 542 | } 543 | 544 | if (myArgs.showTrace) { 545 | nix::loggerSettings.showTrace.assign(true); 546 | } 547 | 548 | nix::Sync state_; 549 | 550 | /* Start a collector thread per worker process. */ 551 | std::vector threads; 552 | std::condition_variable wakeup; 553 | threads.reserve(myArgs.nrWorkers); 554 | for (size_t i = 0; i < myArgs.nrWorkers; i++) { 555 | threads.emplace_back( 556 | [&state_, &wakeup] -> void { collector(state_, wakeup); }); 557 | } 558 | 559 | for (auto &thread : threads) { 560 | thread.join(); 561 | } 562 | 563 | auto state(state_.lock()); 564 | 565 | if (state->exc) { 566 | std::rethrow_exception(state->exc); 567 | } 568 | 569 | if (myArgs.constituents) { 570 | handleConstituents(state->jobs, myArgs); 571 | } 572 | }); 573 | } 574 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # GNU General Public License 2 | 3 | _Version 3, 29 June 2007_ _Copyright © 2007 Free Software Foundation, Inc. 4 | <>_ 5 | 6 | Everyone is permitted to copy and distribute verbatim copies of this license 7 | document, but changing it is not allowed. 8 | 9 | ## Preamble 10 | 11 | The GNU General Public License is a free, copyleft license for software and 12 | other kinds of works. 13 | 14 | The licenses for most software and other practical works are designed to take 15 | away your freedom to share and change the works. By contrast, the GNU General 16 | Public License is intended to guarantee your freedom to share and change all 17 | versions of a program--to make sure it remains free software for all its users. 18 | We, the Free Software Foundation, use the GNU General Public License for most of 19 | our software; it applies also to any other work released this way by its 20 | authors. You can apply it to your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not price. Our 23 | General Public Licenses are designed to make sure that you have the freedom to 24 | distribute copies of free software (and charge for them if you wish), that you 25 | receive source code or can get it if you want it, that you can change the 26 | software or use pieces of it in new free programs, and that you know you can do 27 | these things. 28 | 29 | To protect your rights, we need to prevent others from denying you these rights 30 | or asking you to surrender the rights. Therefore, you have certain 31 | responsibilities if you distribute copies of the software, or if you modify it: 32 | responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether gratis or for a 35 | fee, you must pass on to the recipients the same freedoms that you received. You 36 | must make sure that they, too, receive or can get the source code. And you must 37 | show them these terms so they know their rights. 38 | 39 | Developers that use the GNU GPL protect your rights with two steps: **(1)** 40 | assert copyright on the software, and **(2)** offer you this License giving you 41 | legal permission to copy, distribute and/or modify it. 42 | 43 | For the developers' and authors' protection, the GPL clearly explains that there 44 | is no warranty for this free software. For both users' and authors' sake, the 45 | GPL requires that modified versions be marked as changed, so that their problems 46 | will not be attributed erroneously to authors of previous versions. 47 | 48 | Some devices are designed to deny users access to install or run modified 49 | versions of the software inside them, although the manufacturer can do so. This 50 | is fundamentally incompatible with the aim of protecting users' freedom to 51 | change the software. The systematic pattern of such abuse occurs in the area of 52 | products for individuals to use, which is precisely where it is most 53 | unacceptable. Therefore, we have designed this version of the GPL to prohibit 54 | the practice for those products. If such problems arise substantially in other 55 | domains, we stand ready to extend this provision to those domains in future 56 | versions of the GPL, as needed to protect the freedom of users. 57 | 58 | Finally, every program is threatened constantly by software patents. States 59 | should not allow patents to restrict development and use of software on 60 | general-purpose computers, but in those that do, we wish to avoid the special 61 | danger that patents applied to a free program could make it effectively 62 | proprietary. To prevent this, the GPL assures that patents cannot be used to 63 | render the program non-free. 64 | 65 | The precise terms and conditions for copying, distribution and modification 66 | follow. 67 | 68 | ## TERMS AND CONDITIONS 69 | 70 | ### 0. Definitions 71 | 72 | “This License” refers to version 3 of the GNU General Public License. 73 | 74 | “Copyright” also means copyright-like laws that apply to other kinds of works, 75 | such as semiconductor masks. 76 | 77 | “The Program” refers to any copyrightable work licensed under this License. Each 78 | licensee is addressed as “you”. “Licensees” and “recipients” may be individuals 79 | or organizations. 80 | 81 | To “modify” a work means to copy from or adapt all or part of the work in a 82 | fashion requiring copyright permission, other than the making of an exact copy. 83 | The resulting work is called a “modified version” of the earlier work or a work 84 | “based on” the earlier work. 85 | 86 | A “covered work” means either the unmodified Program or a work based on the 87 | Program. 88 | 89 | To “propagate” a work means to do anything with it that, without permission, 90 | would make you directly or secondarily liable for infringement under applicable 91 | copyright law, except executing it on a computer or modifying a private copy. 92 | Propagation includes copying, distribution (with or without modification), 93 | making available to the public, and in some countries other activities as well. 94 | 95 | To “convey” a work means any kind of propagation that enables other parties to 96 | make or receive copies. Mere interaction with a user through a computer network, 97 | with no transfer of a copy, is not conveying. 98 | 99 | An interactive user interface displays “Appropriate Legal Notices” to the extent 100 | that it includes a convenient and prominently visible feature that **(1)** 101 | displays an appropriate copyright notice, and **(2)** tells the user that there 102 | is no warranty for the work (except to the extent that warranties are provided), 103 | that licensees may convey the work under this License, and how to view a copy of 104 | this License. If the interface presents a list of user commands or options, such 105 | as a menu, a prominent item in the list meets this criterion. 106 | 107 | ### 1. Source Code 108 | 109 | The “source code” for a work means the preferred form of the work for making 110 | modifications to it. “Object code” means any non-source form of a work. 111 | 112 | A “Standard Interface” means an interface that either is an official standard 113 | defined by a recognized standards body, or, in the case of interfaces specified 114 | for a particular programming language, one that is widely used among developers 115 | working in that language. 116 | 117 | The “System Libraries” of an executable work include anything, other than the 118 | work as a whole, that **(a)** is included in the normal form of packaging a 119 | Major Component, but which is not part of that Major Component, and **(b)** 120 | serves only to enable use of the work with that Major Component, or to implement 121 | a Standard Interface for which an implementation is available to the public in 122 | source code form. A “Major Component”, in this context, means a major essential 123 | component (kernel, window system, and so on) of the specific operating system 124 | (if any) on which the executable work runs, or a compiler used to produce the 125 | work, or an object code interpreter used to run it. 126 | 127 | The “Corresponding Source” for a work in object code form means all the source 128 | code needed to generate, install, and (for an executable work) run the object 129 | code and to modify the work, including scripts to control those activities. 130 | However, it does not include the work's System Libraries, or general-purpose 131 | tools or generally available free programs which are used unmodified in 132 | performing those activities but which are not part of the work. For example, 133 | Corresponding Source includes interface definition files associated with source 134 | files for the work, and the source code for shared libraries and dynamically 135 | linked subprograms that the work is specifically designed to require, such as by 136 | intimate data communication or control flow between those subprograms and other 137 | parts of the work. 138 | 139 | The Corresponding Source need not include anything that users can regenerate 140 | automatically from other parts of the Corresponding Source. 141 | 142 | The Corresponding Source for a work in source code form is that same work. 143 | 144 | ### 2. Basic Permissions 145 | 146 | All rights granted under this License are granted for the term of copyright on 147 | the Program, and are irrevocable provided the stated conditions are met. This 148 | License explicitly affirms your unlimited permission to run the unmodified 149 | Program. The output from running a covered work is covered by this License only 150 | if the output, given its content, constitutes a covered work. This License 151 | acknowledges your rights of fair use or other equivalent, as provided by 152 | copyright law. 153 | 154 | You may make, run and propagate covered works that you do not convey, without 155 | conditions so long as your license otherwise remains in force. You may convey 156 | covered works to others for the sole purpose of having them make modifications 157 | exclusively for you, or provide you with facilities for running those works, 158 | provided that you comply with the terms of this License in conveying all 159 | material for which you do not control copyright. Those thus making or running 160 | the covered works for you must do so exclusively on your behalf, under your 161 | direction and control, on terms that prohibit them from making any copies of 162 | your copyrighted material outside their relationship with you. 163 | 164 | Conveying under any other circumstances is permitted solely under the conditions 165 | stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 166 | 167 | ### 3. Protecting Users' Legal Rights From Anti-Circumvention Law 168 | 169 | No covered work shall be deemed part of an effective technological measure under 170 | any applicable law fulfilling obligations under article 11 of the WIPO copyright 171 | treaty adopted on 20 December 1996, or similar laws prohibiting or restricting 172 | circumvention of such measures. 173 | 174 | When you convey a covered work, you waive any legal power to forbid 175 | circumvention of technological measures to the extent such circumvention is 176 | effected by exercising rights under this License with respect to the covered 177 | work, and you disclaim any intention to limit operation or modification of the 178 | work as a means of enforcing, against the work's users, your or third parties' 179 | legal rights to forbid circumvention of technological measures. 180 | 181 | ### 4. Conveying Verbatim Copies 182 | 183 | You may convey verbatim copies of the Program's source code as you receive it, 184 | in any medium, provided that you conspicuously and appropriately publish on each 185 | copy an appropriate copyright notice; keep intact all notices stating that this 186 | License and any non-permissive terms added in accord with section 7 apply to the 187 | code; keep intact all notices of the absence of any warranty; and give all 188 | recipients a copy of this License along with the Program. 189 | 190 | You may charge any price or no price for each copy that you convey, and you may 191 | offer support or warranty protection for a fee. 192 | 193 | ### 5. Conveying Modified Source Versions 194 | 195 | You may convey a work based on the Program, or the modifications to produce it 196 | from the Program, in the form of source code under the terms of section 4, 197 | provided that you also meet all of these conditions: 198 | 199 | - **a)** The work must carry prominent notices stating that you modified it, and 200 | giving a relevant date. 201 | - **b)** The work must carry prominent notices stating that it is released under 202 | this License and any conditions added under section 7. This requirement 203 | modifies the requirement in section 4 to “keep intact all notices”. 204 | - **c)** You must license the entire work, as a whole, under this License to 205 | anyone who comes into possession of a copy. This License will therefore apply, 206 | along with any applicable section 7 additional terms, to the whole of the 207 | work, and all its parts, regardless of how they are packaged. This License 208 | gives no permission to license the work in any other way, but it does not 209 | invalidate such permission if you have separately received it. 210 | - **d)** If the work has interactive user interfaces, each must display 211 | Appropriate Legal Notices; however, if the Program has interactive interfaces 212 | that do not display Appropriate Legal Notices, your work need not make them do 213 | so. 214 | 215 | A compilation of a covered work with other separate and independent works, which 216 | are not by their nature extensions of the covered work, and which are not 217 | combined with it such as to form a larger program, in or on a volume of a 218 | storage or distribution medium, is called an “aggregate” if the compilation and 219 | its resulting copyright are not used to limit the access or legal rights of the 220 | compilation's users beyond what the individual works permit. Inclusion of a 221 | covered work in an aggregate does not cause this License to apply to the other 222 | parts of the aggregate. 223 | 224 | ### 6. Conveying Non-Source Forms 225 | 226 | You may convey a covered work in object code form under the terms of sections 4 227 | and 5, provided that you also convey the machine-readable Corresponding Source 228 | under the terms of this License, in one of these ways: 229 | 230 | - **a)** Convey the object code in, or embodied in, a physical product 231 | (including a physical distribution medium), accompanied by the Corresponding 232 | Source fixed on a durable physical medium customarily used for software 233 | interchange. 234 | - **b)** Convey the object code in, or embodied in, a physical product 235 | (including a physical distribution medium), accompanied by a written offer, 236 | valid for at least three years and valid for as long as you offer spare parts 237 | or customer support for that product model, to give anyone who possesses the 238 | object code either **(1)** a copy of the Corresponding Source for all the 239 | software in the product that is covered by this License, on a durable physical 240 | medium customarily used for software interchange, for a price no more than 241 | your reasonable cost of physically performing this conveying of source, or 242 | **(2)** access to copy the Corresponding Source from a network server at no 243 | charge. 244 | - **c)** Convey individual copies of the object code with a copy of the written 245 | offer to provide the Corresponding Source. This alternative is allowed only 246 | occasionally and noncommercially, and only if you received the object code 247 | with such an offer, in accord with subsection 6b. 248 | - **d)** Convey the object code by offering access from a designated place 249 | (gratis or for a charge), and offer equivalent access to the Corresponding 250 | Source in the same way through the same place at no further charge. You need 251 | not require recipients to copy the Corresponding Source along with the object 252 | code. If the place to copy the object code is a network server, the 253 | Corresponding Source may be on a different server (operated by you or a third 254 | party) that supports equivalent copying facilities, provided you maintain 255 | clear directions next to the object code saying where to find the 256 | Corresponding Source. Regardless of what server hosts the Corresponding 257 | Source, you remain obligated to ensure that it is available for as long as 258 | needed to satisfy these requirements. 259 | - **e)** Convey the object code using peer-to-peer transmission, provided you 260 | inform other peers where the object code and Corresponding Source of the work 261 | are being offered to the general public at no charge under subsection 6d. 262 | 263 | A separable portion of the object code, whose source code is excluded from the 264 | Corresponding Source as a System Library, need not be included in conveying the 265 | object code work. 266 | 267 | A “User Product” is either **(1)** a “consumer product”, which means any 268 | tangible personal property which is normally used for personal, family, or 269 | household purposes, or **(2)** anything designed or sold for incorporation into 270 | a dwelling. In determining whether a product is a consumer product, doubtful 271 | cases shall be resolved in favor of coverage. For a particular product received 272 | by a particular user, “normally used” refers to a typical or common use of that 273 | class of product, regardless of the status of the particular user or of the way 274 | in which the particular user actually uses, or expects or is expected to use, 275 | the product. A product is a consumer product regardless of whether the product 276 | has substantial commercial, industrial or non-consumer uses, unless such uses 277 | represent the only significant mode of use of the product. 278 | 279 | “Installation Information” for a User Product means any methods, procedures, 280 | authorization keys, or other information required to install and execute 281 | modified versions of a covered work in that User Product from a modified version 282 | of its Corresponding Source. The information must suffice to ensure that the 283 | continued functioning of the modified object code is in no case prevented or 284 | interfered with solely because modification has been made. 285 | 286 | If you convey an object code work under this section in, or with, or 287 | specifically for use in, a User Product, and the conveying occurs as part of a 288 | transaction in which the right of possession and use of the User Product is 289 | transferred to the recipient in perpetuity or for a fixed term (regardless of 290 | how the transaction is characterized), the Corresponding Source conveyed under 291 | this section must be accompanied by the Installation Information. But this 292 | requirement does not apply if neither you nor any third party retains the 293 | ability to install modified object code on the User Product (for example, the 294 | work has been installed in ROM). 295 | 296 | The requirement to provide Installation Information does not include a 297 | requirement to continue to provide support service, warranty, or updates for a 298 | work that has been modified or installed by the recipient, or for the User 299 | Product in which it has been modified or installed. Access to a network may be 300 | denied when the modification itself materially and adversely affects the 301 | operation of the network or violates the rules and protocols for communication 302 | across the network. 303 | 304 | Corresponding Source conveyed, and Installation Information provided, in accord 305 | with this section must be in a format that is publicly documented (and with an 306 | implementation available to the public in source code form), and must require no 307 | special password or key for unpacking, reading or copying. 308 | 309 | ### 7. Additional Terms 310 | 311 | “Additional permissions” are terms that supplement the terms of this License by 312 | making exceptions from one or more of its conditions. Additional permissions 313 | that are applicable to the entire Program shall be treated as though they were 314 | included in this License, to the extent that they are valid under applicable 315 | law. If additional permissions apply only to part of the Program, that part may 316 | be used separately under those permissions, but the entire Program remains 317 | governed by this License without regard to the additional permissions. 318 | 319 | When you convey a copy of a covered work, you may at your option remove any 320 | additional permissions from that copy, or from any part of it. (Additional 321 | permissions may be written to require their own removal in certain cases when 322 | you modify the work.) You may place additional permissions on material, added by 323 | you to a covered work, for which you have or can give appropriate copyright 324 | permission. 325 | 326 | Notwithstanding any other provision of this License, for material you add to a 327 | covered work, you may (if authorized by the copyright holders of that material) 328 | supplement the terms of this License with terms: 329 | 330 | - **a)** Disclaiming warranty or limiting liability differently from the terms 331 | of sections 15 and 16 of this License; or 332 | - **b)** Requiring preservation of specified reasonable legal notices or author 333 | attributions in that material or in the Appropriate Legal Notices displayed by 334 | works containing it; or 335 | - **c)** Prohibiting misrepresentation of the origin of that material, or 336 | requiring that modified versions of such material be marked in reasonable ways 337 | as different from the original version; or 338 | - **d)** Limiting the use for publicity purposes of names of licensors or 339 | authors of the material; or 340 | - **e)** Declining to grant rights under trademark law for use of some trade 341 | names, trademarks, or service marks; or 342 | - **f)** Requiring indemnification of licensors and authors of that material by 343 | anyone who conveys the material (or modified versions of it) with contractual 344 | assumptions of liability to the recipient, for any liability that these 345 | contractual assumptions directly impose on those licensors and authors. 346 | 347 | All other non-permissive additional terms are considered “further restrictions” 348 | within the meaning of section 10. If the Program as you received it, or any part 349 | of it, contains a notice stating that it is governed by this License along with 350 | a term that is a further restriction, you may remove that term. If a license 351 | document contains a further restriction but permits relicensing or conveying 352 | under this License, you may add to a covered work material governed by the terms 353 | of that license document, provided that the further restriction does not survive 354 | such relicensing or conveying. 355 | 356 | If you add terms to a covered work in accord with this section, you must place, 357 | in the relevant source files, a statement of the additional terms that apply to 358 | those files, or a notice indicating where to find the applicable terms. 359 | 360 | Additional terms, permissive or non-permissive, may be stated in the form of a 361 | separately written license, or stated as exceptions; the above requirements 362 | apply either way. 363 | 364 | ### 8. Termination 365 | 366 | You may not propagate or modify a covered work except as expressly provided 367 | under this License. Any attempt otherwise to propagate or modify it is void, and 368 | will automatically terminate your rights under this License (including any 369 | patent licenses granted under the third paragraph of section 11). 370 | 371 | However, if you cease all violation of this License, then your license from a 372 | particular copyright holder is reinstated **(a)** provisionally, unless and 373 | until the copyright holder explicitly and finally terminates your license, and 374 | **(b)** permanently, if the copyright holder fails to notify you of the 375 | violation by some reasonable means prior to 60 days after the cessation. 376 | 377 | Moreover, your license from a particular copyright holder is reinstated 378 | permanently if the copyright holder notifies you of the violation by some 379 | reasonable means, this is the first time you have received notice of violation 380 | of this License (for any work) from that copyright holder, and you cure the 381 | violation prior to 30 days after your receipt of the notice. 382 | 383 | Termination of your rights under this section does not terminate the licenses of 384 | parties who have received copies or rights from you under this License. If your 385 | rights have been terminated and not permanently reinstated, you do not qualify 386 | to receive new licenses for the same material under section 10. 387 | 388 | ### 9. Acceptance Not Required for Having Copies 389 | 390 | You are not required to accept this License in order to receive or run a copy of 391 | the Program. Ancillary propagation of a covered work occurring solely as a 392 | consequence of using peer-to-peer transmission to receive a copy likewise does 393 | not require acceptance. However, nothing other than this License grants you 394 | permission to propagate or modify any covered work. These actions infringe 395 | copyright if you do not accept this License. Therefore, by modifying or 396 | propagating a covered work, you indicate your acceptance of this License to do 397 | so. 398 | 399 | ### 10. Automatic Licensing of Downstream Recipients 400 | 401 | Each time you convey a covered work, the recipient automatically receives a 402 | license from the original licensors, to run, modify and propagate that work, 403 | subject to this License. You are not responsible for enforcing compliance by 404 | third parties with this License. 405 | 406 | An “entity transaction” is a transaction transferring control of an 407 | organization, or substantially all assets of one, or subdividing an 408 | organization, or merging organizations. If propagation of a covered work results 409 | from an entity transaction, each party to that transaction who receives a copy 410 | of the work also receives whatever licenses to the work the party's predecessor 411 | in interest had or could give under the previous paragraph, plus a right to 412 | possession of the Corresponding Source of the work from the predecessor in 413 | interest, if the predecessor has it or can get it with reasonable efforts. 414 | 415 | You may not impose any further restrictions on the exercise of the rights 416 | granted or affirmed under this License. For example, you may not impose a 417 | license fee, royalty, or other charge for exercise of rights granted under this 418 | License, and you may not initiate litigation (including a cross-claim or 419 | counterclaim in a lawsuit) alleging that any patent claim is infringed by 420 | making, using, selling, offering for sale, or importing the Program or any 421 | portion of it. 422 | 423 | ### 11. Patents 424 | 425 | A “contributor” is a copyright holder who authorizes use under this License of 426 | the Program or a work on which the Program is based. The work thus licensed is 427 | called the contributor's “contributor version”. 428 | 429 | A contributor's “essential patent claims” are all patent claims owned or 430 | controlled by the contributor, whether already acquired or hereafter acquired, 431 | that would be infringed by some manner, permitted by this License, of making, 432 | using, or selling its contributor version, but do not include claims that would 433 | be infringed only as a consequence of further modification of the contributor 434 | version. For purposes of this definition, “control” includes the right to grant 435 | patent sublicenses in a manner consistent with the requirements of this License. 436 | 437 | Each contributor grants you a non-exclusive, worldwide, royalty-free patent 438 | license under the contributor's essential patent claims, to make, use, sell, 439 | offer for sale, import and otherwise run, modify and propagate the contents of 440 | its contributor version. 441 | 442 | In the following three paragraphs, a “patent license” is any express agreement 443 | or commitment, however denominated, not to enforce a patent (such as an express 444 | permission to practice a patent or covenant not to sue for patent infringement). 445 | To “grant” such a patent license to a party means to make such an agreement or 446 | commitment not to enforce a patent against the party. 447 | 448 | If you convey a covered work, knowingly relying on a patent license, and the 449 | Corresponding Source of the work is not available for anyone to copy, free of 450 | charge and under the terms of this License, through a publicly available network 451 | server or other readily accessible means, then you must either **(1)** cause the 452 | Corresponding Source to be so available, or **(2)** arrange to deprive yourself 453 | of the benefit of the patent license for this particular work, or **(3)** 454 | arrange, in a manner consistent with the requirements of this License, to extend 455 | the patent license to downstream recipients. “Knowingly relying” means you have 456 | actual knowledge that, but for the patent license, your conveying the covered 457 | work in a country, or your recipient's use of the covered work in a country, 458 | would infringe one or more identifiable patents in that country that you have 459 | reason to believe are valid. 460 | 461 | If, pursuant to or in connection with a single transaction or arrangement, you 462 | convey, or propagate by procuring conveyance of, a covered work, and grant a 463 | patent license to some of the parties receiving the covered work authorizing 464 | them to use, propagate, modify or convey a specific copy of the covered work, 465 | then the patent license you grant is automatically extended to all recipients of 466 | the covered work and works based on it. 467 | 468 | A patent license is “discriminatory” if it does not include within the scope of 469 | its coverage, prohibits the exercise of, or is conditioned on the non-exercise 470 | of one or more of the rights that are specifically granted under this License. 471 | You may not convey a covered work if you are a party to an arrangement with a 472 | third party that is in the business of distributing software, under which you 473 | make payment to the third party based on the extent of your activity of 474 | conveying the work, and under which the third party grants, to any of the 475 | parties who would receive the covered work from you, a discriminatory patent 476 | license **(a)** in connection with copies of the covered work conveyed by you 477 | (or copies made from those copies), or **(b)** primarily for and in connection 478 | with specific products or compilations that contain the covered work, unless you 479 | entered into that arrangement, or that patent license was granted, prior to 28 480 | March 2007. 481 | 482 | Nothing in this License shall be construed as excluding or limiting any implied 483 | license or other defenses to infringement that may otherwise be available to you 484 | under applicable patent law. 485 | 486 | ### 12. No Surrender of Others' Freedom 487 | 488 | If conditions are imposed on you (whether by court order, agreement or 489 | otherwise) that contradict the conditions of this License, they do not excuse 490 | you from the conditions of this License. If you cannot convey a covered work so 491 | as to satisfy simultaneously your obligations under this License and any other 492 | pertinent obligations, then as a consequence you may not convey it at all. For 493 | example, if you agree to terms that obligate you to collect a royalty for 494 | further conveying from those to whom you convey the Program, the only way you 495 | could satisfy both those terms and this License would be to refrain entirely 496 | from conveying the Program. 497 | 498 | ### 13. Use with the GNU Affero General Public License 499 | 500 | Notwithstanding any other provision of this License, you have permission to link 501 | or combine any covered work with a work licensed under version 3 of the GNU 502 | Affero General Public License into a single combined work, and to convey the 503 | resulting work. The terms of this License will continue to apply to the part 504 | which is the covered work, but the special requirements of the GNU Affero 505 | General Public License, section 13, concerning interaction through a network 506 | will apply to the combination as such. 507 | 508 | ### 14. Revised Versions of this License 509 | 510 | The Free Software Foundation may publish revised and/or new versions of the GNU 511 | General Public License from time to time. Such new versions will be similar in 512 | spirit to the present version, but may differ in detail to address new problems 513 | or concerns. 514 | 515 | Each version is given a distinguishing version number. If the Program specifies 516 | that a certain numbered version of the GNU General Public License “or any later 517 | version” applies to it, you have the option of following the terms and 518 | conditions either of that numbered version or of any later version published by 519 | the Free Software Foundation. If the Program does not specify a version number 520 | of the GNU General Public License, you may choose any version ever published by 521 | the Free Software Foundation. 522 | 523 | If the Program specifies that a proxy can decide which future versions of the 524 | GNU General Public License can be used, that proxy's public statement of 525 | acceptance of a version permanently authorizes you to choose that version for 526 | the Program. 527 | 528 | Later license versions may give you additional or different permissions. 529 | However, no additional obligations are imposed on any author or copyright holder 530 | as a result of your choosing to follow a later version. 531 | 532 | ### 15. Disclaimer of Warranty 533 | 534 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. 535 | EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER 536 | PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER 537 | EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 538 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE 539 | QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE 540 | DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 541 | 542 | ### 16. Limitation of Liability 543 | 544 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY 545 | COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS 546 | PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, 547 | INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE 548 | THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED 549 | INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE 550 | PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY 551 | HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 552 | 553 | ### 17. Interpretation of Sections 15 and 16 554 | 555 | If the disclaimer of warranty and limitation of liability provided above cannot 556 | be given local legal effect according to their terms, reviewing courts shall 557 | apply local law that most closely approximates an absolute waiver of all civil 558 | liability in connection with the Program, unless a warranty or assumption of 559 | liability accompanies a copy of the Program in return for a fee. 560 | 561 | _END OF TERMS AND CONDITIONS_ 562 | 563 | ## How to Apply These Terms to Your New Programs 564 | 565 | If you develop a new program, and you want it to be of the greatest possible use 566 | to the public, the best way to achieve this is to make it free software which 567 | everyone can redistribute and change under these terms. 568 | 569 | To do so, attach the following notices to the program. It is safest to attach 570 | them to the start of each source file to most effectively state the exclusion of 571 | warranty; and each file should have at least the “copyright” line and a pointer 572 | to where the full notice is found. 573 | 574 | 575 | Copyright (C) 576 | 577 | This program is free software: you can redistribute it and/or modify 578 | it under the terms of the GNU General Public License as published by 579 | the Free Software Foundation, either version 3 of the License, or 580 | (at your option) any later version. 581 | 582 | This program is distributed in the hope that it will be useful, 583 | but WITHOUT ANY WARRANTY; without even the implied warranty of 584 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 585 | GNU General Public License for more details. 586 | 587 | You should have received a copy of the GNU General Public License 588 | along with this program. If not, see . 589 | 590 | Also add information on how to contact you by electronic and paper mail. 591 | 592 | If the program does terminal interaction, make it output a short notice like 593 | this when it starts in an interactive mode: 594 | 595 | Copyright (C) 596 | This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. 597 | This is free software, and you are welcome to redistribute it 598 | under certain conditions; type 'show c' for details. 599 | 600 | The hypothetical commands `show w` and `show c` should show the appropriate 601 | parts of the General Public License. Of course, your program's commands might be 602 | different; for a GUI interface, you would use an “about box”. 603 | 604 | You should also get your employer (if you work as a programmer) or school, if 605 | any, to sign a “copyright disclaimer” for the program, if necessary. For more 606 | information on this, and how to apply and follow the GNU GPL, see 607 | <>. 608 | 609 | The GNU General Public License does not permit incorporating your program into 610 | proprietary programs. If your program is a subroutine library, you may consider 611 | it more useful to permit linking proprietary applications with the library. If 612 | this is what you want to do, use the GNU Lesser General Public License instead 613 | of this License. But first, please read 614 | <>. 615 | --------------------------------------------------------------------------------