├── test ├── return-self-1.nix ├── return-self-2.nix ├── storeSrc │ └── contents ├── samplePackage │ ├── local-src │ ├── upstream-src │ ├── nix │ │ ├── wrangle.json │ │ └── default.nix │ ├── unbuildable.nix │ ├── dep.nix │ ├── dep-user.nix │ ├── exposeGit.nix │ ├── versionSrc.nix │ └── attrs.nix ├── fakeNixpkgs.nix ├── overlay.nix ├── unit.sh ├── nix │ ├── default.nix │ └── wrangle.json ├── nix-impure.sh ├── all.sh ├── piep.nix ├── import-drv-check-identity.sh ├── test.nix ├── override-src.sh ├── gup-readme.nix ├── integration.sh └── unit.nix ├── src ├── Main.hs └── Wrangle │ ├── Util.hs │ ├── Splice.hs │ ├── Fetch.hs │ ├── Source.hs │ └── Cmd.hs ├── .gitignore ├── nix ├── wrangle.nix.gup ├── wrangle.json ├── overrideSrc.nix ├── nixImpure.nix ├── unpackArchive.nix ├── wrangle.nix ├── default.nix ├── exportLocalGit.nix ├── importDrv.nix └── api.nix ├── example ├── .gitignore ├── 00-bootstrap.gup ├── nix │ └── default.nix ├── 04-splice.gup ├── 02-update.gup ├── 03-local-override.gup ├── .doc_setup.sh ├── 01-setup.gup └── default.nix ├── bin └── nix-wrangle ├── shell.nix ├── test.nix ├── default.nix ├── nix-wrangle.cabal ├── README.md.gup └── README.md /test/return-self-1.nix: -------------------------------------------------------------------------------- 1 | { self }: self 2 | -------------------------------------------------------------------------------- /test/return-self-2.nix: -------------------------------------------------------------------------------- 1 | { self }: self 2 | -------------------------------------------------------------------------------- /test/storeSrc/contents: -------------------------------------------------------------------------------- 1 | source contents 2 | -------------------------------------------------------------------------------- /test/samplePackage/local-src: -------------------------------------------------------------------------------- 1 | upstream source 2 | -------------------------------------------------------------------------------- /test/samplePackage/upstream-src: -------------------------------------------------------------------------------- 1 | upstream source 2 | -------------------------------------------------------------------------------- /src/Main.hs: -------------------------------------------------------------------------------- 1 | module Main (main) where 2 | import Wrangle.Cmd 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | result* 2 | dist* 3 | .ghc.environment.* 4 | /test/default.nix 5 | -------------------------------------------------------------------------------- /test/fakeNixpkgs.nix: -------------------------------------------------------------------------------- 1 | { overlays }: 2 | { 3 | testMessage = "fake nixpkgs!"; 4 | } 5 | -------------------------------------------------------------------------------- /nix/wrangle.nix.gup: -------------------------------------------------------------------------------- 1 | #!bash -eu 2 | gup -u ../nix-wrangle.cabal 3 | cabal2nix ../ > "$1" 4 | -------------------------------------------------------------------------------- /test/samplePackage/nix/wrangle.json: -------------------------------------------------------------------------------- 1 | { "wrangle": { "apiversion": 1 } 2 | , "sources": {} 3 | } 4 | -------------------------------------------------------------------------------- /test/samplePackage/unbuildable.nix: -------------------------------------------------------------------------------- 1 | { }: 2 | abort "this package intentionally unbuildable" 3 | -------------------------------------------------------------------------------- /example/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !*.gup 3 | !nix/ 4 | !nix/default.nix 5 | !.doc_setup.sh 6 | !default.nix 7 | -------------------------------------------------------------------------------- /test/overlay.nix: -------------------------------------------------------------------------------- 1 | self: super: 2 | 3 | { 4 | gup = super.callPackage ./gup-readme.nix {}; 5 | } 6 | -------------------------------------------------------------------------------- /test/samplePackage/dep.nix: -------------------------------------------------------------------------------- 1 | {lib, stdenv}: 2 | stdenv.mkDerivation { 3 | name="dep"; 4 | passthru.depProvided = true; 5 | } 6 | -------------------------------------------------------------------------------- /test/unit.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eux 3 | nix-instantiate --show-trace --eval "$(dirname "$0")/unit.nix" --strict --read-write-mode 4 | -------------------------------------------------------------------------------- /test/nix/default.nix: -------------------------------------------------------------------------------- 1 | { lib, nix-wrangle }: 2 | { 3 | apiMembers = lib.concatStringsSep "," (lib.sort (a: b: a < b) (lib.attrNames (nix-wrangle.api {}))); 4 | } 5 | -------------------------------------------------------------------------------- /test/samplePackage/dep-user.nix: -------------------------------------------------------------------------------- 1 | { lib, stdenv, dep }: 2 | stdenv.mkDerivation { 3 | name="dep-user"; 4 | passthru = { 5 | inherit (dep) depProvided; 6 | }; 7 | } 8 | -------------------------------------------------------------------------------- /test/samplePackage/nix/default.nix: -------------------------------------------------------------------------------- 1 | { stdenv }: stdenv.mkDerivation { 2 | name = "samplePackage"; 3 | src = ../upstream-src; 4 | buildCommand = "cat $src > $out"; 5 | } 6 | -------------------------------------------------------------------------------- /test/nix-impure.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eux 3 | set -o pipefail 4 | here="$(dirname "$0")" 5 | drv="$(nix-instantiate "$here/test.nix" --show-trace -A impure --quiet)" 6 | "$here/../bin/nix-impure" "$drv" 7 | 8 | -------------------------------------------------------------------------------- /test/samplePackage/exposeGit.nix: -------------------------------------------------------------------------------- 1 | { pkgs, git }: 2 | # using pkgs.stdenv to get around the fact that 3 | # an unbuildable `stdenv` is injected 4 | pkgs.stdenv.mkDerivation { 5 | name="upstreamGit"; 6 | passthru.git = git; 7 | } 8 | -------------------------------------------------------------------------------- /bin/nix-wrangle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | here="$(dirname "$0")" 4 | exe="$(find "$here/../"dist* -type f -name nix-wrangle)" 5 | [ -n "$exe" ] || (echo "nix-wrangle executable not found"; exit 1) 6 | set -x 7 | exec "$exe" "$@" 8 | -------------------------------------------------------------------------------- /example/00-bootstrap.gup: -------------------------------------------------------------------------------- 1 | #!bash -eu 2 | . .doc_setup.sh 3 | 4 | rm -f nix/wrangle*.json 5 | 6 | set -x 7 | nix-build --expr 'import (builtins.fetchTarball "https://github.com/timbertson/nix-wrangle/archive/v1.tar.gz")' 8 | result/bin/nix-wrangle --help 9 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {}}: 2 | (pkgs.callPackage ./default.nix {}).env.overrideAttrs (o: { 3 | nativeBuildInputs = o.nativeBuildInputs ++ [ 4 | pkgs.haskellPackages.cabal-install 5 | pkgs.haskellPackages.ghcid 6 | pkgs.cabal2nix 7 | ]; 8 | }) 9 | -------------------------------------------------------------------------------- /example/nix/default.nix: -------------------------------------------------------------------------------- 1 | { stdenv, piep }: 2 | stdenv.mkDerivation { 3 | name="sample.txt"; 4 | src = ../.; 5 | buildCommand = '' 6 | cat > "$out" < {}); 3 | fetchFromGitHub { 4 | owner = "timbertson"; 5 | repo = "gup"; 6 | rev = "master"; 7 | # sha256 = "1pwnmlq2pgkkln9sgz4wlb9dqlqw83bkf105qljnlvggc21zm3pv"; 8 | sha256 = null; 9 | } 10 | 11 | -------------------------------------------------------------------------------- /test/samplePackage/versionSrc.nix: -------------------------------------------------------------------------------- 1 | { 2 | type = "github"; 3 | fetch = { 4 | "owner" = "timbertson"; 5 | "repo" = "version"; 6 | "rev" = "version-0.13.1"; 7 | "sha256" = "056l8m0xxl1h7x4fd1hf754w8q4sibpqnwgnbk5af5i66399az61"; 8 | }; 9 | nix = "nix/"; 10 | } 11 | -------------------------------------------------------------------------------- /example/04-splice.gup: -------------------------------------------------------------------------------- 1 | #!bash -eu 2 | . .doc_setup.sh 3 | 4 | [ -f nix/wrangle.json ] || gup 01-setup 5 | 6 | set -x 7 | comment_REM 'Splice the `nix-wrangle` source into nix/default.nix' 8 | nix-wrangle splice nix/default.nix --name nix-wrangle --output nix/public.nix 9 | cat nix/public.nix 10 | -------------------------------------------------------------------------------- /nix/wrangle.json: -------------------------------------------------------------------------------- 1 | { 2 | "sources": { 3 | "self": { 4 | "fetch": { 5 | "ref": "HEAD", 6 | "relativePath": "." 7 | }, 8 | "ref": "HEAD", 9 | "relativePath": ".", 10 | "type": "git-local" 11 | } 12 | }, 13 | "wrangle": { 14 | "apiversion": 1 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/samplePackage/attrs.nix: -------------------------------------------------------------------------------- 1 | { 2 | sources = [{ 3 | wrangle.apiversion = 1; 4 | sources.version = import ./versionSrc.nix; 5 | }]; 6 | nix = ({ pkgs, version, custom }: 7 | (pkgs.callPackage ./default.nix {}).overrideAttrs (o: { passthru = { inherit custom; }; })); 8 | args = { custom = "attr!"; }; 9 | } 10 | -------------------------------------------------------------------------------- /test/all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ux 3 | here="$(dirname "$0")" 4 | st=0 5 | function run() { 6 | if ! "$here/$1"; then 7 | echo -e "*** FAILED: $1 ***\n" 8 | st=1 9 | else 10 | echo -e "Ok: $1\n" 11 | fi 12 | } 13 | run "unit.sh" 14 | run "import-drv-check-identity.sh" 15 | run "nix-impure.sh" 16 | run "override-src.sh" 17 | run "integration.sh" 18 | exit "$st" 19 | -------------------------------------------------------------------------------- /example/02-update.gup: -------------------------------------------------------------------------------- 1 | #!bash -eu 2 | 3 | [ -f nix/wrangle.json ] || gup 01-setup 4 | 5 | . .doc_setup.sh 6 | 7 | # fake an old rev 8 | sed -i -e 's/d805330386553c5784ac7ef48ff38aea716575dc/d805330386553c5784ac7ef48ff38aea716575de/' nix/wrangle.json 9 | set -x 10 | comment_REM 'Time to update the `piep` dependency (this re-resolves `master`, or whatever ref is configured)' 11 | nix-wrangle update piep 12 | -------------------------------------------------------------------------------- /test/piep.nix: -------------------------------------------------------------------------------- 1 | { lib, fetchFromGitHub, buildPythonPackage, pygments, nose, nix-update-source }: 2 | buildPythonPackage rec { 3 | version = "0.9.2"; 4 | src = fetchFromGitHub { 5 | owner = "timbertson"; 6 | repo = "piep"; 7 | rev = "version-0.9.2"; 8 | sha256 = "1q7lzi3lggw8by4pkh5ckkjv68xqbjahrkxgdw89jxdlyd0wq5in"; 9 | }; 10 | name = "piep-${version}"; 11 | propagatedBuildInputs = [ pygments ]; 12 | checkInputs = [ nose ]; 13 | } 14 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | # Stripped down version of default-nix which doesn't dynamically fetch nix-wrangle 2 | let 3 | systemNixpkgs = import {}; 4 | fallback = val: dfl: if val == null then dfl else val; 5 | in 6 | { pkgs ? null, args ? {}, ... }@provided: 7 | let 8 | _pkgs = fallback pkgs systemNixpkgs; 9 | _wrangle = _pkgs.callPackage ./nix { enableSplice = false; }; 10 | in 11 | (_wrangle.api { pkgs = _pkgs; }).inject { provided = provided; path = ./.; } 12 | -------------------------------------------------------------------------------- /test/import-drv-check-identity.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | set -o pipefail 4 | here="$(dirname "$0")" 5 | set -x 6 | drv="$(nix-instantiate "$here/test.nix" -A gup --quiet)" 7 | outputDrv="$(nix-instantiate "$here/test.nix" -A importDrv --argstr drvPath "$drv" --quiet)" 8 | if [ "$drv" != "$outputDrv" ]; then 9 | echo "Error! files differ" 10 | set -x 11 | diff <(nix show-derivation "$drv") <(nix show-derivation "$outputDrv") 12 | else 13 | echo "OK" 14 | fi 15 | -------------------------------------------------------------------------------- /test/test.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {}}: 2 | with pkgs; 3 | with callPackage ../nix/default.nix {}; 4 | let 5 | in 6 | { 7 | impure = nixImpure { name = "testing"; } '' 8 | touch $out; 9 | ls -l $out 10 | '' ; 11 | 12 | importDrv = { drvPath }: importDrv drvPath; 13 | 14 | exportLocalGit = { commit ? null, ref ? null, unpack ? false, workingChanges ? false }: 15 | exportLocalGit { inherit commit ref unpack workingChanges; dir = ../.; }; 16 | 17 | gup = callPackage ./gup-readme.nix {}; 18 | } 19 | -------------------------------------------------------------------------------- /nix/overrideSrc.nix: -------------------------------------------------------------------------------- 1 | { lib }: 2 | { src, drv, version ? null, warn ? true }: 3 | let 4 | optionalVersion = if version == null then {} else { inherit version; }; 5 | override = if drv ? overrideAttrs 6 | then drv.overrideAttrs 7 | else lib.overrideDerivation drv; 8 | in 9 | if lib.isDerivation drv 10 | then override (super: { inherit src; } // optionalVersion) 11 | else ( 12 | if warn then lib.warn "overrideSrc: ${builtins.typeOf drv} is not a derivation, ignoring src ${builtins.toString src}" drv 13 | else drv 14 | ) 15 | -------------------------------------------------------------------------------- /test/nix/wrangle.json: -------------------------------------------------------------------------------- 1 | { 2 | "sources": { 3 | "nix-wrangle": { 4 | "fetch": { 5 | "owner": "timbertson", 6 | "repo": "nix-wrangle", 7 | "rev": "a0dd57880896322f3a2f42239bb017489e0d8fbf", 8 | "sha256": "1pbjyfi7q7475ky0azg9x2c5y7jg9hj81z0awlxngz5cy5r43q6w" 9 | }, 10 | "nix": "default.nix", 11 | "owner": "timbertson", 12 | "ref": "v1", 13 | "repo": "nix-wrangle", 14 | "type": "github" 15 | } 16 | }, 17 | "wrangle": { 18 | "apiversion": 1 19 | } 20 | } -------------------------------------------------------------------------------- /example/03-local-override.gup: -------------------------------------------------------------------------------- 1 | #!bash -eu 2 | [ -f nix/wrangle.json ] || gup 01-setup 3 | 4 | . .doc_setup.sh 5 | 6 | rm -f nix/wrangle-local.json 7 | 8 | # fake an old rev 9 | sed -i -e 's/d805330386553c5784ac7ef48ff38aea716575dc/d805330386553c5784ac7ef48ff38aea716575de/' nix/wrangle.json 10 | set -x 11 | comment_REM 'Let'\''s develop against my local checkout of `piep`.\n# Local overrides are stored in nix/wrangle-local.json, which you shouldn'\''t commit' 12 | nix-wrangle add --local piep --type git-local --path ~/dev/python/piep --nix nix/ 13 | nix-build 14 | cat result 15 | -------------------------------------------------------------------------------- /nix/nixImpure.nix: -------------------------------------------------------------------------------- 1 | { stdenv, nix }: 2 | 3 | attrs: script: 4 | let 5 | defaults = { 6 | name = "nix-impure-script"; 7 | }; 8 | overrides = { 9 | buildCommand = '' 10 | echo "This derivation cannot be built, it is intended for nix-impure" 11 | exit 1 12 | ''; 13 | shellHook = '' 14 | _outbase="$(mktemp -d)" 15 | out="$_outbase/$name" 16 | trap 'rm -rf "$_outbase"' EXIT 17 | exec 3>&1 18 | exec 1>&2 19 | ${script} >&2 20 | ${nix}/bin/nix-store --add "$out" >&3 21 | ''; 22 | }; 23 | in 24 | stdenv.mkDerivation (defaults // attrs // overrides) 25 | -------------------------------------------------------------------------------- /example/.doc_setup.sh: -------------------------------------------------------------------------------- 1 | set -eu 2 | gup -u nix/default.nix 3 | export PS4="\n$ " 4 | exec > >(sed -E -e '/_REM/d' | cat -s | tee "$1") 5 | exec 2>&1 6 | 7 | function nix-build { 8 | # The first (quiet) build is to ensure we're not showing any build output 9 | { env nix-build "$@" 2>&1; } > /dev/null 2>&1 10 | { env nix-build "$@" 2>&1; } 2>/dev/null 11 | } 12 | 13 | function nix-wrangle { 14 | # Make sure we always use the latest built version 15 | { env ../bin/nix-wrangle "$@"; } 2>/dev/null 16 | } 17 | 18 | function comment_REM { 19 | { echo -e -n '#' "$@"; } 2>/dev/null 20 | } 21 | -------------------------------------------------------------------------------- /example/01-setup.gup: -------------------------------------------------------------------------------- 1 | #!bash -eu 2 | rm -f nix/wrangle*.json 3 | . .doc_setup.sh 4 | 5 | set -x 6 | comment_REM 'Here'\''s our derivation, nix/default.nix. It expects a `piep` argument, and uses a relative path for `src`:' 7 | cat nix/default.nix 8 | comment_REM 'Initialize nix/wrangle.json (pass `--pkgs nixos-unstable` to pin nixpkgs version):' 9 | nix-wrangle init 10 | comment_REM 'Now provide the `piep` dependency from a github repo, and build it:' 11 | nix-wrangle add piep timbertson/piep --nix nix/ 12 | nix-build 13 | comment_REM 'And here'\''s the result, with injected source and `piep` dependency:' 14 | cat result 15 | -------------------------------------------------------------------------------- /nix/unpackArchive.nix: -------------------------------------------------------------------------------- 1 | { stdenv }: 2 | path: 3 | stdenv.mkDerivation { 4 | name = "unpack-1"; 5 | buildCommand = '' 6 | srctmp="$out/__nix_unpack_src" 7 | mkdir -p "$srctmp" 8 | cd "$srctmp" 9 | echo "Unpacking source archive: ${path}" 10 | runOneHook unpackCmd "${path}" 11 | numdirs="$(ls -1 | wc -l)" 12 | [ "$numdirs" -eq 1 ] || (echo "expected exactly one directory" >&2; exit 1) 13 | chmod -R u+w -- "$out" 14 | ls -1 | while read root; do 15 | ls -1 "$root" | while read f; do 16 | mv "$root/$f" "$out/" 17 | done 18 | done 19 | cd "$out" 20 | rm -r "$srctmp" 21 | ''; 22 | allowSubstitutes = false; 23 | } 24 | -------------------------------------------------------------------------------- /test/override-src.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | here="$(dirname "$0")" 4 | 5 | gitHeadExport="$("$here/../bin/nix-export-git" --ref HEAD)" 6 | [ -n "$gitHeadExport" ] 7 | gitShaExport="$("$here/../bin/nix-export-git" --commit 0fa3135a596c76cb5c3bae1f9521ea84ef84fef5)" 8 | [ -n "$gitShaExport" ] 9 | gitMasterExport="$("$here/../bin/nix-export-git" --ref master --unpack)" 10 | [ -n "$gitMasterExport" ] 11 | gitWorkingCopyExport="$("$here/../bin/nix-export-git" --working-changes)" 12 | [ -n "$gitWorkingCopyExport" ] 13 | 14 | built="$("$here/../bin/nix-override-src" --src "$gitMasterExport" -A gup "$here/test.nix")" 15 | grep -q "just thoughts right now" "$built" 16 | -------------------------------------------------------------------------------- /test/gup-readme.nix: -------------------------------------------------------------------------------- 1 | { stdenv, fetchFromGitHub, nix-update-source, lib, python, which, pychecker ? null }: 2 | stdenv.mkDerivation rec { 3 | version = "TODO"; 4 | name="gup-${version}"; 5 | src = fetchFromGitHub rec { 6 | owner = "timbertson"; 7 | repo = "gup"; 8 | rev = "version-${meta.version}"; 9 | sha256 = "1pwnmlq2pgkkln9sgz4wlb9dqlqw83bkf105qljnlvggc21zm3pv"; 10 | meta = { 11 | version = "0.7.0"; 12 | update = { current, version }: { 13 | rev = "version-${version}"; 14 | }; 15 | updateManual = { current, version }: { 16 | rev = fetchFromGitHub.impure (current // { sha256 = null; }); 17 | }; 18 | }; 19 | }; 20 | buildCommand = "cat $src/README.md > $out"; 21 | } 22 | -------------------------------------------------------------------------------- /nix/wrangle.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, aeson, aeson-pretty, base, bytestring, data-fix 2 | , directory, exceptions, filepath, hashable, hnix, megaparsec, mtl 3 | , optparse-applicative, prettyprinter, process, regex-tdfa, stdenv 4 | , strict, string-qq, text, unordered-containers 5 | }: 6 | mkDerivation { 7 | pname = "nix-wrangle"; 8 | version = "1.0.0"; 9 | src = ./..; 10 | isLibrary = false; 11 | isExecutable = true; 12 | executableHaskellDepends = [ 13 | aeson aeson-pretty base bytestring data-fix directory exceptions 14 | filepath hashable hnix megaparsec mtl optparse-applicative 15 | prettyprinter process regex-tdfa strict string-qq text 16 | unordered-containers 17 | ]; 18 | license = "unknown"; 19 | hydraPlatforms = stdenv.lib.platforms.none; 20 | } 21 | -------------------------------------------------------------------------------- /test/integration.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | set -o pipefail 4 | here="$(dirname "$0")" 5 | # ensure default-nix is updated 6 | cd "$here" 7 | ../bin/nix-wrangle default-nix 8 | set -x 9 | 10 | function expected_output { 11 | cat < {}).callPackage ../nix {}' 40 | } 41 | 42 | diff <(expected_output) <(actual_output_local) 43 | diff <(expected_output) <(actual_output_github) 44 | -------------------------------------------------------------------------------- /example/default.nix: -------------------------------------------------------------------------------- 1 | # Note: This file is generated by nix-wrangle 2 | # It can be regenerated with `nix-wrangle default-nix` 3 | let 4 | systemNixpkgs = import {}; 5 | fallback = val: dfl: if val == null then dfl else val; 6 | makeFetchers = pkgs: { 7 | github = pkgs.fetchFromGitHub; 8 | url = builtins.fetchTarball; 9 | }; 10 | fetch = pkgs: source: 11 | (builtins.getAttr source.type (makeFetchers pkgs)) source.fetch; 12 | sourcesJson = (builtins.fromJSON (builtins.readFile ./nix/wrangle.json)).sources; 13 | wrangleJson = sourcesJson.nix-wrangle or (abort "No nix-wrangle entry in nix/wrangle.json"); 14 | in 15 | { pkgs ? null, nix-wrangle ? null, ... }@provided: 16 | let 17 | _pkgs = fallback pkgs (if builtins.hasAttr "pkgs" sourcesJson 18 | then fetch systemNixpkgs sourcesJson.pkgs else systemNixpkgs); 19 | _wrangle = fallback nix-wrangle (_pkgs.callPackage "${fetch _pkgs wrangleJson}/${wrangleJson.nix}" {}); 20 | in 21 | (_wrangle.api { pkgs = _pkgs; }).inject { inherit provided; path = ./.; } 22 | 23 | 24 | -------------------------------------------------------------------------------- /nix-wrangle.cabal: -------------------------------------------------------------------------------- 1 | cabal-version: 2.0 2 | 3 | name: nix-wrangle 4 | version: 1.0.0 5 | build-type: Simple 6 | 7 | flag splice 8 | description: enable `splice` subcommand (depends on hnix) 9 | default: True 10 | 11 | executable nix-wrangle 12 | main-is: Main.hs 13 | other-modules: 14 | Wrangle.Cmd 15 | Wrangle.Source 16 | Wrangle.Util 17 | Wrangle.Fetch 18 | hs-source-dirs: 19 | ./src 20 | ghc-options: -Wall -Werror -fno-warn-name-shadowing -fno-warn-missing-signatures 21 | if flag(splice) 22 | cpp-options: -DENABLE_SPLICE 23 | build-depends: 24 | hnix ^>= 0.6.1 25 | , data-fix 26 | , megaparsec 27 | , prettyprinter 28 | other-modules: 29 | Wrangle.Splice 30 | 31 | build-depends: 32 | -- basics 33 | base 34 | , bytestring 35 | , text 36 | , string-qq 37 | 38 | -- util 39 | , filepath 40 | , directory 41 | , exceptions 42 | 43 | -- JSON / hashmap 44 | , aeson 45 | , aeson-pretty 46 | , hashable 47 | , unordered-containers 48 | 49 | -- Cmd 50 | , mtl 51 | , optparse-applicative 52 | 53 | -- Fetch 54 | , regex-tdfa 55 | , process 56 | , strict 57 | 58 | 59 | default-language: Haskell2010 60 | other-extensions: CPP 61 | -------------------------------------------------------------------------------- /nix/default.nix: -------------------------------------------------------------------------------- 1 | { stdenv, lib, git, callPackage, makeWrapper, fetchFromGitHub, haskellPackages, enableSplice ? false, self ? ../. }: 2 | # ./wrangle.nix is the vanilla cabal2nix output, so we wrap it here: 3 | let 4 | splice = 5 | let ret = if enableSplice then { 6 | flag = "+splice"; 7 | filterDeps = x: true; 8 | } else { 9 | flag = "-splice"; 10 | filterDeps = drv: if drv == null then false else 11 | builtins.trace "filtercheck: ${if drv == null then "NULL" else "${drv.pname} v${drv.version or "???"}"}" (drv.pname != "hnix"); # could exclude more, but hnix is the big one 12 | }; in 13 | builtins.trace "splice flag: ${ret.flag}" ret; 14 | 15 | # This is a bit silly. Since `haskellPackages.mkDerivation` embeds so much that's hard to override, 16 | # we fake `mkDerivation` to capture the raw arguments (generated by `cabal2nix`). 17 | # Then we can filter them, _then_ pass them on to the real mkDerivation: 18 | wrapped = haskellPackages.callPackage ./wrangle.nix { mkDerivation = args: 19 | # callPackage merges in some extra attrs to whatever we return, so nest this in `args` 20 | { inherit args; }; 21 | }; 22 | in 23 | 24 | haskellPackages.mkDerivation (let o = wrapped.args; in o // rec { 25 | src = self; 26 | executableHaskellDepends = lib.filter splice.filterDeps (o.executableHaskellDepends); 27 | buildDepends = (o.buildDepends or []) ++ [ makeWrapper ]; 28 | configureFlags = (o.configureFlags or []) ++ ["--flags=${splice.flag}"]; 29 | postInstall = '' 30 | mkdir -p "$out/share" 31 | cp -r "$src/nix" "$out/share/nix" 32 | wrapProgram $out/bin/nix-wrangle \ 33 | --prefix PATH : ${git}/bin \ 34 | --set NIX_WRANGLE_DATA "$out/share" 35 | $out/bin/nix-wrangle installcheck 36 | ''; 37 | passthru = { 38 | api = args: callPackage (../nix + "/api.nix") args; 39 | }; 40 | }) 41 | -------------------------------------------------------------------------------- /nix/exportLocalGit.nix: -------------------------------------------------------------------------------- 1 | { bash, lib, stdenv, git }: 2 | { path, commit ? null, ref ? null, workingChanges ? false }: 3 | let 4 | pathStr = toString path; in 5 | let 6 | path = pathStr; # shadow, so we can't accidentally import `path` 7 | resolveRef = ref: let 8 | candidates = [ 9 | "refs/heads/" 10 | "refs/tags/" 11 | "refs/remotes/" 12 | "" 13 | ]; 14 | prefix = "${path}/.git"; 15 | fullPath = scope: "${prefix}/${scope}${ref}"; 16 | fullCandidates = map fullPath candidates; 17 | refPath = 18 | let found = lib.findFirst builtins.pathExists null fullCandidates; in 19 | if found == null 20 | then abort "No git ref candidates found in ${builtins.toJSON fullCandidates}" 21 | else found; 22 | indirectPrefix = "ref: "; 23 | resolve = refPath: 24 | let result = lib.removeSuffix "\n" (builtins.readFile refPath); in 25 | builtins.trace "Dereferenced git ref ${refPath} to ${result}" ( 26 | if (lib.hasPrefix indirectPrefix result) 27 | then resolve ("${prefix}/${lib.removePrefix indirectPrefix result}") 28 | else result 29 | ); 30 | in resolve refPath; 31 | 32 | commitRequired = abort "exactly one of commit, ref or workingChanges required"; 33 | urlPath = /. + path; 34 | 35 | commitDrv = builtins.fetchGit { url = urlPath; rev = commit; }; 36 | refDrv = builtins.fetchGit { url = urlPath; rev = (resolveRef ref); inherit ref; }; 37 | workspaceDrv = builtins.fetchGit { url = urlPath; }; 38 | in 39 | (lib.findFirst (x: x.condition == true) { drv = commitRequired; } [ 40 | { condition = commit != null && ref == null && !workingChanges; drv = commitDrv; } 41 | { condition = commit == null && ref != null && !workingChanges; drv = refDrv; } 42 | { condition = commit == null && ref == null && workingChanges; drv = workspaceDrv; } 43 | ]).drv 44 | -------------------------------------------------------------------------------- /nix/importDrv.nix: -------------------------------------------------------------------------------- 1 | { nix, runCommand, lib }: 2 | 3 | # Takes a nix derivation path (must be in the store), and 4 | # converts it to a derivation expression. 5 | # Note that the result is not a `stdenv.mkDerivation` with 6 | # all the helpers that entails (like overrideAttrs), but 7 | # is a low-level call to `derivation` 8 | # 9 | # TODO: this likely doesn't suppot multi-output drvs, we'd 10 | # need more smarts around `outputs`. 11 | drvPath: 12 | with builtins; with lib; 13 | let 14 | # TODO: this introduces import-from-derivation, is there some 15 | # way to shift this to eval time? 16 | jsonFile = 17 | assert (isStorePath drvPath); 18 | runCommand "drv.json" {} '' 19 | ${nix}/bin/nix show-derivation ${toString drvPath} > "$out" 20 | ''; 21 | 22 | drvJson = importJSON jsonFile; 23 | 24 | # The JSON has a single toplevel key of the .drv path 25 | rawDrv = getAttr (toString drvPath) drvJson; 26 | 27 | outputs = attrNames rawDrv.outputs; 28 | 29 | filteredEnv = filterAttrs (k: v: 30 | !(elem k outputs) 31 | ) rawDrv.env; 32 | 33 | # We need to produce a derivation with the same inputSrcs and inputDrvs, 34 | # which we don't get just by copying the attributes. 35 | # We know there's always a `builder` attribute, so we manually build 36 | # a string with all the original context of the derivation: 37 | builderWithCtx = with lib; 38 | let 39 | getAllAttrs = src: attrs: map (name: getAttr name src) attrs; 40 | importInputs = attrs: 41 | concatLists ( 42 | mapAttrsToList 43 | # each attr is an attrset with key = path-to-drv and value = list of outputs (attributes) 44 | (name: outputs: getAllAttrs (import name) outputs) 45 | attrs 46 | ); 47 | addContextFrom = orig: dest: 48 | # warn ("Adding context from: ${orig}") 49 | (lib.addContextFrom orig dest); 50 | in 51 | foldr addContextFrom filteredEnv.builder ((importInputs rawDrv.inputDrvs) ++ (map storePath rawDrv.inputSrcs)); 52 | 53 | drvAttrs = filteredEnv // { 54 | inherit outputs; 55 | inherit (rawDrv) args; 56 | builder = builderWithCtx; 57 | }; 58 | in 59 | derivation drvAttrs 60 | -------------------------------------------------------------------------------- /src/Wrangle/Util.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE LambdaCase #-} 2 | 3 | module Wrangle.Util where 4 | 5 | import Prelude hiding (error) 6 | import Control.Monad.Catch 7 | import Data.String (IsString, fromString) 8 | import System.Exit (exitFailure) 9 | import System.IO.Unsafe (unsafePerformIO) 10 | import qualified System.Environment as Env 11 | 12 | -- Awkward workaround for not knowing the type of a string literal 13 | s :: String -> String 14 | s = id 15 | 16 | abort :: String -> IO a 17 | abort msg = do 18 | putStrLn msg 19 | exitFailure 20 | 21 | liftEither :: MonadThrow m => Exception a => Either a b -> m b 22 | liftEither (Left err) = throwM err 23 | liftEither (Right x) = return x 24 | 25 | liftMaybe :: MonadThrow m => Exception a => a -> Maybe b -> m b 26 | liftMaybe err Nothing = throwM err 27 | liftMaybe _ (Just x) = return x 28 | 29 | -- XXX this should just use `liftMaybe`, but that has a weird 30 | -- TypeFamily restriction that e ~ SomeError, which breaks things 31 | toRight :: a -> Maybe b -> Either a b 32 | toRight err Nothing = Left err 33 | toRight _ (Just x) = Right x 34 | 35 | mapLeft :: (a -> c) -> Either a b -> Either c b 36 | mapLeft fn (Left x) = Left (fn x) 37 | mapLeft _ (Right x) = Right x 38 | 39 | orElse (Just x) _ = x 40 | orElse Nothing x = x 41 | 42 | optList (Just x) = [x] 43 | optList Nothing = [] 44 | 45 | orEither (Right x) _ = x 46 | orEither (Left _) dfl = dfl 47 | 48 | orTry (Right x) _ = (Right x) 49 | orTry (Left _) alt = alt 50 | 51 | debugLn :: String -> IO () 52 | debugLn = case unsafePerformIO (Env.lookupEnv "WRANGLE_DEBUG") of 53 | (Just "1") -> putStrLn . ("[debug]: " <>) 54 | (Just _) -> noop 55 | Nothing -> noop 56 | where 57 | noop _ = return () 58 | 59 | infoLn :: String -> IO () 60 | infoLn = putStrLn 61 | 62 | errorLn :: String -> IO () 63 | errorLn = putStrLn . ("[error]: " <>) 64 | 65 | tap :: (a -> IO ()) -> IO a -> IO a 66 | tap action x = x >>= (\x -> action x >> return x) 67 | 68 | newtype AppError = AppError String 69 | instance Show AppError where show (AppError s) = s 70 | instance Semigroup AppError where (<>) (AppError a) (AppError b) = AppError $ a <> b 71 | instance Exception AppError 72 | instance IsString AppError where fromString = AppError 73 | 74 | prefixAppError :: String -> Either AppError a -> Either AppError a 75 | prefixAppError p = mapLeft ((AppError p) <>) 76 | -------------------------------------------------------------------------------- /README.md.gup: -------------------------------------------------------------------------------- 1 | #!bash -eu 2 | set -o pipefail 3 | # vim: set syntax=markdown: 4 | gup --always 5 | dest="$1" 6 | function include { 7 | echo >&2 "Including $1" 8 | gup -u "$1" 9 | echo '```bash' >> "$dest" 10 | cat "$1" >> "$dest" 11 | echo '```' >> "$dest" 12 | } 13 | 14 | echo '' > "$1" 15 | 16 | set -x 17 | 18 | cat >> "$dest" <<"EOF" 19 | # nix-wrangle is deprecated 20 | 21 | `nix-wrangle` is featureful but somewhat complex to use and maintain. I've decided to use the more basic [niv](https://github.com/nmattia/niv) myself rather than maintaining nix-wrangle, I suggest others do the same. 22 | 23 | ## Purpose: 24 | 25 | Nix-wrangle aims to be a swiss-army knife for working with nix dependencies. 26 | It works best with dependencies that include their own nix derivations, although using it for fetching plain archives works too. 27 | 28 | ### Goals: 29 | 30 | * Simple usage should be _idiomatic_ and portable, with no specific references to nix-wrangle's API 31 | * Keeping sources updated (and seeing their current state) should be trivial 32 | * Support local development across multiple related repositories 33 | 34 | ## Get it: 35 | 36 | EOF 37 | include example/00-bootstrap 38 | cat >> "$dest" <<"EOF" 39 | 40 | ## Basic functionality: 41 | 42 | nix-wrangle maintains a set of sources, which are typically dependencies. The following example shows the basic setup: 43 | 44 | EOF 45 | include example/01-setup 46 | cat >> "$dest" <<"EOF" 47 | 48 | Note that the `piep` dependency is built (by using `pkgs.callPackage` on the nix path within the source), which gives you the actual derivation, not simply the source code. This is one important difference compared to [niv][]. 49 | 50 | Sources are typically used for project dependencies, but there are three special sources: 51 | 52 | - 'nix-wrangle': (added automatically) used for bootstrapping in 'default.nix' 53 | - 'self': used to override the toplevel derivation's 'src' attribute when building 'default.nix'. Automatically added by `nix-wrangle init` as a `git-local` source if there's a `.git` directory present. 54 | - 'pkgs': (optional) used to pin the exact version of 'nixpkgs'. Added automatically by `nix-wrangle init` if you pass `--pkgs nixpkgs-unstable` (or any other branch name from https://github.com/NixOS/nixpkgs-channels) 55 | 56 | # Features: 57 | 58 | Nix-wrangle is purpose built to solve a range of specific use cases which come up when developing with nix: 59 | 60 | ## Update from specification 61 | 62 | e.g. when using a git-based dependency, you can specify a branch instead of a specific commit. When you `update`, that branch will be re-resolved: 63 | 64 | EOF 65 | include example/02-update 66 | cat >> "$dest" <<"EOF" 67 | 68 | You can also make use of templating, e.g. for a URL dependency you can use the URL `'http://example.com/libfoo/libfoo-.tgz'`. When updating, you can pass `--version NEW_VERSION` to update it. 69 | 70 | ## 'src' injection 71 | 72 | When building, a `self` dependency (if present) will be used to provide the `src` for the toplevel derivation. This lets you use nix-wrangle's various source types (e.g git-local, which isn't available as a `nixpkgs` builtin) and automatic generation (e.g. sha256 digests). 73 | 74 | nix-wrangle will also inject the relevant `src` into each of your dependencies. Let's say you import commit `x` of the `piep` dependency, with a nix expression in it. It would be impossible for commit `x` of `piep` to refer to commit `x` as its `src` attribute. The best it could do is to refer to the parent of commit `x`, although it may often just refer to the most recently released version. Both of these would be counter-productive - you'd be importing the `derivation` at `x`, but building source code from _some other version_ out of your control. `nix-wrangle` automatically overrides the `src` of imported dependencies so that the version you _import_ is also the source code you _build_. 75 | 76 | ## Local overrides 77 | 78 | When working on both a library and an application that uses it, it's common to want to try out working changes before publishing them. This is easy with local sources: 79 | 80 | EOF 81 | include example/03-local-override 82 | cat >> "$dest" <<"EOF" 83 | 84 | This uses the local version of a dependency for building, but kept separate from the "public" version of your dependency specificaion. 85 | 86 | ## Splicing `src` to produce a self-contained derivation 87 | 88 | nix-wrangle was built so that your base derivation (`nix/default.nix`) can be idiomatic - it doesn't need to reference `nix-wrangle` at all, and its dependencies are injected as arguments, just like regular derivations in `nixpkgs`. The one way in which they aren't idiomatic is the `src` attribute, since in nixpkgs this typically refers to a remote repository or tarball. 89 | 90 | So there's also the `splice` command. This injects the current value of a fetched source (defaulting to `public`) into an existing nix file to create a self-contained derivation. This is perfect for promoting your in-tree derivation (with source provided by nix-wrangle) into a derivation suitable for inclusion in `nixpkgs`, where it includes its own `src` and all dependencies are provided by the caller. 91 | 92 | EOF 93 | include example/04-splice 94 | cat >> "$dest" <<"EOF" 95 | 96 | # Source types 97 | 98 | - `url`: any archive URL. May contain `` which is resolved on `update`. 99 | - `git`: takes a `url` and `rev` (which can be a branch, tag or commit). `rev` is resolved to a concrete commit on initial add, and on `update`. 100 | - `github`: takes an `owner`, `repo` and `rev` (as for `git`) 101 | - `git-local`: takes a path (can be relative) and an optional `rev` (can be a branch, tag or `HEAD`). `rev` is resolved _at evaluation time_. If `rev` is not provided, you'll get the _working changes_ in the given workspace (but not any excluded or untracked files). 102 | - `path`: path (can be relative or absolute) 103 | 104 | ---- 105 | 106 | # Hacking 107 | 108 | There are two options for development: 109 | 110 | From within 'nix-shell', use 'cabal v1-build' 111 | 112 | Alternatively, from within 'nix-shell -p haskellPackages.cabal-install' you can use 'cabal new-build'. This bypasses the nix infrastructure entirely and fetches its own copy of dependencies, but may be convenient when adjusting packages or pinning dependencies to specific versions. 113 | 114 | # Similar tools 115 | 116 | ### niv: 117 | 118 | nix-wrangle was heavily inspired by [niv][] (the command line tool was even based off the niv source code). 119 | The main differences are: 120 | 121 | - nix-wrangle dependencies are derivations, not just source code. 122 | - nix-wrangle attempts to let you write idiomatic nix without explicitly referencing any files or functions provided by nix-wrangle. 123 | - nix-wrangle has a number of extra features not provided by niv: `splice`, `self` injection and local overlays. 124 | - nix-wrangle is more featureful, but also more complex 125 | 126 | ### nix-flakes: 127 | 128 | nix-wrangle (like [niv][]) is similar in spirit to [nix flakes](https://gist.github.com/edolstra/40da6e3a4d4ee8fd019395365e0772e7), but there's no actual implementation of flakes yet. My hope is that any standard solution would be able to support nix-wrangle style workflows. 129 | 130 | [niv]: https://github.com/nmattia/niv 131 | 132 | EOF 133 | -------------------------------------------------------------------------------- /test/unit.nix: -------------------------------------------------------------------------------- 1 | with import {}; 2 | with builtins; 3 | with lib; 4 | let 5 | api = (callPackage ../nix/api.nix {}); 6 | internal = api.internal; 7 | 8 | wrangleHeader = { apiversion = 1; }; 9 | addHeader = j: j // { wrangle = wrangleHeader; }; 10 | eq = name: a: b: 11 | [ name (a == b) ("${toJSON a} != ${toJSON b}") ]; 12 | 13 | testFailure = test: 14 | let 15 | result = if elemAt test 1 then [] else [ "${name}${suffix}" ]; 16 | name = elemAt test 0; 17 | # third element is an optional message 18 | suffix = if length test == 2 then "" else ":\n ${elemAt test 2}"; 19 | testOnly = builtins.getEnv "TEST_ONLY"; 20 | runTest = if testOnly == "" then true else name == testOnly; 21 | logTest = 22 | if builtins.getEnv "WRANGLE_DEBUG" == "1" then 23 | lib.warn "Executing test case: ${elemAt test 0}" 24 | else result: result; 25 | in 26 | if runTest then logTest result else []; 27 | 28 | versionSrc = import samplePackage/versionSrc.nix; 29 | 30 | version = addHeader { 31 | sources = { 32 | version = versionSrc; 33 | }; 34 | }; 35 | 36 | versionNoImport = { 37 | type = "github"; 38 | fetch = { 39 | "owner" = "timbertson"; 40 | "repo" = "version"; 41 | "rev" = "version-0.13.1"; 42 | "sha256" = "056l8m0xxl1h7x4fd1hf754w8q4sibpqnwgnbk5af5i66399az61"; 43 | }; 44 | }; 45 | 46 | fakeNixpkgs = addHeader { 47 | sources = { 48 | nixpkgs = { 49 | type = "path"; 50 | fetch = { path = (toString ./.); }; 51 | nix = "fakeNixpkgs.nix"; 52 | }; 53 | }; 54 | }; 55 | 56 | makeImport = internal.makeImport { 57 | settings = { path = null; }; 58 | inherit pkgs; 59 | }; 60 | 61 | tests = [ 62 | ["implPath is path" (isString (makeImport "name" versionSrc).nix)] 63 | 64 | ((result: eq "returns source if nix is unset" result.src result.drv) 65 | (makeImport "name" versionNoImport)) 66 | 67 | ["nix is modifiable" (hasSuffix "/foo.nix" (makeImport "name" (versionSrc // {nix = "foo.nix";})).nix)] 68 | 69 | ["src is derivation" (isDerivation (makeImport "name" versionSrc).src)] 70 | 71 | (eq "passthru name" (makeImport "name" versionSrc).name "name") 72 | 73 | (eq "passthru attrs" (makeImport "name" versionSrc).attrs versionSrc) 74 | 75 | ["importScope produces valid derivations" 76 | (isDerivation (internal.importScope pkgs { 77 | versionOverride = (makeImport "versionOverride" versionSrc); 78 | }).versionOverride)] 79 | 80 | ( 81 | let myPkgs = internal.importScope pkgs { 82 | myVersion = makeImport "myVersion" versionSrc; 83 | }; in 84 | eq "importScope provides imports to callPackage invocations" 85 | (myPkgs.callPackage ({ myVersion, curl, pkgs }: [myVersion.name curl.name myPkgs.curl.name pkgs.curl.name]) {}) 86 | [myPkgs.myVersion.name curl.name curl.name curl.name] 87 | ) 88 | 89 | ["makes derivations" (isDerivation (api.derivations { sources = [ version ]; }).version)] 90 | 91 | (eq "provides inputs to be used by other packages" ( 92 | (api.importFrom { sources = [ 93 | (addHeader { sources = { 94 | dep = { 95 | type="_passthru"; 96 | fetch = null; 97 | nix = ./samplePackage/dep.nix; 98 | }; 99 | dep-user = { 100 | type="_passthru"; 101 | fetch = null; 102 | nix = ./samplePackage/dep-user.nix; 103 | }; 104 | }; }) 105 | ]; }).sources.dep-user.drv.depProvided) 106 | true) 107 | 108 | (eq "doesn't override imports globally" 109 | # import a bunch of named sources that `git` depends on, 110 | # then assert that the resulting git is equal to the 111 | # upstream one, because its dependencies haven't changed 112 | (api.importFrom { sources = 113 | let impl = { 114 | type="_passthru"; 115 | fetch = null; 116 | nix = ./samplePackage/unbuildable.nix; 117 | }; in 118 | [(addHeader { sources = { 119 | curl = impl; 120 | openssl = impl; 121 | zlib = impl; 122 | openssh = impl; 123 | stdenv = impl; 124 | upstreamGit = { 125 | type="_passthru"; 126 | fetch = null; 127 | nix = ./samplePackage/exposeGit.nix; 128 | }; 129 | }; })]; 130 | }).sources.upstreamGit.drv.git 131 | pkgs.git) 132 | 133 | (eq "normalizes non-store paths without adding them to the store" ( 134 | internal.expandRelativePathWithoutImporting ./. "." 135 | ) ("${builtins.getEnv "PWD"}/test") ) 136 | 137 | (eq "normalizes store paths" ( 138 | internal.expandRelativePath curl "." 139 | ) curl.outPath ) 140 | 141 | (eq "imports from git when path is not a store path" ( 142 | let result = ((internal.makeFetchers { path = ./storeSrc; }) 143 | .git-local { relativePath = "../.."; ref="HEAD"; }); in 144 | result 145 | ) (builtins.fetchGit { url = ../.; ref="HEAD"; }) ) 146 | 147 | (eq "uses store path directly path is a store path" ( 148 | # "${x}" copies path x into the store 149 | let result = ((internal.makeFetchers { path = "${./storeSrc}"; }) 150 | .git-local { relativePath = "."; ref="HEAD"; }); in 151 | [(typeOf result) (isDerivation result) result] 152 | ) ["string" false "${./storeSrc}"]) 153 | 154 | (eq "importFrom merges packages, not recursively" ( 155 | (api.importFrom { sources = [ 156 | (addHeader { sources = { 157 | first = { type="git"; key = "value1"; }; 158 | both = { type="git"; key1 = "key1value1"; key2 = "key2value1";}; 159 | }; }) 160 | (addHeader { sources = { 161 | both = { type="git"; key2 = "key2value2"; key3 = "key3value2"; }; 162 | second = { type="git"; key = "value2"; }; 163 | }; }) 164 | ]; }).sourceAttrs) 165 | { 166 | first = { type="git"; key = "value1"; }; 167 | both = { type="git"; key2 = "key2value2"; key3 = "key3value2"; }; 168 | second = { type="git"; key = "value2"; }; 169 | }) 170 | 171 | (eq "allows overriding of individual package invocations" "injected" (api.derivations { 172 | sources = [ version ]; 173 | extend = nodes: { 174 | version = { 175 | call = { pkgs, path }: ((pkgs.callPackage path {}).overrideAttrs (o: { 176 | passthru = { extra = "injected"; }; 177 | })); 178 | }; 179 | }; 180 | }).version.extra) 181 | 182 | (eq "inject works with just a path" ./samplePackage/upstream-src 183 | (api.inject { path = ./samplePackage; }).src) 184 | 185 | (eq "inject works with an attrset and no `self`" ./samplePackage/upstream-src ( 186 | api.inject { 187 | sources = [ version ]; 188 | nix = ({ pkgs, version }: pkgs.callPackage ./samplePackage/nix {}); 189 | } 190 | ).src) 191 | 192 | (eq "inject overrides src if `self` is given" "${./samplePackage/local-src}" ( 193 | api.inject { 194 | sources = [ version (addHeader { 195 | sources.self = { type = "path"; fetch = { path = ./samplePackage/local-src; }; }; 196 | })]; 197 | nix = ({ pkgs, version }: pkgs.callPackage ./samplePackage/nix {}); 198 | } 199 | ).src) 200 | 201 | (eq "inject uses provides packages in callPackage scope" ( 202 | api.inject { 203 | sources = []; 204 | nix = ({ foo, self }: foo); 205 | provided = { foo = "hello from FOO!"; }; 206 | }) 207 | ("hello from FOO!") 208 | ) 209 | 210 | (eq "makeImport provides sources individually via `self`" ( 211 | let imports = internal.importsOfJson {path = null;} { 212 | version = versionSrc // { 213 | nix = ./return-self-1.nix; 214 | }; 215 | curl = { 216 | type = "_passthru"; 217 | fetch = curl.src; 218 | nix = ./return-self-2.nix; 219 | }; 220 | }; in [imports.version.drv imports.curl.drv]) 221 | [ (makeImport "name" versionSrc).src curl.src ]) 222 | 223 | ]; 224 | failures = concatMap testFailure tests; 225 | in 226 | if (length failures) == 0 then "OK" else abort (lib.concatStringsSep "\n" failures) 227 | -------------------------------------------------------------------------------- /src/Wrangle/Splice.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DerivingStrategies #-} 2 | {-# LANGUAGE NamedFieldPuns #-} 3 | {-# LANGUAGE GeneralizedNewtypeDeriving #-} 4 | {-# LANGUAGE OverloadedStrings #-} 5 | {-# LANGUAGE ScopedTypeVariables #-} 6 | {-# LANGUAGE TupleSections #-} 7 | 8 | module Wrangle.Splice where 9 | 10 | import Prelude hiding (error) 11 | import Wrangle.Util 12 | import Data.Maybe (fromMaybe) 13 | import Data.List.NonEmpty (NonEmpty((:|))) 14 | import Data.Fix 15 | -- import System.IO.Unsafe 16 | import Nix.Expr hiding (stripAnnotation) 17 | import Nix.Parser (Result(..), parseNixTextLoc) 18 | import qualified Data.HashMap.Strict as HMap 19 | import qualified Wrangle.Source as Source 20 | import qualified Nix.Expr as Expr 21 | import qualified Nix.Expr.Shorthands as Shorthands 22 | import qualified Nix.Pretty as Pretty 23 | import qualified Text.Megaparsec as Megaparsec 24 | import qualified Data.Text as T 25 | import qualified Data.Text.IO as T 26 | import qualified Data.Text.Prettyprint.Doc as Doc 27 | import qualified Data.Text.Prettyprint.Doc.Render.Text as Doc 28 | 29 | data Opts = Opts { 30 | input :: FilePath, 31 | output :: Maybe FilePath, 32 | depName :: String 33 | } 34 | 35 | data Indent = Indent { 36 | tabs :: Int, 37 | spaces :: Int 38 | } 39 | 40 | load :: FilePath -> IO T.Text 41 | load = T.readFile 42 | 43 | parse :: T.Text -> Result NExprLoc 44 | parse = parseNixTextLoc 45 | 46 | getExn :: Result NExprLoc -> IO NExprLoc 47 | getExn (Success x) = return x 48 | getExn (Failure f) = abort $ show f 49 | 50 | repr :: NExprLoc -> String 51 | repr expr = show $ stripAnnotation expr 52 | 53 | stripAnnotation :: NExprLoc -> NExpr 54 | stripAnnotation = Expr.stripAnnotation 55 | pretty = Pretty.prettyNix 56 | 57 | nixFetcherName = Source.fetcherNameNix . Source.fetchType . Source.sourceSpec 58 | 59 | nixOfSrc :: Source.PackageSpec -> Either AppError (NExpr, NExpr) 60 | nixOfSrc src = build <$> nixFetcherName src 61 | where 62 | build nixFetcher = (Fix . NSym . T.pack $ nixFetcher, nixAttrs fetchAttrs) 63 | fetchAttrs = HMap.toList $ Source.fetchAttrs src 64 | var :: String -> NAttrPath NExpr 65 | var s = (StaticKey (T.pack s)) :| [] 66 | nixAttrs :: [(String, String)] -> NExpr 67 | nixAttrs items = Fix $ NSet $ map strBinding items 68 | strBinding :: (String, String) -> Binding NExpr 69 | strBinding (key, val) = NamedVar (var key) (Shorthands.mkStr (T.pack val)) nullPos 70 | 71 | replaceSourceLoc :: T.Text -> Source.PackageSpec -> (Maybe (Fix NExprF), SrcSpan) -> Either AppError T.Text 72 | replaceSourceLoc orig src (originalFetcherFn, span) = render <$> nixOfSrc src where 73 | render (defaultFetcherFn, fetcherArgs) = T.unlines $ 74 | (take (startLine-1) origLines) 75 | ++ [ 76 | partialStart <> 77 | -- "<<<" <> (T.pack $ show $ spanBegin span) <> 78 | srcText <> 79 | -- (T.pack $ show $ spanEnd span) <> ">>>" <> 80 | partialEnd 81 | ] ++ (drop (endLine) origLines) 82 | where 83 | origLines = T.lines orig 84 | megaparsecTabWidth = unPos Megaparsec.defaultTabWidth 85 | nixIndentWidth = 2 -- hardcoded in hnix 86 | 87 | colWidth tabWidth '\t' = tabWidth 88 | colWidth _ _ = 1 89 | isTab = (== '\t') 90 | isSpace = (== ' ') 91 | 92 | -- TODO surely megaparsec must have some niceness for this... 93 | columnToIndex :: Int -> T.Text -> Int 94 | columnToIndex col text = columnToIndex' 0 0 (T.unpack text) 95 | where 96 | columnToIndex' _pos index [] = index 97 | columnToIndex' pos index (char: remainder) = 98 | if pos >= col 99 | then index 100 | else columnToIndex' (pos + (colWidth megaparsecTabWidth) char) (index+1) remainder 101 | 102 | partialStart = T.take (columnToIndex (startCol-1) line) line where line = origLines !! (startLine - 1) 103 | partialEnd = T.drop (columnToIndex (endCol-1) line) line where line = origLines !! (endLine - 1) 104 | startLine = (unPos . sourceLine . spanBegin) span 105 | startCol = (unPos . sourceColumn . spanBegin) span 106 | endLine = (unPos . sourceLine . spanEnd) span 107 | endCol = (unPos . sourceColumn . spanEnd) span 108 | 109 | -- TODO: warn if `originalFetcherFn` name differs meaningfully from `defaultFetcherFn` 110 | -- (e.g. pkgs.fetchurl should be fine for fetchurl, but not fetchgit) 111 | srcExpr = Fix $ NBinary NApp (fromMaybe (defaultFetcherFn) originalFetcherFn) fetcherArgs 112 | layoutOpts = Doc.defaultLayoutOptions { Doc.layoutPageWidth = Doc.AvailablePerLine 1 1.0} 113 | prettyPrint = Doc.renderStrict . Doc.removeTrailingWhitespace . Doc.layoutPretty layoutOpts 114 | srcDoc = Pretty.prettyNix srcExpr 115 | srcTextRaw = fixupFinalLine $ prettyPrint $ Doc.nest (tabIndent + spaceIndent) srcDoc 116 | where 117 | tabIndent = nixIndentWidth * (tabs leadingIndent) 118 | spaceIndent = (spaces leadingIndent) 119 | srcText = T.intercalate "\n" (replaceLeadingIndents (T.lines srcTextRaw)) 120 | 121 | replaceLeadingIndents lines = map replaceLeadingIndent lines 122 | replaceLeadingIndent = if tabs leadingIndent == 0 then id else injectTabs 123 | 124 | leadingIndent = Indent { tabs = T.length tabs, spaces = T.length spaces } 125 | where 126 | (tabs, afterTabs) = T.span isTab partialStart 127 | (spaces, _) = T.span isSpace afterTabs 128 | 129 | injectTabs line = indent <> text 130 | where 131 | (leadingSpaces, text) = T.span isSpace line 132 | indentWidth = T.length leadingSpaces 133 | numTabs = indentWidth `quot` nixIndentWidth 134 | numSpaces = indentWidth - (nixIndentWidth * numTabs) 135 | indent = (T.replicate numTabs "\t") <> (T.replicate numSpaces " ") 136 | 137 | -- hnix pretty-print aligns closing brace with attributes, which looks weird. 138 | -- This always operates on pretty-printed nix, so indent is 2 spaces 139 | fixupFinalLine srcText = fixupFinalLine' (T.lines srcText) 140 | where 141 | fixupFinalLine' [] = "" 142 | fixupFinalLine' [last] = reduceIndent last 143 | fixupFinalLine' (line : tail) = line <> "\n" <> (fixupFinalLine' tail) 144 | 145 | reduceIndent line = newSpaces <> remainder where 146 | newSpaces = T.drop nixIndentWidth leadingSpaces 147 | (leadingSpaces, remainder) = T.span isSpace line 148 | 149 | extractSourceLocs :: Fix NExprLocF -> [(Maybe (Fix NExprF), SrcSpan)] 150 | extractSourceLocs expr = 151 | foldMap extractSources (unFix expr) where 152 | dbg :: String -> a -> a 153 | -- dbg s x = unsafePerformIO (putStrLn s >> return x) 154 | dbg _ x = x 155 | extractSources :: Fix NExprLocF -> [(Maybe (Fix NExprF), SrcSpan)] 156 | extractSources expr = 157 | case value of 158 | (NSet bindings) -> dbg "NSet bindings" $ extractSourceBindings bindings 159 | (NRecSet bindings) -> dbg "NRecSet bindings" $ extractSourceBindings bindings 160 | other -> foldMap extractSources other -- dbg ("nothing interesting -- " ++ (show other)) [] 161 | where 162 | node = (getCompose . unFix) expr 163 | value = annotated node 164 | 165 | extractSourceBindings :: [Binding (Fix NExprLocF)] -> [(Maybe (Fix NExprF), SrcSpan)] 166 | extractSourceBindings bindings = concatMap apply bindings where 167 | apply :: Binding (Fix NExprLocF) -> [(Maybe (Fix NExprF), SrcSpan)] 168 | apply (NamedVar ((StaticKey "src") :| []) value _pos) = 169 | (extractSourceFn value, sourcePos) : extractSourceLocs value 170 | where 171 | sourcePos = (annotation . getCompose . unFix) value 172 | apply _ = dbg "non-srv binding" [] 173 | 174 | extractSourceFn :: Fix NExprLocF -> Maybe (Fix NExprF) 175 | extractSourceFn expr = case nexpr of 176 | (NBinary NApp fn _args) -> Just (Expr.stripAnnotation fn) 177 | _ -> Nothing 178 | where 179 | nexpr = annotated . getCompose . unFix $ expr 180 | 181 | 182 | -------------------------------------------------------------------------------- /nix/api.nix: -------------------------------------------------------------------------------- 1 | { pkgs, lib, stdenv, fetchFromGitHub, fetchurl, fetchgit }: 2 | with lib; 3 | let 4 | _nixpkgs = pkgs; 5 | infoLn = msg: ret: builtins.trace ("[wrangle] " + msg) ret; 6 | debugLn = if builtins.getEnv "WRANGLE_DEBUG" == "1" then infoLn else (msg: ret: ret); 7 | utils = rec { 8 | # sub-tools implemented in their own nix files 9 | exportLocalGit = _nixpkgs.callPackage ./exportLocalGit.nix { }; 10 | overrideSrc = _nixpkgs.callPackage ./overrideSrc.nix { }; 11 | }; 12 | 13 | ensureFunction = nixPath: if isFunction nixPath 14 | then nixPath 15 | else ( 16 | let imported = import nixPath; in 17 | if isFunction imported 18 | then imported 19 | else abort "Imported expression ${nixPath} is not a function" 20 | ); 21 | 22 | augmentGitLocalArgs = { path, commit ? null, ref ? null }@args: 23 | if commit == null && ref == null then (args // { workingChanges = true; }) else args; 24 | 25 | # exposed for testing 26 | internal = with api; rec { 27 | 28 | expandRelativePath = base: relative: 29 | if base == null 30 | then abort "relativePath only supported when using `inject` with a path" 31 | # toPath normalizes `/some-path/.` into simply `/some-path` 32 | else with builtins; toPath "${base}/${relative}"; 33 | 34 | expandRelativePathWithoutImporting = base: relative: 35 | with builtins; 36 | # both toStrings are needed to prevent depending 37 | # on the base / final paths (which would import 38 | # them into the store) 39 | toString (expandRelativePath (toString base) relative); 40 | 41 | makeFetchers = { path }: 42 | let withRelativePath = fn: args: 43 | # inject support for `relativePath` as long as 44 | # we were invoked with a base path 45 | if args ? relativePath then ( 46 | # if we're fetching anything from a path which is already in the store, 47 | # short-circuit and just use that path 48 | let 49 | fromStore = isStorePath path; 50 | expandPath = if fromStore 51 | then expandRelativePath 52 | else expandRelativePathWithoutImporting; 53 | fullPath = expandPath path args.relativePath; 54 | in 55 | if fromStore 56 | then debugLn "Imported from store, resolved ${args.relativePath} to ${fullPath}" fullPath 57 | else debugLn "Calling fetcher with resolved path ${fullPath} " ( 58 | fn ({ path = fullPath; } // (filterAttrs (n: v: n != "relativePath") args)) 59 | ) 60 | ) else ( 61 | fn args 62 | ); 63 | in { 64 | github = fetchFromGitHub; 65 | url = fetchurl; 66 | git = fetchgit; 67 | git-local = withRelativePath (args: exportLocalGit (augmentGitLocalArgs args)); 68 | path = withRelativePath ({ path }: "${path}"); 69 | _passthru = arg: arg; # used in tests 70 | }; 71 | 72 | importScope = pkgs: attrs: let 73 | sources = mapAttrs (name: node: node.src) attrs; 74 | nodes = attrValues attrs; 75 | derivations = mapAttrs (name: node: node.drv) attrs; 76 | # This is a bit weird: for each invocation of callPackage, if it's a known 77 | # wrangle it gets `self` injected as appropriate. 78 | newScope = scope: let call = pkgs.newScope scope; in 79 | pkg: args: let 80 | pkgFn = ensureFunction pkg; 81 | node = findFirst (node: node.nix == pkg) null nodes; 82 | selfNeeded = (functionArgs pkgFn) ? "self" && (! args ? "self"); 83 | overrideSelf = if node == null 84 | then debugLn 85 | "Importing a function accepting `self`, but it does not match a known wrangle path" 86 | args 87 | else debugLn 88 | "Injecting node-specific `self`" 89 | (args // { self = node.src; }); 90 | callArgs = if selfNeeded then overrideSelf else args; 91 | in call pkgFn callArgs; 92 | in 93 | lib.makeScope newScope (self: pkgs // derivations); 94 | 95 | makeImport = { settings, pkgs }: name: attrs: 96 | let 97 | fetchers = makeFetchers settings; 98 | fetcher = attrs.type; 99 | fetchArgs = attrs.fetch; 100 | src = if builtins.hasAttr fetcher fetchers 101 | then (builtins.getAttr fetcher fetchers) fetchArgs 102 | else abort "Unknown fetcher: ${fetcher}" 103 | ; 104 | nixAttr = attrs.nix or null; 105 | nix = if nixAttr == null then null else ( 106 | if isString nixAttr then "${src}/${nixAttr}" 107 | else nixAttr # used only in tests 108 | ); 109 | version = attrs.version or (fetchArgs.ref or null); 110 | 111 | defaultCall = { pkgs, path }: pkgs.callPackage path {}; 112 | callImpl = attrs.call or defaultCall; 113 | 114 | callWith = args: 115 | # If attrs.nix == null, we return the source instead of a derivation 116 | if nix == null 117 | then src 118 | else overrideSrc { 119 | inherit src version; 120 | drv = callImpl args; 121 | }; 122 | nixDesc = if (isString nix || builtins.isPath nix) then nix else src; 123 | drv = debugLn "calling ${nixDesc}" (callWith { inherit pkgs; path = nix; }); 124 | in 125 | infoLn "${if nix == null then "Providing source" else "Importing derivation"} ${name} (${fetcher}) from ${nixDesc}" 126 | { inherit name attrs src version nix drv; }; 127 | 128 | importsOfJson = settings: json: 129 | # build a package scope with all imported packages present, 130 | # allowing packages in the set to depend on each other 131 | let 132 | imports = (mapAttrs (makeImport { 133 | inherit settings; 134 | pkgs = importScope pkgs imports; 135 | }) json); 136 | in 137 | imports; 138 | }; 139 | 140 | api = with internal; with utils; utils // (rec { 141 | inherit internal; 142 | 143 | importJsonSrc = path: 144 | let attrs = if isAttrs path 145 | then path 146 | else debugLn "Loading ${path}" (importJSON path); 147 | in 148 | assert attrs.wrangle.apiversion == 1; attrs; 149 | 150 | importFrom = { 151 | path ? null, 152 | sources ? null, 153 | extend ? null, 154 | }: 155 | let 156 | jsonList = map importJsonSrc ( 157 | if sources != null then sources else ( 158 | if path == null 159 | then (abort "path or sources required") 160 | else ( 161 | let 162 | p = builtins.toString path; 163 | candidates = [ 164 | "${p}/nix/wrangle.json" 165 | "${p}/nix/wrangle-local.json" 166 | ]; 167 | present = filter builtins.pathExists candidates; 168 | in 169 | if (length present == 0) 170 | then lib.warn "No files found in candidates:\n - ${concatStringsSep "\n - " candidates}" present 171 | else present 172 | ) 173 | ) 174 | ); 175 | # For convenience we drop everything but `sources` at this stage. 176 | # We could return those at the toplevel, but this lets us add more 177 | # attributes later if needed. 178 | jsonSourcesList = map (j: j.sources) jsonList; 179 | jsonSources = lib.foldl (a: b: a // b) {} jsonSourcesList; 180 | jsonSourcesExtended = if extend == null then jsonSources else ( 181 | # extend only acts on `sources`, not the full attrset 182 | recursiveUpdate jsonSources (extend jsonSources) 183 | ); 184 | in 185 | { 186 | # persist sourcesAttrs for testing / debug purposes 187 | sourceAttrs = jsonSourcesExtended; 188 | # map `sources` into imports instead of plain attrs 189 | sources = importsOfJson { inherit path; } jsonSourcesExtended; 190 | }; 191 | 192 | derivations = args: mapAttrs (name: node: node.drv) (importFrom args).sources; 193 | 194 | inject = { 195 | # inject args 196 | nix ? null, # optional if `path` given 197 | provided ? {}, 198 | 199 | # callPackage args 200 | args ? {}, 201 | 202 | # importFrom args 203 | path ? null, # required if `nix` not given 204 | sources ? null, 205 | extend ? null, 206 | }: 207 | let 208 | pathStr = 209 | # if `path` is not already in the store, we coerce it to a string 210 | # so we don't auto-import it. Requires `--option build-use-chroot false` 211 | if path == null then path else (if isStorePath path then path else builtins.toString path); 212 | nixPath = (if nix != null then nix else "${pathStr}/nix"); 213 | imports = importFrom { inherit path sources extend; }; 214 | 215 | # pull args out of provided 216 | _args = if (args == {} && provided ? args) then provided.args else args; 217 | _provided = filterAttrs (n: v: ! elem n ["args"]) provided; 218 | 219 | # If arguments are explicitly provided, use them in preference to 220 | # local sources. This is used in recursive wrangle, to 221 | # override a dependency. Note that `provided` defaults pkgs & nix-wrangle to `null` 222 | extantProvided = filterAttrs (n: v: v != null) _provided; 223 | callWithProvided = pkgs.newScope ((internal.importScope pkgs imports.sources) // extantProvided); 224 | nixFunction = ensureFunction nixPath; 225 | base = callWithProvided nixPath _args; 226 | selfSrc = imports.sources.self or null; 227 | in 228 | (if selfSrc == null then base else 229 | overrideSrc { 230 | inherit (selfSrc) src version; 231 | drv = base; 232 | # Skip warning if function explicitly accepts `self`, it 233 | # probably knows what it's doing. 234 | warn = ! ((lib.functionArgs nixFunction) ? "self"); 235 | } 236 | ); 237 | }); 238 | in api 239 | -------------------------------------------------------------------------------- /src/Wrangle/Fetch.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NamedFieldPuns #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | 4 | module Wrangle.Fetch where 5 | 6 | import Prelude hiding (error) 7 | import Control.Exception (toException) 8 | import Control.Monad.Except (throwError) 9 | import Control.Applicative (liftA2) 10 | import Data.Aeson (toJSON) 11 | import Data.List (intercalate) 12 | import Data.List.NonEmpty (NonEmpty(..)) 13 | import Data.Maybe (mapMaybe, listToMaybe) 14 | import Data.Char (isSpace) 15 | import Text.Regex.TDFA 16 | import System.Environment (lookupEnv, getExecutablePath) 17 | import System.FilePath.Posix ((), takeDirectory) 18 | import Wrangle.Util 19 | import Wrangle.Source 20 | import qualified System.IO.Strict as H 21 | import qualified Data.Aeson.KeyMap as AMap 22 | import qualified Data.Aeson.Key as Key 23 | import qualified Data.List.NonEmpty as NonEmpty 24 | import qualified Data.Text as T 25 | import qualified System.Process as P 26 | import qualified System.Directory as Dir 27 | 28 | prefetch :: PackageName -> PackageSpec -> IO PackageSpec 29 | prefetch name pkg = do 30 | infoLn $ "fetching " <> (show src) 31 | fetchAttrs <- AMap.fromList <$> resolveAttrs src 32 | debugLn $ "Prefetch results: " <> show fetchAttrs 33 | return $ pkg { fetchAttrs = fetchAttrs } 34 | where 35 | 36 | digestKey = "hash" 37 | existing key = AMap.lookup key (fetchAttrs pkg) 38 | src = sourceSpec pkg 39 | render = renderTemplate (packageAttrs pkg) 40 | 41 | updateDigestIf :: Bool -> [String] -> [Kv] -> IO [Kv] 42 | updateDigestIf cond path attrs = case existing digestKey of 43 | (Just (S digest)) | not cond -> return $ (digestKey, S digest) : attrs 44 | _ -> addDigest path attrs 45 | 46 | addDigest :: [String] -> [Kv] -> IO [Kv] 47 | addDigest path attrs = prefix <$> (log $ prefetchSriHash (fetchType src) attrs) where 48 | prefix d = (digestKey, S (asString d)) : attrs 49 | log = tap (\d -> do 50 | infoLn $ "Resolved " <> (intercalate " -> " (asString name : path)) 51 | infoLn $ " - "<>(Key.toString digestKey)<>":" <> asString d) 52 | 53 | gitCommonAttrs :: GitCommon -> [Kv] 54 | gitCommonAttrs common = if fetchSubmodules common 55 | then [(fetchSubmodulesKeyJSON, B True)] 56 | else [] 57 | 58 | resolveGit :: String -> Template -> IO (String, GitRevision) 59 | resolveGit repo ref = do 60 | ref <- liftEither $ render ref 61 | commit <- revision <$> resolveGitRef repo ref 62 | return (ref, commit) 63 | 64 | addGitDigest :: String -> GitRevision -> GitCommon -> [Kv] -> IO [Kv] 65 | addGitDigest ref commit common attrs = 66 | updateDigestIf (Just (S $ asString commit) /= existing "rev") 67 | [ref, asString commit] 68 | (attrs <> gitCommonAttrs common) 69 | 70 | resolveAttrs :: SourceSpec -> IO [Kv] 71 | resolveAttrs (Github (GithubSpec { ghOwner, ghRepo, ghRef, ghCommon })) = do 72 | (ref, commit) <- resolveGit repo ghRef 73 | addGitDigest ref commit ghCommon [ 74 | ("owner", S ghOwner), 75 | ("repo", S ghRepo), 76 | ("rev", S $ asString commit)] 77 | where repo = "https://github.com/"<>ghOwner<>"/"<>ghRepo<>".git" 78 | 79 | resolveAttrs (Git (GitSpec { gitUrl, gitRef, gitCommon })) = do 80 | (ref, commit) <- resolveGit gitUrl gitRef 81 | addGitDigest ref commit gitCommon [("url", S gitUrl), ("rev", S $ asString commit)] 82 | 83 | resolveAttrs (Url (UrlSpec { url })) = do 84 | renderedUrl <- liftEither $ render url 85 | addDigest [renderedUrl] [("url", S renderedUrl)] 86 | 87 | -- *Local require no prefetching: 88 | resolveAttrs (GitLocal (GitLocalSpec { glPath, glRef, glCommon })) = do 89 | ref <- liftEither $ sequence $ render <$> glRef 90 | return $ optList (refAttr <$> ref) <> toKvPairs glPath <> gitCommonAttrs glCommon 91 | where 92 | refAttr ref = ("ref", S ref) 93 | 94 | resolveAttrs (Path p) = do 95 | return $ toKvPairs p 96 | 97 | data ResolvedRef = ResolvedRef { 98 | revision :: GitRevision, 99 | ref :: String 100 | } deriving Show 101 | 102 | data OutputStream = Stdout | Stderr 103 | 104 | runProcessOutput :: OutputStream -> P.CreateProcess -> IO String 105 | runProcessOutput src p = P.withCreateProcess p read where 106 | read _stdin stdout stderr proc = do 107 | h <- liftMaybe (toException $ AppError $ srcDesc<>" handle is null") handle 108 | contents <- H.hGetContents h 109 | _ <- P.waitForProcess proc 110 | return contents 111 | where 112 | (srcDesc, handle) = case src of 113 | Stdout -> ("stdout", stdout) 114 | Stderr -> ("stderr", stderr) 115 | 116 | -- Git ref is resolved though: 117 | -- $ git ls-remote ~/dev/ocaml/gup/ 118 | -- $ git ls-remote https://github.com/timbertson/gup.git 119 | -- Output: lines of `SHAref` 120 | resolveGitRef :: String -> String -> IO ResolvedRef 121 | resolveGitRef remote refName = do 122 | debugLn $ "Resolving git reference: "<>showRef 123 | refs <- getRefs 124 | sequence_ $ map (debugLn . show) refs 125 | tap logResult $ liftEither $ toRight missing $ firstMatch (refs <> identityRef) 126 | where 127 | showRef = remote<>"#"<>refName 128 | missing = AppError $ "Couldn't resolve ref "<>showRef 129 | logResult result = debugLn $ "Resolved to: "<> show result 130 | 131 | identityRef = 132 | if isValidSha 133 | then [ResolvedRef { 134 | revision = GitRevision refName, 135 | ref = refName 136 | }] 137 | else [] 138 | where 139 | shaRegex = "^[0-9a-fA-F]{40}$" :: String 140 | isValidSha = refName =~ shaRegex :: Bool 141 | 142 | getRefs :: IO [ResolvedRef] 143 | getRefs = do 144 | lines <- T.lines . T.pack <$> runProcessOutput Stdout processSpec 145 | return $ mapMaybe parseLine lines 146 | 147 | processSpec = (P.proc "git" ["ls-remote", remote]) { 148 | P.std_in = P.NoStream, 149 | P.std_out = P.CreatePipe, 150 | P.std_err = P.Inherit 151 | } 152 | 153 | parseLine :: T.Text -> Maybe ResolvedRef 154 | parseLine line = if rev == "" 155 | then Nothing 156 | else Just (ResolvedRef { revision = GitRevision (T.unpack rev), ref = T.unpack ref }) 157 | where 158 | (rev, remainder) = T.break isSpace line 159 | ref = T.strip remainder 160 | 161 | firstMatch refs = listToMaybe (matchingRefs refs) 162 | matchingRefs :: [ResolvedRef] -> [ResolvedRef] 163 | matchingRefs refs = concatMap (\c -> filter (matches c) refs) candidates where 164 | candidates = [refName, "refs/tags/"<>refName, "refs/heads/"<>refName] 165 | matches :: String -> ResolvedRef -> Bool 166 | matches candidate = ((==) candidate) . ref 167 | 168 | dummyHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" 169 | 170 | nixBuildCommand :: NixApiContext -> FetchType -> [Kv] -> NonEmpty String 171 | nixBuildCommand (NixApiContext { apiNix, projectRoot }) fetchType attrs 172 | = exe :| args 173 | where 174 | fetcherName = fetcherNameWrangle fetchType 175 | fetchJSON = encodeOnelineString . toJSON . AMap.fromList $ attrs 176 | fetchExpr = intercalate "\n" [ 177 | "{fetchJSON, apiPath, path}:", 178 | "let api = (import {}).callPackage apiPath {}; in", 179 | "(api.internal.makeFetchers { inherit path; })."<>fetcherName<>" (builtins.fromJSON fetchJSON)"] 180 | exe = "nix-build" 181 | args = [ 182 | "--no-out-link", 183 | "--show-trace", 184 | "--argstr", "fetchJSON", fetchJSON, 185 | "--argstr", "path", projectRoot, 186 | "--argstr", "apiPath", apiNix, 187 | "--expr", fetchExpr] 188 | 189 | data NixApiContext = NixApiContext { 190 | apiNix :: FilePath, 191 | projectRoot :: FilePath 192 | } 193 | 194 | globalApiContext :: IO NixApiContext 195 | globalApiContext = do 196 | declaredDataDir <- lookupEnv "NIX_WRANGLE_DATA" 197 | dataDir <- case declaredDataDir of 198 | (Just dir) -> return dir 199 | Nothing -> (\base -> (takeDirectory base) ".." ".." "..") <$> getExecutablePath 200 | debugLn $ "using NIX_WRANGLE_DATA directory " <> dataDir 201 | let apiNix = dataDir "nix" "api.nix" 202 | projectRoot <- Dir.getCurrentDirectory 203 | return $ NixApiContext { apiNix, projectRoot } 204 | 205 | -- This supports arbitrary prefetching without worrying about nix-prefetch-*. 206 | -- It's slightly inefficient since it results in two downloads of a file, 207 | -- but it's very reliable regardless of fetch method. 208 | prefetchSriHash :: FetchType -> [Kv] -> IO SriHash 209 | prefetchSriHash fetchType attrs = do 210 | apiContext <- globalApiContext 211 | let cmd = nixBuildCommand apiContext fetchType $ ("hash", S dummyHash) : attrs 212 | debugLn $ "+ " <> (show $ NonEmpty.toList cmd) 213 | errText <- runProcessOutput Stderr (processSpec cmd) 214 | sequence_ $ map debugLn $ lines errText 215 | liftEither $ extractExpectedDigest errText 216 | where 217 | processSpec cmd = (P.proc (NonEmpty.head cmd) (NonEmpty.tail cmd)) { 218 | P.std_in = P.NoStream, 219 | P.std_out = P.NoStream, 220 | P.std_err = P.CreatePipe } 221 | 222 | -- Thanks https://github.com/seppeljordan/nix-prefetch-github/blob/cd9708fcdf033874451a879ac5fe68d7df930b7e/src/nix_prefetch_github/__init__.py#L124 223 | -- For the future, note SRI: https://github.com/NixOS/nix/commit/6024dc1d97212130c19d3ff5ce6b1d102837eee6 224 | -- and https://github.com/NixOS/nix/commit/5e6fa9092fb5be722f3568c687524416bc746423 225 | extractExpectedDigest :: String -> Either AppError SriHash 226 | extractExpectedDigest output = SriHash <$> ( 227 | (singleResult $ subMatches nix_sri)) 228 | where 229 | subMatches :: String -> [String] 230 | subMatches pat = concat $ drop 1 <$> ((output =~ pat) :: [[String]]) 231 | 232 | nix_sri = " got: +(sha256-[^\n]+)" 233 | 234 | singleResult [result] = Right result 235 | singleResult _ = Left . AppError $ "Unable to detect resulting digest from nix-build output:\n\n" <> output 236 | 237 | renderTemplate :: StringMap -> Template -> Either AppError String 238 | renderTemplate attrs fullText = render (asString fullText) where 239 | render :: String -> Either AppError String 240 | render ('<':str) = do 241 | case span (/= '>') str of 242 | (key, '>':rest) -> 243 | liftA2 (<>) value (render rest) 244 | where 245 | value = toRight notFound $ AMap.lookup (Key.fromString key) attrs 246 | notFound = AppError $ "Missing key `"<> key <>"` in template: " <> (asString fullText) 247 | _ -> throwError . AppError $ "Value contains an unterminated key: " <> (asString fullText) 248 | render (c:str) = (c:) <$> render str 249 | render [] = Right [] 250 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # nix-wrangle is deprecated 3 | 4 | `nix-wrangle` is featureful but somewhat complex to use and maintain. I've decided to use the more basic [niv](https://github.com/nmattia/niv) myself rather than maintaining nix-wrangle, I suggest others do the same. 5 | 6 | ## Purpose: 7 | 8 | Nix-wrangle aims to be a swiss-army knife for working with nix dependencies. 9 | It works best with dependencies that include their own nix derivations, although using it for fetching plain archives works too. 10 | 11 | ### Goals: 12 | 13 | * Simple usage should be _idiomatic_ and portable, with no specific references to nix-wrangle's API 14 | * Keeping sources updated (and seeing their current state) should be trivial 15 | * Support local development across multiple related repositories 16 | 17 | ## Get it: 18 | 19 | ```bash 20 | 21 | $ nix-build --expr 'import (builtins.fetchTarball "https://github.com/timbertson/nix-wrangle/archive/v1.tar.gz")' 22 | trace: [wrangle] Importing self (git-local) from /nix/store/1d2h4gqmryw30wydxad8ynjcxpkd9avb-w2bvbxf0xrk1fcml8976vrr2g9i019q9-source/default.nix 23 | /nix/store/3ksik1ydgzvwjd2dgh4sj4l2ry1nzd04-nix-wrangle-0.0.0 24 | 25 | $ result/bin/nix-wrangle --help 26 | Nix-wrangle - source & dependency manager for Nix projects 27 | 28 | Usage: nix-wrangle COMMAND 29 | 30 | Available options: 31 | -h,--help Show this help text 32 | 33 | Available commands: 34 | init Initialize nix-wrangle 35 | add Add a source 36 | rm Remove one or more sources 37 | update Update one or more sources 38 | splice Splice current `public` source into a .nix document 39 | show Show source details 40 | ls list sources 41 | default-nix Generate default.nix 42 | ``` 43 | 44 | ## Basic functionality: 45 | 46 | nix-wrangle maintains a set of sources, which are typically dependencies. The following example shows the basic setup: 47 | 48 | ```bash 49 | 50 | # Here's our derivation, nix/default.nix. It expects a `piep` argument, and uses a relative path for `src`: 51 | $ cat nix/default.nix 52 | { stdenv, piep }: 53 | stdenv.mkDerivation { 54 | name="sample.txt"; 55 | src = ../.; 56 | buildCommand = '' 57 | cat > "$out" < v1 -> 79287edfaa7bd065888e26eba429d2e2710c2f51 70 | - sha256:0fk4bd4lkkmxr2z75vwsj5b2f7xlg8ybjmd51aandpz3minz03f5 71 | Writing: nix/wrangle.json 72 | Writing: default.nix 73 | 74 | # Now provide the `piep` dependency from a github repo, and build it: 75 | $ nix-wrangle add piep timbertson/piep --nix nix/ 76 | Adding "piep" // PackageSpec {sourceSpec = Github (GithubSpec {ghOwner = "timbertson", ghRepo = "piep", ghRef = Template "master"}), fetchAttrs = fromList [], packageAttrs = fromList [("nix","nix/")]} 77 | fetching Github (GithubSpec {ghOwner = "timbertson", ghRepo = "piep", ghRef = Template "master"}) 78 | Resolved piep -> master -> d805330386553c5784ac7ef48ff38aea716575dc 79 | - sha256:1q7lzi3lggw8by4pkh5ckkjv68xqbjahrkxgdw89jxdlyd0wq5in 80 | Writing: nix/wrangle.json 81 | 82 | $ nix-build 83 | trace: [wrangle] Importing piep (github) from /nix/store/ax68rn4b8dc4lrcfqq4rhx2fcwdr807a-source/nix/ 84 | these derivations will be built: 85 | /nix/store/kqdazszslqwyks5ckl26c4y3ya7jhkfk-sample.txt.drv 86 | building '/nix/store/kqdazszslqwyks5ckl26c4y3ya7jhkfk-sample.txt.drv'... 87 | /nix/store/y8kwj1phi0m2byynr9dw741yyvn32vk1-sample.txt 88 | 89 | # And here's the result, with injected source and `piep` dependency: 90 | $ cat result 91 | Sample derivation, built with: 92 | - piep: /nix/store/6w23yj4hyabxzwcgnl0d3xjr261ywrvr-python2.7-piep-0.8.1 93 | - src: /nix/store/hiazl86hdzmlnjzkcxs6hn1p7hlkx9fi-example 94 | ``` 95 | 96 | Note that the `piep` dependency is built (by using `pkgs.callPackage` on the nix path within the source), which gives you the actual derivation, not simply the source code. This is one important difference compared to [niv][]. 97 | 98 | Sources are typically used for project dependencies, but there are three special sources: 99 | 100 | - 'nix-wrangle': (added automatically) used for bootstrapping in 'default.nix' 101 | - 'self': used to override the toplevel derivation's 'src' attribute when building 'default.nix'. Automatically added by `nix-wrangle init` as a `git-local` source if there's a `.git` directory present. 102 | - 'pkgs': (optional) used to pin the exact version of 'nixpkgs'. Added automatically by `nix-wrangle init` if you pass `--pkgs nixpkgs-unstable` (or any other branch name from https://github.com/NixOS/nixpkgs-channels) 103 | 104 | # Features: 105 | 106 | Nix-wrangle is purpose built to solve a range of specific use cases which come up when developing with nix: 107 | 108 | ## Update from specification 109 | 110 | e.g. when using a git-based dependency, you can specify a branch instead of a specific commit. When you `update`, that branch will be re-resolved: 111 | 112 | ```bash 113 | 114 | # Time to update the `piep` dependency (this re-resolves `master`, or whatever ref is configured) 115 | $ nix-wrangle update piep 116 | Updating nix/wrangle.json ... 117 | - updating "piep"... 118 | fetching Github (GithubSpec {ghOwner = "timbertson", ghRepo = "piep", ghRef = Template "master"}) 119 | Resolved piep -> master -> d805330386553c5784ac7ef48ff38aea716575dc 120 | - sha256:1q7lzi3lggw8by4pkh5ckkjv68xqbjahrkxgdw89jxdlyd0wq5in 121 | Writing: nix/wrangle.json 122 | ``` 123 | 124 | You can also make use of templating, e.g. for a URL dependency you can use the URL `'http://example.com/libfoo/libfoo-.tgz'`. When updating, you can pass `--version NEW_VERSION` to update it. 125 | 126 | ## 'src' injection 127 | 128 | When building, a `self` dependency (if present) will be used to provide the `src` for the toplevel derivation. This lets you use nix-wrangle's various source types (e.g git-local, which isn't available as a `nixpkgs` builtin) and automatic generation (e.g. sha256 digests). 129 | 130 | nix-wrangle will also inject the relevant `src` into each of your dependencies. Let's say you import commit `x` of the `piep` dependency, with a nix expression in it. It would be impossible for commit `x` of `piep` to refer to commit `x` as its `src` attribute. The best it could do is to refer to the parent of commit `x`, although it may often just refer to the most recently released version. Both of these would be counter-productive - you'd be importing the `derivation` at `x`, but building source code from _some other version_ out of your control. `nix-wrangle` automatically overrides the `src` of imported dependencies so that the version you _import_ is also the source code you _build_. 131 | 132 | ## Local overrides 133 | 134 | When working on both a library and an application that uses it, it's common to want to try out working changes before publishing them. This is easy with local sources: 135 | 136 | ```bash 137 | 138 | # Let's develop against my local checkout of `piep`. 139 | # Local overrides are stored in nix/wrangle-local.json, which you shouldn't commit 140 | $ nix-wrangle add --local piep --type git-local --path /home/tim/dev/python/piep --nix nix/ 141 | Adding "piep" // PackageSpec {sourceSpec = GitLocal (GitLocalSpec {glPath = FullPath "/home/tim/dev/python/piep", glRef = Nothing}), fetchAttrs = fromList [], packageAttrs = fromList [("nix","nix/")]} 142 | fetching GitLocal (GitLocalSpec {glPath = FullPath "/home/tim/dev/python/piep", glRef = Nothing}) 143 | Writing: nix/wrangle-local.json 144 | 145 | $ nix-build 146 | trace: [wrangle] Importing piep (git-local) from /nix/store/ba4y245q521hk8nyf4f272zgnb7js5v7-source/nix/ 147 | these derivations will be built: 148 | /nix/store/y0gjhrcidms9k62sb4wcsq8x14y094p7-sample.txt.drv 149 | building '/nix/store/y0gjhrcidms9k62sb4wcsq8x14y094p7-sample.txt.drv'... 150 | /nix/store/jhcdh3fc5nr41v660xw50pyb6xa48hzi-sample.txt 151 | 152 | $ cat result 153 | Sample derivation, built with: 154 | - piep: /nix/store/zlnz9al4hykj9f1mx9018l1xl4fd5x6w-python2.7-piep-0.9.2 155 | - src: /nix/store/5099nhjdbz5bb5yhc0bd34ady2ymp14m-example 156 | ``` 157 | 158 | This uses the local version of a dependency for building, but kept separate from the "public" version of your dependency specificaion. 159 | 160 | ## Splicing `src` to produce a self-contained derivation 161 | 162 | **Note:** this requires you pass `--arg args '{ enableSplice = true; }'` to nix-build; the `hnix` dependency required for this is not included by default. If importing the nix expression some other way, include `args = { enableSplice = true; }` in the call arguments. 163 | 164 | nix-wrangle was built so that your base derivation (`nix/default.nix`) can be idiomatic - it doesn't need to reference `nix-wrangle` at all, and its dependencies are injected as arguments, just like regular derivations in `nixpkgs`. The one way in which they aren't idiomatic is the `src` attribute, since in nixpkgs this typically refers to a remote repository or tarball. 165 | 166 | So there's also the `splice` command. This injects the current value of a fetched source (defaulting to `public`) into an existing nix file to create a self-contained derivation. This is perfect for promoting your in-tree derivation (with source provided by nix-wrangle) into a derivation suitable for inclusion in `nixpkgs`, where it includes its own `src` and all dependencies are provided by the caller. 167 | 168 | ```bash 169 | 170 | # Splice the `nix-wrangle` source into nix/default.nix 171 | $ nix-wrangle splice nix/default.nix --name nix-wrangle --output nix/public.nix 172 | Updating nix/wrangle.json ... 173 | - updating "nix-wrangle"... 174 | fetching Github (GithubSpec {ghOwner = "timbertson", ghRepo = "nix-wrangle", ghRef = Template "v1"}) 175 | ... (unchanged) 176 | Writing: nix/wrangle.json 177 | Updating nix/wrangle-local.json ... 178 | Writing: nix/wrangle-local.json 179 | Writing: nix/public.nix 180 | 181 | $ cat nix/public.nix 182 | { stdenv, piep }: 183 | stdenv.mkDerivation { 184 | name="sample.txt"; 185 | src = fetchFromGitHub { 186 | rev = "2066dd8a382ee974cdbfd109f37a9be1d04f8481"; 187 | sha256 = "13gfwk6lx8yn0gfxyfrfkqiz1yzicg6qq5219l5fb517n3da5chq"; 188 | repo = "nix-wrangle"; 189 | owner = "timbertson"; 190 | }; 191 | buildCommand = '' 192 | cat > "$out" <` which is resolved on `update`. 204 | - `git`: takes a `url` and `rev` (which can be a branch, tag or commit). `rev` is resolved to a concrete commit on initial add, and on `update`. 205 | - `github`: takes an `owner`, `repo` and `rev` (as for `git`) 206 | - `git-local`: takes a path (can be relative) and an optional `rev` (can be a branch, tag or `HEAD`). `rev` is resolved _at evaluation time_. If `rev` is not provided, you'll get the _working changes_ in the given workspace (but not any excluded or untracked files). 207 | - `path`: path (can be relative or absolute) 208 | 209 | ---- 210 | 211 | # Hacking 212 | 213 | There are two options for development: 214 | 215 | From within 'nix-shell', use 'cabal v1-build' 216 | 217 | Alternatively, from within 'nix-shell -p haskellPackages.cabal-install' you can use 'cabal new-build'. This bypasses the nix infrastructure entirely and fetches its own copy of dependencies, but may be convenient when adjusting packages or pinning dependencies to specific versions. 218 | 219 | # Similar tools 220 | 221 | ### niv: 222 | 223 | nix-wrangle was heavily inspired by [niv][] (the command line tool was even based off the niv source code). 224 | The main differences are: 225 | 226 | - nix-wrangle dependencies are derivations, not just source code. 227 | - nix-wrangle attempts to let you write idiomatic nix without explicitly referencing any files or functions provided by nix-wrangle. 228 | - nix-wrangle has a number of extra features not provided by niv: `splice`, `self` injection and local overlays. 229 | - nix-wrangle is more featureful, but also more complex 230 | 231 | ### nix-flakes: 232 | 233 | nix-wrangle (like [niv][]) is similar in spirit to [nix flakes](https://github.com/tweag/rfcs/blob/flakes/rfcs/0049-flakes.md), but the implementation is still experimental. My hope is that any standard solution would be able to support nix-wrangle style workflows. 234 | 235 | [niv]: https://github.com/nmattia/niv 236 | 237 | -------------------------------------------------------------------------------- /src/Wrangle/Source.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DerivingStrategies #-} 2 | {-# LANGUAGE NamedFieldPuns #-} 3 | {-# LANGUAGE GeneralizedNewtypeDeriving #-} 4 | {-# LANGUAGE OverloadedStrings #-} 5 | {-# LANGUAGE ScopedTypeVariables #-} 6 | {-# LANGUAGE TupleSections #-} 7 | 8 | module Wrangle.Source where 9 | 10 | import Prelude hiding (error) 11 | import Control.Monad 12 | import Control.Exception (Exception) 13 | import Data.Bifoldable (bifoldMap) 14 | import Control.Applicative ((<|>)) 15 | import Data.Aeson hiding (eitherDecodeFileStrict, encodeFile) 16 | import Data.Aeson.Types (typeMismatch, Parser) 17 | import Data.Hashable (Hashable) 18 | import System.FilePath (()) 19 | import Wrangle.Util 20 | import Data.List (intercalate) 21 | import Data.List.NonEmpty (NonEmpty(..)) 22 | import qualified Data.List.NonEmpty as NonEmpty 23 | import qualified Data.Aeson as Aeson 24 | import qualified Data.Aeson.Encode.Pretty as AesonPretty 25 | import qualified Data.ByteString as B 26 | import qualified Data.ByteString.Lazy as L 27 | import qualified Data.HashMap.Strict as HMap 28 | import qualified Data.Aeson.KeyMap as AMap 29 | import qualified Data.Aeson.Key as Key 30 | import qualified Data.Text as T 31 | import qualified Data.Text.Encoding as E 32 | import qualified Data.Text.Lazy as TL 33 | import qualified Data.Text.Lazy.Encoding as TLE 34 | import qualified Options.Applicative as Opts 35 | import qualified System.Directory as Dir 36 | 37 | {- 38 | - Terminology: 39 | - 40 | - SourceFile : a wrangle(/-local).json file 41 | - Packages: the loaded contents of a SourceFile, as a map of PackageName -> PackageSpec 42 | - PackageSpec: an entry in Packages 43 | - SourceSpec: an (impure) fetch spec for some source code. Resolved to specific `fetch` information on update. 44 | -} 45 | 46 | latestApiVersion = 1 47 | 48 | data StringOrBool = S String | B Bool deriving Eq 49 | instance Show StringOrBool where 50 | show (S x) = show x 51 | show (B x) = show x 52 | 53 | type Kv = (Key, StringOrBool) 54 | 55 | type StringMap = AMap.KeyMap String 56 | 57 | type KvMap = AMap.KeyMap StringOrBool 58 | 59 | fetchKeyJSON = "fetch" :: Key 60 | typeKeyJSON = "type" :: Key 61 | versionKeyJSON = "version" :: Key 62 | sourcesKeyJSON = "sources" :: Key 63 | wrangleKeyJSON = "wrangle" :: Key 64 | apiversionKeyJSON = "apiversion" :: Key 65 | fetchSubmodulesKeyJSON = "fetchSubmodules" :: Key 66 | 67 | wrangleHeaderJSON :: Aeson.Value 68 | wrangleHeaderJSON = 69 | Object $ AMap.singleton apiversionKeyJSON (toJSON latestApiVersion) 70 | 71 | class ToKvPairs t where 72 | toKvPairs :: t -> [Kv] 73 | toKvMap :: t -> KvMap 74 | toKvMap = AMap.fromList . toKvPairs 75 | 76 | class AsString t where 77 | asString :: t -> String 78 | 79 | newtype Template = Template String deriving (Show, Eq, FromJSON, ToJSON) 80 | 81 | instance AsString Template where 82 | asString (Template s) = s 83 | 84 | newtype GitRevision = GitRevision String deriving (Show, Eq, FromJSON, ToJSON) 85 | 86 | instance AsString GitRevision where 87 | asString (GitRevision s) = s 88 | 89 | newtype Sha256 = Sha256 String deriving (Show, Eq, FromJSON, ToJSON) 90 | 91 | newtype SriHash = SriHash String deriving (Show, Eq, FromJSON, ToJSON) 92 | 93 | instance AsString Sha256 where 94 | asString (Sha256 s) = s 95 | 96 | instance AsString SriHash where 97 | asString (SriHash s) = s 98 | 99 | data GitCommon = GitCommon { 100 | fetchSubmodules :: Bool 101 | } deriving (Show, Eq) 102 | 103 | defaultGitCommon = GitCommon { fetchSubmodules = False } 104 | 105 | data FetchType 106 | = FetchGithub 107 | | FetchGit 108 | | FetchUrl UrlFetchType 109 | | FetchGitLocal 110 | | FetchPath 111 | 112 | allFetchTypes = [ FetchGithub, FetchGit, FetchUrl FetchArchive, FetchUrl FetchFile, FetchGitLocal, FetchPath ] 113 | validTypesDoc = "Valid types: " <> (intercalate "|" $ map fetcherNameWrangle allFetchTypes) 114 | 115 | parseFetchType :: String -> Either String FetchType 116 | parseFetchType s = case s of 117 | "github" -> Right FetchGithub 118 | "git" -> Right FetchGit 119 | "url" -> Right $ FetchUrl FetchArchive 120 | "file" -> Right $ FetchUrl FetchFile 121 | "path" -> Right FetchPath 122 | "git-local" -> Right FetchGitLocal 123 | t -> Left $ "Unsupported type: '" <> t <> "'. "<>validTypesDoc 124 | 125 | fetchType :: SourceSpec -> FetchType 126 | fetchType spec = case spec of 127 | (Github _) -> FetchGithub 128 | (Url (UrlSpec { urlType })) -> FetchUrl urlType 129 | (Git _) -> FetchGit 130 | (GitLocal _) -> FetchGitLocal 131 | (Path _) -> FetchPath 132 | 133 | fetcherNameNix :: FetchType -> Either AppError String 134 | fetcherNameNix f = case f of 135 | FetchGithub -> Right "fetchFromGitHub" 136 | FetchGit -> Right "fetchgit" 137 | (FetchUrl FetchArchive) -> Right "fetchzip" 138 | (FetchUrl FetchFile) -> Right "fetchurl" 139 | -- These two don't have a nix fetcher, which means they 140 | -- aren't supported in `splice`, but they also require 141 | -- no prefetching 142 | FetchGitLocal -> err 143 | FetchPath -> err 144 | where err = Left $ AppError $ "No plain-nix fetcher for type '"<>fetcherNameWrangle f<>"'" 145 | 146 | fetcherNameWrangle :: FetchType -> String 147 | fetcherNameWrangle f = case f of 148 | FetchGithub -> "github" 149 | FetchGit -> "git" 150 | FetchGitLocal -> "git-local" 151 | FetchPath -> "path" 152 | (FetchUrl typ) -> asString typ 153 | 154 | data GithubSpec = GithubSpec { 155 | ghOwner :: String, 156 | ghRepo :: String, 157 | ghRef :: Template, 158 | ghCommon :: GitCommon 159 | } deriving (Show, Eq) 160 | 161 | instance ToKvPairs GithubSpec where 162 | toKvPairs GithubSpec { ghOwner, ghRepo, ghRef } = 163 | [ 164 | ("owner", S ghOwner), 165 | ("repo", S ghRepo), 166 | ("ref", S $ asString ghRef) 167 | ] 168 | 169 | instance FromJSON StringOrBool where 170 | parseJSON (Bool v) = pure (B v) 171 | parseJSON (String v) = pure (S (T.unpack v)) 172 | parseJSON v = typeMismatch "String/Boolean" v 173 | 174 | instance ToJSON StringOrBool where 175 | toJSON (B v) = toJSON v 176 | toJSON (S v) = toJSON v 177 | 178 | data UrlFetchType = FetchArchive | FetchFile deriving (Show, Eq) 179 | 180 | instance AsString UrlFetchType where 181 | asString FetchArchive = "url" 182 | asString FetchFile = "file" 183 | 184 | instance ToJSON UrlFetchType where 185 | toJSON = toJSON . asString 186 | 187 | instance FromJSON UrlFetchType where 188 | parseJSON (String "file") = pure FetchFile 189 | parseJSON (String "url") = pure FetchArchive 190 | parseJSON v = typeMismatch "\"file\" or \"url\"" v 191 | 192 | data UrlSpec = UrlSpec { 193 | urlType :: UrlFetchType, 194 | url :: Template 195 | } deriving (Show, Eq) 196 | 197 | instance ToKvPairs UrlSpec where 198 | toKvPairs UrlSpec { urlType = _urlType, url } = 199 | [ ("url", S $ asString url) ] 200 | 201 | data GitSpec = GitSpec { 202 | gitUrl :: String, 203 | gitRef :: Template, 204 | gitCommon :: GitCommon 205 | } deriving (Show, Eq) 206 | 207 | instance ToKvPairs GitSpec where 208 | toKvPairs GitSpec { gitUrl, gitRef } = 209 | [ 210 | ("url", S gitUrl), 211 | ("ref", S $ asString gitRef) 212 | ] 213 | 214 | data LocalPath 215 | = FullPath FilePath 216 | | RelativePath FilePath 217 | deriving (Show, Eq) 218 | 219 | instance ToKvPairs LocalPath where 220 | toKvPairs (FullPath p) = [("path", S p)] 221 | toKvPairs (RelativePath p) = [("relativePath", S p)] 222 | 223 | data GitLocalSpec = GitLocalSpec { 224 | glPath :: LocalPath, 225 | glRef :: Maybe Template, 226 | glCommon :: GitCommon 227 | } deriving (Show, Eq) 228 | 229 | instance ToKvPairs GitLocalSpec where 230 | toKvPairs GitLocalSpec { glPath, glRef } = 231 | (toKvPairs glPath) <> optList (refAttr <$> glRef) where 232 | refAttr ref = ("ref", S $ asString ref) 233 | 234 | data SourceSpec 235 | = Github GithubSpec 236 | | Url UrlSpec 237 | | Git GitSpec 238 | | GitLocal GitLocalSpec 239 | | Path LocalPath 240 | deriving (Show, Eq) 241 | 242 | instance ToKvPairs SourceSpec where 243 | toKvPairs (Github f) = toKvPairs f 244 | toKvPairs (Url f) = toKvPairs f 245 | toKvPairs (Git f) = toKvPairs f 246 | toKvPairs (GitLocal f) = toKvPairs f 247 | toKvPairs (Path f) = toKvPairs f 248 | 249 | parseBoolFromString :: String -> Parser Bool 250 | parseBoolFromString "true" = pure True 251 | parseBoolFromString "false" = pure False 252 | parseBoolFromString other = fail $ "Not a boolean: " <> other 253 | 254 | -- TODO return (SourceSpec, remainingAttrs)? 255 | parseSourceSpecObject :: Value -> Object -> Parser SourceSpec 256 | parseSourceSpecObject fetcher attrs = parseFetcher fetcher >>= parseSpec 257 | where 258 | parseFetcher :: Value -> Parser FetchType 259 | parseFetcher json = parseJSON json >>= bifoldMap invalid pure . parseFetchType 260 | 261 | parseSpec :: FetchType -> Parser SourceSpec 262 | parseSpec fetcher = case fetcher of 263 | FetchGithub -> 264 | build <$> owner <*> repo <*> refRequired <*> gitCommon where 265 | build ghOwner ghRepo ghRef ghCommon = Github $ GithubSpec { 266 | ghOwner, ghRepo, ghRef, ghCommon } 267 | FetchGit -> build <$> url <*> refRequired <*> gitCommon where 268 | build gitUrl gitRef gitCommon = Git $ GitSpec { gitUrl, gitRef, gitCommon } 269 | FetchGitLocal -> 270 | build <$> path <*> refOpt <*> gitCommon where 271 | build glPath glRef glCommon = GitLocal $ GitLocalSpec { glPath, glRef, glCommon } 272 | FetchPath -> Path <$> path 273 | (FetchUrl t) -> buildUrl t <$> url 274 | 275 | gitCommon :: Parser GitCommon 276 | gitCommon = build <$> fetchSubmodulesOpt where 277 | build fetchSubmodules = GitCommon { fetchSubmodules } 278 | 279 | owner = attrs .: "owner" 280 | repo = attrs .: "repo" 281 | url = attrs .: "url" 282 | path = (FullPath <$> attrs .: "path") <|> (RelativePath <$> attrs .: "relativePath") 283 | refRequired :: Parser Template = attrs .: "ref" 284 | refOpt :: Parser (Maybe Template) = attrs .:? "ref" 285 | 286 | fetchSubmodulesOpt :: Parser Bool 287 | fetchSubmodulesOpt = do 288 | (strOpt :: Maybe String) <- attrs .:? fetchSubmodulesKeyJSON 289 | (boolOpt :: Maybe Bool) <- traverse parseBoolFromString strOpt 290 | return $ maybe (fetchSubmodules defaultGitCommon) id boolOpt 291 | 292 | buildUrl urlType url = Url $ UrlSpec { urlType, url = Template url } 293 | invalid v = fail $ "Unable to parse SourceSpec from: " <> (encodePrettyString v) 294 | 295 | data PackageSpec = PackageSpec { 296 | sourceSpec :: SourceSpec, 297 | fetchAttrs :: KvMap, 298 | packageAttrs :: StringMap 299 | } deriving (Show, Eq) 300 | 301 | instance FromJSON PackageSpec where 302 | parseJSON = withObject "PackageSpec" (\attrs -> do 303 | (fetchJSON, attrs) <- attrs `extract` fetchKeyJSON 304 | (fetcher, attrs) <- attrs `extract` typeKeyJSON 305 | let fetchAttrs = (parseJSON fetchJSON) 306 | let packageAttrs = (parseJSON (Object attrs)) 307 | build <$> (parseSourceSpecObject fetcher attrs) 308 | <*> fetchAttrs <*> packageAttrs 309 | ) where 310 | extract obj key = pairWithout key obj <$> obj .: key 311 | pairWithout key obj v = (v, AMap.delete key obj) 312 | build sourceSpec fetchAttrs packageAttrs = PackageSpec { 313 | sourceSpec, fetchAttrs, packageAttrs 314 | } 315 | 316 | instance ToJSON PackageSpec where 317 | toJSON PackageSpec { sourceSpec, fetchAttrs, packageAttrs } = 318 | toJSON 319 | . AMap.insert typeKeyJSON (toJSON . fetcherNameWrangle . fetchType $ sourceSpec) 320 | . AMap.insert fetchKeyJSON (toJSON fetchAttrs) 321 | $ (AMap.map toJSON (AMap.map S packageAttrs <> (toKvMap sourceSpec))) 322 | 323 | newtype Packages = Packages 324 | { unPackages :: HMap.HashMap PackageName PackageSpec } 325 | deriving newtype (Show) 326 | 327 | emptyPackages = Packages HMap.empty 328 | 329 | add :: Packages -> PackageName -> PackageSpec -> Packages 330 | add s name spec = Packages $ HMap.insert name spec $ unPackages s 331 | 332 | remove :: Packages -> PackageName -> Packages 333 | remove s name = Packages $ HMap.delete name $ unPackages s 334 | 335 | instance FromJSON Packages where 336 | parseJSON = withObject "document" $ \obj -> 337 | (obj .: wrangleKeyJSON >>= checkHeader) >> 338 | Packages <$> (obj .: sourcesKeyJSON >>= withObject "sources" parsePackageSpecs) 339 | where 340 | parsePackageSpecs attrs = HMap.fromList <$> mapM parseItem (AMap.toList attrs) 341 | parseItem :: (Key, Value) -> Parser (PackageName, PackageSpec) 342 | parseItem (k,v) = (PackageName $ Key.toString k,) <$> parseJSON v 343 | checkHeader = withObject "Wrangle Header" $ \obj -> 344 | (obj .: apiversionKeyJSON >>= checkApiVersion) 345 | checkApiVersion v = 346 | if v == (Number latestApiVersion) 347 | then pure () 348 | else fail ("unsupported API version: " <> (TL.unpack . TLE.decodeUtf8 $ Aeson.encode v)) 349 | 350 | instance ToJSON Packages where 351 | toJSON (Packages s) = toJSON $ 352 | HMap.fromList [ 353 | (sourcesKeyJSON, toJSON s), 354 | (wrangleKeyJSON, wrangleHeaderJSON) 355 | ] 356 | 357 | liftResult :: Result a -> Either AppError a 358 | liftResult (Error err) = Left $ AppError err 359 | liftResult (Success x) = Right x 360 | 361 | -- Note: doesn't update `fetch` attrs 362 | updatePackageSpec :: PackageSpec -> StringMap -> Either AppError PackageSpec 363 | updatePackageSpec original attrs = mergedJSON >>= liftResult <$> fromJSON where 364 | -- Going via JSON is a little hacky, but 365 | -- we've already got a nice to/from JSON code path 366 | mergedJSON = case (toJSON original, toJSON attrs) of 367 | (Object orig, Object add) -> Right $ Object $ AMap.union add orig 368 | (_, _) -> Left $ "Expected JSON object" -- should be impossible 369 | 370 | loadSourceFile :: SourceFile -> IO Packages 371 | loadSourceFile source = do 372 | debugLn $ "Reading sources: " ++ sourcePath 373 | contents <- eitherDecodeFileStrict sourcePath 374 | liftEither $ mapLeft invalidSourceDocument contents 375 | where 376 | sourcePath = pathOfSource source 377 | invalidSourceDocument reason = AppError $ unlines [ 378 | "Cannot load " <> sourcePath, 379 | "This file should be a JSON map with toplevel objects `sources` and `wrangle`.", 380 | reason] 381 | 382 | loadSources :: NonEmpty SourceFile -> IO (NonEmpty Packages) 383 | loadSources sources = do 384 | debugLn $ "Loading sources: " ++ (show sources) 385 | traverse loadSourceFile sources 386 | 387 | merge :: NonEmpty Packages -> Packages 388 | merge packages = do 389 | -- TODO check order 390 | Packages $ foldr HMap.union HMap.empty (unPackages <$> packages) 391 | 392 | writeSourceFile :: SourceFile -> Packages -> IO () 393 | writeSourceFile sourceFile packages = 394 | encodeFile (pathOfSource sourceFile) packages 395 | 396 | newtype NotFound = NotFound (String, [String]) 397 | instance Show NotFound where 398 | show (NotFound (key, keys)) = 399 | "key `" <> key <> "` not found in " <> (show keys) 400 | 401 | instance Exception NotFound 402 | 403 | lookup :: PackageName -> Packages -> Either NotFound PackageSpec 404 | lookup pkg (Packages packages) = toRight err (HMap.lookup pkg packages) where 405 | err = (NotFound (show pkg, asString <$> HMap.keys packages)) 406 | 407 | keys :: Packages -> [PackageName] 408 | keys = HMap.keys . unPackages 409 | 410 | member :: Packages -> PackageName -> Bool 411 | member = (flip HMap.member) . unPackages 412 | 413 | defaultSourceFileCandidates :: [SourceFile] 414 | defaultSourceFileCandidates = [ DefaultSource, LocalSource ] 415 | 416 | doesSourceExist = Dir.doesFileExist . pathOfSource 417 | 418 | detectDefaultSources :: IO (Maybe (NonEmpty SourceFile)) 419 | detectDefaultSources = tap log $ NonEmpty.nonEmpty <$> fileList where 420 | log sources = debugLn $ "Detected default sources:" <> show sources 421 | fileList = filterM doesSourceExist defaultSourceFileCandidates 422 | 423 | configuredSources :: Maybe (NonEmpty SourceFile) -> IO (Maybe (NonEmpty SourceFile)) 424 | configuredSources Nothing = detectDefaultSources 425 | configuredSources explicitSources@(Just _) = return explicitSources 426 | 427 | newtype PackageName = PackageName { unPackageName :: String } 428 | deriving newtype (Eq, Hashable, FromJSONKey, ToJSONKey, Show) 429 | 430 | instance AsString PackageName where asString = unPackageName 431 | 432 | parsePackageName :: Opts.Parser PackageName 433 | parsePackageName = PackageName <$> 434 | Opts.argument Opts.str (Opts.metavar "PACKAGE") 435 | 436 | eitherDecodeFileStrict :: (FromJSON a) => FilePath -> IO (Either String a) 437 | eitherDecodeFileStrict = fmap Aeson.eitherDecodeStrict . B.readFile 438 | 439 | encodePretty :: (ToJSON a) => a -> L.ByteString 440 | encodePretty = AesonPretty.encodePretty' (AesonPretty.defConfig { 441 | AesonPretty.confIndent = AesonPretty.Spaces 2, 442 | AesonPretty.confCompare = AesonPretty.compare 443 | }) 444 | 445 | encodePrettyString :: (ToJSON a) => a -> String 446 | encodePrettyString = stringOfLazy . encodePretty 447 | 448 | encodeOnelineString :: (ToJSON a) => a -> String 449 | encodeOnelineString = stringOfLazy . Aeson.encode 450 | 451 | stringOfLazy :: L.ByteString -> String 452 | stringOfLazy = TL.unpack . TLE.decodeUtf8 453 | 454 | encodeFile :: (ToJSON a) => FilePath -> a -> IO () 455 | encodeFile path json = writeFileLazyBytestring path (encodePretty json <> "\n") 456 | 457 | writeFileLazyBytestring :: FilePath -> L.ByteString -> IO () 458 | writeFileLazyBytestring path contents = do 459 | debugLn $ "Writing contents: " <> (show contents) 460 | writeFileContents' L.writeFile path contents 461 | 462 | writeFileText :: FilePath -> T.Text -> IO () 463 | writeFileText path text = do 464 | debugLn $ "Writing contents: " <> (show text) 465 | writeFileContents' (\path -> B.writeFile path . E.encodeUtf8) path text 466 | 467 | writeFileContents' :: (FilePath -> a -> IO ()) -> FilePath -> a -> IO () 468 | writeFileContents' writer path contents = do 469 | infoLn $ "Writing: " <> path 470 | writer tmpPath contents 471 | Dir.renameFile tmpPath path 472 | where tmpPath = path <> ".tmp" :: FilePath 473 | 474 | data SourceFile 475 | = DefaultSource 476 | | LocalSource 477 | | NamedSource String 478 | deriving Show 479 | 480 | pathOfSource :: SourceFile -> FilePath 481 | pathOfSource source = case source of 482 | DefaultSource -> "nix" "wrangle.json" 483 | LocalSource -> "nix" "wrangle-local.json" 484 | NamedSource path -> path 485 | -------------------------------------------------------------------------------- /src/Wrangle/Cmd.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE CPP #-} 2 | {-# LANGUAGE DerivingStrategies #-} 3 | {-# LANGUAGE GeneralizedNewtypeDeriving #-} 4 | {-# LANGUAGE LambdaCase #-} 5 | {-# LANGUAGE NamedFieldPuns #-} 6 | {-# LANGUAGE ScopedTypeVariables #-} 7 | {-# LANGUAGE OverloadedStrings #-} 8 | {-# LANGUAGE QuasiQuotes #-} 9 | {-# LANGUAGE TupleSections #-} 10 | {-# LANGUAGE ViewPatterns #-} 11 | 12 | module Wrangle.Cmd where 13 | 14 | import Prelude hiding (error) 15 | import Control.Applicative 16 | import Control.Monad 17 | import Control.Monad.Except (throwError) 18 | import Control.Monad.Catch (throwM) 19 | import Control.Monad.State 20 | import Data.Char (toUpper) 21 | import Data.Maybe (fromMaybe, isJust) 22 | import Data.List (partition, intercalate, intersperse) 23 | import Data.List.NonEmpty (NonEmpty(..)) 24 | import System.Exit (exitFailure) 25 | import Wrangle.Source (PackageName(..), StringMap, asString) 26 | import Wrangle.Util 27 | import Data.Aeson.Key (Key) 28 | import qualified Data.List.NonEmpty as NonEmpty 29 | import qualified Data.HashMap.Strict as HMap 30 | import qualified Data.Aeson.KeyMap as AMap 31 | import qualified Data.Aeson.Key as Key 32 | import qualified Data.String.QQ as QQ 33 | import qualified Data.Text as T 34 | import qualified Data.Text.Encoding as TE 35 | import qualified Data.ByteString as B 36 | import qualified Wrangle.Fetch as Fetch 37 | import qualified Wrangle.Source as Source 38 | import qualified System.Directory as Dir 39 | #ifdef ENABLE_SPLICE 40 | import qualified Wrangle.Splice as Splice 41 | #endif 42 | import qualified Options.Applicative as Opts 43 | import qualified Options.Applicative.Help.Pretty as Doc 44 | import qualified System.FilePath.Posix as PosixPath 45 | 46 | main :: IO () 47 | main = join $ Opts.customExecParser prefs opts where 48 | opts = Opts.info (parseCommand <**> Opts.helper) $ mconcat desc 49 | prefs = Opts.prefs Opts.showHelpOnEmpty 50 | desc = 51 | [ Opts.fullDesc 52 | , Opts.header "Nix-wrangle - source & dependency manager for Nix projects" 53 | ] 54 | 55 | parseCommand :: Opts.Parser (IO ()) 56 | parseCommand = Opts.subparser ( 57 | Opts.command "init" parseCmdInit <> 58 | Opts.command "add" parseCmdAdd <> 59 | Opts.command "rm" parseCmdRm <> 60 | Opts.command "update" parseCmdUpdate <> 61 | #ifdef ENABLE_SPLICE 62 | Opts.command "splice" parseCmdSplice <> 63 | #endif 64 | Opts.command "show" parseCmdShow <> 65 | Opts.command "ls" parseCmdLs <> 66 | Opts.command "default-nix" parseCmdDefaultNix 67 | ) <|> Opts.subparser ( 68 | (Opts.command "installcheck" 69 | (subcommand "postinstall check" (pure cmdInstallCheck) [])) 70 | <> Opts.internal 71 | ) 72 | 73 | subcommand desc action infoMod = 74 | Opts.info 75 | (Opts.helper <*> action) $ 76 | mconcat ([ 77 | Opts.fullDesc, 78 | Opts.progDesc desc 79 | ] ++ infoMod) 80 | 81 | docLines :: [Doc.Doc] -> Doc.Doc 82 | docLines lines = foldr (<>) Doc.empty (intersperse Doc.hardline lines) 83 | softDocLines lines = foldr (<>) Doc.empty (intersperse Doc.softline lines) 84 | 85 | examplesDoc ex = Opts.footerDoc $ Just $ docLines ["Examples:", Doc.indent 2 $ docLines ex] 86 | 87 | newtype CommonOpts = CommonOpts { 88 | sources :: Maybe (NonEmpty Source.SourceFile) 89 | } deriving newtype Show 90 | 91 | parseCommon :: Opts.Parser CommonOpts 92 | parseCommon = 93 | build <$> parseSources <*> parseLocal <*> parsePublic 94 | where 95 | build src a b = CommonOpts { sources = NonEmpty.nonEmpty (src <> a <> b) } 96 | parseSources = many $ Source.NamedSource <$> Opts.strOption 97 | ( Opts.long "source" <> 98 | Opts.short 's' <> 99 | Opts.metavar "SOURCE.json" <> 100 | Opts.help "Specify wrangle.json file to operate on" 101 | ) 102 | parseLocal = Opts.flag [] [Source.LocalSource] 103 | ( Opts.long "local" <> 104 | Opts.help "use nix/wrangle-local.json" 105 | ) 106 | parsePublic = Opts.flag [] [Source.DefaultSource] 107 | ( Opts.long "public" <> 108 | Opts.help "use nix/wrangle.json" 109 | ) 110 | 111 | parseName :: Opts.Parser Source.PackageName 112 | parseName = Source.PackageName <$> Opts.argument Opts.str (Opts.metavar "NAME") 113 | 114 | parseNames :: Opts.Parser (Maybe (NonEmpty Source.PackageName)) 115 | parseNames = NonEmpty.nonEmpty <$> many parseName 116 | 117 | (|>) a fn = fn a 118 | 119 | lookupAttr :: Key -> StringMap -> (Maybe String, StringMap) 120 | lookupAttr key map = (AMap.lookup key map, map) 121 | 122 | consumeAttr :: Key -> StringMap -> (Maybe String, StringMap) 123 | consumeAttr key map = (AMap.lookup key map, AMap.delete key map) 124 | 125 | attrRequired :: Key -> String 126 | attrRequired key = "--"<> (Key.toString key) <> " required" 127 | 128 | consumeRequiredAttr :: Key -> StringMap -> (Either String String, StringMap) 129 | consumeRequiredAttr key map = require $ consumeAttr key map where 130 | -- this error message is a little presumptuous... 131 | require (value, map) = (toRight (attrRequired key) value, map) 132 | 133 | type StringMapState a = StateT StringMap (Either String) a 134 | 135 | consumeOptionalAttrT :: Key -> StringMapState (Maybe String) 136 | consumeOptionalAttrT key = state $ consumeAttr key 137 | 138 | lookupOptionalAttrT :: Key -> StringMapState (Maybe String) 139 | lookupOptionalAttrT key = state $ lookupAttr key 140 | 141 | consumeAttrT :: Key -> StringMapState String 142 | consumeAttrT key = StateT consume where 143 | consume :: StringMap -> Either String (String, StringMap) 144 | consume = reshape . consumeRequiredAttr key 145 | reshape (result, map) = (\result -> (result, map)) <$> result 146 | 147 | defaultGitRef = "master" 148 | 149 | data ParsedAttrs = ParsedAttrs StringMap 150 | 151 | instance Show ParsedAttrs where 152 | show (ParsedAttrs attrs) = show attrs 153 | 154 | isEmptyAttrs :: ParsedAttrs -> Bool 155 | isEmptyAttrs (ParsedAttrs attrs) = AMap.empty == attrs 156 | 157 | extractAttrs :: PackageAttrsMode -> Maybe Source.PackageName -> ParsedAttrs -> StringMap 158 | extractAttrs mode nameOpt (ParsedAttrs attrs) = canonicalizeNix withDefaultNix where 159 | withDefaultNix = case mode of 160 | PackageAttrsForAdd -> addDefaultNix nameOpt attrs 161 | _ -> attrs 162 | 163 | -- drop nix attribute it if it's explicitly `"false"` 164 | canonicalizeNix attrs = case AMap.lookup key attrs of 165 | Just "false" -> AMap.delete key attrs 166 | _ -> attrs 167 | where key = "nix" 168 | 169 | -- add default nix attribute, unless it's the `self` package 170 | addDefaultNix nameOpt attrs = case (nameOpt, AMap.lookup key attrs) of 171 | (Just (Source.PackageName "self"), Nothing) -> attrs 172 | (_, Just _) -> attrs 173 | (_, Nothing) -> AMap.insert key defaultDepNixPath attrs 174 | where key = "nix" 175 | 176 | 177 | processAdd :: Maybe PackageName -> Maybe String -> ParsedAttrs -> Either AppError (Maybe PackageName, Source.PackageSpec) 178 | processAdd nameOpt source attrs = mapLeft AppError $ build nameOpt source attrs 179 | where 180 | build :: Maybe PackageName -> Maybe String -> ParsedAttrs -> Either String (Maybe PackageName, Source.PackageSpec) 181 | build nameOpt source parsedAttrs = evalStateT 182 | (build' nameOpt source) 183 | (extractAttrs PackageAttrsForAdd nameOpt parsedAttrs) 184 | 185 | build' :: Maybe PackageName -> Maybe String -> StringMapState (Maybe PackageName, Source.PackageSpec) 186 | build' nameOpt sourceOpt = typ >>= \case 187 | Source.FetchGithub -> buildGithub sourceOpt nameOpt 188 | (Source.FetchUrl urlType) -> withName nameOpt $ buildUrl urlType sourceOpt 189 | Source.FetchPath -> withName nameOpt $ buildLocalPath sourceOpt 190 | Source.FetchGitLocal -> withName nameOpt $ buildGitLocal sourceOpt 191 | Source.FetchGit -> withName nameOpt $ buildGit sourceOpt 192 | where 193 | typ :: StringMapState Source.FetchType 194 | typ = (consumeAttrT "type" <|> pure "github") >>= lift . Source.parseFetchType 195 | 196 | withName :: Maybe PackageName -> StringMapState a -> StringMapState (Maybe PackageName, a) 197 | withName name = fmap (\snd -> (name, snd)) 198 | 199 | packageSpec :: Source.SourceSpec -> StringMapState Source.PackageSpec 200 | packageSpec sourceSpec = state $ \attrs -> (Source.PackageSpec { 201 | Source.sourceSpec, 202 | Source.packageAttrs = attrs, 203 | Source.fetchAttrs = AMap.empty 204 | }, AMap.empty) 205 | 206 | buildPathOpt :: StringMapState (Maybe Source.LocalPath) 207 | buildPathOpt = fmap pathOfString <$> consumeOptionalAttrT "path" where 208 | 209 | buildPath :: Maybe String -> StringMapState Source.LocalPath 210 | buildPath source = 211 | buildPathOpt >>= \path -> lift $ 212 | toRight "--path or source required" (path <|> (pathOfString <$> source)) 213 | 214 | pathOfString :: String -> Source.LocalPath 215 | pathOfString path = if PosixPath.isAbsolute path 216 | then Source.FullPath path 217 | else Source.RelativePath path 218 | 219 | buildLocalPath :: Maybe String -> StringMapState Source.PackageSpec 220 | buildLocalPath source = do 221 | path <- buildPath source 222 | packageSpec (Source.Path path) 223 | 224 | buildGitCommon :: StringMapState Source.GitCommon 225 | buildGitCommon = do 226 | fetchSubmodulesStr <- lookupOptionalAttrT Source.fetchSubmodulesKeyJSON 227 | fetchSubmodules <- lift $ case fetchSubmodulesStr of 228 | Just "true" -> Right True 229 | Just "false" -> Right False 230 | Nothing -> Right False 231 | Just other -> Left ("fetchSubmodules: expected Bool, got: " ++ (other)) 232 | return $ Source.GitCommon { Source.fetchSubmodules } 233 | 234 | buildGit :: Maybe String -> StringMapState Source.PackageSpec 235 | buildGit source = do 236 | urlArg <- consumeOptionalAttrT "url" 237 | gitRef <- consumeOptionalAttrT "ref" 238 | gitUrl <- lift $ toRight 239 | ("--url or source required") 240 | (urlArg <|> source) 241 | gitCommon <- buildGitCommon 242 | packageSpec $ Source.Git $ Source.GitSpec { 243 | Source.gitUrl, Source.gitCommon, 244 | Source.gitRef = Source.Template (gitRef `orElse` defaultGitRef) 245 | } 246 | 247 | buildGitLocal :: Maybe String -> StringMapState Source.PackageSpec 248 | buildGitLocal source = do 249 | glPath <- buildPath source 250 | ref <- consumeOptionalAttrT "ref" 251 | glCommon <- buildGitCommon 252 | packageSpec $ Source.GitLocal $ Source.GitLocalSpec { 253 | Source.glPath, Source.glCommon, 254 | Source.glRef = Source.Template <$> ref 255 | } 256 | 257 | buildUrl :: Source.UrlFetchType -> Maybe String -> StringMapState Source.PackageSpec 258 | buildUrl urlType source = do 259 | urlAttr <- consumeOptionalAttrT "url" 260 | url <- lift $ toRight "--url or souce required" (urlAttr <|> source) 261 | packageSpec $ Source.Url Source.UrlSpec { 262 | Source.urlType = urlType, 263 | Source.url = Source.Template url 264 | } 265 | 266 | parseGithubSource :: Maybe PackageName -> String -> Either String (PackageName, String, String) 267 | parseGithubSource name source = case span (/= '/') source of 268 | (owner, '/':repo) -> Right (fromMaybe (PackageName repo) name, owner, repo) 269 | _ -> throwError ("`" <> source <> "` doesn't look like a github repo") 270 | 271 | buildGithub :: Maybe String -> Maybe PackageName -> StringMapState (Maybe PackageName, Source.PackageSpec) 272 | buildGithub source name = do 273 | (name, ghOwner, ghRepo) <- identity 274 | ref <- consumeOptionalAttrT "ref" 275 | ghCommon <- buildGitCommon 276 | withName (Just name) $ packageSpec $ Source.Github Source.GithubSpec { 277 | Source.ghOwner, 278 | Source.ghRepo, 279 | Source.ghCommon, 280 | Source.ghRef = Source.Template . fromMaybe "master" $ ref 281 | } 282 | where 283 | explicitSource (owner, repo) = (fromMaybe (PackageName repo) name, owner, repo) 284 | 285 | identity :: StringMapState (PackageName, String, String) 286 | identity = do 287 | owner <- consumeOptionalAttrT "owner" 288 | repo <- consumeOptionalAttrT "repo" 289 | lift $ buildIdentity owner repo 290 | 291 | buildIdentity :: Maybe String -> Maybe String -> Either String (PackageName, String, String) 292 | buildIdentity owner repo = case (fromAttrs, fromSource, fromNameAsSource) of 293 | (Just fromAttrs, Nothing, _) -> Right fromAttrs 294 | (Nothing, Just fromSource, _) -> fromSource 295 | (Nothing, Nothing, Just fromName) -> fromName 296 | (Nothing, Nothing, Nothing) -> throwError "name, source or --owner/--repo required" 297 | (Just _, Just _, _) -> throwError "use source or --owner/--repo, not both" 298 | where 299 | ownerAndRepo :: Maybe (String, String) = (,) <$> owner <*> repo 300 | fromAttrs :: Maybe (PackageName, String, String) = explicitSource <$> ownerAndRepo 301 | fromSource = parseGithubSource name <$> source 302 | fromNameAsSource = parseGithubSource Nothing <$> unPackageName <$> name 303 | 304 | parseAdd :: Opts.Parser (Either AppError (PackageName, Source.PackageSpec)) 305 | parseAdd = build 306 | <$> Opts.optional parseName 307 | <*> Opts.optional parseSource 308 | <*> parsePackageAttrs PackageAttrsForAdd 309 | where 310 | parseSource = Opts.argument Opts.str (Opts.metavar "SOURCE") 311 | build :: Maybe PackageName -> Maybe String -> ParsedAttrs -> Either AppError (PackageName, Source.PackageSpec) 312 | build nameOpt source attrs = do 313 | (name, package) <- processAdd nameOpt source attrs 314 | name <- toRight (AppError "--name required") name 315 | return (name, package) 316 | 317 | data PackageAttrsMode = PackageAttrsForAdd | PackageAttrsForUpdate | PackageAttrsForSlice 318 | 319 | parsePackageAttrs :: PackageAttrsMode -> Opts.Parser ParsedAttrs 320 | parsePackageAttrs mode = ParsedAttrs . AMap.fromList <$> many parseAttribute where 321 | parseAttribute :: Opts.Parser (Key, String) 322 | parseAttribute = 323 | Opts.option (Opts.maybeReader parseKeyVal) 324 | ( Opts.long "attr" <> 325 | Opts.short 'a' <> 326 | Opts.metavar "KEY=VAL" <> 327 | Opts.help "Set the package spec attribute to " 328 | ) <|> shortcutAttributes <|> 329 | (("type",) <$> Opts.strOption 330 | ( Opts.long "type" <> 331 | Opts.short 't' <> 332 | Opts.metavar "TYPE" <> 333 | Opts.help ("The source type. "<> Source.validTypesDoc) 334 | )) 335 | 336 | -- Parse "key=val" into ("key", "val") 337 | parseKeyVal :: String -> Maybe (Key, String) 338 | parseKeyVal str = case span (/= '=') str of 339 | (key, '=':val) -> Just (Key.fromString key, val) 340 | _ -> Nothing 341 | 342 | -- Shortcuts for known attributes 343 | shortcutAttributes :: Opts.Parser (Key, String) 344 | shortcutAttributes = foldr (<|>) empty $ mkShortcutAttribute <$> shortcuts 345 | where 346 | shortcuts = case mode of 347 | PackageAttrsForAdd -> allShortcuts 348 | PackageAttrsForUpdate -> allShortcuts 349 | PackageAttrsForSlice -> sourceShortcuts 350 | allShortcuts = ("nix", "all") : sourceShortcuts 351 | sourceShortcuts = [ 352 | ("ref", "github / git / git-local"), 353 | ("fetchSubmodules", "github / git / git-local"), 354 | ("owner", "github"), 355 | ("repo", "github"), 356 | ("url", "url / file / git"), 357 | ("path", "git-local"), 358 | ("version", "all")] 359 | 360 | mkShortcutAttribute :: (String, String) -> Opts.Parser (Key, String) 361 | mkShortcutAttribute (attr, types) = 362 | (Key.fromString attr,) <$> Opts.strOption 363 | ( Opts.long attr <> 364 | Opts.metavar (toUpper <$> attr) <> 365 | Opts.help 366 | ( 367 | "Equivalent to --attr " <> attr <> "=" <> (toUpper <$> attr) <> 368 | ", used for source type " <> types 369 | ) 370 | ) 371 | 372 | ------------------------------------------------------------------------------- 373 | -- Show 374 | ------------------------------------------------------------------------------- 375 | parseCmdShow :: Opts.ParserInfo (IO ()) 376 | parseCmdShow = subcommand "Show source details" (cmdShow <$> parseCommon <*> parseNames) [] 377 | 378 | cmdShow :: CommonOpts -> Maybe (NonEmpty PackageName) -> IO () 379 | cmdShow opts names = 380 | do 381 | sourceFiles <- requireConfiguredSources $ sources opts 382 | sequence_ $ map showPkgs (NonEmpty.toList sourceFiles) where 383 | showPkgs :: Source.SourceFile -> IO () 384 | showPkgs sourceFile = do 385 | putStrLn $ " - "<>Source.pathOfSource sourceFile<>":" 386 | packages <- Source.loadSourceFile sourceFile 387 | putStrLn $ Source.encodePrettyString (filterPackages names packages) 388 | 389 | filterPackages Nothing p = Source.unPackages p 390 | filterPackages (Just names) p = HMap.filterWithKey pred (Source.unPackages p) where 391 | pred name _ = elem name names 392 | 393 | parseCmdLs :: Opts.ParserInfo (IO ()) 394 | parseCmdLs = subcommand "list sources" (cmdLs <$> parseCommon) [] 395 | 396 | cmdLs :: CommonOpts -> IO () 397 | cmdLs opts = 398 | do 399 | sourceFiles <- requireConfiguredSources $ sources opts 400 | sources <- Source.loadSources sourceFiles 401 | putStrLn $ 402 | intercalate "\n" $ 403 | map (\s -> " - "<> asString s) $ 404 | HMap.keys $ Source.unPackages $ 405 | Source.merge $ sources 406 | 407 | requireConfiguredSources :: Maybe (NonEmpty Source.SourceFile) -> IO (NonEmpty Source.SourceFile) 408 | requireConfiguredSources sources = 409 | Source.configuredSources sources >>= 410 | (liftMaybe (AppError "No wrangle JSON files found")) 411 | 412 | ------------------------------------------------------------------------------- 413 | -- Init 414 | ------------------------------------------------------------------------------- 415 | data InitOpts = InitOpts { 416 | nixpkgsChannel :: Maybe String 417 | } 418 | parseCmdInit :: Opts.ParserInfo (IO ()) 419 | parseCmdInit = subcommand "Initialize nix-wrangle" ( 420 | cmdInit <$> parseInit) [] 421 | where 422 | parseInit = Opts.optional (Opts.strOption 423 | ( Opts.long "pkgs" <> 424 | Opts.short 'p' <> 425 | Opts.metavar "CHANNEL" <> 426 | Opts.help ("Pin nixpkgs to CHANNEL") 427 | )) 428 | 429 | cmdInit :: Maybe String -> IO () 430 | cmdInit nixpkgs = do 431 | isGit <- Dir.doesPathExist ".git" 432 | debugLn $ "isGit ? " <> (show isGit) 433 | addMultiple OverwriteSource NoAutoInit (Right (wrangleSpec : (selfSpecs isGit ++ nixpkgsSpecs))) commonOpts 434 | updateDefaultNix defaultNixOptsDefault 435 | where 436 | commonOpts = CommonOpts { sources = Nothing } 437 | wrangleSpec = (PackageName "nix-wrangle", Source.PackageSpec { 438 | Source.sourceSpec = Source.Github Source.GithubSpec { 439 | Source.ghOwner = "timbertson", 440 | Source.ghRepo = "nix-wrangle", 441 | Source.ghCommon = Source.defaultGitCommon, 442 | Source.ghRef = Source.Template "v1" 443 | }, 444 | Source.fetchAttrs = AMap.empty, 445 | Source.packageAttrs = AMap.fromList [("nix", "nix")] 446 | }) 447 | nixpkgsSpecs = case nixpkgs of 448 | Nothing -> [] 449 | Just channel -> [(PackageName "pkgs", Source.PackageSpec { 450 | Source.sourceSpec = Source.Github Source.GithubSpec { 451 | Source.ghOwner = "NixOS", 452 | Source.ghRepo = "nixpkgs-channels", 453 | Source.ghCommon = Source.defaultGitCommon, 454 | Source.ghRef = Source.Template channel 455 | }, 456 | Source.fetchAttrs = AMap.empty, 457 | Source.packageAttrs = AMap.fromList [("nix", defaultDepNixPath)] 458 | })] 459 | 460 | selfSpecs isGit = 461 | if isGit then [ 462 | (PackageName "self", Source.PackageSpec { 463 | Source.sourceSpec = Source.GitLocal Source.GitLocalSpec { 464 | Source.glPath = Source.RelativePath ".", 465 | Source.glRef = Nothing, 466 | Source.glCommon = Source.defaultGitCommon 467 | }, 468 | Source.fetchAttrs = AMap.empty, 469 | Source.packageAttrs = AMap.empty 470 | }) 471 | ] else [] 472 | 473 | ------------------------------------------------------------------------------- 474 | -- Add 475 | ------------------------------------------------------------------------------- 476 | 477 | data AddMode = AddSource | OverwriteSource | AddIfMissing 478 | 479 | data AutoInit = AutoInit | NoAutoInit 480 | 481 | parseCmdAdd :: Opts.ParserInfo (IO ()) 482 | parseCmdAdd = subcommand "Add a source" (cmdAdd <$> parseAddMode <*> parseAdd <*> parseCommon) 483 | [ examplesDoc [ 484 | "nix-wrangle add timbertson/opam2nix-packages", 485 | "nix-wrangle add pkgs nixos/nixpkgs-channels --ref nixos-unstable", 486 | "nix-wrangle add pkgs nixos/nixpkgs-channels --ref nixos-unstable", 487 | "nix-wrangle add pkgs --owner nixos --repo nixpkgs-channels --ref nixos-unstable", 488 | "nix-wrangle add --type git-local self .." 489 | ]] 490 | where 491 | parseAddMode = Opts.flag AddSource OverwriteSource 492 | (Opts.long "replace" <> Opts.help "Replace existing source") 493 | 494 | addMultiple :: AddMode -> AutoInit -> Either AppError [(PackageName, Source.PackageSpec)] -> CommonOpts -> IO () 495 | addMultiple addMode autoInit addOpts opts = 496 | do 497 | addSpecs <- liftEither $ addOpts 498 | configuredSources <- Source.configuredSources $ sources opts 499 | let sourceFile = NonEmpty.head <$> configuredSources 500 | debugLn $ "sourceFile: " <> show sourceFile 501 | source <- loadOrInit autoInit sourceFile 502 | debugLn $ "source: " <> show source 503 | let (sourceFile, inputSource) = source 504 | let baseSource = fromMaybe (Source.emptyPackages) inputSource 505 | modifiedSource <- foldM addSingle baseSource addSpecs 506 | Dir.createDirectoryIfMissing True $ PosixPath.takeDirectory (Source.pathOfSource sourceFile) 507 | Source.writeSourceFile sourceFile modifiedSource 508 | where 509 | addSingle :: Source.Packages -> (PackageName, Source.PackageSpec) -> IO Source.Packages 510 | addSingle base (name, inputSpec) = do 511 | shouldAdd' <- shouldAdd addMode name base 512 | if shouldAdd' then do 513 | putStrLn $ "Adding " <> show name <> " // " <> show inputSpec 514 | spec <- Fetch.prefetch name inputSpec 515 | return $ Source.add base name spec 516 | else 517 | return base 518 | 519 | loadOrInit :: AutoInit -> Maybe Source.SourceFile -> IO (Source.SourceFile, Maybe Source.Packages) 520 | -- TODO: arrows? 521 | loadOrInit AutoInit Nothing = do 522 | let source = Source.DefaultSource 523 | infoLn $ Source.pathOfSource source <> " does not exist, initializing..." 524 | cmdInit Nothing 525 | loadOrInit NoAutoInit (Just source) 526 | 527 | loadOrInit NoAutoInit Nothing = return (Source.DefaultSource, Nothing) 528 | 529 | loadOrInit _ (Just f) = do 530 | exists <- Source.doesSourceExist f 531 | loaded <- sequence $ if exists 532 | then Just $ Source.loadSourceFile f 533 | else Nothing 534 | return (f, loaded) 535 | 536 | shouldAdd :: AddMode -> PackageName -> Source.Packages -> IO Bool 537 | shouldAdd mode name@(PackageName nameStr) existing = 538 | if Source.member existing name then 539 | case mode of 540 | AddSource -> throwM $ AppError $ nameStr <> " already present, use --replace to replace it" 541 | OverwriteSource -> infoLn ("Replacing existing " <> nameStr) >> return True 542 | AddIfMissing -> infoLn ("Not replacing existing " <> nameStr) >> return False 543 | else return True 544 | 545 | cmdAdd :: AddMode -> Either AppError (PackageName, Source.PackageSpec) -> CommonOpts -> IO () 546 | cmdAdd addMode addOpt opts = addMultiple addMode AutoInit ((\x -> [x]) <$> addOpt) opts 547 | 548 | ------------------------------------------------------------------------------- 549 | -- Rm 550 | ------------------------------------------------------------------------------- 551 | parseCmdRm :: Opts.ParserInfo (IO ()) 552 | parseCmdRm = subcommand "Remove one or more sources" (cmdRm <$> parseNames <*> parseCommon) [] 553 | 554 | cmdRm :: Maybe (NonEmpty PackageName) -> CommonOpts -> IO () 555 | cmdRm maybeNames opts = do 556 | packageNames <- liftMaybe (AppError "at least one name required") maybeNames 557 | alterPackagesNamed (Just packageNames) opts updateSingle where 558 | updateSingle :: Source.Packages -> PackageName -> IO Source.Packages 559 | updateSingle packages name = do 560 | infoLn $ " - removing " <> (show name) <> "..." 561 | return $ Source.remove packages name 562 | 563 | ------------------------------------------------------------------------------- 564 | -- Update 565 | ------------------------------------------------------------------------------- 566 | parseCmdUpdate :: Opts.ParserInfo (IO ()) 567 | parseCmdUpdate = subcommand "Update one or more sources" 568 | (cmdUpdate <$> parseNames <*> parsePackageAttrs PackageAttrsForUpdate <*> parseCommon) 569 | [ examplesDoc [ 570 | "nix-wrangle update pkgs --ref nixpkgs-unstable", 571 | "nix-wrangle update gup --nix nix/" 572 | ]] 573 | 574 | cmdUpdate :: Maybe (NonEmpty PackageName) -> ParsedAttrs -> CommonOpts -> IO () 575 | cmdUpdate packageNamesOpt parsedAttrs opts = 576 | -- Update must either specify no attributes (update everything to latest version) 577 | -- or specify one or more explicit package names 578 | if isJust packageNamesOpt || isEmptyAttrs parsedAttrs 579 | then alterPackagesNamed packageNamesOpt opts updateSingle 580 | else throwM $ AppError ( 581 | "You must explicitly list dependency names when modifying attributes (" <> show parsedAttrs <> ")" 582 | ) where 583 | 584 | updateSingle :: Source.Packages -> PackageName -> IO Source.Packages 585 | updateSingle packages name = do 586 | infoLn $ " - updating " <> (show name) <> "..." 587 | original <- liftEither $ Source.lookup name packages 588 | debugLn $ "original: " <> show original 589 | let updateAttrs = extractAttrs PackageAttrsForUpdate (Just name) parsedAttrs 590 | debugLn $ "updateAttrs: " <> show updateAttrs 591 | newSpec <- liftEither $ Source.updatePackageSpec original updateAttrs 592 | fetched <- Fetch.prefetch name newSpec 593 | if fetched == original 594 | then infoLn " ... (unchanged)" 595 | else return () 596 | return $ Source.add packages name fetched 597 | 598 | -- shared by update/rm 599 | -- TODO: pass actual source, since it is always Just 600 | processPackagesNamed :: Maybe (NonEmpty PackageName) -> CommonOpts 601 | -> (Source.SourceFile -> Source.Packages -> [PackageName] -> IO ())-> IO () 602 | processPackagesNamed packageNamesOpt opts process = do 603 | sourceFiles <- requireConfiguredSources $ sources opts 604 | sources <- sequence $ loadSource <$> sourceFiles 605 | checkMissingKeys (snd <$> sources) 606 | sequence_ $ traverseSources <$> sources 607 | where 608 | checkMissingKeys :: NonEmpty Source.Packages -> IO () 609 | checkMissingKeys sources = case missingKeys of 610 | [] -> return () 611 | _ -> fail $ "No such packages: " <> show missingKeys 612 | where 613 | (_, missingKeys) = partitionPackageNames $ Source.merge sources 614 | 615 | partitionPackageNames :: Source.Packages -> ([PackageName], [PackageName]) 616 | partitionPackageNames sources = case packageNamesOpt of 617 | Nothing -> (Source.keys sources, []) 618 | (Just names) -> partition (Source.member sources) (NonEmpty.toList names) 619 | 620 | traverseSources :: (Source.SourceFile, Source.Packages) -> IO () 621 | traverseSources (sourceFile, sources) = do 622 | let (packageNames, _) = partitionPackageNames sources 623 | debugLn $ "Package names: " <> (show packageNames) 624 | process sourceFile sources packageNames 625 | 626 | -- shared by update/rm 627 | alterPackagesNamed :: Maybe (NonEmpty PackageName) -> CommonOpts -> (Source.Packages -> PackageName -> IO Source.Packages)-> IO () 628 | alterPackagesNamed packageNamesOpt opts updateSingle = 629 | processPackagesNamed packageNamesOpt opts $ \sourceFile sources packageNames -> do 630 | infoLn $ "Updating "<> Source.pathOfSource sourceFile <> " ..." 631 | updated <- foldM updateSingle sources packageNames 632 | Source.writeSourceFile sourceFile updated 633 | 634 | loadSource :: Source.SourceFile -> IO (Source.SourceFile, Source.Packages) 635 | loadSource f = (,) f <$> Source.loadSourceFile f 636 | 637 | #ifdef ENABLE_SPLICE 638 | ------------------------------------------------------------------------------- 639 | -- Splice 640 | ------------------------------------------------------------------------------- 641 | data SpliceOutput = SpliceOutput FilePath | SpliceReplace 642 | data SpliceOpts = SpliceOpts { 643 | spliceName :: Maybe PackageName, 644 | spliceAttrs :: StringMap, 645 | spliceInput :: FilePath, 646 | spliceOutput :: SpliceOutput, 647 | spliceUpdate :: Bool 648 | } 649 | 650 | parseCmdSplice :: Opts.ParserInfo (IO ()) 651 | parseCmdSplice = subcommand "Splice current `self` source into a .nix document" 652 | (cmdSplice <$> parseSplice <*> parseCommon) [ 653 | Opts.footerDoc $ Just $ docLines [ 654 | softDocLines [ 655 | "This command generates a copy of the input .nix file, with", 656 | "the `src` attribute replaced with the current fetcher for", 657 | "the source named `public`."], 658 | "", 659 | softDocLines [ 660 | "This allows you to build a standalone", 661 | ".nix file for publishing (e.g. to nixpkgs itself)" ], 662 | "", 663 | softDocLines [ 664 | "If your source does not come from an existing wrangle.json,", 665 | "you can pass it in explicitly as attributes, like with", 666 | "`nix-wrangle add` (i.e. --type, --repo, --owner, --url, etc)"] 667 | ]] 668 | where 669 | parseSplice = build <$> parseInput <*> parseOutput <*> parseName <*> parsePackageAttrs ParsePackageAttrsSource <*> parseUpdate where 670 | build spliceInput spliceOutput spliceName spliceAttrs spliceUpdate = 671 | SpliceOpts { spliceInput, spliceOutput, spliceName, spliceAttrs, spliceUpdate } 672 | parseInput = Opts.argument Opts.str (Opts.metavar "SOURCE") 673 | parseName = Opts.optional (PackageName <$> Opts.strOption 674 | ( Opts.long "name" <> 675 | Opts.short 'n' <> 676 | Opts.metavar "NAME" <> 677 | Opts.help ("Source name to use (default: public)") 678 | )) 679 | parseOutput = explicitOutput <|> replaceOutput 680 | replaceOutput = Opts.flag' SpliceReplace 681 | ( Opts.long "replace" <> 682 | Opts.short 'r' <> 683 | Opts.help "Overwrite input file" 684 | ) 685 | explicitOutput = SpliceOutput <$> (Opts.strOption 686 | ( Opts.long "output" <> 687 | Opts.short 'o' <> 688 | Opts.metavar "DEST" <> 689 | Opts.help ("Destination file") 690 | )) 691 | parseUpdate = Opts.flag True False 692 | ( Opts.long "no-update" <> 693 | Opts.help "Don't fetch the latest version of `public` before splicing" 694 | ) 695 | 696 | cmdSplice :: SpliceOpts -> CommonOpts -> IO () 697 | cmdSplice (SpliceOpts { spliceName, spliceAttrs, spliceInput, spliceOutput, spliceUpdate}) opts = do 698 | fileContents <- Splice.load spliceInput 699 | let expr = Splice.parse fileContents 700 | expr <- Splice.getExn expr 701 | -- putStrLn $ show $ expr 702 | let existingSrcSpans = Splice.extractSourceLocs expr 703 | srcSpan <- case existingSrcSpans of 704 | [single] -> return single 705 | other -> fail $ "No single source found in " ++ (show other) 706 | self <- getPublic 707 | debugLn $ "got source: " <> show self 708 | replacedText <- liftEither $ Splice.replaceSourceLoc fileContents self srcSpan 709 | Source.writeFileText outputPath replacedText 710 | 711 | where 712 | outputPath = case spliceOutput of 713 | SpliceOutput p -> p 714 | SpliceReplace -> spliceInput 715 | 716 | getPublic :: IO Source.PackageSpec 717 | getPublic = 718 | if HMap.null spliceAttrs then do 719 | sourceFiles <- requireConfiguredSources $ sources opts 720 | sources <- Source.merge <$> Source.loadSources sourceFiles 721 | let name = (spliceName `orElse` PackageName "public") 722 | if spliceUpdate then 723 | cmdUpdate (Just $ name :| []) HMap.empty opts 724 | else 725 | return () 726 | liftEither $ Source.lookup name sources 727 | else do 728 | -- For splicing, we support a subset of `add` arguments. We don't 729 | -- accept a name or source, only explicit spliceAttrs 730 | infoLn $ "Splicing anonymous source from attributes: " <> show spliceAttrs 731 | self <- liftEither $ snd <$> processAdd Nothing Nothing spliceAttrs 732 | Fetch.prefetch (PackageName "self") self 733 | #endif 734 | -- ^ ENABLE_SPLICE 735 | 736 | ------------------------------------------------------------------------------- 737 | -- default-nix 738 | ------------------------------------------------------------------------------- 739 | parseCmdDefaultNix :: Opts.ParserInfo (IO ()) 740 | parseCmdDefaultNix = subcommand "Generate default.nix" 741 | (pure cmdDefaultNix) [ 742 | Opts.footerDoc $ Just $ 743 | "Typically this only needs to be done once, though it" <> 744 | " may be necessary if you have a very old default.nix" 745 | ] 746 | 747 | cmdDefaultNix :: IO () 748 | cmdDefaultNix = updateDefaultNix (DefaultNixOpts { force = True }) 749 | 750 | data DefaultNixOpts = DefaultNixOpts { 751 | force :: Bool 752 | } 753 | defaultNixOptsDefault = DefaultNixOpts { force = False } 754 | 755 | updateDefaultNix :: DefaultNixOpts -> IO () 756 | updateDefaultNix (DefaultNixOpts { force }) = do 757 | continue <- if force then return True else shouldWriteFile 758 | if continue then Source.writeFileText path contents 759 | else infoLn $ "Note: not replacing existing "<>path<>", run `nix-wrangle default-nix` to explicitly override" 760 | where 761 | path = "default.nix" 762 | markerText :: T.Text = "# Note: This file is generated by nix-wrangle" 763 | contents :: T.Text 764 | contents = T.unlines [ 765 | markerText, 766 | "# It can be regenerated with `nix-wrangle default-nix`", 767 | defaultNixContents ] 768 | 769 | shouldWriteFile :: IO Bool 770 | shouldWriteFile = do 771 | exists <- Dir.doesFileExist path 772 | if exists then 773 | (T.isInfixOf markerText) <$> TE.decodeUtf8 <$> B.readFile path 774 | else 775 | return True 776 | 777 | defaultDepNixPath = "default.nix" 778 | 779 | defaultNixContents = T.strip [QQ.s| 780 | let 781 | systemNixpkgs = import {}; 782 | fallback = val: dfl: if val == null then dfl else val; 783 | makeFetchers = pkgs: { 784 | github = pkgs.fetchFromGitHub; 785 | url = builtins.fetchTarball; 786 | }; 787 | fetch = pkgs: source: 788 | (builtins.getAttr source.type (makeFetchers pkgs)) source.fetch; 789 | sourcesJson = (builtins.fromJSON (builtins.readFile ./nix/wrangle.json)).sources; 790 | wrangleJson = sourcesJson.nix-wrangle or (abort "No nix-wrangle entry in nix/wrangle.json"); 791 | in 792 | { pkgs ? null, nix-wrangle ? null, ... }@provided: 793 | let 794 | _pkgs = fallback pkgs ( 795 | if builtins.hasAttr "pkgs" sourcesJson 796 | then import (fetch systemNixpkgs sourcesJson.pkgs) {} else systemNixpkgs 797 | ); 798 | _wrangle = fallback nix-wrangle (_pkgs.callPackage "${fetch _pkgs wrangleJson}/${wrangleJson.nix}" {}); 799 | in 800 | (_wrangle.api { pkgs = _pkgs; }).inject { inherit provided; path = ./.; } 801 | |] 802 | 803 | cmdInstallCheck :: IO () 804 | cmdInstallCheck = do 805 | apiContext <- Fetch.globalApiContext 806 | let apiPath = Fetch.apiNix apiContext 807 | infoLn $ "checking for nix API at "<>apiPath 808 | apiExists <- Dir.doesFileExist apiPath 809 | if not apiExists 810 | then exitFailure 811 | else return () 812 | infoLn "ok" 813 | --------------------------------------------------------------------------------