├── README.md ├── .envrc ├── yarn2nix ├── .gitignore ├── Setup.hs ├── Main.hs ├── tests │ ├── README.md │ ├── Test.hs │ └── TestNpmjsPackage.hs ├── Repl.hs ├── LICENSE ├── src │ ├── Distribution │ │ ├── Nixpkgs │ │ │ └── Nodejs │ │ │ │ ├── Utils.hs │ │ │ │ ├── FromPackage.hs │ │ │ │ ├── ResolveLockfile.hs │ │ │ │ ├── License.hs │ │ │ │ ├── Cli.hs │ │ │ │ └── OptimizedNixOutput.hs │ │ └── Nodejs │ │ │ └── Package.hs │ └── Nix │ │ └── Expr │ │ └── Additions.hs ├── yarn2nix.nix ├── nix-lib │ ├── buildNodePackage.nix │ └── default.nix ├── package.yaml ├── README.md ├── yarn2nix.cabal ├── CHANGELOG.md └── NodePackageTool.hs ├── .gitignore ├── cabal.project ├── nix-tests ├── left-pad │ └── package.json ├── deps.nix ├── test-overriding.nix ├── vendor │ └── runTestsuite.nix └── default.nix ├── yarn-lock ├── tests │ ├── Test.hs │ ├── TestMultiKeyedMap.hs │ ├── TestParse.hs │ └── TestFile.hs ├── yarn-lock.nix ├── package.yaml ├── LICENSE ├── yarn-lock.cabal ├── src │ ├── Yarn │ │ ├── Lock │ │ │ ├── Helpers.hs │ │ │ ├── Types.hs │ │ │ ├── File.hs │ │ │ └── Parse.hs │ │ └── Lock.hs │ └── Data │ │ └── MultiKeyedMap.hs └── CHANGELOG.md ├── nixpkgs-pinned.nix ├── shell.nix ├── LICENSE ├── default.nix ├── haskell-pkgs.nix ├── .github └── workflows │ └── main.yml └── .hlint.yaml /README.md: -------------------------------------------------------------------------------- 1 | yarn2nix/README.md -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | eval "$(lorri direnv)" 2 | -------------------------------------------------------------------------------- /yarn2nix/.gitignore: -------------------------------------------------------------------------------- 1 | /tests/nix-tests/result* 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /dist-newstyle 2 | /result* 3 | /.ninja 4 | -------------------------------------------------------------------------------- /cabal.project: -------------------------------------------------------------------------------- 1 | packages: 2 | ./yarn2nix 3 | ./yarn-lock 4 | -------------------------------------------------------------------------------- /yarn2nix/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /nix-tests/left-pad/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "left-pad", 3 | "version": "0.1.2" 4 | } 5 | -------------------------------------------------------------------------------- /yarn2nix/Main.hs: -------------------------------------------------------------------------------- 1 | module Main where 2 | 3 | import Distribution.Nixpkgs.Nodejs.Cli (cli) 4 | 5 | main :: IO () 6 | main = cli 7 | -------------------------------------------------------------------------------- /yarn2nix/tests/README.md: -------------------------------------------------------------------------------- 1 | # yarn2nix testsuite 2 | 3 | To run the haskell tests: 4 | 5 | $ cabal test 6 | 7 | To run the nix tests: 8 | 9 | $ 10 | -------------------------------------------------------------------------------- /yarn2nix/tests/Test.hs: -------------------------------------------------------------------------------- 1 | module Main where 2 | 3 | import Test.Tasty 4 | import qualified TestNpmjsPackage as Npmjs 5 | 6 | main :: IO () 7 | main = defaultMain $ testGroup "tests" 8 | [ Npmjs.tests 9 | ] 10 | -------------------------------------------------------------------------------- /yarn-lock/tests/Test.hs: -------------------------------------------------------------------------------- 1 | module Main where 2 | 3 | import Test.Tasty 4 | import qualified TestParse as Parse 5 | import qualified TestFile as File 6 | import qualified TestMultiKeyedMap as MKM 7 | 8 | main :: IO () 9 | main = defaultMain $ testGroup "tests" 10 | [ Parse.tests 11 | , File.tests 12 | , MKM.tests 13 | ] 14 | -------------------------------------------------------------------------------- /nixpkgs-pinned.nix: -------------------------------------------------------------------------------- 1 | let 2 | # nixos unstable 2022-03-27 3 | rev = "1f57d3e7224290eebda23fa1c79718d6b8361574"; 4 | sha256 = "0l81fiqgh6fyz9j3y9fd5v7lqzc40f8j5mj9kac3sprs8dqwravq"; 5 | in 6 | 7 | import (builtins.fetchTarball { 8 | url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz"; 9 | inherit sha256; 10 | }) 11 | -------------------------------------------------------------------------------- /nix-tests/deps.nix: -------------------------------------------------------------------------------- 1 | # this is a file that has the same interface 2 | # as the files generated by yarn2nix, but uses 3 | # local files instead of downloading them 4 | { fetchurl, fetchgit }: 5 | self: super: 6 | 7 | let 8 | buildNodePackage = super._buildNodePackage; 9 | 10 | localFilePackage = key: version: path: deps: 11 | buildNodePackage { 12 | inherit key version; 13 | nodeBuildInputs = deps; 14 | src = path; 15 | }; 16 | 17 | in { 18 | "left-pad" = localFilePackage "left-pad" "0.1.2" ./left-pad []; 19 | } 20 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import ./nixpkgs-pinned.nix {} }: 2 | 3 | let 4 | haskellPackages = import ./haskell-pkgs.nix { 5 | inherit pkgs; 6 | }; 7 | 8 | # fetch a github tarball, at evaluation time 9 | githubSrc = { 10 | owner, 11 | repo, 12 | revision, 13 | sha256 14 | }: 15 | builtins.fetchTarball { 16 | url = "https://github.com/${owner}/${repo}/archive/${revision}.tar.gz"; 17 | inherit sha256; 18 | }; 19 | 20 | in 21 | haskellPackages.shellFor { 22 | packages = hps: [ 23 | hps.yarn2nix 24 | hps.yarn-lock 25 | ]; 26 | withHoogle = true; 27 | buildInputs = [ 28 | pkgs.ninja 29 | pkgs.cabal-install 30 | pkgs.hpack 31 | pkgs.ghcid 32 | haskellPackages.haskell-language-server 33 | ]; 34 | } 35 | -------------------------------------------------------------------------------- /yarn-lock/yarn-lock.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, ansi-wl-pprint, base, containers, either, hpack 2 | , lib, megaparsec, neat-interpolation, quickcheck-instances, tasty 3 | , tasty-hunit, tasty-quickcheck, tasty-th, text 4 | }: 5 | mkDerivation { 6 | pname = "yarn-lock"; 7 | version = "0.6.5"; 8 | src = ./.; 9 | libraryHaskellDepends = [ base containers either megaparsec text ]; 10 | libraryToolDepends = [ hpack ]; 11 | testHaskellDepends = [ 12 | ansi-wl-pprint base containers either megaparsec neat-interpolation 13 | quickcheck-instances tasty tasty-hunit tasty-quickcheck tasty-th 14 | text 15 | ]; 16 | prePatch = "hpack"; 17 | homepage = "https://github.com/Profpatsch/yarn2nix#readme"; 18 | description = "Represent and parse yarn.lock files"; 19 | license = lib.licenses.mit; 20 | } 21 | -------------------------------------------------------------------------------- /yarn2nix/Repl.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NoImplicitPrelude, LambdaCase, RecordWildCards, OverloadedStrings #-} 2 | module Repl where 3 | 4 | import Prelude () 5 | import Distribution.Nixpkgs.Nodejs.ResolveLockfile 6 | import Distribution.Nixpkgs.Nodejs.OptimizedNixOutput 7 | import Protolude 8 | import Yarn.Lock 9 | import Yarn.Lock.Types 10 | import Control.Concurrent.Chan 11 | import Text.PrettyPrint.ANSI.Leijen (putDoc) 12 | import Nix.Pretty (prettyNix) 13 | 14 | yarn = "./yl" 15 | 16 | ps = do 17 | Right res <- makeitso 18 | putDoc $ prettyNix $ mkPackageSet $ convertLockfile res 19 | 20 | makeitso = do 21 | Right f <- parseFile yarn 22 | ch <- newChan 23 | thrd <- forkIO $ forever $ do 24 | readChan ch >>= \case 25 | FileRemote{..} -> pass 26 | GitRemote{..} -> print $ "Downloading " <> gitRepoUrl 27 | lf <- resolveLockfileStatus ch f 28 | killThread thrd 29 | pure lf 30 | -------------------------------------------------------------------------------- /yarn-lock/package.yaml: -------------------------------------------------------------------------------- 1 | name: yarn-lock 2 | version: 0.6.5 3 | github: Profpatsch/yarn2nix 4 | license: MIT 5 | license-file: LICENSE 6 | synopsis: Represent and parse yarn.lock files 7 | description: 8 | Types and parser for the lock file format of the npm successor yarn. 9 | All modules should be imported qualified. 10 | author: Profpatsch 11 | maintainer: mail@profpatsch.de 12 | category: Data 13 | extra-source-files: 14 | - CHANGELOG.md 15 | 16 | ghc-options: 17 | - -Wall 18 | 19 | dependencies: 20 | - base == 4.* 21 | - containers 22 | - text 23 | - megaparsec >= 7 && < 10 24 | - either >= 4 && < 6 25 | 26 | library: 27 | source-dirs: src 28 | 29 | tests: 30 | yarn-lock-tests: 31 | main: Test.hs 32 | source-dirs: tests 33 | dependencies: 34 | - yarn-lock 35 | - ansi-wl-pprint >= 0.6 36 | - tasty >= 0.11 37 | - tasty-th >= 0.1.7 38 | - tasty-hunit >= 0.9 39 | - tasty-quickcheck >= 0.8 40 | - quickcheck-instances == 0.3.* 41 | - neat-interpolation >= 0.3 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Profpatsch 2 | MIT LICENSE 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. -------------------------------------------------------------------------------- /yarn2nix/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Profpatsch 2 | MIT LICENSE 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. -------------------------------------------------------------------------------- /yarn-lock/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Profpatsch 2 | MIT LICENSE 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. -------------------------------------------------------------------------------- /yarn2nix/src/Distribution/Nixpkgs/Nodejs/Utils.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NoImplicitPrelude, OverloadedStrings, RecordWildCards, LambdaCase #-} 2 | {-| 3 | Description: Misc utils 4 | -} 5 | module Distribution.Nixpkgs.Nodejs.Utils where 6 | import Protolude 7 | import Nix.Expr 8 | import qualified Yarn.Lock.Types as YLT 9 | 10 | -- | Representation of a PackageKey as nix attribute name. 11 | packageKeyToSymbol :: YLT.PackageKey -> Text 12 | packageKeyToSymbol (YLT.PackageKey{..}) = 13 | packageKeyNameToSymbol name <> "@" <> npmVersionSpec 14 | {-# INLINABLE packageKeyToSymbol #-} 15 | 16 | -- | Representation of a PackageKeyName as nix attribute name. 17 | packageKeyNameToSymbol :: YLT.PackageKeyName -> Text 18 | packageKeyNameToSymbol = \case 19 | YLT.SimplePackageKey n -> n 20 | YLT.ScopedPackageKey scope n -> "@" <> scope <> "/" <> n 21 | {-# INLINABLE packageKeyNameToSymbol #-} 22 | 23 | -- | Return a 'Binding' if 'Just' (or none if 'Nothing') 24 | -- for 'mkRecSet' and 'mkNonRecSet'. 25 | attrSetMay :: Text -> Maybe NExpr -> [Binding NExpr] 26 | attrSetMay k v = maybeToList $ (k $=) <$> v 27 | {-# INLINABLE attrSetMay #-} 28 | 29 | -- | Convenience shortcut for @'attrSetMay' x (mkStr \<$\> y)@. 30 | attrSetMayStr :: Text -> Maybe Text -> [Binding NExpr] 31 | attrSetMayStr k = attrSetMay k . fmap mkStr 32 | {-# INLINABLE attrSetMayStr #-} 33 | -------------------------------------------------------------------------------- /nix-tests/test-overriding.nix: -------------------------------------------------------------------------------- 1 | { pkgs, nixLib, yarn2nix }: 2 | 3 | nixLib.linkNodeDeps { 4 | name = "test"; 5 | dependencies = 6 | let allDeps = 7 | nixLib.buildNodeDeps (pkgs.lib.composeExtensions 8 | (pkgs.callPackage ./deps.nix {}) 9 | (self: super: { 10 | # we are able to override an existing package 11 | # TODO don’t use the { name, drv } version 12 | # of _buildNodePackage here. 13 | left-pad = { 14 | key = { scope = ""; name = "left-pad"; }; 15 | drv = super.left-pad.drv.overrideAttrs (old: { 16 | buildPhase = ''echo OVERRIDDEN!''; 17 | }); 18 | }; 19 | # we can also add a new package 20 | right-pad = super._buildNodePackage { 21 | key = "right-pad"; 22 | version = "0.0.0.1"; 23 | src = pkgs.runCommand "right-pad-src" {} '' 24 | mkdir $out 25 | echo '{ "name": "right-pad", "version": "0.0.0.1" }' \ 26 | > $out/package.json 27 | ''; 28 | nodeBuildInputs = [ self.left-pad ]; 29 | # which reference the other package 30 | postInstall = '' 31 | echo left-pad package.json 32 | cat $out/node_modules/left-pad/package.json 33 | ''; 34 | }; 35 | })); 36 | in [ 37 | allDeps.left-pad 38 | allDeps.right-pad 39 | ]; 40 | } 41 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import ./nixpkgs-pinned.nix {} }: 2 | 3 | let 4 | lib = pkgs.lib; 5 | 6 | haskellPackages = import ./haskell-pkgs.nix { 7 | inherit pkgs; 8 | }; 9 | 10 | static = pkgs.haskell.lib.justStaticExecutables haskellPackages.yarn2nix; 11 | 12 | yarn2nix = pkgs.stdenv.mkDerivation { 13 | name = "yarn2nix"; 14 | src = "${pkgs.nix-gitignore.gitignoreSource [ 15 | ".git/" 16 | ] ./yarn2nix}"; 17 | outputs = [ "bin" "doc" "out" ]; 18 | phases = [ "unpackPhase" "installPhase" "fixupPhase" ]; 19 | installPhase = '' 20 | install -D --target-directory=$bin/bin ${static}/bin/* 21 | ${pkgs.skawarePackages.cleanPackaging.commonFileActions { 22 | noiseFiles = [ 23 | ".gitignore" 24 | "tests/*" 25 | "src/*" 26 | "Main.hs" 27 | "Setup.hs" 28 | "package.yaml" 29 | "yarn2nix.cabal" 30 | ".envrc" 31 | "Repl.hs" 32 | "yarn2nix.nix" 33 | "NodePackageTool.hs" 34 | "nix-lib" 35 | ]; 36 | docFiles = [ 37 | "README.md" 38 | "LICENSE" 39 | "CHANGELOG.md" 40 | ]; 41 | }} "$doc/share/yarn2nix" 42 | ${pkgs.skawarePackages.cleanPackaging.checkForRemainingFiles} 43 | ''; 44 | 45 | passthru.nixLib = import ./yarn2nix/nix-lib { 46 | inherit lib pkgs; 47 | inherit yarn2nix; 48 | }; 49 | }; 50 | 51 | in yarn2nix 52 | -------------------------------------------------------------------------------- /yarn2nix/yarn2nix.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, aeson, aeson-better-errors, async-pool, base 2 | , bytestring, containers, data-fix, directory, filepath, hnix 3 | , hpack, lib, mtl, neat-interpolation, optparse-applicative 4 | , prettyprinter, process, protolude, regex-tdfa, scientific, stm 5 | , tasty, tasty-hunit, tasty-quickcheck, tasty-th, text 6 | , transformers, unix, unordered-containers, yarn-lock 7 | }: 8 | mkDerivation { 9 | pname = "yarn2nix"; 10 | version = "0.10.1"; 11 | src = ./.; 12 | isLibrary = true; 13 | isExecutable = true; 14 | libraryHaskellDepends = [ 15 | aeson aeson-better-errors async-pool base bytestring containers 16 | data-fix directory filepath hnix mtl optparse-applicative 17 | prettyprinter process protolude regex-tdfa scientific stm text 18 | transformers unordered-containers yarn-lock 19 | ]; 20 | libraryToolDepends = [ hpack ]; 21 | executableHaskellDepends = [ 22 | aeson aeson-better-errors async-pool base bytestring containers 23 | data-fix directory filepath hnix mtl optparse-applicative 24 | prettyprinter process protolude regex-tdfa scientific stm text 25 | transformers unix unordered-containers yarn-lock 26 | ]; 27 | testHaskellDepends = [ 28 | aeson aeson-better-errors async-pool base bytestring containers 29 | data-fix directory filepath hnix mtl neat-interpolation 30 | optparse-applicative prettyprinter process protolude regex-tdfa 31 | scientific stm tasty tasty-hunit tasty-quickcheck tasty-th text 32 | transformers unordered-containers yarn-lock 33 | ]; 34 | prePatch = "hpack"; 35 | homepage = "https://github.com/Profpatsch/yarn2nix#readme"; 36 | description = "Convert yarn.lock files to nix expressions"; 37 | license = lib.licenses.mit; 38 | } 39 | -------------------------------------------------------------------------------- /yarn-lock/tests/TestMultiKeyedMap.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, TemplateHaskell, QuasiQuotes, NamedFieldPuns, ViewPatterns #-} 2 | module TestMultiKeyedMap (tests) where 3 | 4 | import qualified Data.List as List 5 | import qualified Data.List.NonEmpty as NE 6 | import Test.Tasty (TestTree) 7 | import Test.Tasty.TH 8 | import Test.Tasty.QuickCheck 9 | import Test.QuickCheck.Instances () -- orphans! 10 | 11 | import qualified Data.MultiKeyedMap as MKM 12 | import Data.Data (Proxy (Proxy)) 13 | import Data.Function (on) 14 | import Data.Foldable (foldl') 15 | 16 | emptyMkm :: (Ord k) => MKM.MKMap k v 17 | emptyMkm = MKM.mkMKMap (Proxy :: Proxy Int) 18 | 19 | fromList :: [((NE.NonEmpty Int), v)] -> MKM.MKMap Int v 20 | fromList = MKM.fromList (Proxy :: Proxy Int) 21 | 22 | prop_equality :: Property 23 | prop_equality = 24 | forAll (resize 5 arbitrary :: Gen [(NE.NonEmpty Int, Int)]) 25 | $ \map1 -> forAll (resize 2 arbitrary :: Gen [(NE.NonEmpty Int, Int)]) 26 | $ \map2 -> 27 | -- equality of contents also applies to the MKM 28 | (map1 == map2) === (fromList map1 == fromList map2) 29 | -- force the contents to be the same, should always be equal 30 | 31 | -- | inserting the same value is idempotent 32 | prop_insertIdempotent :: Int -> Int -> Property 33 | prop_insertIdempotent key val = do 34 | let insVal = MKM.insert key val 35 | (insVal emptyMkm) === (insVal (insVal emptyMkm)) 36 | 37 | -- | inserting the same values in a different order 38 | -- results in the same map 39 | prop_insertShuffled :: [(Int, Int)] -> Property 40 | prop_insertShuffled xs = 41 | let xs' = List.nubBy ((==) `on` fst) xs 42 | insVals = foldl' (\m (k, v) -> MKM.insert k v m) emptyMkm 43 | in forAll (shuffle xs') 44 | $ \ys -> insVals xs' === insVals ys 45 | 46 | tests :: TestTree 47 | tests = $(testGroupGenerator) 48 | -------------------------------------------------------------------------------- /haskell-pkgs.nix: -------------------------------------------------------------------------------- 1 | { pkgs }: 2 | 3 | let 4 | inherit (pkgs) lib; 5 | 6 | licensesJson = pkgs.writeText "licenses.json" 7 | (builtins.toJSON (lib.filterAttrs (n: v: v ? spdxId) lib.licenses)); 8 | 9 | minimalHaskellSource = root: extra: 10 | builtins.path { 11 | path = root; 12 | name = "${builtins.baseNameOf root}-source"; 13 | filter = path: type: 14 | lib.any (p: lib.hasPrefix (toString root + "/" + p) path) ([ 15 | "package.yaml" 16 | "LICENSE" 17 | "CHANGELOG.md" 18 | "src" 19 | "tests" 20 | ] ++ extra); 21 | }; 22 | in 23 | 24 | pkgs.haskellPackages.override { 25 | overrides = 26 | (self: super: { 27 | yarn-lock = 28 | let 29 | pkg = self.callPackage ./yarn-lock/yarn-lock.nix { 30 | inherit (pkgs) hpack; 31 | }; 32 | in pkgs.haskell.lib.overrideCabal pkg (old: { 33 | version = "git"; 34 | src = minimalHaskellSource ./yarn-lock []; 35 | }); 36 | 37 | yarn2nix = 38 | let 39 | pkg = self.callPackage ./yarn2nix/yarn2nix.nix { 40 | inherit (pkgs) hpack; 41 | }; 42 | in pkgs.haskell.lib.overrideCabal pkg (old: { 43 | version = "git"; 44 | src = minimalHaskellSource ./yarn2nix [ 45 | "Main.hs" 46 | "NodePackageTool.hs" 47 | ]; 48 | 49 | prePatch = '' 50 | # we depend on the git prefetcher 51 | substituteInPlace \ 52 | src/Distribution/Nixpkgs/Nodejs/ResolveLockfile.hs \ 53 | --replace '"nix-prefetch-git"' \ 54 | '"${pkgs.nix-prefetch-git.override { git = pkgs.gitMinimal; }}/bin/nix-prefetch-git"' 55 | sed -i '/license-data/a \ <> O.value "${licensesJson}"' \ 56 | src/Distribution/Nixpkgs/Nodejs/Cli.hs 57 | '' + old.prePatch or ""; 58 | }); 59 | }); 60 | } 61 | -------------------------------------------------------------------------------- /yarn2nix/nix-lib/buildNodePackage.nix: -------------------------------------------------------------------------------- 1 | { lib, stdenv, linkNodeDeps, nodejs, yarn2nix }: 2 | { key # { scope: String, name: String } 3 | , version # String 4 | , src # Drv 5 | , nodeBuildInputs # Listof { key: { scope: String, name: String }, drv : Drv } 6 | , ... }@args: 7 | 8 | # since we skip the build phase, pre and post will not work 9 | # the caller gives them with no buildPhase 10 | assert (args ? preBuild || args ? postBuild) -> args ? buildPhase; 11 | # same for configurePhase 12 | assert (args ? preConfigure || args ? postConfigure) -> args ? configurePhase; 13 | 14 | with lib; 15 | 16 | let 17 | # TODO: scope should be more structured somehow. :( 18 | packageName = 19 | if key.scope == "" 20 | then "${key.name}-${version}" 21 | else "${key.scope}-${key.name}-${version}"; 22 | 23 | in stdenv.mkDerivation ((removeAttrs args [ "key" "nodeBuildInputs" ]) // { 24 | name = packageName; 25 | inherit version src; 26 | 27 | buildInputs = [ nodejs ]; 28 | 29 | configurePhase = args.configurePhase or "true"; 30 | # skip the build phase except when given as attribute 31 | dontBuild = !(args ? buildPhase); 32 | 33 | # TODO: maybe we can enable tests? 34 | doCheck = false; 35 | 36 | installPhase = '' 37 | runHook preInstall 38 | mkdir $out 39 | 40 | # a npm package is just the tarball extracted to $out 41 | cp -r . $out 42 | 43 | # the binaries should be executable (TODO: always on?) 44 | ${yarn2nix}/bin/node-package-tool \ 45 | set-bin-exec-flag \ 46 | --package $out 47 | 48 | # then a node_modules folder is created for all its dependencies 49 | ${if nodeBuildInputs != [] 50 | then '' 51 | rm -rf $out/node_modules 52 | ln -sT "${linkNodeDeps { 53 | name = packageName; 54 | dependencies = nodeBuildInputs; 55 | }}" $out/node_modules 56 | '' else ""} 57 | 58 | runHook postInstall 59 | ''; 60 | 61 | dontStrip = true; # stolen from npm2nix 62 | 63 | }) 64 | -------------------------------------------------------------------------------- /yarn2nix/package.yaml: -------------------------------------------------------------------------------- 1 | name: yarn2nix 2 | version: 0.10.1 3 | github: Profpatsch/yarn2nix 4 | license: MIT 5 | license-file: LICENSE 6 | synopsis: Convert yarn.lock files to nix expressions 7 | description: Convert @yarn.lock@ files to nix expressions. See @yarn2nix@ executable. Contains a nix library to call the generated nix files in @nix-lib/@. Library functions and module names might be restructured in the future. 8 | author: Profpatsch 9 | maintainer: mail@profpatsch.de 10 | category: Distribution, Nix 11 | 12 | extra-source-files: 13 | - LICENSE 14 | - README.md 15 | - nix-lib/* 16 | 17 | ghc-options: 18 | - -Wall 19 | 20 | dependencies: 21 | - aeson >= 2.0 22 | - aeson-better-errors >= 0.9.1.1 23 | - async-pool == 0.9.* 24 | - base == 4.* 25 | - bytestring == 0.10.* 26 | - containers >= 0.5 && < 0.7 27 | - data-fix >= 0.0.7 && < 0.4 28 | - directory == 1.3.* 29 | - filepath == 1.4.* 30 | - hnix >= 0.6 && < 0.15 31 | - mtl == 2.2.* 32 | - prettyprinter >= 1.2 && < 1.8 33 | - process >= 1.4 34 | - protolude ^>= 0.3 35 | - regex-tdfa ^>= 1.3 36 | - stm > 2.4.0 && < 2.6.0.0 37 | - scientific > 0.3.3.0 && < 0.4 38 | - text == 1.2.* 39 | - transformers == 0.5.* 40 | - unordered-containers == 0.2.* 41 | - yarn-lock == 0.6.* 42 | - optparse-applicative >= 0.16 && < 0.17 43 | 44 | library: 45 | source-dirs: src 46 | 47 | executables: 48 | yarn2nix: 49 | main: Main.hs 50 | dependencies: 51 | - yarn2nix 52 | node-package-tool: 53 | main: NodePackageTool.hs 54 | dependencies: 55 | - optparse-applicative >= 0.13 56 | - unix == 2.7.* 57 | - yarn2nix 58 | 59 | tests: 60 | yarn2nix-tests: 61 | main: Test.hs 62 | source-dirs: tests 63 | dependencies: 64 | - neat-interpolation >= 0.3 && < 0.6 65 | - protolude ^>= 0.3 66 | - tasty >= 0.11 && < 1.5 67 | - tasty-hunit >= 0.9 && < 0.11 68 | - tasty-quickcheck >= 0.8 && < 0.11 69 | - tasty-th == 0.1.7.* 70 | - yarn2nix 71 | -------------------------------------------------------------------------------- /yarn-lock/yarn-lock.cabal: -------------------------------------------------------------------------------- 1 | cabal-version: 1.12 2 | 3 | -- This file has been generated from package.yaml by hpack version 0.34.4. 4 | -- 5 | -- see: https://github.com/sol/hpack 6 | 7 | name: yarn-lock 8 | version: 0.6.5 9 | synopsis: Represent and parse yarn.lock files 10 | description: Types and parser for the lock file format of the npm successor yarn. All modules should be imported qualified. 11 | category: Data 12 | homepage: https://github.com/Profpatsch/yarn2nix#readme 13 | bug-reports: https://github.com/Profpatsch/yarn2nix/issues 14 | author: Profpatsch 15 | maintainer: mail@profpatsch.de 16 | license: MIT 17 | license-file: LICENSE 18 | build-type: Simple 19 | extra-source-files: 20 | CHANGELOG.md 21 | 22 | source-repository head 23 | type: git 24 | location: https://github.com/Profpatsch/yarn2nix 25 | 26 | library 27 | exposed-modules: 28 | Data.MultiKeyedMap 29 | Yarn.Lock 30 | Yarn.Lock.File 31 | Yarn.Lock.Helpers 32 | Yarn.Lock.Parse 33 | Yarn.Lock.Types 34 | other-modules: 35 | Paths_yarn_lock 36 | hs-source-dirs: 37 | src 38 | ghc-options: -Wall 39 | build-depends: 40 | base ==4.* 41 | , containers 42 | , either >=4 && <6 43 | , megaparsec >=7 && <10 44 | , text 45 | default-language: Haskell2010 46 | 47 | test-suite yarn-lock-tests 48 | type: exitcode-stdio-1.0 49 | main-is: Test.hs 50 | other-modules: 51 | TestFile 52 | TestMultiKeyedMap 53 | TestParse 54 | Paths_yarn_lock 55 | hs-source-dirs: 56 | tests 57 | ghc-options: -Wall 58 | build-depends: 59 | ansi-wl-pprint >=0.6 60 | , base ==4.* 61 | , containers 62 | , either >=4 && <6 63 | , megaparsec >=7 && <10 64 | , neat-interpolation >=0.3 65 | , quickcheck-instances ==0.3.* 66 | , tasty >=0.11 67 | , tasty-hunit >=0.9 68 | , tasty-quickcheck >=0.8 69 | , tasty-th >=0.1.7 70 | , text 71 | , yarn-lock 72 | default-language: Haskell2010 73 | -------------------------------------------------------------------------------- /yarn-lock/src/Yarn/Lock/Helpers.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Yarn.Lock.Helpers 3 | Description : Helpers for modifying Lockfiles 4 | Maintainer : Profpatsch 5 | Stability : experimental 6 | 7 | Freshly parsed 'Lockfile's are often not directly usable 8 | e.g. because they still can contain cycles. This module 9 | provides helpers for modifying them. 10 | -} 11 | module Yarn.Lock.Helpers 12 | ( decycle 13 | ) where 14 | 15 | import qualified Data.List as L 16 | import GHC.Stack (HasCallStack) 17 | 18 | import qualified Data.MultiKeyedMap as MKM 19 | 20 | import Yarn.Lock.Types 21 | import Data.Foldable (foldl') 22 | 23 | 24 | -- | Takes a 'Lockfile' and removes dependency cycles. 25 | -- 26 | -- Node packages often contain those and the yarn lockfile 27 | -- does not yet eliminate them, which may lead to infinite 28 | -- recursions. 29 | -- 30 | -- Invariant: Every dependency entry in each package in the 31 | -- 'Lockfile' *must* point to an existing key, otherwise 32 | -- the function crashes. 33 | decycle :: HasCallStack => Lockfile -> Lockfile 34 | decycle lf = goFold [] lf (MKM.keys lf) 35 | -- TODO: probably rewrite with State 36 | where 37 | -- | fold over all package keys, passing the lockfile 38 | goFold seen lf' pkeys = 39 | foldl' (\lf'' pkey -> go (pkey:seen) lf'') lf' pkeys 40 | -- | We get a stack of already seen packages 41 | -- and filter out any dependencies we already saw. 42 | go :: [PackageKey] -> Lockfile -> Lockfile 43 | go seen@(we:_) lf' = 44 | let ourPkg = lf' MKM.! we 45 | -- old deps minus the already seen ones 46 | -- TODO make handling of opt pkgs less of a duplication 47 | newDeps = dependencies ourPkg L.\\ seen 48 | newOptDeps = optionalDependencies ourPkg L.\\ seen 49 | -- we update the pkg with the cleaned dependencies 50 | lf'' = MKM.insert we (ourPkg { dependencies = newDeps 51 | , optionalDependencies = newOptDeps }) lf' 52 | -- finally we do the same for all remaining deps 53 | in goFold seen lf'' $ newDeps ++ newOptDeps 54 | go [] _ = error "should not happen!" 55 | 56 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: CI 4 | 5 | # Controls when the action will run. Triggers the workflow on push or pull request 6 | # events but only for the master branch 7 | on: 8 | push: 9 | branches: [ master ] 10 | pull_request: 11 | branches: [ master ] 12 | 13 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 14 | jobs: 15 | # This workflow contains a single job called "build" 16 | nix-tests: 17 | # The type of runner that the job will run on 18 | runs-on: ubuntu-latest 19 | 20 | # Steps represent a sequence of tasks that will be executed as part of the job 21 | steps: 22 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 23 | - uses: actions/checkout@v2 24 | 25 | - uses: cachix/install-nix-action@v16 26 | 27 | - uses: cachix/cachix-action@v10 28 | with: 29 | name: yarn2nix 30 | signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' 31 | 32 | # TODO: move into nix script 33 | - name: check cabal file up to date (run hpack) 34 | run: | 35 | nix-shell \ 36 | -E 'with import ./nixpkgs-pinned.nix {}; mkShell { buildInputs = [ hpack execline ]; }' \ 37 | --run ' 38 | execline-cd yarn2nix hpack \ 39 | && \ 40 | execline-cd yarn-lock hpack \ 41 | && \ 42 | git diff --exit-code 43 | ' 44 | 45 | # TODO: move into nix script 46 | - name: check nix files up to date (run cabal2nix) 47 | run: | 48 | nix-shell \ 49 | -E 'with import ./nixpkgs-pinned.nix {}; mkShell { buildInputs = [ hpack execline cabal2nix ]; }' \ 50 | --run ' 51 | execline-cd yarn2nix redirfd -w 1 yarn2nix.nix cabal2nix . \ 52 | && \ 53 | execline-cd yarn-lock redirfd -w 1 yarn-lock.nix cabal2nix . \ 54 | && \ 55 | git diff --exit-code 56 | ' 57 | 58 | # Runs a single command using the runners shell 59 | - name: Build nix-tests 60 | run: env NIX_PATH= nix-build nix-tests/default.nix 61 | -------------------------------------------------------------------------------- /yarn2nix/src/Distribution/Nixpkgs/Nodejs/FromPackage.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NoImplicitPrelude, OverloadedStrings, RecordWildCards #-} 2 | {-| 3 | Description: Generate nix expression for 'NP.Package' 4 | -} 5 | module Distribution.Nixpkgs.Nodejs.FromPackage 6 | ( genTemplate 7 | ) where 8 | 9 | import Protolude 10 | 11 | import Nix.Expr 12 | import Nix.Expr.Additions 13 | 14 | import Distribution.Nixpkgs.Nodejs.Utils (packageKeyToSymbol, attrSetMayStr, attrSetMay) 15 | import qualified Distribution.Nodejs.Package as NP 16 | import qualified Distribution.Nixpkgs.Nodejs.License as NL 17 | import qualified Yarn.Lock.Types as YLT 18 | import qualified Data.Aeson.KeyMap as KeyMap 19 | import qualified Data.Aeson.Key as Key 20 | 21 | depsToPkgKeys :: NP.Dependencies -> [YLT.PackageKey] 22 | depsToPkgKeys deps = 23 | deps 24 | & KeyMap.toList 25 | <&> first Key.toText 26 | <&> toPkgKey 27 | where 28 | toPkgKey (k, v) = 29 | YLT.PackageKey (NP.parsePackageKeyName k) v 30 | 31 | -- | generate a nix expression that translates your package.nix 32 | -- 33 | -- and can serve as template for manual adjustments 34 | genTemplate :: Maybe NL.LicensesBySpdxId -> NP.Package -> NExpr 35 | genTemplate licSet NP.Package{..} = 36 | -- reserved for possible future arguments (to prevent breakage) 37 | simpleParamSet [] 38 | ==> Param nodeDepsSym 39 | ==> (mkNonRecSet 40 | [ "key" $= packageKeyToSet (NP.parsePackageKeyName name) 41 | , "version" $= mkStr version 42 | , "nodeBuildInputs" $= (letE "a" (mkSym nodeDepsSym) 43 | $ mkList (map (pkgDep "a") depPkgKeys)) 44 | , "meta" $= (mkNonRecSet 45 | $ attrSetMayStr "description" description 46 | <> attrSetMay "license" (NL.nodeLicenseToNixpkgs <$> license <*> licSet) 47 | <> attrSetMayStr "homepage" homepage) 48 | ]) 49 | where 50 | -- TODO: The devDependencies are only needed for the build 51 | -- and probably also only from packages not stemming from 52 | -- a npm registry (e.g. a git package). It would be cool 53 | -- if these dependencies were gone in the final output. 54 | -- See https://github.com/Profpatsch/yarn2nix/issues/5 55 | depPkgKeys = depsToPkgKeys (dependencies <> devDependencies) 56 | pkgDep depsSym pk = mkSym depsSym !!. packageKeyToSymbol pk 57 | nodeDepsSym = "allDeps" 58 | packageKeyToSet (YLT.SimplePackageKey n) = 59 | packageKeyToSet $ YLT.ScopedPackageKey "" n 60 | packageKeyToSet (YLT.ScopedPackageKey s n) = mkNonRecSet $ 61 | [ bindTo "name" $ mkStrQ [ StrQ n ] 62 | , bindTo "scope" $ mkStrQ [ StrQ s ] 63 | ] 64 | -------------------------------------------------------------------------------- /nix-tests/vendor/runTestsuite.nix: -------------------------------------------------------------------------------- 1 | { pkgs }: 2 | 3 | # Copied from the code I wrote for https://code.tvl.fyi/tree/nix/runTestsuite/default.nix?id=c8e888c1d2c6dfe60a835d1810ab57d87d097e93 4 | # with the typechecking parts removed. 5 | 6 | # Run a nix testsuite. 7 | # 8 | # The tests are simple assertions on the nix level, 9 | # and can use derivation outputs if IfD is enabled. 10 | # 11 | # You build a testsuite by bundling assertions into 12 | # “it”s and then bundling the “it”s into a testsuite. 13 | # 14 | # Running the testsuite will abort evaluation if 15 | # any assertion fails. 16 | # 17 | # Example: 18 | # 19 | # runTestsuite "myFancyTestsuite" [ 20 | # (it "does an assertion" [ 21 | # (assertEq "42 is equal to 42" "42" "42") 22 | # (assertEq "also 23" 23 23) 23 | # ]) 24 | # (it "frmbls the brlbr" [ 25 | # (assertEq true false) 26 | # ]) 27 | # ] 28 | # 29 | # will fail the second it group because true is not false. 30 | 31 | let 32 | lib = pkgs.lib; 33 | 34 | # rewrite the builtins.partition result 35 | # to use `ok` and `err` instead of `right` and `wrong`. 36 | partitionTests = pred: xs: 37 | let res = builtins.partition pred xs; 38 | in { 39 | ok = res.right; 40 | err = res.wrong; 41 | }; 42 | 43 | # assert that left and right values are equal 44 | assertEq = 45 | (desc: left: right: 46 | if left == right 47 | then { yep = { test = desc; }; } 48 | else { nope = { 49 | test = desc; 50 | inherit left right; 51 | }; 52 | }); 53 | 54 | # Annotate a bunch of asserts with a descriptive name 55 | it = desc: asserts: { 56 | it-desc = desc; 57 | inherit asserts; 58 | }; 59 | 60 | # Run a bunch of its and check whether all asserts are yep. 61 | # If not, abort evaluation with `throw` 62 | # and print the result of the test suite. 63 | # 64 | # Takes a test suite name as first argument. 65 | runTestsuite = 66 | (name: itResults: 67 | let 68 | goodIt = it: { 69 | inherit (it) it-desc; 70 | asserts = partitionTests (ass: 71 | if ass ? yep then true 72 | else if ass ? nope then false 73 | else abort "assertion result should be either yep or nope" 74 | ) it.asserts; 75 | }; 76 | goodIts = partitionTests (it: (goodIt it).asserts.err == []); 77 | res = goodIts itResults; 78 | in 79 | if res.err == [] 80 | then {} 81 | # TODO(Profpatsch): pretty printing of results 82 | # and probably also somewhat easier to read output 83 | else res); 84 | 85 | in { 86 | inherit 87 | assertEq 88 | it 89 | runTestsuite 90 | ; 91 | } 92 | -------------------------------------------------------------------------------- /yarn2nix/src/Nix/Expr/Additions.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE TupleSections, OverloadedStrings #-} 2 | {-| 3 | Description: Additional functions that should probably be in @hnix@ 4 | 5 | Nix generation helpers. No guarantee of stability (internal!). 6 | -} 7 | module Nix.Expr.Additions 8 | ( stringKey, ($$=), dynamicKey, inheritStatic 9 | , simpleParamSet, multiParam 10 | , (!!.) 11 | , StrQ(..), mkStrQ, mkStrQI 12 | ) where 13 | 14 | import Data.Fix (Fix(..)) 15 | import Data.Text (Text) 16 | import Data.String (IsString(..)) 17 | 18 | import Nix.Expr 19 | import Text.Regex.TDFA.Text () 20 | import Text.Regex.TDFA ((=~)) 21 | 22 | -- hnix helpers 23 | -- TODO submit upstream 24 | 25 | -- | Make a binding, but have the key be a string, not symbol. 26 | stringKey :: Text -> NExpr -> Binding NExpr 27 | stringKey k v = NamedVar (pure $ dynamicKey k) v nullPos 28 | -- | Infix version of 'stringKey'. 29 | ($$=) :: Text -> NExpr -> Binding NExpr 30 | ($$=) = stringKey 31 | infixr 2 $$= 32 | 33 | -- | Make a dynamic key name that is only enclosed in double quotes (no antiquotes). 34 | dynamicKey :: Text -> NKeyName NExpr 35 | dynamicKey k = DynamicKey $ Plain $ DoubleQuoted [Plain k] 36 | 37 | -- | Inherit the given list of symbols. 38 | inheritStatic :: [Text] -> Binding e 39 | inheritStatic names = inherit (map StaticKey names) nullPos 40 | 41 | -- | shortcut to create a list of closed params, like @{ foo, bar, baz }:@ 42 | simpleParamSet :: [Text] -> Params NExpr 43 | simpleParamSet prms = mkParamset (fmap (, Nothing) prms) False 44 | 45 | -- | shortcut to create a list of multiple params, like @a: b: c:@ 46 | multiParam :: [Text] -> NExpr -> NExpr 47 | multiParam ps expr = foldr mkFunction expr $ map Param ps 48 | 49 | -- TODO: switch over to !. when 50 | -- https://github.com/jwiegley/hnix/commit/8b4c137a3b125f52bb78039a9d201492032b38e8 51 | -- goes upstream 52 | -- | Like '!.', but automatically convert plain strings to static keys. 53 | (!!.) :: NExpr -> Text -> NExpr 54 | aset !!. k = Fix 55 | $ NSelect aset 56 | (pure $ (if isPlainSymbol k then StaticKey else dynamicKey) k) Nothing 57 | where 58 | -- the nix lexer regex for IDs (symbols) is 59 | -- [a-zA-Z\_][a-zA-Z0-9\_\'\-]* 60 | isPlainSymbol :: Text -> Bool 61 | isPlainSymbol s = s =~ ("^[a-zA-Z_][a-zA-Z0-9_'-]*$" :: Text) 62 | infixl 8 !!. 63 | 64 | 65 | -- | String quotation, either a plain string (S) or antiquoted (A) 66 | data StrQ = StrQ !Text | AntiQ !NExpr 67 | instance IsString StrQ where 68 | fromString = StrQ . fromString 69 | 70 | -- 71 | mkStrQtmpl :: ([Antiquoted Text NExpr] -> NString NExpr) -> [StrQ] -> NExpr 72 | mkStrQtmpl strtr = Fix . NStr . strtr . map trans 73 | where trans (StrQ t) = Plain t 74 | trans (AntiQ r) = Antiquoted r 75 | 76 | mkStrQ, mkStrQI :: [StrQ] -> NExpr 77 | -- | Create a double-quoted string from a list of antiquotes/plain strings. 78 | mkStrQ = mkStrQtmpl DoubleQuoted 79 | -- | Create a single-quoted string from a list of antiquotes/plain strings. 80 | mkStrQI = mkStrQtmpl (Indented 2) 81 | -------------------------------------------------------------------------------- /.hlint.yaml: -------------------------------------------------------------------------------- 1 | # HLint configuration file 2 | # https://github.com/ndmitchell/hlint 3 | # Run `hlint --default` to see the example configuration file. 4 | ########################## 5 | 6 | # Ignore some builtin hints 7 | 8 | # often functions are more readable with explicit arguments 9 | - ignore: {name: Eta reduce} 10 | 11 | # these redundancy warnings are just completely irrelevant 12 | - ignore: {name: Redundant bracket} 13 | - ignore: {name: Move brackets to avoid $} 14 | - ignore: {name: Redundant $} 15 | - ignore: {name: Redundant do} 16 | - ignore: {name: Redundant pure} 17 | - ignore: {name: Redundant <&>} 18 | 19 | # allow case-matching on bool, because why not 20 | - ignore: {name: Use if} 21 | 22 | # hlint cannot distinguish actual newtypes from data types 23 | # that accidentally have only one field 24 | # (but might have more in the future). 25 | # Since it’s a mostly irrelevant runtime optimization, we don’t care. 26 | - ignore: {name: Use newtype instead of data} 27 | 28 | # these lead to harder-to-read/more implicit code 29 | - ignore: {name: Use fmap} 30 | - ignore: {name: Use tuple-section} 31 | - ignore: {name: Use fromMaybe} 32 | - ignore: {name: Use const} 33 | - ignore: {name: Replace case with maybe} 34 | - ignore: {name: Replace case with fromMaybe} 35 | - ignore: {name: Avoid lambda} 36 | - ignore: {name: Use curry} 37 | - ignore: {name: Use uncurry} 38 | 39 | # list comprehensions are a seldomly used part of the Haskell language 40 | # and they introduce syntactic overhead that is usually not worth the conciseness 41 | - ignore: {name: Use list comprehension} 42 | 43 | # multiple maps in a row are usually used for clarity, 44 | # and the compiler will optimize them away, thank you very much. 45 | - ignore: {name: Use map once} 46 | - ignore: {name: Fuse mapMaybe/map} 47 | 48 | # this is silly, why would I use a special function if I can just (heh) `== Nothing` 49 | - ignore: {name: Use isNothing} 50 | 51 | # The duplication heuristic is not very smart 52 | # and more annoying than helpful. 53 | # see https://github.com/ndmitchell/hlint/issues/1009 54 | - ignore: {name: Reduce duplication} 55 | 56 | # Stops the pattern match trick 57 | - ignore: {name: Use record patterns} 58 | - ignore: {name: Use null} 59 | - ignore: {name: Use uncurry} 60 | 61 | # we don’t want void, see below 62 | - ignore: {name: Use void} 63 | 64 | - functions: 65 | 66 | # disallow Enum instance functions, they are partial 67 | - {name: succ, within: [Relude.Extra.Enum]} 68 | - {name: pred, within: [Relude.Extra.Enum]} 69 | - {name: toEnum, within: []} 70 | - {name: fromEnum, within: []} 71 | - {name: enumFrom, within: []} 72 | - {name: enumFromThen, within: []} 73 | - {name: enumFromThenTo, within: []} 74 | - {name: BoundedEnumFrom, within: []} 75 | - {name: BoundedEnumFromThen, within: []} 76 | 77 | # if you want to use randomIO, make sure to add a TypeApplication 78 | # and create a small helper function in a specialized module, 79 | # then add that module to this list. 80 | - {name: randomIO, within: []} 81 | 82 | # `void` discards its argument and is polymorphic, 83 | # thus making it brittle in the face of code changes. 84 | # (see https://tech.freckle.com/2020/09/23/void-is-a-smell/) 85 | # Use an explicit `_ <- …` instead. 86 | - {name: void, within: []} 87 | 88 | # Make restricted functions into an error if found 89 | - error: {name: Avoid restricted function, see comment in .hlint.yaml} 90 | -------------------------------------------------------------------------------- /yarn-lock/src/Yarn/Lock/Types.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DeriveFunctor, OverloadedStrings #-} 2 | {-| 3 | Module : Yarn.Lock.Types 4 | Description : Types for yarn.lock files 5 | Maintainer : Profpatsch 6 | Stability : experimental 7 | -} 8 | module Yarn.Lock.Types where 9 | 10 | import qualified Data.MultiKeyedMap as MKM 11 | import qualified Data.List.NonEmpty as NE 12 | import Data.Data (Proxy (Proxy)) 13 | import Data.Text (Text) 14 | import qualified Data.Text as Text 15 | 16 | -- | Yarn lockfile. 17 | -- 18 | -- It is a multi-keyed map (each value can be referenced by multiple keys). 19 | -- This is achieved by using an intermediate key @ik@. 20 | -- 21 | -- Attention: Might be changed to a newtype in a future release. 22 | type Lockfile = MKM.MKMap PackageKey Package 23 | -- TODO newtype Lockfile = Lockfile (MKM.MKMap PackageKey Package) 24 | 25 | -- | Proxy type for our MKMap intermediate key 26 | lockfileIkProxy :: Proxy Int 27 | lockfileIkProxy = Proxy 28 | 29 | -- | Key that indexes package for a specific version. 30 | data PackageKey = PackageKey 31 | { name :: PackageKeyName -- ^ package name 32 | , npmVersionSpec :: Text 33 | -- ^ tring that specifies the version of a package; 34 | -- sometimes a npm semver, sometimes an arbitrary string 35 | } deriving (Show, Eq, Ord) 36 | 37 | -- | The name of a package. They can be scoped, see 38 | -- | for an explanation. 39 | data PackageKeyName 40 | = SimplePackageKey Text 41 | -- ^ just a package name 42 | | ScopedPackageKey Text Text 43 | -- ^ a scope and a package name (e.g. @types/foobar) 44 | deriving (Show, Eq, Ord) 45 | 46 | -- | Try to parse a string into a package key name (scoped or not). 47 | parsePackageKeyName :: Text -> Maybe PackageKeyName 48 | parsePackageKeyName n = case Text.stripPrefix "@" n of 49 | Nothing -> Just $ SimplePackageKey n 50 | Just sc -> case Text.breakOn "/" sc of 51 | (_, "") -> Nothing 52 | (scope, pkg) -> Just $ ScopedPackageKey scope (Text.drop 1 pkg) 53 | 54 | -- | Something with a list of 'PackageKey's pointing to it. 55 | data Keyed a = Keyed (NE.NonEmpty PackageKey) a 56 | deriving (Show, Eq, Ord, Functor) 57 | 58 | -- | The actual npm package with dependencies and a way to download. 59 | data Package = Package 60 | { version :: Text -- ^ resolved, specific version 61 | , remote :: Remote 62 | , dependencies :: [PackageKey] -- ^ list of dependencies 63 | , optionalDependencies :: [PackageKey] -- ^ list of optional dependencies 64 | } deriving (Eq, Show) 65 | 66 | -- | Information on where to download the package. 67 | data Remote 68 | = FileRemote 69 | { fileUrl :: Text -- ^ URL to a remote file 70 | , fileSha1 :: Text -- ^ sha1 hash of the file (attached to the link) 71 | 72 | } 73 | | FileRemoteNoIntegrity 74 | { fileNoIntegrityUrl :: Text -- ^ URL to a remote file 75 | } 76 | | GitRemote 77 | { gitRepoUrl :: Text -- ^ valid git remote URL 78 | , gitRev :: Text -- ^ git tree-ish (commit, branch, &c.) 79 | } 80 | -- this is a bit of an oddidity, but what isn’t 81 | | FileLocal 82 | { fileLocalPath :: Text -- ^ (relative) path to file on the local machine 83 | , fileLocalSha1 :: Text -- ^ sha1 hash of the file (attached to the link) 84 | } 85 | | FileLocalNoIntegrity 86 | { fileLocalNoIntegrityPath :: Text -- ^ (relative) path to file on the local machine 87 | } deriving (Eq, Show) 88 | -------------------------------------------------------------------------------- /yarn-lock/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/) 6 | and this project adheres to [Semantic Versioning](http://semver.org/). 7 | 8 | ## [0.6.5] - 2021-06-26 9 | 10 | - yarn-lock: Remove `protolude` dependency in order to add to `stackage`. 11 | - Relax upper bound of `hnix`. 12 | 13 | ## [0.6.4] - 2021-03-30 14 | 15 | - Repository changed to https://github.com/Profpatsch/yarn2nix 16 | 17 | ## [0.6.3] - 2021-03-29 18 | 19 | - Relax upper bounds of `protolude` and `megaparsec`. 20 | 21 | ## [0.6.2] - 2019-12-22 22 | 23 | - `megaparsec` `0.8` is compatible as well as `0.7`. 24 | - Support for local and remote file source remotes without hashes. Some old versions of might have these. `FileRemoteNoIntegrity` and `FileLocalNoIntegrity`. 25 | 26 | ## [0.6.1] - 2019-11-03 27 | 28 | - Update `megaparsec` dependency to `0.7`. 29 | 30 | ## [0.6.0] - 2018-12-19 31 | 32 | ### Added 33 | 34 | - Support for local dependencies (`resolved` field starts with `file:`) 35 | 36 | ### Fixed 37 | 38 | - Semigroup is superclass of Monoid 39 | 40 | ## [0.5.0] - 2018-06-12 41 | 42 | ### Changed 43 | 44 | - `PackageKey`s now correctly parse scoped npm names 45 | - This means `Text` is now a sum of `SimplePackageKey`/`ScopedPackageKey` 46 | 47 | ### Fixed 48 | 49 | - `PackageKey`s with versions containing `@` parse correctly 50 | - Like for example `git+ssh:git@github.com` links for some git packages 51 | 52 | ## [0.4.1] - 2018-05-15 53 | 54 | ### Changed 55 | 56 | - Update the parser to `megaparsec 6.*` 57 | - Raise `protolude` minimal version to `0.2.*` 58 | 59 | ### Fixed 60 | 61 | - Import missing quickcheck `NonEmpty` instances from `quickcheck-orphans` 62 | 63 | ## [0.4.0] - 2017-10-07 64 | 65 | ### Changed 66 | - MKMap functions `fromList` and `toList` only take non-empty key lists 67 | 68 | ## [0.3.4] - 2017-10-04 69 | 70 | ### Fixed 71 | - Support for Protolude 0.2.* ((<>) is exported from Monoid, not Semigroup) 72 | 73 | ## [0.3.3] - 2017-10-04 74 | 75 | ### Fixed 76 | - Remove (broken) support for megaparsec 6.* 77 | 78 | ## [0.3.2] - 2017-10-02 79 | 80 | ### Fixed 81 | - Support parsing packages with `@` in the package name 82 | 83 | ## [0.3.1] - 2017-08-16 84 | 85 | ### Added 86 | - Functor, Foldable and Traversable instances for MKMap 87 | 88 | ### Fixed 89 | - Remote URL parsing strips more unneeded elements 90 | 91 | ## [0.3] - 2017-08-16 92 | 93 | This is a major overhaul, changing nearly every part of the API 94 | and the implementation! 95 | 96 | ### Added 97 | - Support for multiple kinds of remote. 98 | - Heuristics for parsing git and file remotes. 99 | - Helpful, local error messages if the parsing goes wrong somewhere 100 | - A convenience function for doing all parsing steps at once 101 | - A pretty printer for error messages 102 | - Tests for all parsing logic 103 | - Tests for simple invariants in the multi-keyed map implementation 104 | 105 | ### Changed 106 | - Split the code into multiple modules. 107 | - Rewrote the parser to have a separate AST parsing step. 108 | 109 | ## [0.2] - 2017-05-21 110 | 111 | ### Added 112 | - A multi-keyed map module. 113 | - `decycle` function for removing npm dependency cycles. 114 | 115 | ### Changed 116 | - Lockfile type is now a multi-keyed map. 117 | 118 | 119 | ## [0.1] - 2017-04-18 120 | 121 | ### Added 122 | - Parser for `yarn.lock` files generated by yarn. 123 | - Data types representing the yarn file. 124 | - Lockfile type that is a simple `Map`. 125 | 126 | 127 | -------------------------------------------------------------------------------- /yarn-lock/src/Yarn/Lock.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE LambdaCase, OverloadedStrings, RecordWildCards #-} 2 | {-| 3 | Module : Yarn.Lock 4 | Description : High-level parser of yarn.lock files 5 | Maintainer : Profpatsch 6 | Stability : experimental 7 | 8 | The improves on npm, 9 | because it writes @yarn.lock@ files that contain a complete 10 | version resolution of all dependencies. This way a deterministic 11 | deployment can be guaranteed. 12 | -} 13 | module Yarn.Lock 14 | ( T.Lockfile 15 | , parseFile, parse 16 | -- * Errors 17 | , prettyLockfileError 18 | , LockfileError(..), PackageErrorInfo(..) 19 | ) where 20 | 21 | import qualified Data.Text as T 22 | import qualified Data.List.NonEmpty as NE 23 | import qualified Text.Megaparsec as MP 24 | import qualified Data.Either.Validation as V 25 | 26 | import qualified Yarn.Lock.Types as T 27 | import qualified Yarn.Lock.File as File 28 | import qualified Yarn.Lock.Parse as Parse 29 | import Data.Text (Text) 30 | import Data.Functor ((<&>)) 31 | import Control.Monad ((>=>)) 32 | import qualified Data.Text as Text 33 | import qualified Data.Text.IO as Text.IO 34 | import Data.Bifunctor (first, Bifunctor (bimap)) 35 | 36 | -- | Everything that can go wrong when parsing a 'Lockfile'. 37 | data LockfileError 38 | = ParseError Text 39 | -- ^ The initial parsing step failed 40 | | PackageErrors (NE.NonEmpty PackageErrorInfo) 41 | -- ^ a package could not be parsed from the AST 42 | deriving (Show, Eq) 43 | 44 | -- | Information about package parsing errors. 45 | data PackageErrorInfo = PackageErrorInfo 46 | { srcPos :: MP.SourcePos 47 | -- ^ the position of the package in the original file 48 | , convErrs :: NE.NonEmpty File.ConversionError 49 | -- ^ list of reasons for failure 50 | } deriving (Show, Eq) 51 | 52 | -- | Convenience function, combining all parsing steps. 53 | -- 54 | -- The resulting 'Lockfile' structure might not yet be optimal, 55 | -- see 'File.fromPackages'. 56 | parseFile :: FilePath -- ^ file to read 57 | -> IO (Either LockfileError T.Lockfile) 58 | parseFile fp = Text.IO.readFile fp <&> parse fp 59 | 60 | -- | For when you want to provide only the file contents. 61 | parse :: FilePath -- ^ name of the input file, used for the parser 62 | -> Text -- ^ content of a @yarn.lock@ 63 | -> Either LockfileError T.Lockfile 64 | parse fp = astParse fp >=> toPackages >=> toLockfile 65 | 66 | -- | Pretty print a parsing error with sane default formatting. 67 | prettyLockfileError :: LockfileError -> Text 68 | prettyLockfileError = \case 69 | (ParseError t) -> "Error while parsing the yarn.lock:\n" 70 | <> T.unlines (indent 2 (T.lines t)) 71 | (PackageErrors errs) -> "Some packages could not be made sense of:\n" 72 | <> T.unlines (NE.toList $ indent 2 (errs >>= errText)) 73 | where 74 | indent :: Functor f => Int -> f Text -> f Text 75 | indent i = fmap (T.replicate i " " <>) 76 | errText PackageErrorInfo{..} = 77 | (pure $ "Package at " <> (Text.pack $ MP.sourcePosPretty srcPos) <> ":") 78 | <> indent 2 (fmap convErrText convErrs) 79 | convErrText = \case 80 | (File.MissingField t) -> "Field " <> qu t <> " is missing." 81 | (File.WrongType{..}) -> "Field " <> qu fieldName 82 | <> " should be of type " <> fieldType <> "." 83 | (File.UnknownRemoteType) -> "We don’t know this remote type." 84 | qu t = "\"" <> t <> "\"" 85 | 86 | -- helpers 87 | astParse :: FilePath -> Text -> Either LockfileError [Parse.Package] 88 | astParse fp = first (ParseError . Text.pack . MP.errorBundlePretty) 89 | . MP.parse Parse.packageList fp 90 | 91 | toPackages :: [Parse.Package] -> Either LockfileError [T.Keyed T.Package] 92 | toPackages = first PackageErrors . V.validationToEither 93 | . traverse validatePackage 94 | 95 | validatePackage :: Parse.Package 96 | -> V.Validation (NE.NonEmpty PackageErrorInfo) (T.Keyed T.Package) 97 | validatePackage (T.Keyed keys (pos, fields)) = V.eitherToValidation 98 | $ bimap (pure . PackageErrorInfo pos) (T.Keyed keys) 99 | $ File.astToPackage fields 100 | 101 | toLockfile :: [T.Keyed T.Package] -> Either LockfileError T.Lockfile 102 | toLockfile = pure . File.fromPackages 103 | 104 | -------------------------------------------------------------------------------- /nix-tests/default.nix: -------------------------------------------------------------------------------- 1 | # TODO: pkgs ad yarn2nix reference files from outside of the cabal package 2 | { pkgs ? import ../nixpkgs-pinned.nix {} 3 | , nixLibPath ? ../yarn2nix/nix-lib 4 | , yarn2nix ? import ../default.nix { inherit pkgs; } 5 | }: 6 | let 7 | nixLib = pkgs.callPackage nixLibPath { 8 | inherit yarn2nix; 9 | }; 10 | 11 | inherit (import vendor/runTestsuite.nix { inherit pkgs; }) 12 | runTestsuite 13 | it 14 | assertEq 15 | ; 16 | 17 | # small test package.json 18 | my-package-json = pkgs.writeText "package.json" (builtins.toJSON { 19 | name = "my-package"; 20 | version = "1.5.3"; 21 | license = "MIT"; 22 | }); 23 | 24 | # very simple package depending on moment.js 25 | readme-example = rec { 26 | momentjsVersion = "^2.27.0"; 27 | yarn-lock = pkgs.writeText "yarn.lock" '' 28 | moment@^2.27.0: 29 | version "2.27.0" 30 | resolved "https://registry.yarnpkg.com/moment/-/moment-2.27.0.tgz#8bff4e3e26a236220dfe3e36de756b6ebaa0105d" 31 | integrity sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ== 32 | ''; 33 | package-json = pkgs.writeText "package.json" (builtins.toJSON { 34 | name = "readme-example"; 35 | version = "0.1.0"; 36 | dependencies = { 37 | moment = momentjsVersion; 38 | }; 39 | }); 40 | src = pkgs.runCommandLocal "readme-example" {} '' 41 | mkdir -p $out 42 | cp ${yarn-lock} $out/yarn.lock 43 | cp ${package-json} $out/package.json 44 | ''; 45 | }; 46 | 47 | # generates nix expression for license with a given spdx id and imports it 48 | spdxLicenseSet = spdx: 49 | let 50 | packageJson = pkgs.writeText "package.json" (builtins.toJSON { 51 | name = "license-test-${spdx}"; 52 | version = "0.1.0"; 53 | license = spdx; 54 | }); 55 | tpl = nixLib.callPackageJson packageJson {} {}; 56 | in tpl.meta.license; 57 | 58 | # test suite 59 | tests = runTestsuite "yarn2nix" [ 60 | (it "checks the template output" 61 | (let tmpl = nixLib.callPackageJson my-package-json {} {}; 62 | in [ 63 | # TODO: this is a naïve match, might want to create a better test 64 | (assertEq "template" tmpl { 65 | key = { 66 | name = "my-package"; 67 | scope = ""; 68 | }; 69 | version = "1.5.3"; 70 | nodeBuildInputs = []; 71 | meta = { 72 | license = pkgs.lib.licenses.mit; 73 | }; 74 | }) 75 | ])) 76 | (it "checks the readme example" 77 | (let 78 | tmpl = nixLib.callPackageJson readme-example.package-json {}; 79 | deps = nixLib.callYarnLock readme-example.yarn-lock {}; 80 | pkg = nixLib.buildNodePackage ({ inherit (readme-example) src; } // 81 | tmpl (nixLib.buildNodeDeps deps)); 82 | in [ 83 | # TODO we can probably check more here, but this seems like a 84 | # good sanity check, since its mostly about building successfully 85 | (assertEq "momentjs linked" (builtins.readDir "${pkg}/node_modules") { 86 | ".bin" = "directory"; 87 | "moment" = "symlink"; 88 | })])) 89 | (it "checks license conversion" 90 | (builtins.map 91 | ({spdx, set}: 92 | assertEq 93 | "the same license set for ${spdx}" 94 | (spdxLicenseSet spdx) 95 | set 96 | ) 97 | [ 98 | { spdx = "AGPL-3.0-only"; set = pkgs.lib.licenses.agpl3Only; } 99 | { spdx = "GPL-3.0-or-later"; set = pkgs.lib.licenses.gpl3Plus; } 100 | { spdx = "MIT"; set = pkgs.lib.licenses.mit; } 101 | { spdx = "BSD-3-Clause"; set = pkgs.lib.licenses.bsd3; } 102 | { spdx = "ISC"; set = pkgs.lib.licenses.isc; } 103 | { spdx = "UNLICENSED"; set = pkgs.lib.licenses.unfree; } 104 | # Check that anything else is kept as is 105 | { spdx = "See LICENSE.txt"; set = "See LICENSE.txt"; } 106 | ] 107 | ) 108 | ) 109 | ]; 110 | 111 | # small helper that checks the output of tests 112 | # and pretty-prints errors if there were any 113 | runTests = pkgs.runCommandLocal "run-tests" { 114 | testOutput = builtins.toJSON tests; 115 | passAsFile = [ "testOutput" ]; 116 | } 117 | (if tests == {} 118 | then ''touch $out'' 119 | else '' 120 | echo "ERROR: some tests failed:" >&2 121 | cat "$testOutputPath" | ${pkgs.jq}/bin/jq >&2 122 | exit 1 123 | ''); 124 | 125 | in { 126 | inherit runTests; 127 | testOverriding = import ./test-overriding.nix { 128 | inherit pkgs nixLib yarn2nix; 129 | }; 130 | } 131 | -------------------------------------------------------------------------------- /yarn2nix/src/Distribution/Nixpkgs/Nodejs/ResolveLockfile.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, TupleSections, ScopedTypeVariables, ViewPatterns, RecordWildCards, NoImplicitPrelude, LambdaCase, NamedFieldPuns, GeneralizedNewtypeDeriving, DeriveFunctor #-} 2 | -- TODO: remove exts 3 | {-| 4 | Description: IO-based resolving of missing hashes 5 | 6 | Resolving a 'YLT.Lockfile' and generating all necessary data (e.g. hashes), so that it can be converted to a nix expression. Might need IO & network access to succeed. 7 | -} 8 | module Distribution.Nixpkgs.Nodejs.ResolveLockfile 9 | ( resolveLockfileStatus 10 | , ResolverConfig(..) 11 | , Resolved(..), ResolvedLockfile 12 | ) where 13 | 14 | import Protolude hiding (toS) 15 | import Protolude.Conv (toS) 16 | import qualified Control.Monad.Trans.Except as E 17 | import Data.ByteString.Lazy () 18 | import qualified Data.List.NonEmpty as NE 19 | import qualified Data.MultiKeyedMap as MKM 20 | import qualified Data.Aeson as Aeson 21 | import qualified Data.Aeson.Types as AesonT 22 | import qualified System.Process as Process 23 | 24 | import qualified Control.Concurrent.Async.Pool as Async 25 | import qualified Control.Monad.STM as STM 26 | 27 | import qualified Yarn.Lock.Types as YLT 28 | 29 | nixPrefetchGitPath :: FilePath 30 | nixPrefetchGitPath = "nix-prefetch-git" 31 | 32 | maxFetchers :: Int 33 | maxFetchers = 5 34 | 35 | data ResolverConfig 36 | = ResolverConfig 37 | { resolveOffline :: Bool -- ^ If @True@, 'resolveLockfileStatus' will throw an 38 | -- error in case resolving a hash requires network 39 | -- access (for when it started in a nix build) 40 | } 41 | 42 | -- | A thing whose hash is already known (“resolved”). 43 | -- 44 | -- Only packages with known hashes are truly “locked”. 45 | data Resolved a = Resolved 46 | { hashSum :: Text 47 | , resolved :: a 48 | } deriving (Show, Eq, Functor) 49 | 50 | -- | In order to write a nix file, all packages need to know their shasums first. 51 | type ResolvedLockfile = MKM.MKMap YLT.PackageKey (Resolved YLT.Package) 52 | 53 | -- | Resolve all packages by downloading their sources if necessary. 54 | -- 55 | -- Respects 'runOffline' from 'RunConfig': If it is 'True', it throws 56 | -- an error as soon as it would need to download something which is the 57 | -- case for 'YLT.GitRemote'. 58 | resolveLockfileStatus :: ResolverConfig -> (Chan YLT.Remote) -> YLT.Lockfile 59 | -> IO (Either (NE.NonEmpty Text) ResolvedLockfile) 60 | resolveLockfileStatus cfg msgChan lf = Async.withTaskGroup maxFetchers $ \taskGroup -> do 61 | job <- STM.atomically $ Async.mapReduce taskGroup 62 | $ fmap (\(ks, pkg) -> (:[]) <$> (E.runExceptT $ do 63 | liftIO $ writeChan msgChan (YLT.remote pkg) 64 | res <- resolve pkg 65 | pure (ks, res))) 66 | $ MKM.toList lf 67 | resolved <- Async.wait job 68 | case partitionEithers resolved of 69 | (x:xs, _ ) -> pure $ Left $ x NE.:| xs 70 | (_ , ys) -> pure $ Right $ MKM.fromList YLT.lockfileIkProxy ys 71 | 72 | where 73 | resolve :: YLT.Package -> E.ExceptT Text IO (Resolved YLT.Package) 74 | resolve pkg = case YLT.remote pkg of 75 | YLT.FileRemote{..} -> pure $ r fileSha1 76 | YLT.FileLocal{..} -> pure $ r fileLocalSha1 77 | YLT.GitRemote{..} -> if cfg & resolveOffline 78 | then E.throwE $ "Refusing to resolve \"git+" 79 | <> gitRepoUrl <> "#" <> gitRev 80 | <> "\" because --offline is set" 81 | else r <$> fetchFromGit gitRepoUrl gitRev 82 | YLT.FileRemoteNoIntegrity{..} -> E.throwE 83 | $ "The remote " 84 | <> fileNoIntegrityUrl 85 | <> " does not specify a sha1 hash in the yarn.lock file, which we don’t support (yet)" 86 | YLT.FileLocalNoIntegrity{..} -> E.throwE 87 | $ "The local file " 88 | <> fileLocalNoIntegrityPath 89 | <> " does not specify a sha1 hash in the yarn.lock file, which we don’t support (yet)" 90 | where 91 | r sha = Resolved { hashSum = sha, resolved = pkg } 92 | 93 | fetchFromGit :: Text -> Text -> E.ExceptT Text IO Text 94 | fetchFromGit repo rev = do 95 | res <- liftIO $ Process.readProcessWithExitCode nixPrefetchGitPath 96 | ["--url", toS repo, "--rev", toS rev, "--hash", "sha256"] "" 97 | case res of 98 | ((ExitFailure _), _, err) -> E.throwE $ toS err 99 | (ExitSuccess, out, _) -> E.ExceptT . pure 100 | $ first (\decErr -> "parsing json output failed:\n" 101 | <> toS decErr <> "\nThe output was:\n" <> toS out) 102 | $ do val <- Aeson.eitherDecode' (toS out) 103 | AesonT.parseEither 104 | (Aeson.withObject "PrefetchOutput" (Aeson..: "sha256")) val 105 | -------------------------------------------------------------------------------- /yarn2nix/src/Distribution/Nixpkgs/Nodejs/License.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NoImplicitPrelude, OverloadedStrings, RecordWildCards, FlexibleInstances, GeneralizedNewtypeDeriving #-} 2 | {-# LANGUAGE LambdaCase #-} 3 | {-# LANGUAGE MultiWayIf #-} 4 | {-# LANGUAGE DerivingStrategies #-} 5 | {-# LANGUAGE TypeApplications #-} 6 | {-| 7 | Description: Convert @package.json@ license fields to nixpkgs license attribute sets 8 | -} 9 | module Distribution.Nixpkgs.Nodejs.License 10 | ( -- * Conversion Logic 11 | nodeLicenseToNixpkgs 12 | -- * License Lookup Table 13 | , LicensesBySpdxId 14 | ) where 15 | 16 | import Protolude 17 | 18 | import qualified Data.Aeson as A 19 | import qualified Nix.Expr as Nix 20 | import qualified Data.Map.Strict as Map 21 | import qualified Data.Aeson.BetterErrors as Json 22 | import qualified Data.Scientific as Scientific 23 | 24 | -- newtype to circumvent the default instance: we don't want 25 | -- the key of the JSON object to be the key of the HashMap, 26 | -- but one of its values (spdxId). 27 | -- | Lookup table from SPDX identifier (as 'Text') to 'NixpkgsLicense'. 28 | newtype LicensesBySpdxId 29 | = LicensesBySpdxId (Map Text NixpkgsLicense) 30 | deriving stock (Show, Eq) 31 | deriving newtype (Semigroup, Monoid) 32 | 33 | -- | Representation of a nixpkgs license set as found in 34 | -- @lib.licenses@. There doesn't seem to be a strict 35 | -- definition of what is required and what is optional, 36 | -- the distribution of 'Maybe' and non-'Maybe' values 37 | -- is based on the current situation in @lib/licenses.nix@. 38 | data NixpkgsLicense 39 | = NixpkgsLicense ([(Text, LicenseValue)]) 40 | deriving stock (Show, Eq) 41 | data LicenseValue 42 | = LText Text 43 | | LBool Bool 44 | | LInt Int 45 | deriving stock (Show, Eq) 46 | 47 | -- | Static version of @lib.licenses.unfree@, 48 | -- so @UNLICENSED@ can be handled correctly 49 | -- even if no lookup table is provided. 50 | -- 51 | -- TODO: this will go out of sync with the nixpkgs definitions every once in a while, how to fix? 52 | unfreeLicense :: NixpkgsLicense 53 | unfreeLicense = NixpkgsLicense $ [ 54 | ("shortName", LText "unfree") 55 | , ("deprecated", LBool False) 56 | , ("fullName", LText "Unfree") 57 | , ("redistributable", LBool False) 58 | , ("free", LBool False) 59 | ] 60 | 61 | instance A.FromJSON LicensesBySpdxId where 62 | parseJSON = Json.toAesonParser identity ((Json.forEachInObject $ \_key -> do 63 | Json.keyMay "spdxId" Json.asText 64 | >>= \case 65 | Nothing -> pure Nothing 66 | Just spdxId -> do 67 | spdxLicense <- (Json.eachInObject $ Json.withValue $ \case 68 | A.String t -> Right $ LText t 69 | A.Bool b -> Right $ LBool b 70 | A.Number s -> case Scientific.toBoundedInteger @Int s of 71 | Just i -> Right $ LInt i 72 | Nothing -> Left $ "Not an integer: " <> (s & show) 73 | A.Null -> Left "Cannot parse Null as license value for now" 74 | A.Object _ -> Left "Cannot parse Object as license value for now" 75 | A.Array _ -> Left "Cannot parse Array as license value for now") 76 | <&> NixpkgsLicense 77 | pure $ Just (spdxId, spdxLicense) 78 | ) 79 | <&> catMaybes 80 | <&> Map.fromList 81 | <&> LicensesBySpdxId 82 | ) 83 | 84 | 85 | -- | Build nix attribute set for given 'NixpkgsLicense'. 86 | -- 87 | -- The resulting nix value of @nixpkgsLicenseExpression x@ 88 | -- should be equal to @lib.licenses.@ for the 89 | -- same version of nixpkgs used. 90 | nixpkgsLicenseExpression :: NixpkgsLicense -> Nix.NExpr 91 | nixpkgsLicenseExpression (NixpkgsLicense m) = 92 | m 93 | <&> second licenseValueToNExpr 94 | & Nix.attrsE 95 | 96 | licenseValueToNExpr :: LicenseValue -> Nix.NExpr 97 | licenseValueToNExpr = \case 98 | LText t -> Nix.mkStr t 99 | LInt i -> Nix.mkInt (i & fromIntegral @Int @Integer) 100 | LBool b -> Nix.mkBool b 101 | 102 | -- | Implements the logic for converting from an (optional) 103 | -- @package.json@ @license@ field to a nixpkgs @meta.license@ 104 | -- set. Since support for multiple licenses is poor in nixpkgs 105 | -- at the moment, we don't attempt to convert SPDX expressions 106 | -- like @(ISC OR GPL-3.0-only)@. 107 | -- 108 | -- See for 109 | -- details on npm's @license@ field. 110 | nodeLicenseToNixpkgs :: Text -> LicensesBySpdxId -> Nix.NExpr 111 | nodeLicenseToNixpkgs nodeLicense licSet = do 112 | if nodeLicense == "UNLICENSED" 113 | then nixpkgsLicenseExpression unfreeLicense 114 | else case lookupSpdxId nodeLicense licSet of 115 | Nothing -> Nix.mkStr nodeLicense 116 | Just license -> license 117 | 118 | -- | Lookup function for 'LicensesBySpdxId' which directly returns a 'NExpr'. 119 | -- This function only looks up by SPDX identifier and does not take 120 | -- npm-specific quirks into account. 121 | -- 122 | -- Use 'nodeLicenseToNixpkgs' when dealing with the @license@ field 123 | -- of a npm-ish javascript package. 124 | lookupSpdxId :: Text -> LicensesBySpdxId -> Maybe Nix.NExpr 125 | lookupSpdxId lic (LicensesBySpdxId licSet) = 126 | licSet 127 | & Map.lookup lic 128 | <&> nixpkgsLicenseExpression 129 | -------------------------------------------------------------------------------- /yarn2nix/README.md: -------------------------------------------------------------------------------- 1 | 2 | ATTENTION: You are not looking at the `yarn2nix` as packaged in `nixpkgs`. This is an alternative implementation, with different tradeoffs. 3 | For an overview of the history of these tools and current options, see [this nixpkgs issue](https://github.com/NixOS/nixpkgs/issues/20637). 4 | 5 | I currently don’t have much time to maintain this project, it should work for some cases, but has not been “battle-tested” very much. 6 | 7 | Alternative, more active projects: 8 | 9 | * https://github.com/canva-public/js2nix (handles dependency cycles correctly, does not require flakes) 10 | * https://github.com/nix-community/dream2nix (requires flakes) 11 | 12 | 13 | # yarn2nix 14 | 15 | ``` 16 | yarn2nix [--offline] [path/to/yarn.lock] 17 | 18 | Convert a `yarn.lock` into a synonymous nix expression. 19 | If no path is given, search for `./yarn.lock`. 20 | If --offline is given, abort if figuring out a hash 21 | requires network access. 22 | 23 | yarn2nix --template [path/to/package.json] 24 | 25 | Generate a package template nix-expression for your `package.json`. 26 | ``` 27 | 28 | ## Features 29 | 30 | - Purely transform `yarn.lock` files into very minimal, line-diffable nix expressions. 31 | - Nix is used to its fullest. Every package is a derivation, whole dependency 32 | subtrees are shared in an optimal way, even between projects. 33 | - The ability to resolve git dependencies by prefetching their repos and including the hashes. 34 | - Completely local transformation if there are no git dependencies (can be used inside nix-build, no large file check-in). 35 | - Extremely fast. 36 | - Nice code that can be easily extended, new repositories introduced, adapt to new versions of the `yarn.lock` format. 37 | - Comes with a [nix library][nix-lib] that uses the power of overlays to make overriding dependencies possible. 38 | - POWERED BY [HNIX](https://github.com/haskell-nix/hnix)™ since before it was cool. 39 | 40 | Probably a few more. 41 | 42 | ## Example Output 43 | 44 | The [CodiMD server](https://github.com/codimd/server) is an elaborate npm package with hundreds of 45 | dependencies. `yarn2nix` flawlessly parses the current (2020-07) `yarn.lock` 46 | file distributed with the project, including resolving their manual git forks of 47 | multiple npm packages: 48 | 49 | ``` 50 | $ yarn2nix ~/tmp/server/yarn.lock | wc 51 | 7320 22701 399111 52 | $ wc ~/tmp/server/yarn.lock 53 | 11938 18615 500078 /home/lukas/tmp/server/yarn.lock 54 | ``` 55 | 56 | The output of this conversion [can be seen 57 | here](https://gist.github.com/sternenseemann/0c253305350b2406e38c700b840869f2). Also 58 | note that [git dependencies are resolved 59 | correctly](https://gist.github.com/sternenseemann/0c253305350b2406e38c700b840869f2#file-codimd-dependencies-nix-L2086-L2087). 60 | 61 | Pushing it through the provided [library of nix 62 | functions][nix-lib], we get a complete build of CodiMD's 63 | dependencies, using the project template (generated with `--template`), we also 64 | build the CodiMD server. Included executables will be in `node_modules/.bin` as expected and 65 | correctly link to their respective library paths in the nix store, for example: 66 | 67 | ``` 68 | $ /nix/store/zs9jk7yhdxsasn26m0903fq89cmyllzv-CodiMD-1.6.0/node_modules/.bin/markdown-it -v 69 | 10.0.0 70 | $ readlink /nix/store/zs9jk7yhdxsasn26m0903fq89cmyllzv-CodiMD-1.6.0/node_modules/.bin/markdown-it 71 | /nix/store/bgas2l5izznq1b61a3jyf3gpb73x8chn-markdown-it-10.0.0/bin/markdown-it.js 72 | ``` 73 | 74 | [nix-lib]: ./nix-lib/default.nix 75 | 76 | ## Building `yarn2nix` 77 | 78 | ``` 79 | $ nix-build 80 | $ result/bin/yarn2nix 81 | ``` 82 | 83 | ## Using the generated nix files to build a project 84 | 85 | **Note:** This is a temporary interface. Ideally, the library will be in nixpkgs 86 | and yarn2nix will be callable from inside the build (so the resulting nix files 87 | don’t have to be checked in). 88 | 89 | Once you have the `yarn2nix` binary, use it to generate nix files for the 90 | `yarn.lock` file and the `package.json`: 91 | 92 | ```shell 93 | $ yarn2nix ./jsprotect/yarn.lock > npm-deps.nix 94 | $ yarn2nix --template ./jsproject/package.json > npm-package.nix 95 | ``` 96 | 97 | Then use the library to assemble the generated files in a `default.nix`: 98 | 99 | ```nix 100 | let 101 | pkgs = import {}; 102 | yarn2nix = import /path/to/yarn2nix {}; 103 | nixLib = yarn2nix.nixLib; 104 | 105 | in 106 | nixLib.buildNodePackage 107 | ( { src = nixLib.removePrefixes [ "node_modules" ] ./.; } // 108 | nixLib.callTemplate ./npm-package.nix 109 | (nixLib.buildNodeDeps (pkgs.callPackage ./npm-deps.nix {}))) 110 | ``` 111 | 112 | Finally, run `nix-build`, and voilà, in `./result/` you find the project with 113 | all its dependencies correctly linked to their corresponding `node_modules` 114 | folder, recursively. 115 | 116 | ## Using private package repository 117 | 118 | Since `yarn2nix` uses standard `fetchurl` to download packages, 119 | it is possible to authenticate by overriding `fetchurl` 120 | to use the access credentials in `/etc/nix/netrc`. 121 | 122 | Refer to the [Enterprise NixOS Wiki article](https://nixos.wiki/wiki/Enterprise) 123 | for instructions. 124 | 125 | ## Development 126 | 127 | ``` 128 | $ nix-shell 129 | nix-shell> ninja 130 | nix-shell> cabal build yarn2nix 131 | ``` 132 | -------------------------------------------------------------------------------- /yarn2nix/yarn2nix.cabal: -------------------------------------------------------------------------------- 1 | cabal-version: 1.12 2 | 3 | -- This file has been generated from package.yaml by hpack version 0.34.6. 4 | -- 5 | -- see: https://github.com/sol/hpack 6 | 7 | name: yarn2nix 8 | version: 0.10.1 9 | synopsis: Convert yarn.lock files to nix expressions 10 | description: Convert @yarn.lock@ files to nix expressions. See @yarn2nix@ executable. Contains a nix library to call the generated nix files in @nix-lib/@. Library functions and module names might be restructured in the future. 11 | category: Distribution, Nix 12 | homepage: https://github.com/Profpatsch/yarn2nix#readme 13 | bug-reports: https://github.com/Profpatsch/yarn2nix/issues 14 | author: Profpatsch 15 | maintainer: mail@profpatsch.de 16 | license: MIT 17 | license-file: LICENSE 18 | build-type: Simple 19 | extra-source-files: 20 | LICENSE 21 | README.md 22 | nix-lib/buildNodePackage.nix 23 | nix-lib/default.nix 24 | 25 | source-repository head 26 | type: git 27 | location: https://github.com/Profpatsch/yarn2nix 28 | 29 | library 30 | exposed-modules: 31 | Distribution.Nixpkgs.Nodejs.Cli 32 | Distribution.Nixpkgs.Nodejs.FromPackage 33 | Distribution.Nixpkgs.Nodejs.License 34 | Distribution.Nixpkgs.Nodejs.OptimizedNixOutput 35 | Distribution.Nixpkgs.Nodejs.ResolveLockfile 36 | Distribution.Nixpkgs.Nodejs.Utils 37 | Distribution.Nodejs.Package 38 | Nix.Expr.Additions 39 | other-modules: 40 | Paths_yarn2nix 41 | hs-source-dirs: 42 | src 43 | ghc-options: -Wall 44 | build-depends: 45 | aeson >=2.0 46 | , aeson-better-errors >=0.9.1.1 47 | , async-pool ==0.9.* 48 | , base ==4.* 49 | , bytestring ==0.10.* 50 | , containers >=0.5 && <0.7 51 | , data-fix >=0.0.7 && <0.4 52 | , directory ==1.3.* 53 | , filepath ==1.4.* 54 | , hnix >=0.6 && <0.15 55 | , mtl ==2.2.* 56 | , optparse-applicative ==0.16.* 57 | , prettyprinter >=1.2 && <1.8 58 | , process >=1.4 59 | , protolude ==0.3.* 60 | , regex-tdfa ==1.3.* 61 | , scientific >0.3.3.0 && <0.4 62 | , stm >2.4.0 && <2.6.0.0 63 | , text ==1.2.* 64 | , transformers ==0.5.* 65 | , unordered-containers ==0.2.* 66 | , yarn-lock ==0.6.* 67 | default-language: Haskell2010 68 | 69 | executable node-package-tool 70 | main-is: NodePackageTool.hs 71 | other-modules: 72 | Paths_yarn2nix 73 | ghc-options: -Wall 74 | build-depends: 75 | aeson >=2.0 76 | , aeson-better-errors >=0.9.1.1 77 | , async-pool ==0.9.* 78 | , base ==4.* 79 | , bytestring ==0.10.* 80 | , containers >=0.5 && <0.7 81 | , data-fix >=0.0.7 && <0.4 82 | , directory ==1.3.* 83 | , filepath ==1.4.* 84 | , hnix >=0.6 && <0.15 85 | , mtl ==2.2.* 86 | , optparse-applicative >=0.13 87 | , prettyprinter >=1.2 && <1.8 88 | , process >=1.4 89 | , protolude ==0.3.* 90 | , regex-tdfa ==1.3.* 91 | , scientific >0.3.3.0 && <0.4 92 | , stm >2.4.0 && <2.6.0.0 93 | , text ==1.2.* 94 | , transformers ==0.5.* 95 | , unix ==2.7.* 96 | , unordered-containers ==0.2.* 97 | , yarn-lock ==0.6.* 98 | , yarn2nix 99 | default-language: Haskell2010 100 | 101 | executable yarn2nix 102 | main-is: Main.hs 103 | other-modules: 104 | Paths_yarn2nix 105 | ghc-options: -Wall 106 | build-depends: 107 | aeson >=2.0 108 | , aeson-better-errors >=0.9.1.1 109 | , async-pool ==0.9.* 110 | , base ==4.* 111 | , bytestring ==0.10.* 112 | , containers >=0.5 && <0.7 113 | , data-fix >=0.0.7 && <0.4 114 | , directory ==1.3.* 115 | , filepath ==1.4.* 116 | , hnix >=0.6 && <0.15 117 | , mtl ==2.2.* 118 | , optparse-applicative ==0.16.* 119 | , prettyprinter >=1.2 && <1.8 120 | , process >=1.4 121 | , protolude ==0.3.* 122 | , regex-tdfa ==1.3.* 123 | , scientific >0.3.3.0 && <0.4 124 | , stm >2.4.0 && <2.6.0.0 125 | , text ==1.2.* 126 | , transformers ==0.5.* 127 | , unordered-containers ==0.2.* 128 | , yarn-lock ==0.6.* 129 | , yarn2nix 130 | default-language: Haskell2010 131 | 132 | test-suite yarn2nix-tests 133 | type: exitcode-stdio-1.0 134 | main-is: Test.hs 135 | other-modules: 136 | TestNpmjsPackage 137 | Paths_yarn2nix 138 | hs-source-dirs: 139 | tests 140 | ghc-options: -Wall 141 | build-depends: 142 | aeson >=2.0 143 | , aeson-better-errors >=0.9.1.1 144 | , async-pool ==0.9.* 145 | , base ==4.* 146 | , bytestring ==0.10.* 147 | , containers >=0.5 && <0.7 148 | , data-fix >=0.0.7 && <0.4 149 | , directory ==1.3.* 150 | , filepath ==1.4.* 151 | , hnix >=0.6 && <0.15 152 | , mtl ==2.2.* 153 | , neat-interpolation >=0.3 && <0.6 154 | , optparse-applicative ==0.16.* 155 | , prettyprinter >=1.2 && <1.8 156 | , process >=1.4 157 | , protolude ==0.3.* 158 | , regex-tdfa ==1.3.* 159 | , scientific >0.3.3.0 && <0.4 160 | , stm >2.4.0 && <2.6.0.0 161 | , tasty >=0.11 && <1.5 162 | , tasty-hunit >=0.9 && <0.11 163 | , tasty-quickcheck >=0.8 && <0.11 164 | , tasty-th ==0.1.7.* 165 | , text ==1.2.* 166 | , transformers ==0.5.* 167 | , unordered-containers ==0.2.* 168 | , yarn-lock ==0.6.* 169 | , yarn2nix 170 | default-language: Haskell2010 171 | -------------------------------------------------------------------------------- /yarn-lock/tests/TestParse.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, TemplateHaskell, QuasiQuotes, NamedFieldPuns, ViewPatterns #-} 2 | module TestParse (tests) where 3 | 4 | import qualified Data.Map as Map 5 | import qualified Data.List.NonEmpty as NE 6 | import Test.Tasty (TestTree) 7 | import Test.Tasty.TH 8 | import Test.Tasty.HUnit 9 | import NeatInterpolation 10 | import qualified Text.Megaparsec as MP 11 | import qualified Data.Char as Ch 12 | 13 | import Yarn.Lock.Types 14 | import Yarn.Lock.Parse 15 | import Data.Text (Text) 16 | import qualified Data.Text as Text 17 | import Control.Monad (void) 18 | 19 | startComment :: Text 20 | startComment = [text| 21 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 22 | # yarn lockfile v1 23 | dummy-package@foo: 24 | version: foo 25 | |] 26 | 27 | case_startCommentEmptyPackageList :: Assertion 28 | case_startCommentEmptyPackageList = do 29 | parseSuccess packageList startComment 30 | >>= \((Keyed keys _) : _) -> do 31 | assertBool "only foo" 32 | (keys == pure (PackageKey (SimplePackageKey "dummy-package") "foo")) 33 | 34 | nonsenseEntry :: Text 35 | nonsenseEntry = [text| 36 | foobar@~1.2.3, xyz@hehe: 37 | field1 "°§ℓ»«UAIERNT" 38 | field2 "nopedidope" 39 | |] 40 | 41 | case_NonsenseASTPackageEntry :: Assertion 42 | case_NonsenseASTPackageEntry = do 43 | parseSuccess packageEntry nonsenseEntry 44 | >>= \(Keyed keys (_, PackageFields fields)) -> do 45 | assertBool "two keys" (length keys == 2) 46 | assertBool "two fields" (length fields == 2) 47 | assertBool "field1 member" (Map.member "field1" fields) 48 | assertBool "field2 member" (Map.member "field2" fields) 49 | Map.lookup "field1" fields @=? (Just (Left "°§ℓ»«UAIERNT")) 50 | 51 | nestedPackageExample :: Text 52 | nestedPackageExample = [text| 53 | readable-stream@1.0, "readable-stream@>=1.0.33-1 <1.1.0-0": 54 | dependencies: 55 | core-util-is "~1.0.0" 56 | is.array "" 57 | "@types/string_decoder" "~0.10.x" 58 | johnny-dep 2.3.4 59 | |] 60 | 61 | nestedFieldExample :: Text 62 | nestedFieldExample = [text| 63 | dependencies: 64 | core-util-is "~1.0.0" 65 | is.array "" 66 | "@types/string_decoder" "~0.10.x" 67 | johnny-dep 2.3.4 68 | |] 69 | 70 | case_nestedField :: Assertion 71 | case_nestedField = do 72 | void $ parseSuccess nestedField nestedFieldExample 73 | 74 | case_NestedPackage :: Assertion 75 | case_NestedPackage = do 76 | assertBool "there is unicode" (all Ch.isAscii (Text.unpack nestedPackageExample :: [Char])) 77 | parseSuccess packageEntry nestedPackageExample 78 | >>= \(Keyed _ (_, PackageFields fields)) -> do 79 | case Map.lookup "dependencies" fields of 80 | (Nothing) -> assertFailure "where’s the key" 81 | (Just (Left s)) -> do 82 | assertFailure $ Text.unpack (s <> "should be a nested package") 83 | (Just (Right (PackageFields nested))) -> do 84 | assertEqual "nested keys" 4 $ length nested 85 | assertEqual "dep exists" (Just (Left "2.3.4")) 86 | $ Map.lookup "johnny-dep" nested 87 | assertEqual "scoped packages start with @" (Just (Left "~0.10.x")) 88 | $ Map.lookup "@types/string_decoder" nested 89 | 90 | case_PackageField :: IO () 91 | case_PackageField = do 92 | let goodField = "myfield12 \"abc\"" 93 | badField = "badbad \"abc" 94 | okayishField = "f abc" 95 | parseFailure field badField 96 | parseSuccess field goodField 97 | >>= \(key, val) -> do 98 | key @=? "myfield12" 99 | val @=? (Left "abc") 100 | parseSuccess field okayishField 101 | >>= \(key, val) -> do 102 | key @=? "f" 103 | val @=? (Left "abc") 104 | 105 | case_PackageKey :: Assertion 106 | case_PackageKey = do 107 | let key = "foo@^1.3.4, bar@blafoo234, xnu@, @types/foo@:\n" 108 | parseSuccess packageKeys key 109 | >>= \keys -> do 110 | keys @?= NE.fromList 111 | [ PackageKey (SimplePackageKey "foo") "^1.3.4" 112 | , PackageKey (SimplePackageKey "bar") "blafoo234" 113 | -- yes, the version can be empty … 114 | , PackageKey (SimplePackageKey "xnu") "" 115 | -- and yes, package names can contain `@` 116 | , PackageKey (ScopedPackageKey "types" "foo") "" 117 | ] 118 | 119 | 120 | -- | PackageKeys can contain arbitrary stuff apparently 121 | case_complexKey :: Assertion 122 | case_complexKey = do 123 | parseSuccess packageKeys 124 | "\"mango-components@git+ssh://git@github.com:stuff/#fe234\":" 125 | >>= \((PackageKey name version) NE.:| []) -> do 126 | assertEqual "complexKey name" 127 | (SimplePackageKey "mango-components") name 128 | assertEqual "complexKey version" 129 | "git+ssh://git@github.com:stuff/#fe234" 130 | version 131 | parseSuccess packageKeys 132 | "\"@types/mango-components@git@github\":" 133 | >>= \((PackageKey name version) NE.:| []) -> do 134 | assertEqual "complexKeyScoped name" 135 | (ScopedPackageKey "types" "mango-components") name 136 | assertEqual "complexKeyScoped version" "git@github" version 137 | 138 | 139 | -- HELPERS 140 | 141 | parseSuccess :: Parser a -> Text -> IO a 142 | parseSuccess parser string = do 143 | case MP.parse parser "" string of 144 | (Right a) -> pure a 145 | (Left err) -> do 146 | _ <- assertFailure ("parse should succeed, but: \n" 147 | <> MP.errorBundlePretty err 148 | <> "for input\n" <> Text.unpack string <> "\n\"") 149 | error "not reached" 150 | 151 | parseFailure :: Parser a -> Text -> IO () 152 | parseFailure parser string = do 153 | case MP.parseMaybe parser string of 154 | Nothing -> pure () 155 | (Just _) -> assertFailure "parse should have failed" 156 | 157 | tests :: TestTree 158 | tests = $(testGroupGenerator) 159 | -------------------------------------------------------------------------------- /yarn2nix/nix-lib/default.nix: -------------------------------------------------------------------------------- 1 | { lib, pkgs 2 | # TODO: temporary, to make overwriting yarn2nix easy 3 | # TODO: remove static building once RPATHs are fixed 4 | , yarn2nix ? pkgs.haskell.lib.justStaticExecutables 5 | pkgs.haskellPackages.yarn2nix 6 | }: 7 | 8 | let 9 | # Build an attrset of node dependencies suitable for the `nodeBuildInputs` 10 | # argument of `buildNodePackage`. The input is an overlay 11 | # of node packages that call `_buildNodePackage`, like in the 12 | # files generated by `yarn2nix`. 13 | # It is possible to just call it with a generated file, like so: 14 | # `buildNodeDeps (pkgs.callPackage ./npm-deps.nix {})` 15 | # You can also use `lib.composeExtensions` to override packages 16 | # in the set: 17 | # ``` 18 | # buildNodeDeps (lib.composeExtensions 19 | # (pkgs.callPackage ./npm-deps.nix {}) 20 | # (self: super: { pkg = super.pkg.override {…}; })) 21 | # ``` 22 | # TODO: should _buildNodePackage be fixed in here? 23 | buildNodeDeps = nodeDeps: lib.fix 24 | (lib.extends 25 | nodeDeps 26 | (self: { 27 | # The actual function building our packages. 28 | # type: { key: String | { scope: String, name: String } 29 | # , } 30 | # Wraps the invocation in the fix point, to construct the 31 | # list of { key, drv } needed by buildNodePackage 32 | # from the templates. 33 | # It is basically a manual paramorphism, carrying parts of the 34 | # information of the previous layer (the original package key). 35 | # TODO: move that function out of the package set 36 | # and get nice self/super scoping right 37 | _buildNodePackage = { key, ... }@args: 38 | # To keep the generated files shorter, we allow keys to 39 | # be represented as strings if they have no scopes. 40 | # This is the only place where this is accepted, 41 | # but hacky nonetheless. Probably fix with above TODO. 42 | let key' = if builtins.isString key 43 | then { scope = ""; name = key; } 44 | else key; 45 | in { key = key'; 46 | drv = buildNodePackage (args // { key = key'; }); }; 47 | })); 48 | 49 | # Build a package template generated by the `yarn2nix --template` 50 | # utility from a yarn package. The first input is the path to the 51 | # template nix file, the second input is all node dependencies 52 | # needed by the template, in the form generated by `buildNodeDeps`. 53 | callTemplate = yarn2nixTemplate: allDeps: 54 | pkgs.callPackage yarn2nixTemplate {} allDeps; 55 | 56 | 57 | buildNodePackage = import ./buildNodePackage.nix { 58 | inherit linkNodeDeps yarn2nix; 59 | inherit (pkgs) stdenv lib nodejs; 60 | }; 61 | 62 | # Link together a `node_modules` folder that can be used 63 | # by npm’s module system to call dependencies. 64 | # Also link executables of all dependencies into `.bin`. 65 | # TODO: copy manpages & docs as well 66 | # type: { name: String 67 | # , dependencies: ListOf { key: { scope: String, name: String } 68 | # , drv : Drv } } 69 | # -> Drv 70 | linkNodeDeps = {name, dependencies}: 71 | pkgs.runCommand ("${name}-node_modules") { 72 | # This just creates a simple link farm, which should be pretty fast, 73 | # saving us from additional hydra requests for potentially hundreds 74 | # of packages. 75 | allowSubstitutes = false; 76 | # Also tell Hydra it’s not worth copying to a builder. 77 | preferLocalBuild = true; 78 | } '' 79 | mkdir -p $out/.bin 80 | ${lib.concatMapStringsSep "\n" 81 | (dep: 82 | let 83 | hasScope = dep.key.scope != ""; 84 | # scoped packages get another subdirectory for their scope (`@scope/`) 85 | parentfolder = if hasScope 86 | then "$out/@${dep.key.scope}" 87 | else "$out"; 88 | subfolder = "${parentfolder}/${dep.key.name}"; 89 | in '' 90 | echo "linking node dependency ${formatKey dep.key}" 91 | ${ # we need to create the scope folder, otherwise ln fails 92 | lib.optionalString hasScope ''mkdir -p "${parentfolder}"'' } 93 | ln -sT ${dep.drv} "${subfolder}" 94 | ${yarn2nix}/bin/node-package-tool \ 95 | link-bin \ 96 | --to=$out/.bin \ 97 | --package=${subfolder} 98 | '') 99 | dependencies} 100 | ''; 101 | 102 | # Filter out files/directories with one of the given prefix names 103 | # from the given path. 104 | # type: ListOf File -> Path -> Drv 105 | removePrefixes = prfxs: path: 106 | let 107 | hasPrefix = file: prfx: lib.hasPrefix ((builtins.toPath path) + "/" + prfx) file; 108 | hasAnyPrefix = file: lib.any (hasPrefix file) prfxs; 109 | in 110 | builtins.filterSource (file: _: ! (hasAnyPrefix file)) path; 111 | 112 | # `callYarnLock` calls `yarn2nix` to generate a nix representation of 113 | # a `yarn.lock` file and directly imports it. It uses `yarn2nix`'s 114 | # offline mode, so its resolving capabilities are limited, i. e. git 115 | # dependencies are not possible. 116 | # 117 | # Example usage: 118 | # 119 | # ``` 120 | # buildNodeDeps (callYarnLock ./yarn.lock {}) 121 | # ``` 122 | callYarnLock = yarnLock: { name ? "yarn.lock.nix" }: 123 | pkgs.callPackage (pkgs.runCommand name { 124 | # faster to build locally, see also note at linkNodeDeps 125 | allowSubstitutes = false; 126 | preferLocalBuild = true; 127 | } '' 128 | ${yarn2nix}/bin/yarn2nix --offline ${yarnLock} > $out 129 | '') { }; 130 | 131 | # `callPackageJson` calls `yarn2nix --template` to generate a 132 | # nix representation of a `package.json` and directly imports 133 | # it. It returns a function which expects a dependency attrset 134 | # like `callYarnLock` generates. 135 | # 136 | # Example usage: 137 | # 138 | # ``` 139 | # let template = callPackageJson ./package.json {}; 140 | # in buildNodePackage ({ src = ./.; } // 141 | # template (callYarnLock ./yarn.lock {})) 142 | # ``` 143 | callPackageJson = packageJson: { name ? "package.json.nix" }: 144 | pkgs.callPackage (pkgs.runCommand name { 145 | # faster to build locally, see also note at linkNodeDeps 146 | allowSubstitutes = false; 147 | preferLocalBuild = true; 148 | } '' 149 | ${yarn2nix}/bin/yarn2nix --template ${packageJson} > $out 150 | '') {}; 151 | 152 | # format a package key of { scope: String, name: String } 153 | formatKey = { scope, name }: 154 | if scope == "" 155 | then name 156 | else "@${scope}/${name}"; 157 | 158 | in { 159 | inherit buildNodeDeps linkNodeDeps buildNodePackage 160 | callTemplate removePrefixes callYarnLock 161 | callPackageJson; 162 | } 163 | -------------------------------------------------------------------------------- /yarn-lock/src/Data/MultiKeyedMap.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE ExistentialQuantification, NamedFieldPuns, ScopedTypeVariables, RecordWildCards, ApplicativeDo #-} 2 | {-| 3 | Module : Data.MultiKeyedMap 4 | Description : A map with possibly multiple keys per value 5 | Maintainer : Profpatsch 6 | Stability : experimental 7 | 8 | Still very much experimental and missing lots of functions and testing. 9 | 10 | Internally, a 'MKMap' is two maps, a @keyMap@ referencing an intermediate key 11 | (whose type can be chosen freely and which is incremented sequentially), and 12 | a @valueMap@ going from intermediate key to final value. 13 | 14 | A correct implementation guarantees that 15 | 16 | (1) the internal structure can’t be corrupted by operations declared safe 17 | (2) adding and removing keys does not make values inaccessible 18 | (thus leaking memory) and doesn’t insert unnecessary values 19 | -} 20 | module Data.MultiKeyedMap 21 | ( MKMap 22 | , at, (!) 23 | , mkMKMap, fromList, toList 24 | , insert 25 | , flattenKeys, keys, values 26 | ) where 27 | 28 | import qualified Data.Map.Strict as M 29 | import Data.Monoid (All(..)) 30 | import Data.Foldable (foldl') 31 | import qualified Data.List.NonEmpty as NE 32 | import Data.Proxy (Proxy(..)) 33 | import qualified Data.Tuple as Tuple 34 | import GHC.Stack (HasCallStack) 35 | import qualified Text.Show as Show 36 | 37 | -- TODO: add time behaviour of functions to docstrings 38 | 39 | -- | A `Map`-like structure where multiple keys can point 40 | -- to the same value, with corresponding abstracted interface. 41 | -- 42 | -- Internally, we use two maps connected by an intermediate key. 43 | -- The intermediate key (@ik@) can be anything implementing 44 | -- 'Ord' (for 'Map'), 'Bounded' (to get the first value) 45 | -- and 'Enum' (for 'succ'). 46 | data MKMap k v = forall ik. (Ord ik, Enum ik) 47 | => MKMap 48 | { keyMap :: M.Map k ik 49 | , highestIk :: ik 50 | , valMap :: M.Map ik v } 51 | 52 | -- TODO: is it possible without (Ord k)? 53 | instance (Eq k, Ord k, Eq v) => Eq (MKMap k v) where 54 | -- TODO: not sure if that’s correct, add tests 55 | (==) m1@(MKMap { keyMap = km1 56 | , valMap = vm1 }) 57 | m2@(MKMap { keyMap = km2 58 | , valMap = vm2 }) 59 | = getAll $ foldMap All 60 | $ let ks1 = M.keys km1 in 61 | -- shortcut if the length of the value map is not equal 62 | [ length vm1 == length vm2 63 | -- the keys have to be equal (the lists are ascending) 64 | , ks1 == M.keys km2 ] 65 | -- now test whether every key leads to the same value 66 | -- I wonder if there is a more efficient way? 67 | ++ map (\k -> m1 ! k == m2 ! k) ks1 68 | -- we could test whether the values are equal, 69 | -- but if the implementation is correct they should 70 | -- all be reachable from the keys (TODO: add invariants) 71 | -- TODO: can (/=) be implemented more efficient than not.(==)? 72 | 73 | 74 | instance Functor (MKMap k) where 75 | fmap f (MKMap{..}) = MKMap { valMap = fmap f valMap, .. } 76 | {-# INLINE fmap #-} 77 | 78 | -- TODO implement all functions Data.Map also implements for Foldable 79 | instance Foldable (MKMap k) where 80 | foldMap f (MKMap{..}) = foldMap f valMap 81 | {-# INLINE foldMap #-} 82 | 83 | instance Traversable (MKMap k) where 84 | traverse f (MKMap{..}) = do 85 | val <- traverse f valMap 86 | pure $ MKMap{ valMap=val, .. } 87 | {-# INLINE traverse #-} 88 | 89 | -- | Find value at key. Partial. See 'M.!'. 90 | at :: (HasCallStack, Ord k) => MKMap k v -> k -> v 91 | at MKMap{keyMap, valMap} k = valMap M.! (keyMap M.! k) 92 | -- | Operator alias of 'at'. 93 | (!) :: (HasCallStack, Ord k) => MKMap k v -> k -> v 94 | (!) = at 95 | {-# INLINABLE (!) #-} 96 | {-# INLINABLE at #-} 97 | infixl 9 ! 98 | 99 | -- | Create a 'MKMap' given a type for the internally used intermediate key. 100 | mkMKMap :: forall k ik v. (Ord k, Ord ik, Enum ik, Bounded ik) 101 | => (Proxy ik) -- ^ type of intermediate key 102 | -> MKMap k v -- ^ new map 103 | mkMKMap _ = MKMap mempty (minBound :: ik) mempty 104 | {-# INLINE mkMKMap #-} 105 | 106 | instance (Show k, Show v) => Show (MKMap k v) where 107 | showsPrec d m = Show.showString "fromList " . (showsPrec d $ toList m) 108 | 109 | -- | Build a map from a list of key\/value pairs. 110 | fromList :: forall ik k v. (Ord k, Ord ik, Enum ik, Bounded ik) 111 | => (Proxy ik) -- ^ type of intermediate key 112 | -> [(NE.NonEmpty k, v)] -- ^ list of @(key, value)@ 113 | -> MKMap k v -- ^ new map 114 | 115 | -- TODO: it’s probably better to implement with M.fromList 116 | fromList p = foldl' (\m (ks, v) -> newVal ks v m) (mkMKMap p) 117 | 118 | -- | Convert the map to a list of key\/value pairs. 119 | toList :: MKMap k v -> [(NE.NonEmpty k, v)] 120 | toList MKMap{keyMap, valMap} = 121 | map (fmap (valMap M.!) . Tuple.swap) . M.assocs . aggregateIk $ keyMap 122 | where 123 | aggregateIk :: forall k ik. (Ord ik, Enum ik) 124 | => M.Map k ik 125 | -> M.Map ik (NE.NonEmpty k) 126 | aggregateIk = M.foldlWithKey 127 | (\m k ik -> M.insertWith (<>) ik (pure k) m) mempty 128 | 129 | -- | “Unlink” keys that are pointing to the same value. 130 | -- 131 | -- Returns a normal map. 132 | flattenKeys :: (Ord k) => MKMap k v -> M.Map k v 133 | flattenKeys MKMap{keyMap, valMap} = 134 | M.foldlWithKey' (\m k ik -> M.insert k (valMap M.! ik) m) mempty keyMap 135 | 136 | -- | Return a list of all keys. 137 | keys :: (Ord k) => MKMap k v -> [k] 138 | keys = M.keys . flattenKeys 139 | 140 | -- | Return a list of all values. 141 | values :: MKMap k v -> [v] 142 | values (MKMap _ _ valMap) = M.elems valMap 143 | 144 | -- TODO: this is like normal insert, it doesn’t search if the value 145 | -- already exists (where it might want to add the key instead). 146 | -- Of course that would be O(n) in the naive implementation. 147 | -- In that case the keyMap should probably be changed to a bimap. 148 | -- also, naming 149 | -- | Equivalent to 'M.insert', if the key doesn’t exist a new 150 | -- singleton key is added. 151 | insert :: (Ord k) => k -> v -> MKMap k v -> MKMap k v 152 | insert k v m@MKMap{keyMap, highestIk, valMap} = 153 | case M.lookup k keyMap of 154 | Nothing -> ins 155 | Just ik -> upd ik 156 | where 157 | ins = newVal (pure k) v m 158 | upd ik = MKMap { keyMap, highestIk, valMap = M.insert ik v valMap } 159 | 160 | 161 | -- | Helper, assumes there is no such value already. 162 | -- Will leak space otherwise! 163 | -- 164 | -- Insert every key into the keyMap, increase the intermediate counter, 165 | -- insert the value at new intermediate counter. 166 | -- Overwrites all already existing keys! 167 | newVal :: (Ord k) => NE.NonEmpty k -> v -> MKMap k v -> MKMap k v 168 | newVal ks v MKMap{keyMap, highestIk, valMap} = 169 | MKMap { keyMap = foldl' (\m k -> M.insert k next m) keyMap ks 170 | , highestIk = next 171 | , valMap = M.insert next v valMap } 172 | where next = succ highestIk 173 | -------------------------------------------------------------------------------- /yarn2nix/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/) 6 | and this project adheres to [Semantic Versioning](http://semver.org/). 7 | 8 | All changes are documented in more detail in their respective commit messages. 9 | 10 | ## [0.10.1] - 2022-03-28 11 | 12 | ### Fixed 13 | 14 | - Rewrite license parsing from nixpkgs licenses 15 | 16 | It broke in more recent nixpkgs versions, now it should be more resilient 17 | against upstream changes. 18 | 19 | ## [0.10.0] - 2022-03-26 20 | 21 | - Support `aeson-2.0`, requires GHC `9.0` 22 | 23 | Add support for the new `aeson-2.0` API, 24 | but do not make it backward compatible with `1.x`. 25 | 26 | Since nixpkgs is going to fast-forward to 9.x soon, this should not be a problem. 27 | If it is, conditional support for `aeson-1.x` could be added later. 28 | 29 | Since this might be a breaking deal for some, we mark it as breaking change. 30 | 31 | 32 | ## [0.9.0] - 2021-08-24 33 | 34 | ### Added 35 | 36 | - `--offline` flag to abort if network would be required 37 | 38 | If `--offline` is given, yarn2nix will abort with an error message if 39 | `nix-prefetch-git` would be necessary to use 40 | (not all yarn.lock files can be converted to nix without network access) 41 | 42 | - `--template`: add proper SPDX license to nix derivation 43 | 44 | Converts the SPDX license to a nix library license, 45 | so that the template has the correct license. 46 | This could be extended to all node_module dependencies in the future. 47 | 48 | ### Fixed 49 | 50 | - Fix binary name for scoped packages 51 | 52 | Fix short form of bin field (`"bin": "./path/to/bin"`) for scoped package 53 | where the binary name should be the package name without scope 54 | instead of the full name. 55 | 56 | For example for @babel/parser it should generate a `.bin/parser` link 57 | instead of a `.bin/@babel/parser` link. 58 | 59 | - Fix `--template` generation 60 | 61 | Since d607336 buildNodePackage doesn't 62 | accept a name argument anymore, but a key one. Due to an oversight in 63 | that change yarn2nix --template would still generate name attributes 64 | which causes buildNodePackage to fail if directly used with callTemplate 65 | and an automatically generated template. 66 | 67 | - `node-package-tool`: default to no (dev) dependencies if field unparsable 68 | 69 | In the node world you can't depend on anything, especially not that a 70 | certain field of package.json is well-formed. 71 | 72 | The dependencies field is sometimes malformed, probably in most cases it 73 | is safe to default to {}. I have observed this in the wild with JSV 0.4.2 74 | (https://github.com/garycourt/JSV) where dependencies is [] instead of 75 | {} or being missing. I think packages with malformed dependency field 76 | can't reasonably expect their dependencies to be installed correctly. 77 | 78 | We only default to {} in cases where we can expect beyond a reasonable 79 | doubt that there are no dependencies being expressed by the field: 80 | 81 | - Empty array 82 | - Scalars (Number, String, …) 83 | 84 | We fail parsing if the field is malformed and seems to contain 85 | something: 86 | 87 | - Non-empty array 88 | - Object that can't be parsed to Dependencies (i. e. is malformed) 89 | 90 | We use this strategy for both dependencies and devDependencies. 91 | 92 | - `node-package-tool`: create target directory if it doesn’t exist 93 | 94 | Apparently some namespaced packages have binaries which live in a 95 | subdirectory. This would previously crash the build. 96 | 97 | ## [0.8.0] - 2019-12-22 98 | 99 | ### Added 100 | 101 | - Preliminary support for local packages 102 | 103 | To support local packages (see commit for 0.6.0 in yarn-lock), we have to add a new package function to the nix output. 104 | 105 | For now it will only add the path directly, which has the drawback that the nix expression is now required to be in the right position (for relative paths). It should be changed to be user-defined later (i.e. the user passes a function which takes the string of the local path and returns a derivation to the package tarball). 106 | 107 | We don’t use the tarball hash from the lockfile yet to verify the integrity when importing into the nix store. 108 | 109 | ### Changed 110 | 111 | - Bumped `hnix` to `0.6.*` 112 | - Small breaking change, because the pretty print library changed to `prettyprint`. 113 | - Bumped `yarn-lock` to `0.6.2` 114 | 115 | ### Fixed 116 | 117 | - nix-lib: Allow creating a scoped directory if it exists 118 | 119 | ## [0.7.0] - 2018-06-14 120 | 121 | ### Added 122 | 123 | - Support for [scoped npm](https://docs.npmjs.com/misc/scope) packages 124 | - These are used mostly for Typescript annotations, but have been cropping up in other cases as well 125 | - The support is first-class and scoped packages are already recognized by the `yarn.lock` parser; as such every corner case should be supported (if not it’s a bug or missed type signature change) 126 | 127 | ### Changed 128 | 129 | - `nix-lib` 130 | - `buildNodeDeps` now takes an overlay instead of a path; for the generated files this means `pkgs.callPackage ./npm-deps.nix {}` instead of just `./npm-deps.nix` 131 | - Because of scoped package support, the package names are now mostly an attributeset of `{ scope: String, name: String }` where an empty string for `scope` means (360) no scope 132 | - One exception is the `key` argument of `_buildNodePackage`, which accepts a string as well if the package is not scoped, to save on bytes in the generated nix deps file 133 | 134 | 135 | ## [0.6.0] - 2018-05-31 136 | 137 | First bigger packaging project: the `pulp` package manager. It is accessible in the new [`yarn2nix-packages`](https://github.com/Profpatsch/yarn2nix-packages) repository. 138 | 139 | ### Added 140 | 141 | - `npmjs.org` registry shortener for nix output 142 | - Extensive usage documentation for `nix-lib` 143 | - Give every binary in `node_modules/.bin` the executable flag 144 | - Pin `nixpkgs` to `unstable` (makes deterministic `nix-build` possible) 145 | 146 | ### Changed 147 | 148 | - `nix-lib` usage has changed considerably; see `./README.md` 149 | - The templates now include `devDependencies` 150 | - Bump `yarn-lock` to `0.4.1` 151 | - Bump `tasty` to `1.1` (nothing changed) 152 | - Bump `hnix` to `0.5` (some signatures/functions changed) 153 | 154 | ### Fixed 155 | 156 | - Correctly generate npm registry paths containing `/` in package names 157 | - Ignore multiple `package.json` fields if they are mistyped/wrong 158 | - There seem to be no checks on the registry side 159 | - We print warnings instead (which are checked in the test suite) 160 | - Correctly substitute `nix-prefetch-git` to use the store path 161 | - Don’t substitute the `node_modules` link farm derivation 162 | - Errors are printed to `stderr` 163 | 164 | ### Removed 165 | 166 | - Dependency on `either`, `EitherT` was removed and replaced by `ExceptT` rom `transformers` 167 | 168 | 169 | ## [0.5.0] - 2017-12-16 170 | 171 | ### Added 172 | 173 | - First working release 174 | - `0.5.0` to signify that it’s not alpha, but not ready for upstream either 175 | 176 | -------------------------------------------------------------------------------- /yarn2nix/tests/TestNpmjsPackage.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, TemplateHaskell, QuasiQuotes, NoImplicitPrelude, LambdaCase, TypeApplications, RecordWildCards, ScopedTypeVariables #-} 2 | module TestNpmjsPackage (tests) where 3 | 4 | import Protolude 5 | import Test.Tasty (TestTree) 6 | import Test.Tasty.TH 7 | import Test.Tasty.HUnit (Assertion, testCase) 8 | import qualified Test.Tasty.HUnit as HUnit 9 | -- import Test.Tasty.QuickCheck 10 | 11 | import qualified Data.Aeson as A 12 | import qualified Data.Aeson.Types as AT 13 | import qualified Data.Text as Text 14 | 15 | import qualified Distribution.Nodejs.Package as NP 16 | import qualified Data.Aeson.KeyMap as KeyMap 17 | import Data.Aeson (Key) 18 | 19 | assertEqual :: (HasCallStack, Eq a, Show a) => Text -> a -> a -> Assertion 20 | assertEqual t a b = HUnit.assertEqual (toS t) a b 21 | 22 | assertBool :: (HasCallStack) => Text -> Bool -> Assertion 23 | assertBool t b = HUnit.assertBool (toS t) b 24 | 25 | baseAnd :: [(Key, A.Value)] -> A.Value 26 | baseAnd fields = A.Object $ KeyMap.fromList $ 27 | [ ("name", "foopkg") 28 | , ("version", "1.0.2") 29 | ] <> fields 30 | 31 | parseWithWarningsZoom :: (Eq a, Show a) 32 | => Text -> A.Value -> (NP.Package -> a) -> a 33 | -> ([NP.Warning] -> Assertion) 34 | -> Assertion 35 | parseWithWarningsZoom name got zoom want warningPred = 36 | NP.unLoggingPackage <$> parseSuccess got 37 | >>= \(val, warnings) -> do 38 | assertEqual (toS name) want (zoom val) 39 | warningPred warnings 40 | 41 | formatWarnings :: [NP.Warning] -> Text 42 | formatWarnings ws = Text.intercalate ", " (map f ws) 43 | where 44 | f w@(NP.PlainWarning _) = "PlainWarning `" <> NP.formatWarning w <> "`" 45 | f w@(NP.WrongType {}) = "WrongType `" <> NP.formatWarning w <> "`" 46 | 47 | parseZoom :: (Eq a, Show a) 48 | => Text -> A.Value -> (NP.Package -> a) -> a 49 | -> Assertion 50 | parseZoom name got zoom want = 51 | parseWithWarningsZoom name got zoom want 52 | (\ws -> assertBool ("unexpected warnings: " <> formatWarnings ws ) $ null ws) 53 | 54 | data WarningType 55 | = SomePlainWarning 56 | | WrongTypeField 57 | { wrongTypeField :: Text 58 | , wrongTypeDefault :: Maybe () 59 | } 60 | deriving (Show) 61 | 62 | -- TODO: the warning list should be an exact list/set! 63 | hasWarning :: WarningType -> [NP.Warning] -> Assertion 64 | hasWarning t = assertBool ("no such warning: " <> show t) 65 | . any (checkWarningType t) 66 | 67 | checkWarningType :: WarningType -> NP.Warning -> Bool 68 | checkWarningType tp w = case (tp, w) of 69 | (SomePlainWarning, NP.PlainWarning _) -> True 70 | ( WrongTypeField { wrongTypeField = ft 71 | , wrongTypeDefault = deft }, 72 | NP.WrongType { NP.wrongTypeField = f 73 | , NP.wrongTypeDefault = def }) 74 | -> ft == f && case (deft, def) of 75 | (Nothing, Nothing) -> True 76 | (Just (), Just _) -> True 77 | _ -> False 78 | (_, _) -> False 79 | 80 | case_dependencies :: Assertion 81 | case_dependencies = do 82 | parseZoom "dependencies are missing" 83 | (baseAnd [ ]) 84 | NP.dependencies 85 | mempty 86 | 87 | parseZoom "dependencies are empty" 88 | (baseAnd [ ("dependencies", A.object []) ]) 89 | NP.dependencies 90 | mempty 91 | 92 | parseZoom "some dependencies" 93 | (baseAnd [ ("dependencies", A.object 94 | [ ("foo", "1.2.3") 95 | , ("bar", "3.4.0") ]) ]) 96 | NP.dependencies 97 | (KeyMap.fromList 98 | [ ("foo", "1.2.3") 99 | , ("bar", "3.4.0") ]) 100 | 101 | parseWithWarningsZoom "dependencies are an empty list" 102 | (baseAnd [ ("dependencies", A.Array mempty) ]) 103 | NP.dependencies 104 | mempty 105 | (hasWarning $ WrongTypeField 106 | { wrongTypeField = "dependencies" 107 | , wrongTypeDefault = Just () }) 108 | 109 | parseWithWarningsZoom "dependencies is a random scalar" 110 | (baseAnd [ ("dependencies", A.String "hiho") ]) 111 | NP.dependencies 112 | mempty 113 | (hasWarning $ WrongTypeField 114 | { wrongTypeField = "dependencies" 115 | , wrongTypeDefault = Just () }) 116 | 117 | parseFailure (Proxy @NP.LoggingPackage) "dependencies are a non-empty list" 118 | (baseAnd [ ("dependencies", A.Array (pure "foo")) ]) 119 | 120 | case_binPaths :: Assertion 121 | case_binPaths = do 122 | parseZoom ".bin exists with files" 123 | (baseAnd [ ("bin", "./abc") ]) 124 | NP.bin 125 | (NP.BinFiles $ KeyMap.fromList [ ("foopkg", "./abc") ]) 126 | 127 | parseZoom "scoped package" 128 | (baseAnd [ ("name", "@foo/bar"), ("bin", "./abc") ]) 129 | NP.bin 130 | (NP.BinFiles $ KeyMap.fromList [ ("bar", "./abc") ]) 131 | 132 | parseZoom ".directories.bin exists with path" 133 | (baseAnd [ ("directories", A.object [("bin", "./abc")]) ]) 134 | NP.bin 135 | (NP.BinFolder "./abc") 136 | 137 | parseZoom "multiple .bin files are parsed" 138 | (baseAnd [ ("bin", A.object 139 | [ ("one", "./bin/one") 140 | , ("two", "imhere") ]) ]) 141 | NP.bin 142 | (NP.BinFiles $ KeyMap.fromList 143 | [ ("one", "./bin/one") 144 | , ("two", "imhere") ]) 145 | 146 | parseWithWarningsZoom "bin and directories.bin both exist" 147 | (baseAnd [ ("bin", "foo") 148 | , ("directories", A.object 149 | [ ("bin", "foo") ]) ]) 150 | NP.bin 151 | (NP.BinFiles mempty) 152 | (hasWarning SomePlainWarning) 153 | 154 | parseZoom "neither .bin nor .directories.bin exis" 155 | (baseAnd []) 156 | NP.bin 157 | (NP.BinFiles mempty) 158 | 159 | parseWithWarningsZoom ".scripts field has a wrong type" 160 | (baseAnd [ ("scripts", A.object 161 | [ ("foo", A.object []) 162 | , ("bar", "imascript") ]) ]) 163 | NP.scripts 164 | (KeyMap.fromList [ ("bar", "imascript") ]) 165 | (hasWarning (WrongTypeField 166 | { wrongTypeField = "scripts.foo" 167 | , wrongTypeDefault = Nothing })) 168 | 169 | parseSuccess :: (A.FromJSON a) => A.Value -> IO a 170 | parseSuccess v = case A.fromJSON v of 171 | (AT.Error err) -> HUnit.assertFailure err >> panic "not reached" 172 | (AT.Success a) -> pure a 173 | 174 | parseFailure :: forall a. (A.FromJSON a) => Proxy a -> Text -> A.Value -> IO () 175 | parseFailure Proxy msg v = case AT.fromJSON @a v of 176 | -- TODO: check the error? 177 | (AT.Error _) -> pass 178 | (AT.Success _) -> HUnit.assertFailure $ (toS msg) <> ", parse should have failed." 179 | 180 | tests :: TestTree 181 | tests = $(testGroupGenerator) 182 | -------------------------------------------------------------------------------- /yarn-lock/tests/TestFile.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, TemplateHaskell, NamedFieldPuns, ViewPatterns, LambdaCase, RecordWildCards #-} 2 | module TestFile (tests) where 3 | 4 | import qualified Data.List.NonEmpty as NE 5 | import Test.Tasty (TestTree) 6 | import Test.Tasty.TH 7 | import Test.Tasty.HUnit 8 | import qualified Data.Map.Strict as M 9 | 10 | import qualified Yarn.Lock.Types as T 11 | import qualified Yarn.Lock.File as File 12 | import qualified Yarn.Lock.Parse as Parse 13 | import Data.Text (Text) 14 | import Data.Functor ((<&>)) 15 | import Control.Applicative (Alternative (empty)) 16 | 17 | -- TODO: actually use somehow (apart from manual testing) 18 | -- The yarn.lock file should resolve each packageKey exactly once. 19 | -- 20 | -- No pkgname/semver combination should appear twice. That means 21 | -- the lengths of the converted map and the list lists need to match. 22 | -- prop_LockfileSameAmountOfKeys :: [Package] -> Bool 23 | -- prop_LockfileSameAmountOfKeys pl = length (packageListToLockfile pl) 24 | -- == length (concatMap fst pl) 25 | 26 | emptyAst :: [(Text, Either Text Parse.PackageFields)] -> Parse.PackageFields 27 | emptyAst = Parse.PackageFields . M.fromList 28 | 29 | minimalAst :: [(Text, Either Text Parse.PackageFields)] -> Parse.PackageFields 30 | minimalAst = emptyAst . ([("version", Left "0.3")] <>) 31 | 32 | case_gitRemote :: Assertion 33 | case_gitRemote = do 34 | let ref = "abcthisisaref" 35 | ast link_ hasUid = minimalAst $ 36 | [ ("resolved", Left link_) ] 37 | <> hasUid `orEmpty` ("uid", Left ref) 38 | let gitRemIs parsed (url', ref') = parsed 39 | <&> T.remote >>= \case 40 | T.GitRemote{..} -> do 41 | assertEqual "url url" url' gitRepoUrl 42 | assertEqual "url ref" ref' gitRev 43 | a -> assertFailure ("should be GitRemote, is " <> show a) 44 | let url1 = "git://github.com/bla" 45 | astToPackageSuccess (ast (url1 <> "#" <> ref) False) 46 | `gitRemIs` (url1, ref) 47 | let url2 = "https://github.com/bla" 48 | astToPackageSuccess (ast ("git+" <> url2) True) 49 | `gitRemIs` (url2, ref) 50 | 51 | case_fileRemote :: Assertion 52 | case_fileRemote = do 53 | let sha = "helloimref" 54 | good = minimalAst $ 55 | [ ("resolved", Left $ "https://gnu.org/stallmanstoe#" <> sha) ] 56 | goodNoIntegrity = minimalAst $ 57 | [ ("resolved", Left $ "https://gnu.org/stallmanstoe") ] 58 | astToPackageSuccess good 59 | <&> T.remote >>= \case 60 | T.FileRemote{..} -> do 61 | assertEqual "remote url" "https://gnu.org/stallmanstoe" fileUrl 62 | assertEqual "file sha" sha fileSha1 63 | a -> assertFailure ("should be FileRemote, is " <> show a) 64 | astToPackageSuccess goodNoIntegrity 65 | <&> T.remote >>= \case 66 | T.FileRemoteNoIntegrity{..} -> assertEqual "remote url" "https://gnu.org/stallmanstoe" fileNoIntegrityUrl 67 | a -> assertFailure ("should be FileRemote, is " <> show a) 68 | 69 | case_fileLocal :: Assertion 70 | case_fileLocal = do 71 | let good = minimalAst $ 72 | [ ("resolved" 73 | , Left $ "file:../extensions/jupyterlab-toc-0.6.0.tgz#393fe") ] 74 | goodNoIntegrity = minimalAst $ 75 | [ ("resolved" 76 | , Left $ "file:../extensions/jupyterlab-toc-0.6.0.tgz") ] 77 | astToPackageSuccess good 78 | <&> T.remote >>= \case 79 | T.FileLocal{..} -> do 80 | assertEqual "file path" "../extensions/jupyterlab-toc-0.6.0.tgz" fileLocalPath 81 | assertEqual "file sha" "393fe" fileLocalSha1 82 | a -> assertFailure ("should be FileLocal, is " <> show a) 83 | astToPackageSuccess goodNoIntegrity 84 | <&> T.remote >>= \case 85 | T.FileLocalNoIntegrity{..} -> do 86 | assertEqual "file path" "../extensions/jupyterlab-toc-0.6.0.tgz" fileLocalNoIntegrityPath 87 | a -> assertFailure ("should be FileLocal, is " <> show a) 88 | 89 | case_missingField :: Assertion 90 | case_missingField = do 91 | astToPackageFailureWith 92 | (File.MissingField "version" 93 | NE.:| [File.UnknownRemoteType]) $ emptyAst [] 94 | 95 | astToPackageSuccess :: Parse.PackageFields -> IO T.Package 96 | astToPackageSuccess ast = case File.astToPackage ast of 97 | (Left errs) -> do 98 | _ <- assertFailure ("should have succeded, but:\n" <> show errs) 99 | error "not reached" 100 | (Right pkg) -> pure pkg 101 | 102 | astToPackageFailureWith :: (NE.NonEmpty File.ConversionError) 103 | -> Parse.PackageFields -> IO () 104 | astToPackageFailureWith errs ast = case File.astToPackage ast of 105 | (Right _) -> assertFailure "should have failed" 106 | (Left actual) -> assertEqual "errors should be the same" errs actual 107 | 108 | --TODO 109 | {- 110 | data Keys = Keys { a, b, c, y, z :: PackageKey } 111 | keys :: Keys 112 | keys = Keys (pk "a") (pk "b") (pk "c") (pk "y") (pk "z") 113 | where pk n = PackageKey n "0.1" 114 | 115 | data LFs = LFs 116 | { lfNormal, lfEmpty, lfCycle, lfDecycled 117 | , lfComplex, lfComplexD :: Lockfile } 118 | -- | Example lockfiles for tests. 119 | -- These are put into scope in tests by use of @NamedFieldPuns@. 120 | lfs :: LFs 121 | lfs = LFs 122 | { lfNormal = (tlf [pkg' a [b, c], pkg' b [c], pkg' c []]) 123 | , lfEmpty = (tlf []) 124 | , lfCycle = (tlf [pkg' a [b, c], pkg' b [a, c], pkg' c [c]]) 125 | , lfDecycled = (tlf [pkg' a [b, c], pkg' b [ c], pkg' c [ ]]) 126 | , lfComplex = (tlf [pkg [a, z] [a, c], pkg [c, y] [c, a, z]]) 127 | -- Hm, this test is implementation dependent. But the cycles get removed. 128 | , lfComplexD = (tlf [pkg [a, z] [ ], pkg [c, y] [ z]]) 129 | } 130 | where pkg keys_ deps = (keys_, Package "0.1" (RemoteFile "" "") deps []) 131 | pkg' key = pkg [key] 132 | tlf = packageListToLockfile 133 | Keys{a,b,c,y,z} = keys 134 | 135 | -- | Test for the 'decycle' method. 136 | case_decycle :: Assertion 137 | case_decycle = do 138 | -- print lfCycle 139 | lfDecycled @=? (decycle lfCycle) 140 | lfComplexD @=? (decycle lfComplex) 141 | where LFs{lfCycle, lfDecycled, lfComplex, lfComplexD} = lfs 142 | 143 | type PkgMap = Map PackageKey Package 144 | 145 | -- | A lockfile is basically a flat version of a recursive dependency structure. 146 | -- 'Built' resembles the recursive version of said flat structure. 147 | data Built = Built PackageKey [Built] deriving (Eq) 148 | instance Show Built where 149 | show (Built k b) = show $ printBuild b 150 | where printBuild b' = Pr.list 151 | [Pr.tupled [Pr.text . toS $ name k, printBuild b']] 152 | 153 | buildFromMap :: PkgMap -> [Built] 154 | buildFromMap m = map go $ M.keys m 155 | where 156 | go :: PackageKey -> Built 157 | go pk = Built pk $ map go (dependencies $ m M.! pk) 158 | 159 | -- | Checks if the flat lockfile builds a correct recursive structure. 160 | case_built :: Assertion 161 | case_built = do 162 | let 163 | LFs{lfNormal} = lfs 164 | Keys{a,b,c} = keys 165 | bl = Built 166 | ble p = Built p [] 167 | buildFromMap (flattenKeys lfNormal) 168 | @?= [ bl a [bl b [ble c], ble c] 169 | , bl b [ble c] 170 | , ble c] 171 | 172 | 173 | 174 | -} 175 | 176 | tests :: TestTree 177 | tests = $(testGroupGenerator) 178 | 179 | 180 | orEmpty :: Alternative f => Bool -> a -> f a 181 | orEmpty b a = if b then pure a else empty 182 | -------------------------------------------------------------------------------- /yarn2nix/NodePackageTool.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE RecordWildCards, NoImplicitPrelude, LambdaCase, FlexibleContexts, NamedFieldPuns, OverloadedStrings #-} 2 | import Protolude 3 | import qualified Data.Text as T 4 | import qualified Data.Text.IO as TIO 5 | import qualified Control.Monad.Except as ExcT 6 | import qualified Control.Exception as Exc 7 | import qualified System.IO.Error as IOErr 8 | import qualified Data.ByteString.Lazy as BL 9 | import qualified System.FilePath as FP 10 | import qualified System.Directory as Dir 11 | import qualified System.Posix.Files as PosixFiles 12 | import Options.Applicative 13 | 14 | import qualified Distribution.Nodejs.Package as NP 15 | import qualified Data.Aeson.KeyMap as KeyMap 16 | import qualified Data.Aeson.Key as Key 17 | 18 | data Args 19 | = Args 20 | { argsPackageDir :: FilePath 21 | , argsMode :: Mode } 22 | 23 | data Mode 24 | = LinkBin 25 | { linkBinTargetDir :: FilePath } 26 | | SetBinExecFlag 27 | 28 | args :: Parser Args 29 | args = subparser 30 | ( command "link-bin" 31 | (info (modeCommands linkBinSubcommands) 32 | (progDesc "link package dependecies’ bin files")) 33 | <> command "set-bin-exec-flag" 34 | (info (modeCommands setBinExecFlagSubcommands) 35 | (progDesc "make all bin scripts executable")) ) 36 | where 37 | modeCommands modeSubcommands = Args 38 | <$> strOption 39 | ( long "package" 40 | <> metavar "FOLDER" 41 | <> help "folder with the node package" ) 42 | <*> modeSubcommands 43 | linkBinSubcommands = LinkBin 44 | <$> strOption 45 | ( long "to" 46 | <> metavar "LINK_TARGET" 47 | <> help "folder to link to (absolute or relative from package folder)" ) 48 | setBinExecFlagSubcommands = pure SetBinExecFlag 49 | 50 | 51 | type ErrorLogger = ExceptT [Char] IO 52 | 53 | -- | Print a warning to stdout. 54 | warn :: [Text] -> ErrorLogger () 55 | warn ws = for_ ws $ \w -> liftIO $ TIO.hPutStrLn stderr ("Warning: " <> w) 56 | 57 | -- | On Exception rethrow with annotation. 58 | tryIOMsg :: ([Char] -> [Char]) -> IO a -> ErrorLogger a 59 | tryIOMsg errAnn = ExcT.withExceptT (errAnn . Exc.displayException) . tryIO 60 | 61 | 62 | main :: IO () 63 | main = execParser (info (args <**> helper) 64 | (progDesc "Tool for various node package maintenance tasks")) 65 | >>= realMain 66 | 67 | realMain :: Args -> IO () 68 | realMain Args{..} = do 69 | -- basic sanity checks 70 | let packageJsonPath = argsPackageDir FP. "package.json" 71 | unlessM (Dir.doesFileExist packageJsonPath) 72 | $ die $ toS $ packageJsonPath <> " does not exist." 73 | case argsMode of 74 | LinkBin{..} -> do 75 | unlessM (Dir.doesDirectoryExist linkBinTargetDir) 76 | $ die $ toS $ linkBinTargetDir <> " is not a directory." 77 | _ -> pass 78 | 79 | -- parse & decode & run logic 80 | runExceptT 81 | (tryRead packageJsonPath >>= tryDecode packageJsonPath >>= go) 82 | >>= \case 83 | (Left err) -> die $ toS $ "ERROR: " <> err 84 | (Right _) -> pass 85 | 86 | where 87 | tryRead :: FilePath -> ErrorLogger BL.ByteString 88 | tryRead fp = tryIOMsg exc $ BL.readFile fp 89 | where exc e = fp <> " cannot be read:\n" <> e 90 | tryDecode :: FilePath -> BL.ByteString -> ExceptT [Char] IO NP.Package 91 | tryDecode fp fileBs = do 92 | (pkg, warnings) <- NP.unLoggingPackage 93 | <$> ExcT.ExceptT (pure $ first exc $ NP.decode fileBs) 94 | warn $ fmap ((\w -> toS fp <> ": " <> w) . NP.formatWarning) warnings 95 | pure pkg 96 | where exc e = fp <> " cannot be decoded\n" <> toS e 97 | 98 | tryAccess :: IO a -> IO (Maybe a) 99 | tryAccess io = 100 | hush <$> tryJust 101 | (\e -> guard (IOErr.isDoesNotExistError e || 102 | IOErr.isPermissionError e)) 103 | io 104 | 105 | qte s = "\"" <> s <> "\"" 106 | 107 | go :: NP.Package -> ErrorLogger () 108 | go NP.Package{bin} = do 109 | binFiles <- readBinFiles bin 110 | for_ binFiles $ case argsMode of 111 | -- Link all dependency binaries to their target folder 112 | LinkBin{..} -> linkBin linkBinTargetDir 113 | -- Set the executable flag of all package binaries 114 | SetBinExecFlag -> setBinExecFlag . snd 115 | 116 | -- | Read the binary files and return their names & paths. 117 | readBinFiles :: NP.Bin -> ErrorLogger [(Text, FilePath)] 118 | readBinFiles bin = case bin of 119 | -- files with names how they should be linked 120 | (NP.BinFiles bs) -> pure $ (bs & KeyMap.toList <&> first Key.toText) 121 | -- a whole folder where everything should be linked 122 | (NP.BinFolder bf) -> do 123 | dirM <- liftIO $ tryAccess (Dir.listDirectory bf) 124 | case dirM of 125 | Nothing -> do 126 | warn ["Binary folder " <> toS (qte bf) <> " could not be accessed."] 127 | pure [] 128 | (Just dir) -> 129 | pure $ fmap (\f -> (toS f, bf FP. f)) dir 130 | 131 | -- | Canonicalize the path. 132 | canon :: FilePath -> ErrorLogger FilePath 133 | canon fp = tryIOMsg 134 | (\e -> "Couldn’t canonicalize path " <> qte fp <> ": " <> e) 135 | (Dir.canonicalizePath fp) 136 | 137 | -- | Canonicalize relative to our package 138 | -- and ensure that the relative path is not outside. 139 | canonPkg :: FilePath -> ErrorLogger FilePath 140 | canonPkg relPath = do 141 | pkgDir <- canon argsPackageDir 142 | resPath <- canon $ argsPackageDir FP. relPath 143 | when (not $ pkgDir `isPrefixOf` resPath) 144 | $ throwError $ mconcat 145 | [ "The link to executable file " 146 | , qte relPath 147 | , " lies outside of the package folder!\n" 148 | , "That’s a security risk, aborting." ] 149 | pure resPath 150 | 151 | -- | Link a binary file to @targetDir/name@. 152 | -- @relBinPath@ is relative from the package dir. 153 | linkBin :: FilePath -> (Text, FilePath) -> ErrorLogger () 154 | linkBin targetDir_ (name_, relBinPath) = do 155 | binPath <- canonPkg relBinPath 156 | (name, targetDir) <- traverse canon $ 157 | symlinkTarget name_ targetDir_ 158 | tryIOMsg (\e -> "Directory could not be created: " <> e) $ 159 | Dir.createDirectoryIfMissing False targetDir 160 | tryIOMsg (\e -> "Symlink could not be created: " <> e) $ 161 | PosixFiles.createSymbolicLink binPath $ targetDir FP. toS name 162 | 163 | -- | Given a name and a target directory, return 164 | -- the basename and the target (sub) directory 165 | -- of the target file 166 | symlinkTarget :: Text -> FilePath -> (FilePath, FilePath) 167 | symlinkTarget name targetDir = 168 | if "/" `T.isInfixOf` name 169 | then (FP.takeFileName name', targetDir FP. FP.takeDirectory name') 170 | else (name', targetDir) 171 | where name' = T.unpack name 172 | 173 | -- | Set executable flag of the file. 174 | setBinExecFlag :: FilePath -> ErrorLogger () 175 | setBinExecFlag file_ = do 176 | file <- canonPkg file_ 177 | res <- liftIO $ tryAccess $ do 178 | perm <- Dir.getPermissions file 179 | Dir.setPermissions file 180 | $ Dir.setOwnerExecutable True perm 181 | case res of 182 | Nothing -> 183 | warn ["Cannot set executable bit on " <> toS file] 184 | Just () -> pass 185 | -------------------------------------------------------------------------------- /yarn2nix/src/Distribution/Nixpkgs/Nodejs/Cli.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, LambdaCase, NoImplicitPrelude #-} 2 | {-# LANGUAGE TypeApplications #-} 3 | {-| 4 | Description: command line interface 5 | -} 6 | module Distribution.Nixpkgs.Nodejs.Cli 7 | ( cli 8 | , parseArgsPure 9 | ) 10 | where 11 | 12 | import Protolude 13 | import qualified Data.ByteString.Lazy as BL 14 | import qualified Data.Text as T 15 | import qualified Data.Text.IO as TIO 16 | import qualified Options.Applicative as O 17 | import qualified Options.Applicative.Help.Pretty as O (linebreak) 18 | import qualified System.Directory as Dir 19 | import System.Environment (getProgName) 20 | 21 | import qualified Nix.Pretty as NixP 22 | import qualified Prettyprinter.Render.Text as RT 23 | import qualified Yarn.Lock as YL 24 | import qualified Yarn.Lock.Types as YLT 25 | import qualified Yarn.Lock.Helpers as YLH 26 | 27 | import qualified Distribution.Nixpkgs.Nodejs.OptimizedNixOutput as NixOut 28 | import qualified Distribution.Nixpkgs.Nodejs.FromPackage as NodeFP 29 | import qualified Distribution.Nixpkgs.Nodejs.ResolveLockfile as Res 30 | import qualified Distribution.Nodejs.Package as NP 31 | import Distribution.Nixpkgs.Nodejs.ResolveLockfile (ResolverConfig(ResolverConfig, resolveOffline)) 32 | import Distribution.Nixpkgs.Nodejs.License (LicensesBySpdxId) 33 | import qualified Data.Aeson as Json 34 | 35 | 36 | description :: O.InfoMod a 37 | description = O.fullDesc 38 | <> O.progDescDoc (Just $ mconcat $ intersperse O.linebreak 39 | [ "yarn2nix has two modes:" 40 | <> O.linebreak 41 | , "In its default mode (started without --template) it parses a given yarn.lock file" 42 | , "and prints a nix expressions representing it to stdout." 43 | <> O.linebreak 44 | , "If --template is given, it processes a given package.json" 45 | , "and prints a template nix expression for an equivalent nix package." 46 | <> O.linebreak 47 | , "In both modes yarn2nix will take the file as an argument" 48 | , "or read it from stdin if it is missing." 49 | ]) 50 | 51 | -- | Main entry point for @yarn2nix@. 52 | cli :: IO () 53 | cli = parseArgs >>= runAction 54 | 55 | -- | Type of action @yarn2nix@ is performing. 56 | data RunMode 57 | = YarnLock -- ^ Output a nix expression for a @yarn.lock@ 58 | | NodeTemplate -- ^ Output a nix template corresponding to a @package.json@ 59 | deriving (Show, Eq) 60 | 61 | -- | Runtime configuration of @yarn2nix@. Typically this is determined from 62 | -- its command line arguments and valid for the current invocation only. 63 | data RunConfig 64 | = RunConfig 65 | { runMode :: RunMode 66 | , runOffline :: Bool -- ^ If @True@, @yarn2nix@ will fail if it 67 | -- requires network access. Currently this means 68 | -- 'Distribution.Nixpkgs.Nodejs.ResolveLockfile.resolveLockfileStatus' 69 | -- will throw an error in case resolving a hash 70 | -- requires network access. 71 | , runLicensesJson :: Maybe FilePath -- ^ Optional Path to a licenses.json file 72 | -- equivalent to the lib.licenses set from 73 | -- @nixpkgs@. 74 | , runInputFile :: Maybe FilePath -- ^ File to process. If missing the appropriate 75 | -- file for the current mode from the current 76 | -- working directory is used. 77 | } deriving (Show, Eq) 78 | 79 | 80 | fileFor :: RunConfig -> Text 81 | fileFor cfg = 82 | case runMode cfg of 83 | YarnLock -> "yarn.lock" 84 | NodeTemplate -> "package.json" 85 | 86 | parseArgs :: IO RunConfig 87 | parseArgs = O.customExecParser optparsePrefs runConfigParserWithHelp 88 | 89 | parseArgsPure :: [Text] -> IO RunConfig 90 | parseArgsPure args = 91 | args 92 | <&> T.unpack 93 | & O.execParserPure optparsePrefs runConfigParserWithHelp 94 | & O.handleParseResult 95 | 96 | 97 | runAction :: RunConfig -> IO () 98 | runAction cfg = do 99 | file <- fileForConfig 100 | case runMode cfg of 101 | YarnLock -> parseYarn file 102 | NodeTemplate -> parseNode file 103 | where 104 | fileForConfig :: IO FilePath 105 | fileForConfig = 106 | case runInputFile cfg of 107 | Just f -> pure f 108 | Nothing -> Dir.getCurrentDirectory >>= \d -> 109 | Dir.findFile [d] (toS $ fileFor cfg) >>= \case 110 | Nothing -> dieWithUsage 111 | $ "No " <> fileFor cfg <> " found in current directory" 112 | Just path -> pure path 113 | parseYarn :: FilePath -> IO () 114 | parseYarn path = do 115 | fc <- catchCouldNotOpen path $ readFile path 116 | case YL.parse path fc of 117 | Right yarnfile -> toStdout cfg yarnfile 118 | Left err -> die' ("Could not parse " <> toS path <> ":\n" <> YL.prettyLockfileError err) 119 | 120 | parseNode :: FilePath -> IO () 121 | parseNode path = do 122 | BL.readFile path >>= (\case 123 | Right (NP.LoggingPackage (nodeModule, warnings)) -> do 124 | for_ warnings $ TIO.hPutStrLn stderr . NP.formatWarning 125 | 126 | licenseSet <- case cfg & runLicensesJson of 127 | Nothing -> pure Nothing 128 | Just licensesJson -> do 129 | catchCouldNotOpen licensesJson 130 | (BL.readFile licensesJson) 131 | <&> Json.decode @LicensesBySpdxId 132 | 133 | print $ NixP.prettyNix $ NodeFP.genTemplate licenseSet nodeModule 134 | Left err -> die' ("Could not parse " <> toS path <> ":\n" <> show err)) . NP.decode 135 | catchCouldNotOpen :: FilePath -> IO a -> IO a 136 | catchCouldNotOpen path action = action `catch` \e -> 137 | dieWithUsage $ "Could not open " <> toS path <> ":\n" <> show (e :: IOException) 138 | 139 | -- get rid of odd linebreaks by increasing width enough 140 | optparsePrefs :: O.ParserPrefs 141 | optparsePrefs = O.defaultPrefs { O.prefColumns = 100 } 142 | 143 | -- If --template is given, run in NodeTemplate mode, 144 | -- otherwise the default mode YarnLock is used. 145 | runModeParser :: O.Parser RunMode 146 | runModeParser = O.flag YarnLock NodeTemplate $ 147 | O.long "template" 148 | <> O.help "Output a nix package template for a given package.json" 149 | 150 | runConfigParser :: O.Parser RunConfig 151 | runConfigParser = RunConfig 152 | <$> runModeParser 153 | <*> O.switch 154 | (O.long "offline" 155 | <> O.help "Makes yarn2nix fail if network access is required") 156 | <*> O.optional (O.option O.str 157 | (O.long "license-data" 158 | <> O.metavar "FILE" 159 | <> O.help "Path to a license.json equivalent to nixpkgs.lib.licenses" 160 | -- only really interesting for wrapping at build 161 | <> O.internal)) 162 | <*> O.optional (O.argument O.str (O.metavar "FILE")) 163 | 164 | runConfigParserWithHelp :: O.ParserInfo RunConfig 165 | runConfigParserWithHelp = 166 | O.info (runConfigParser <**> O.helper) description 167 | 168 | die' :: Text -> IO a 169 | die' err = putErrText err *> exitFailure 170 | 171 | dieWithUsage :: Text -> IO a 172 | dieWithUsage err = do 173 | putErrText (err <> "\n") 174 | progn <- getProgName 175 | hPutStr stderr 176 | . fst . flip O.renderFailure progn 177 | $ O.parserFailure optparsePrefs 178 | runConfigParserWithHelp (O.ShowHelpText Nothing) mempty 179 | exitFailure 180 | 181 | -- TODO refactor 182 | toStdout :: RunConfig -> YLT.Lockfile -> IO () 183 | toStdout cfg lf = do 184 | ch <- newChan 185 | -- thrd <- forkIO $ forever $ do 186 | -- readChan ch >>= \case 187 | -- FileRemote{..} -> pass 188 | -- GitRemote{..} -> print $ "Downloading " <> gitRepoUrl 189 | let resolverConfig = ResolverConfig { 190 | resolveOffline = cfg & runOffline 191 | } 192 | lf' <- Res.resolveLockfileStatus resolverConfig ch (YLH.decycle lf) >>= \case 193 | Left err -> die' (T.intercalate "\n" $ toList err) 194 | Right res -> pure res 195 | -- killThread thrd 196 | RT.putDoc $ NixP.prettyNix $ NixOut.mkPackageSet $ NixOut.convertLockfile lf' 197 | -------------------------------------------------------------------------------- /yarn-lock/src/Yarn/Lock/File.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Yarn.Lock.File 3 | Description : Convert AST to semantic data structures 4 | Maintainer : Profpatsch 5 | Stability : experimental 6 | 7 | After parsing yarn.lock files in 'Yarn.Lock.Parse', 8 | you want to convert the AST to something with more information 9 | and ultimately get a 'T.Lockfile'. 10 | 11 | @yarn.lock@ files don’t follow a structured approach 12 | (like for example sum types), so information like e.g. 13 | the remote type have to be inferred frome AST values. 14 | -} 15 | {-# LANGUAGE OverloadedStrings, ApplicativeDo, RecordWildCards, NamedFieldPuns #-} 16 | {-# LANGUAGE LambdaCase #-} 17 | module Yarn.Lock.File 18 | ( fromPackages 19 | , astToPackage 20 | -- * Errors 21 | , ConversionError(..) 22 | ) where 23 | 24 | import qualified Data.List.NonEmpty as NE 25 | import qualified Data.Map.Strict as M 26 | import qualified Data.Text as Text 27 | import qualified Data.Either.Validation as V 28 | 29 | import qualified Yarn.Lock.Parse as Parse 30 | import qualified Yarn.Lock.Types as T 31 | import qualified Data.MultiKeyedMap as MKM 32 | import Data.Text (Text) 33 | import Data.Bifunctor (first) 34 | import Control.Monad ((>=>)) 35 | import Control.Applicative ((<|>)) 36 | import Data.Either.Validation (Validation(Success, Failure)) 37 | import Data.Traversable (for) 38 | 39 | -- | Press a list of packages into the lockfile structure. 40 | -- 41 | -- It’s a dumb conversion, you should probably apply 42 | -- the 'Yarn.Lock.Helpers.decycle' function afterwards. 43 | fromPackages :: [T.Keyed T.Package] -> T.Lockfile 44 | fromPackages = MKM.fromList T.lockfileIkProxy 45 | . fmap (\(T.Keyed ks p) -> (ks, p)) 46 | 47 | -- | Possible errors when converting from AST. 48 | data ConversionError 49 | = MissingField Text 50 | -- ^ field is missing 51 | | WrongType { fieldName :: Text, fieldType :: Text } 52 | -- ^ this field has the wrong type 53 | | UnknownRemoteType 54 | -- ^ the remote (e.g. git, tar archive) could not be determined 55 | deriving (Show, Eq) 56 | 57 | -- | Something that can parse the value of a field into type @a@. 58 | data FieldParser a = FieldParser 59 | { parseField :: Either Text Parse.PackageFields -> Maybe a 60 | -- ^ the parsing function (Left is a simple field, Right a nested one) 61 | , parserName :: Text 62 | -- ^ name of this parser (for type errors) 63 | } 64 | 65 | type Val = V.Validation (NE.NonEmpty ConversionError) 66 | 67 | -- | Parse an AST 'PackageFields' to a 'T.Package', which has 68 | -- the needed fields resolved. 69 | astToPackage :: Parse.PackageFields 70 | -> Either (NE.NonEmpty ConversionError) T.Package 71 | astToPackage = V.validationToEither . validate 72 | where 73 | validate :: Parse.PackageFields -> Val T.Package 74 | validate fs = do 75 | version <- getField text "version" fs 76 | remote <- checkRemote fs 77 | dependencies <- getFieldOpt keylist "dependencies" fs 78 | optionalDependencies <- getFieldOpt keylist "optionalDependencies" fs 79 | pure $ T.Package{..} 80 | 81 | -- | Parse a field from a 'PackageFields'. 82 | getField :: FieldParser a -> Text -> Parse.PackageFields -> Val a 83 | getField = getFieldImpl Nothing 84 | -- | Parse an optional field and insert the empty monoid value 85 | getFieldOpt :: Monoid a => FieldParser a -> Text -> Parse.PackageFields -> Val a 86 | getFieldOpt = getFieldImpl (Just mempty) 87 | 88 | getFieldImpl :: Maybe a -> FieldParser a -> Text -> Parse.PackageFields -> Val a 89 | getFieldImpl mopt typeParser fieldName (Parse.PackageFields m)= 90 | first pure $ V.eitherToValidation $ do 91 | case M.lookup fieldName m of 92 | Nothing -> case mopt of 93 | Just opt -> Right opt 94 | Nothing -> Left $ MissingField fieldName 95 | Just val -> 96 | case parseField typeParser val of 97 | Nothing -> Left 98 | (WrongType { fieldName, fieldType = parserName typeParser }) 99 | Just a -> Right a 100 | 101 | -- | Parse a simple field to type 'Text'. 102 | text :: FieldParser Text 103 | text = FieldParser { parseField = either Just (const Nothing) 104 | , parserName = "text" } 105 | 106 | packageKey :: FieldParser T.PackageKeyName 107 | packageKey = FieldParser 108 | { parseField = parseField text >=> T.parsePackageKeyName 109 | , parserName = "package key" } 110 | 111 | -- | Parse a field nested one level to a list of 'PackageKey's. 112 | keylist :: FieldParser [T.PackageKey] 113 | keylist = FieldParser 114 | { parserName = "list of package keys" 115 | , parseField = either (const Nothing) 116 | (\(Parse.PackageFields inner) -> 117 | for (M.toList inner) $ \(k, v) -> do 118 | name <- parseField packageKey (Left k) 119 | npmVersionSpec <- parseField text v 120 | pure $ T.PackageKey { name, npmVersionSpec }) } 121 | 122 | -- | Appling heuristics to the field contents to find the 123 | -- correct remote type. 124 | checkRemote :: Parse.PackageFields -> Val T.Remote 125 | checkRemote fs = 126 | -- any error is replaced by the generic remote error 127 | mToV (pure UnknownRemoteType) 128 | -- implementing the heuristics of searching for types; 129 | -- it should of course not lead to false positives 130 | -- see tests/TestLock.hs 131 | $ checkGit <|> checkFileLocal <|> checkFile 132 | where 133 | mToV :: e -> Maybe a -> V.Validation e a 134 | mToV err mb = case mb of 135 | Nothing -> Failure err 136 | Just a -> Success a 137 | vToM :: Val a -> Maybe a 138 | vToM = \case 139 | Success a -> Just a 140 | Failure _err -> Nothing 141 | 142 | -- | "https://blafoo.com/a/b#alonghash" 143 | -- -> ("https://blafoo.com/a/b", "alonghash") 144 | -- we assume the # can only occur exactly once 145 | findUrlHash :: Text -> (Text, Maybe Text) 146 | findUrlHash url = case Text.splitOn "#" url of 147 | [url'] -> (url', Nothing) 148 | [url', ""] -> (url', Nothing) 149 | [url', hash] -> (url', Just hash) 150 | _ -> error "checkRemote: # should only appear exactly once!" 151 | 152 | checkGit :: Maybe T.Remote 153 | checkGit = do 154 | resolved <- vToM $ getField text "resolved" fs 155 | -- either in uid field or after the hash in the “resolved” URL 156 | (repo, gitRev) <- do 157 | let (repo', mayHash) = findUrlHash resolved 158 | hash <- vToM (getField text "uid" fs) 159 | <|> if any (`Text.isPrefixOf` resolved) ["git+", "git://"] 160 | then mayHash else Nothing 161 | pure (repo', hash) 162 | pure $ T.GitRemote 163 | { T.gitRepoUrl = noPrefix "git+" repo , .. } 164 | 165 | -- | resolved fields that are prefixed with @"file:"@ 166 | checkFileLocal :: Maybe T.Remote 167 | checkFileLocal = do 168 | resolved <- vToM $ getField text "resolved" fs 169 | let (file, mayHash) = findUrlHash resolved 170 | fileLocalPath <- if "file:" `Text.isPrefixOf` file 171 | then Just $ noPrefix "file:" file 172 | else Nothing 173 | case mayHash of 174 | Just hash -> pure (T.FileLocal fileLocalPath hash) 175 | Nothing -> pure (T.FileLocalNoIntegrity fileLocalPath) 176 | 177 | checkFile :: Maybe T.Remote 178 | checkFile = do 179 | resolved <- vToM (getField text "resolved" fs) 180 | let (fileUrl, mayHash) = findUrlHash resolved 181 | case mayHash of 182 | Just hash -> pure (T.FileRemote fileUrl hash) 183 | Nothing -> pure (T.FileRemoteNoIntegrity fileUrl) 184 | 185 | -- | ensure the prefix is removed 186 | noPrefix :: Text -> Text -> Text 187 | noPrefix pref hay = case Text.stripPrefix pref hay of 188 | Nothing -> hay 189 | Just t -> t 190 | -------------------------------------------------------------------------------- /yarn2nix/src/Distribution/Nodejs/Package.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NoImplicitPrelude, OverloadedStrings, RecordWildCards, LambdaCase, TypeApplications #-} 2 | {-| 3 | Description: Parse and make sense of npm’s @package.json@ project files 4 | 5 | They are documented on https://docs.npmjs.com/files/package.json and have a few gotchas. Luckily plain JSON, but the interpretation of certain fields is non-trivial (since they contain a lot of “sugar”). 6 | -} 7 | module Distribution.Nodejs.Package 8 | ( -- * Parsing @package.json@ 9 | LoggingPackage(..), decode 10 | , Warning(..), formatWarning 11 | -- * @package.json@ data 12 | , Package(..) 13 | , Bin(..), Man(..), Dependencies 14 | , parsePackageKeyName 15 | ) where 16 | 17 | import Protolude hiding (packageName) 18 | import Control.Monad (fail) 19 | import qualified Control.Monad.Writer.Lazy as WL 20 | import qualified Data.ByteString.Lazy as BL 21 | import qualified Data.Text as T 22 | import qualified System.FilePath as FP 23 | 24 | import Data.Aeson ((.:), (.:?), (.!=), Key) 25 | import qualified Data.Aeson as A 26 | import qualified Data.Aeson.Types as AT 27 | import qualified Yarn.Lock.Types as YLT 28 | import qualified Data.Aeson.Key as Key 29 | import Data.Aeson.KeyMap (KeyMap) 30 | import qualified Data.Aeson.KeyMap as KeyMap 31 | 32 | -- | npm `package.json`. Not complete. 33 | -- 34 | -- See https://docs.npmjs.com/files/package.json 35 | data Package = Package 36 | { name :: Text 37 | , version :: Text 38 | , description :: Maybe Text 39 | , homepage :: Maybe Text 40 | , private :: Bool 41 | , scripts :: KeyMap Text 42 | , bin :: Bin 43 | , man :: Man 44 | , license :: Maybe Text 45 | , dependencies :: Dependencies 46 | , devDependencies :: Dependencies 47 | } deriving (Show, Eq) 48 | 49 | -- | 'Package' with a potential bunch of parsing warnings. 50 | -- Note the 'A.FromJson' instance. 51 | newtype LoggingPackage = LoggingPackage 52 | { unLoggingPackage :: (Package, [Warning]) } 53 | 54 | -- | Possible warnings from parsing. 55 | data Warning 56 | = WrongType 57 | { wrongTypeField :: Text -- ^ the field which has a wrong type 58 | , wrongTypeDefault :: Maybe Text -- ^ the default value, if used 59 | } 60 | | PlainWarning Text 61 | 62 | -- | The package’s executable files. 63 | data Bin 64 | = BinFiles (KeyMap FilePath) 65 | -- ^ map of files from name to their file path (relative to package path) 66 | | BinFolder FilePath 67 | -- ^ a folder containing all executable files of the project (also relative) 68 | deriving (Show, Eq) 69 | 70 | -- | The package’s manual files. 71 | data Man 72 | = ManFiles (KeyMap FilePath) 73 | -- ^ map of files from name to their file path (relative to package path) 74 | deriving (Show, Eq) 75 | 76 | -- | Dependencies of a package. 77 | type Dependencies = KeyMap Text 78 | 79 | type Warn = WL.WriterT [Warning] AT.Parser 80 | putWarning :: a -> Warning -> Warn a 81 | putWarning a w = WL.writer (a, [w]) 82 | 83 | -- | See https://github.com/npm/normalize-package-data for 84 | -- normalization steps used by npm itself. 85 | instance A.FromJSON LoggingPackage where 86 | parseJSON = A.withObject "Package" $ \v -> fmap LoggingPackage . WL.runWriterT $ do 87 | let 88 | l :: AT.Parser a -> Warn a 89 | l = WL.WriterT . fmap (\a -> (a, [])) 90 | tryWarn :: (AT.FromJSON a, Show a) 91 | => AT.Key -> a -> Warn a 92 | tryWarn field def = 93 | lift (v .:? field .!= def) 94 | <|> putWarning def (WrongType { wrongTypeField = field & Key.toText 95 | , wrongTypeDefault = Just (show def) }) 96 | name <- l $ v .: "name" 97 | version <- l $ v .: "version" 98 | description <- tryWarn "description" Nothing 99 | homepage <- tryWarn "homepage" Nothing 100 | private <- tryWarn "private" False 101 | scripts <- (parseMapText "scripts" =<< (tryWarn "scripts" mempty)) 102 | bin <- parseBin name v 103 | man <- l $ parseMan name v 104 | license <- tryWarn "license" Nothing 105 | dependencies <- tryWarn "dependencies" (AT.Object mempty) 106 | >>= parseDependencies "dependencies" 107 | devDependencies <- tryWarn "devDependencies" (AT.Object mempty) 108 | >>= parseDependencies "devDependencies" 109 | pure Package{..} 110 | where 111 | 112 | parseDependencies :: Text -> AT.Value -> Warn Dependencies 113 | parseDependencies field v = 114 | let 115 | warn = putWarning mempty 116 | $ WrongType 117 | { wrongTypeField = field 118 | , wrongTypeDefault = Just (show (mempty :: Dependencies)) } 119 | in case v of 120 | AT.Array a -> 121 | -- we interpret empty arrays as just confused users 122 | if null a then warn 123 | -- however if the user uses a non-empty array, 124 | -- they probably mean something which we don’t know how to deal with. 125 | else fail 126 | $ "\"" ++ T.unpack field ++ "\" is a non empty array instead of a JSON object" 127 | -- if we get an object here, it's malformed 128 | AT.Object deps -> lift $ traverse (A.parseJSON @Text) deps 129 | -- everything else defaults to mempty and generates a warning 130 | _ -> warn 131 | 132 | parseMapText :: Text -> KeyMap AT.Value 133 | -> Warn (KeyMap Text) 134 | parseMapText fieldPath val = 135 | KeyMap.mapMaybe identity <$> KeyMap.traverseWithKey tryParse val 136 | where 137 | tryParse :: A.Key -> A.Value -> Warn (Maybe Text) 138 | tryParse key el = lift (Just <$> AT.parseJSON el) 139 | <|> putWarning Nothing 140 | (WrongType { wrongTypeField = fieldPath <> "." <> (key & Key.toText) 141 | , wrongTypeDefault = Nothing }) 142 | parseBin :: Text -> AT.Object -> Warn Bin 143 | parseBin packageName v = do 144 | -- check for existence of these fields 145 | binVal <- lift $ optional $ v .: "bin" 146 | dirBinVal <- lift $ optional $ v .: "directories" >>= (.: "bin") 147 | -- now check for all possible cases of the fields 148 | -- see npm documentation for more 149 | case (binVal, dirBinVal) of 150 | (Just _ , Just _) -> 151 | putWarning (BinFiles mempty) $ PlainWarning 152 | "`bin` and `directories.bin` must not exist at the same time, skipping." 153 | -- either "bin" is a direct path, then it’s linked to the package name 154 | (Just (A.String path), _) -> pure $ BinFiles 155 | $ KeyMap.singleton (parsePackageName packageName & Key.fromText) (toS path) 156 | -- or it’s a map from names to paths 157 | (Just (A.Object bins), _) -> lift $ BinFiles 158 | <$> traverse (A.withText "BinPath" (pure.toS)) bins 159 | (Just _ , _) -> fail 160 | $ "`bin` must be a path or a map of names to paths." 161 | (_ , Just (A.String path)) -> pure $ BinFolder $ toS path 162 | (_ , Just _) -> fail 163 | $ "`directories.bin` must be a path." 164 | -- if no executables are given, return an empty set 165 | (Nothing , Nothing) -> pure . BinFiles $ mempty 166 | 167 | -- TODO: parsing should be as thorough as with "bin" 168 | parseMan name v = do 169 | let getMan f = ManFiles . f <$> v .: "man" 170 | extractName :: FilePath -> (Key, FilePath) 171 | extractName file = 172 | let f = T.pack $ FP.takeFileName file 173 | in if name `T.isPrefixOf` f 174 | then (Key.fromText name, file) 175 | else (Key.fromText $ name <> "-" <> f, file) 176 | -- TODO: handle directories.man 177 | (getMan (KeyMap.fromList . map extractName) 178 | <|> getMan (KeyMap.fromList . (:[]) . extractName) 179 | <|> pure (ManFiles mempty)) 180 | 181 | -- | Convenience decoding function. 182 | decode :: BL.ByteString -> Either Text LoggingPackage 183 | decode = first toS . A.eitherDecode 184 | 185 | -- | Convert a @package.json@ parsing warning to plain text. 186 | formatWarning :: Warning -> Text 187 | formatWarning = \case 188 | WrongType{..} -> 189 | "Field \"" 190 | <> wrongTypeField 191 | <> "\" has the wrong type. " 192 | <> (case wrongTypeDefault of 193 | Just def -> "Defaulting to " <> def 194 | Nothing -> "Leaving it out") 195 | <> "." 196 | (PlainWarning t) -> t 197 | 198 | -- | Parse a package name string into a 'YLT.PackageKeyName'. 199 | parsePackageKeyName :: Text -> YLT.PackageKeyName 200 | parsePackageKeyName k = 201 | case YLT.parsePackageKeyName k of 202 | -- we don’t crash on a “wrong” package key to keep this 203 | -- code pure, but assume it’s a simple key instead. 204 | Nothing -> (YLT.SimplePackageKey k) 205 | Just pkn -> pkn 206 | 207 | parsePackageName :: Text -> Text 208 | parsePackageName k = 209 | case parsePackageKeyName k of 210 | YLT.SimplePackageKey n -> n 211 | YLT.ScopedPackageKey _ n -> n 212 | -------------------------------------------------------------------------------- /yarn-lock/src/Yarn/Lock/Parse.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE GeneralizedNewtypeDeriving, OverloadedStrings #-} 2 | {-| 3 | Module : Yarn.Lock.Parse 4 | Description : Parser for yarn.lock files 5 | Maintainer : Profpatsch 6 | Stability : experimental 7 | 8 | This module provides a parser for the AST of @yarn.lock@ files. 9 | -} 10 | module Yarn.Lock.Parse 11 | ( PackageFields(..), Package 12 | -- * Parsing 13 | -- ** Re-export 14 | , Parser 15 | -- ** Parsers 16 | , packageList 17 | , packageEntry 18 | -- * Internal Parsers 19 | , field, nestedField, simpleField 20 | , packageKeys 21 | ) where 22 | 23 | import qualified Data.Char as Ch 24 | import qualified Data.List.NonEmpty as NE 25 | import qualified Data.Text as T 26 | import qualified Data.Map.Strict as M 27 | import Control.Monad (void) 28 | import qualified Text.Megaparsec as MP 29 | import qualified Text.Megaparsec.Char as MP 30 | import qualified Text.Megaparsec.Char.Lexer as MPL 31 | import qualified Yarn.Lock.Types as YLT 32 | import Data.Text (Text) 33 | import Data.Void (Void) 34 | import Data.Map.Strict (Map) 35 | import qualified Data.Text as Text 36 | import Text.Megaparsec (Parsec, Stream (Tokens), ()) 37 | import Text.Megaparsec.Pos (SourcePos) 38 | import Control.Applicative ((<|>)) 39 | 40 | 41 | -- | We use a simple (pure) @Megaparsec@ parser. 42 | type Parser = Parsec Void Text 43 | 44 | -- | The @yarn.lock@ format doesn’t specifically include a fixed scheme, 45 | -- it’s just an unnecessary custom version of a list of fields. 46 | -- 47 | -- An field can either be a string or more fields w/ deeper indentation. 48 | -- 49 | -- The actual conversion to semantic structures needs to be done afterwards. 50 | newtype PackageFields = PackageFields (Map Text (Either Text PackageFields)) 51 | deriving (Show, Eq, Semigroup, Monoid) 52 | 53 | -- | A parsed 'Package' AST has one or more keys, a position in the original files 54 | -- and a collection of fields. 55 | type Package = YLT.Keyed (SourcePos, PackageFields) 56 | 57 | 58 | -- | Parse a complete yarn.lock into an abstract syntax tree, 59 | -- keeping the source positions of each package entry. 60 | packageList :: Parser [Package] 61 | packageList = MP.many $ (MP.skipMany (comment <|> MP.string "\n")) *> packageEntry 62 | where 63 | comment :: Parser (Tokens Text) 64 | comment = MP.char '#' *> MP.takeWhileP Nothing (/= '\n') 65 | 66 | -- | A single Package. 67 | -- 68 | -- Example: 69 | -- 70 | -- @ 71 | -- handlebars@^4.0.4: 72 | -- version "4.0.6" 73 | -- resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.6.tgz#2ce4484850537f9c97a8026d5399b935c4ed4ed7" 74 | -- dependencies: 75 | -- async "^1.4.0" 76 | -- optimist "^0.6.1" 77 | -- source-map "^0.4.4" 78 | -- optionalDependencies: 79 | -- uglify-js "^2.6" 80 | -- " 81 | -- @ 82 | packageEntry :: Parser (YLT.Keyed (SourcePos, PackageFields)) 83 | packageEntry = MP.label "package entry" $ do 84 | pos <- MP.getSourcePos 85 | -- A package entry is a non-indented 86 | (keys, pkgs) <- nonIndented 87 | -- block that has a header of package keys 88 | -- and an indented part that contains fields 89 | $ indentedFieldsWithHeader packageKeys 90 | pure $ YLT.Keyed keys (pos, pkgs) 91 | 92 | -- | The list of PackageKeys that index the same Package 93 | -- 94 | -- @ 95 | -- align-text@^0.1.1, align-text@^0.1.3:\\n 96 | -- @ 97 | packageKeys :: Parser (NE.NonEmpty YLT.PackageKey) 98 | packageKeys = MP.label "package keys" $ do 99 | firstEls <- MP.many (MP.try $ lexeme $ packageKey ":," <* MP.char ',') 100 | lastEl <- packageKey ":" <* MP.char ':' 101 | pure $ NE.fromList $ firstEls <> [lastEl] 102 | 103 | -- | A packageKey is @\\@\@; 104 | -- 105 | -- If the semver contains spaces, it is also quoted with @"@. 106 | packageKey :: [Char] -> Parser YLT.PackageKey 107 | packageKey separators = inString (pkgKey "\"") 108 | -- if no string delimiters is used we need to check for the separators 109 | -- this file format is shit :< 110 | <|> pkgKey separators 111 | "package key" 112 | where 113 | pkgKey :: [Char] -> Parser YLT.PackageKey 114 | pkgKey valueChars = MP.label "package key" $ do 115 | key <- someTextOf (MP.noneOf valueChars) 116 | -- okay, here’s the rub: 117 | -- `@` is used for separation, but package names can also 118 | -- start with the `@` character (so-called “scoped packages”). 119 | -- Furthermore, versions can contain `@` as well. 120 | -- This file format is a pile of elephant shit. 121 | case breakDrop '@' key of 122 | ("", rest) -> case breakDrop '@' rest of 123 | -- scoped key with empty name 124 | ("", _) -> emptyKeyErr key 125 | -- scoped key ("@scope/package") 126 | (scopedName, ver) -> YLT.PackageKey 127 | <$> scoped (T.cons '@' scopedName) <*> pure ver 128 | -- just a simple key 129 | (name, ver) -> pure $ YLT.PackageKey (YLT.SimplePackageKey name) ver 130 | 131 | emptyKeyErr :: Text -> Parser a 132 | emptyKeyErr key = fail 133 | ("packagekey: package name can not be empty (is: " 134 | <> Text.unpack key <> ")") 135 | 136 | -- | Like 'T.breakOn', but drops the separator char. 137 | breakDrop :: Char -> Text -> (Text, Text) 138 | breakDrop c str = case T.breakOn (T.singleton c) str of 139 | (s, "") -> (s, "") 140 | (s, s') -> (s, T.drop 1 s') 141 | 142 | -- | Parses a (scoped) package key and throws an error if misformatted. 143 | scoped n = maybe 144 | (fail $ "packageKey: scoped variable must be of form @scope/package" 145 | <> " (is: " <> Text.unpack n <> ")") 146 | pure $ YLT.parsePackageKeyName n 147 | 148 | -- | Either a simple or a nested field. 149 | field :: Parser (Text, Either Text PackageFields) 150 | field = MP.try nested <|> simple "field" 151 | where 152 | simple = fmap Left <$> simpleField 153 | nested = fmap Right <$> nestedField 154 | 155 | -- | A key-value pair, separated by space. 156 | -- Key any value may be enclosed in "". 157 | -- Returns key and value. 158 | simpleField :: Parser (Text, Text) 159 | simpleField = (,) <$> lexeme (strSymbolChars <|> symbolChars) 160 | -- valueChars may be in Strings or maybe not >: 161 | -- this file format is absolute garbage 162 | <*> (strValueChars <|> valueChars) 163 | "simple field" 164 | where 165 | valueChars, strValueChars :: Parser Text 166 | valueChars = someTextOf (MP.noneOf ("\n\r\"" :: [Char])) 167 | strSymbolChars = inString $ symbolChars 168 | strValueChars = inString $ valueChars 169 | -- as with packageKey semvers, this can be empty 170 | <|> (pure T.empty "an empty value field") 171 | 172 | -- | Similar to a @simpleField@, but instead of a string 173 | -- we get another block with deeper indentation. 174 | nestedField :: Parser (Text, PackageFields) 175 | nestedField = MP.label "nested field" $ 176 | indentedFieldsWithHeader (symbolChars <* MP.char ':') 177 | 178 | 179 | -- internal parsers 180 | 181 | -- | There are two kinds of indented blocks: 182 | -- One where the header is the package 183 | -- and one where the header is already a package field key. 184 | indentedFieldsWithHeader :: Parser a -> Parser (a, PackageFields) 185 | indentedFieldsWithHeader header = indentBlock $ do 186 | -- … block that has a header of package keys 187 | hdr <- header 188 | -- … and an indented part that contains fields 189 | pure $ MPL.IndentSome Nothing 190 | (\fields -> pure (hdr, toPfs fields)) field 191 | where 192 | toPfs :: [(Text, Either Text PackageFields)] -> PackageFields 193 | toPfs = PackageFields . M.fromList 194 | 195 | -- | Characters allowed in key symbols. 196 | -- 197 | -- TODO: those are partly npm package names, so check the allowed symbols, too. 198 | -- 199 | -- Update: npm doesn’t specify the package name format, at all. 200 | -- Apart from the length. 201 | -- Update: According to https://docs.npmjs.com/misc/scope 202 | -- the package name format is “URL-safe characters, no leading dots or underscores” TODO 203 | symbolChars :: Parser Text 204 | symbolChars = MP.label "key symbol" $ someTextOf $ MP.satisfy 205 | (\c -> Ch.isAscii c && 206 | (Ch.isLower c || Ch.isUpper c || Ch.isNumber c || c `elem` special)) 207 | where special = "-_.@/" :: [Char] 208 | 209 | 210 | -- text versions of parsers & helpers 211 | 212 | someTextOf :: Parser Char -> Parser Text 213 | someTextOf c = T.pack <$> MP.some c 214 | 215 | -- | parse everything as inside a string 216 | inString :: Parser a -> Parser a 217 | inString = MP.between (MP.char '"') (MP.char '"') 218 | 219 | -- lexers 220 | 221 | -- | Parse whitespace. 222 | space :: Parser () 223 | space = MPL.space (void MP.spaceChar) 224 | (MPL.skipLineComment "# ") 225 | (void $ MP.satisfy (const False)) 226 | 227 | -- | Parse a lexeme. 228 | lexeme :: Parser a -> Parser a 229 | lexeme = MPL.lexeme space 230 | 231 | -- | Ensure parser is not indented. 232 | nonIndented :: Parser a -> Parser a 233 | nonIndented = MPL.nonIndented space 234 | indentBlock :: Parser (MPL.IndentOpt Parser a b) 235 | -> Parser a 236 | indentBlock = MPL.indentBlock space 237 | -------------------------------------------------------------------------------- /yarn2nix/src/Distribution/Nixpkgs/Nodejs/OptimizedNixOutput.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings, NoImplicitPrelude, GeneralizedNewtypeDeriving, ViewPatterns, RecordWildCards, LambdaCase, NamedFieldPuns #-} 2 | {-| 3 | Description: Generate an optimized nix file from a resolved @YLT.Lockfile@ 4 | 5 | We want to generate a nix file with the following attributes: 6 | 7 | 1. easy to parse by humans 8 | 2. as short as possible 9 | 3. updating the yarn.lock generates diffs that are as short as possible 10 | 11 | Readability means a clear structure, with definitions at the top. 12 | 13 | Reducing the filesize means we can’t duplicate any information and keep identifiers very short. This interferes with readability, but can be amended by giving the full names in the static section and then giving them short identifiers in a second section. 14 | 15 | Nice diffing includes having line-based output (if possible one line per package/dependency), as well as keeping the order of items stable (alphabetically sorting package names and dependencies). 16 | -} 17 | module Distribution.Nixpkgs.Nodejs.OptimizedNixOutput 18 | ( convertLockfile 19 | -- * File Structure 20 | -- $fileStructure 21 | , mkPackageSet 22 | -- * NOTE: fix 23 | -- $noteFix 24 | ) where 25 | 26 | import Protolude 27 | import qualified Data.Map as M 28 | import qualified Data.Text as T 29 | import Data.Fix (Fix(Fix)) 30 | import qualified Data.MultiKeyedMap as MKM 31 | import qualified Data.List as List 32 | import qualified Data.List.NonEmpty as NE 33 | 34 | import Nix.Expr (NExpr, ($=), (==>), (@@)) 35 | import Nix.Expr.Additions (($$=), (!!.), inheritStatic) 36 | import qualified Nix.Expr as N 37 | import qualified Nix.Expr.Additions as NA 38 | 39 | import qualified Yarn.Lock.Types as YLT 40 | 41 | import qualified Distribution.Nixpkgs.Nodejs.ResolveLockfile as Res 42 | import Distribution.Nixpkgs.Nodejs.Utils (packageKeyToSymbol) 43 | 44 | -- | Nix symbol. 45 | newtype NSym = NSym { unNSym :: Text } 46 | deriving (IsString, Ord, Eq) 47 | 48 | -- | Nix input variable. 49 | newtype NVar = NVar NSym 50 | deriving (IsString) 51 | 52 | -- | Builder type for simple antiquoted nix strings. 53 | data AStrVal = V NVar 54 | -- ^ nix antiquoted variable 55 | | T Text 56 | -- ^ normal nix string 57 | 58 | -- | Build a nix string from multiple @AStrVal@s. 59 | antiquote :: [AStrVal] -> NExpr 60 | antiquote vals = Fix . N.NStr . N.DoubleQuoted 61 | $ flip map vals $ \case 62 | T t -> N.Plain t 63 | V (NVar (NSym t)) -> N.Antiquoted $ N.mkSym t 64 | 65 | -- | A registry that we know of and can therefore shorten 66 | -- into a nix function call. 67 | data Registry = Registry 68 | { registrySym :: NSym 69 | -- ^ nix symbol used in the output file 70 | , registryBuilder :: NVar -> NVar -> [AStrVal] 71 | -- ^ constructs a nix function that in turn constructs a repository string; 72 | -- the function takes a package name and package version 73 | } 74 | 75 | data Git = Git 76 | { gitUrl :: Text 77 | , gitRev :: Text } 78 | 79 | -- | Final package reference used in the generated package list. 80 | data PkgRef 81 | -- | reference to another package definition (e.g. @^1.2@ points to @1.2@) 82 | = PkgRef Text 83 | | PkgDefFile (PkgData (Either Text Registry)) 84 | -- ^ actual definition of a file package 85 | | PkgDefFileLocal (PkgData Text) 86 | -- ^ actual definition of a local package (tar.gz file relative to nix expression) 87 | | PkgDefGit (PkgData Git) 88 | -- ^ actual definition of a git package 89 | 90 | -- | Package definition needed for calling the build function. 91 | data PkgData a = PkgData 92 | { pkgDataName :: YLT.PackageKeyName -- ^ package name 93 | , pkgDataVersion :: Text -- ^ package version 94 | , pkgDataUpstream :: a -- ^ points to upstream 95 | , pkgDataHashSum :: Text -- ^ the hash sum of the package 96 | , pkgDataDependencies :: [Text] -- ^ list of dependencies (as resolved nix symbols) 97 | } 98 | 99 | -- | Tuples of prefix string to registry 100 | registries :: [(Text, Registry)] 101 | registries = 102 | [ ( yarnP 103 | , Registry "yarn" 104 | $ \n v -> [T yarnP, V n, T "/-/", V n, T "-", V v, T ".tgz"] ) 105 | , ( npmjsP 106 | , Registry "npm" 107 | $ \n v -> [T npmjsP, V n, T "/-/", V n, T "-", V v, T ".tgz"] ) 108 | 109 | ] 110 | where 111 | yarnP = "https://registry.yarnpkg.com/" 112 | npmjsP = "https://registry.npmjs.org/" 113 | 114 | shortcuts :: M.Map [NSym] NSym 115 | shortcuts = M.fromList 116 | [ (["self"], "s") 117 | , (["registries", "yarn"], "y") 118 | , (["registries", "npm"], "n") 119 | , (["nodeFilePackage"], "f") 120 | , (["nodeFileLocalPackage"], "l") 121 | , (["nodeGitPackage"], "g") 122 | , (["identityRegistry"], "ir") 123 | , (["scopedName"], "sc") 124 | ] 125 | 126 | -- | Find out which registry the given 'YLT.Remote' shortens to. 127 | recognizeRegistry :: YLT.PackageKeyName -- ^ package name 128 | -> Text -- ^ url to file 129 | -> Maybe Registry 130 | -- We don’t shorten scoped key names, because 131 | -- they are handled specially by npm registries and 132 | -- the URLs differ from other packages 133 | recognizeRegistry (YLT.ScopedPackageKey{}) _ = Nothing 134 | recognizeRegistry _ fileUrl = snd <$> foundRegistry 135 | where 136 | -- | Get registry by the prefix of the registry’s URL. 137 | foundRegistry = find predicate registries 138 | predicate :: (Text, Registry) -> Bool 139 | predicate reg = fst reg `T.isPrefixOf` fileUrl 140 | 141 | 142 | -- | Convert a 'Res.ResolvedLockfile' to its final, nix-ready form. 143 | convertLockfile :: Res.ResolvedLockfile -> M.Map Text PkgRef 144 | convertLockfile = M.fromList . foldMap convert . MKM.toList 145 | where 146 | -- | For the list of package keys we generate a @PkgRef@ each 147 | -- and then one actual @PkgDef@. 148 | convert :: (NE.NonEmpty YLT.PackageKey, (Res.Resolved YLT.Package)) 149 | -> [(Text, PkgRef)] 150 | convert (keys, Res.Resolved{ hashSum, resolved=pkg }) = let 151 | -- | Since there might be more than one key name, we choose 152 | -- the one with most entries. 153 | defName = NE.head $ maximumBy (comparing length) $ NE.group $ NE.sort $ fmap YLT.name keys 154 | defSym = packageKeyToSymbol $ YLT.PackageKey 155 | { YLT.name = defName 156 | , YLT.npmVersionSpec = YLT.version pkg } 157 | pkgDataGeneric upstream = PkgData 158 | { pkgDataName = defName 159 | , pkgDataVersion = YLT.version pkg 160 | , pkgDataUpstream = upstream 161 | , pkgDataHashSum = hashSum 162 | , pkgDataDependencies = map packageKeyToSymbol 163 | -- TODO: handle optional dependencies better 164 | $ YLT.dependencies pkg <> YLT.optionalDependencies pkg 165 | } 166 | def = case YLT.remote pkg of 167 | YLT.FileRemote{fileUrl} -> 168 | PkgDefFile $ pkgDataGeneric $ note fileUrl 169 | $ recognizeRegistry defName fileUrl 170 | YLT.FileLocal{fileLocalPath} -> 171 | PkgDefFileLocal $ pkgDataGeneric $ fileLocalPath 172 | YLT.GitRemote{gitRepoUrl, gitRev} -> 173 | PkgDefGit $ pkgDataGeneric $ Git gitRepoUrl gitRev 174 | YLT.FileRemoteNoIntegrity {} -> 175 | panic "programming error, should have thrown an error in ResolveLockfile" 176 | YLT.FileLocalNoIntegrity {} -> 177 | panic "programming error, should have thrown an error in ResolveLockfile" 178 | -- we don’t need another ref indirection 179 | -- if that’s already the name of our def 180 | refNames = List.delete defSym $ toList $ NE.nub 181 | $ fmap packageKeyToSymbol keys 182 | in (defSym, def) : map (\rn -> (rn, PkgRef defSym)) refNames 183 | 184 | 185 | {- $fileStructure 186 | 187 | @ 188 | { fetchgit, fetchurl }: 189 | # self & super: see notes on fix 190 | self: super: 191 | let 192 | # shorten the name of known package registries 193 | registries = { 194 | yarn = n: v: "https://registry.yarnpkg.com/${n}/-/${n}-${v}.tgz"; 195 | }; 196 | 197 | # We want each package definition to be one line, by putting 198 | # the boilerplate into these functions for different remotes. 199 | nodeFilePackage = … 200 | nodeFileLocalPackage = … 201 | nodeGitPackage = … 202 | 203 | # an identity function for e.g. git repos or unknown registries 204 | identityRegistry = url: _: _: url; 205 | 206 | # a way to pass through scoped package names 207 | scopedName = scope: name: { inherit scope name; } 208 | 209 | # shortcut section 210 | s = self; 211 | ir = identityRegistry; 212 | f = nodeFilePackage; 213 | l = nodeFileLocalPackage; 214 | g = nodeGitPackage; 215 | y = registries.yarnpkg; 216 | sc = scopedName; 217 | … 218 | 219 | # the actual package definitions 220 | in { 221 | "accepts@~1.3.3" = s."accepts@1.3.3"; 222 | "accepts@1.3.3" = f "accepts" "1.3.3" y "sha" []; 223 | "@types/accepts@1.3.3" = f (sc "types" "accepts") "1.3.3" y "sha" []; 224 | "babel-core@^6.14.0" = s."babel-core@6.24.1"; 225 | "babel-core@6.24.1" = f "babel-core" "6.24.1" y "a0e457c58ebdbae575c9f8cd75127e93756435d8" [ 226 | s."accepts@~1.3.3" 227 | ]; 228 | "local-package@file:../foo.tgz" = l "local-package" "file:../foo.tgz" ../foo.tgz "thehash" [] 229 | } 230 | @ 231 | -} 232 | 233 | -- | Convert a list of packages prepared with 'convertLockfile' 234 | -- to a nix expression. 235 | mkPackageSet :: M.Map Text PkgRef -> NExpr 236 | mkPackageSet packages = 237 | NA.simpleParamSet ["fetchurl", "fetchgit"] 238 | -- enable self-referencing of packages 239 | -- with string names with a self/super fix 240 | -- see note FIX 241 | ==> N.Param "self" ==> N.Param "super" 242 | ==> N.mkLets 243 | ( [ "registries" $= N.mkNonRecSet (fmap (mkRegistry . snd) registries) 244 | , "nodeFilePackage" $= buildPkgFn 245 | , "nodeFileLocalPackage" $= buildPkgLocalFn 246 | , "nodeGitPackage" $= buildPkgGitFn 247 | , "identityRegistry" $= NA.multiParam ["url", "_", "_"] "url" 248 | , "scopedName" $= 249 | (NA.multiParam ["scope", "name"] 250 | $ N.mkNonRecSet [ inheritStatic ["scope", "name"] ]) 251 | ] 252 | <> fmap mkShortcut (M.toList shortcuts) ) 253 | (N.mkNonRecSet (map mkPkg $ M.toAscList packages)) 254 | where 255 | mkRegistry (Registry{..}) = unNSym registrySym $= 256 | (N.Param "n" ==> N.Param "v" ==> antiquote (registryBuilder "n" "v")) 257 | 258 | concatNSyms :: [NSym] -> NExpr 259 | concatNSyms [] = panic "non-empty shortcut syms!" 260 | concatNSyms (l:ls) = foldl (!!.) (N.mkSym $ unNSym l) (fmap unNSym ls) 261 | mkShortcut :: ([NSym], NSym) -> N.Binding NExpr 262 | mkShortcut (nSyms, short) = unNSym short $= concatNSyms nSyms 263 | -- | Try to shorten sym, otherwise use input. 264 | shorten :: [NSym] -> NExpr 265 | shorten s = case M.lookup s shortcuts of 266 | Nothing -> concatNSyms s 267 | Just sc -> N.mkSym (unNSym sc) 268 | -- | Build function boilerplate the build functions share in common. 269 | buildPkgFnGeneric :: [Text] -> NExpr -> NExpr 270 | buildPkgFnGeneric additionalArguments srcNExpr = 271 | NA.multiParam (["key", "version"] <> additionalArguments <> ["deps"]) 272 | $ ("super" !!. "_buildNodePackage") @@ N.mkNonRecSet 273 | [ inheritStatic ["key", "version"] 274 | , "src" $= srcNExpr 275 | , "nodeBuildInputs" $= "deps" ] 276 | -- | Building a 'YLT.FileRemote' package. 277 | buildPkgFn :: NExpr 278 | buildPkgFn = 279 | buildPkgFnGeneric ["registry", "sha1"] 280 | ("fetchurl" @@ N.mkNonRecSet 281 | [ "url" $= ("registry" @@ "key" @@ "version") 282 | , inheritStatic ["sha1"] ]) 283 | -- | Building a 'YLT.FileLocal' package. 284 | buildPkgLocalFn :: NExpr 285 | buildPkgLocalFn = 286 | buildPkgFnGeneric ["path", "sha1"] 287 | ("builtins.path" @@ N.mkNonRecSet 288 | [ inheritStatic ["path"] 289 | -- TODO: use the sha1 here! (does builtins.path only take sha256?) 290 | -- , "sha256" $= "sha1" 291 | ]) 292 | -- | Building a 'YLT.GitRemote' package. 293 | buildPkgGitFn :: NExpr 294 | buildPkgGitFn = 295 | buildPkgFnGeneric ["url", "rev", "sha256"] 296 | ("fetchgit" @@ N.mkNonRecSet 297 | [ inheritStatic ["url", "rev", "sha256"] ]) 298 | 299 | -- | Create a package definition. 300 | mkPkg :: (Text, PkgRef) -> N.Binding NExpr 301 | mkPkg (key, pkgRef) = key $$= case pkgRef of 302 | PkgRef t -> N.mkSym selfSym !!. t 303 | PkgDefFile pd@PkgData{pkgDataUpstream, pkgDataHashSum} -> 304 | mkDefGeneric pd "nodeFilePackage" 305 | [ either (\url -> shorten ["identityRegistry"] @@ N.mkStr url ) 306 | (\reg -> shorten ["registries", registrySym reg]) 307 | pkgDataUpstream 308 | , N.mkStr pkgDataHashSum ] 309 | PkgDefFileLocal pd@PkgData{pkgDataUpstream = path, pkgDataHashSum} -> 310 | mkDefGeneric pd "nodeFileLocalPackage" [ N.mkPath False (toS path), N.mkStr pkgDataHashSum ] 311 | PkgDefGit pd@PkgData{pkgDataUpstream = Git{..}, pkgDataHashSum} -> 312 | mkDefGeneric pd "nodeGitPackage" 313 | [ N.mkStr gitUrl, N.mkStr gitRev, N.mkStr pkgDataHashSum ] 314 | 315 | -- | The common parts of creating a package definition. 316 | mkDefGeneric :: PkgData a -> NSym -> [NExpr] -> NExpr 317 | mkDefGeneric PkgData{..} buildFnSym additionalArguments = 318 | foldl' (@@) (shorten [buildFnSym]) 319 | $ [ case pkgDataName of 320 | YLT.SimplePackageKey n -> N.mkStr n 321 | YLT.ScopedPackageKey s n -> "sc" @@ N.mkStr s @@ N.mkStr n 322 | , N.mkStr pkgDataVersion ] 323 | <> additionalArguments <> 324 | [ N.mkList $ map (N.mkSym selfSym !!.) pkgDataDependencies ] 325 | 326 | selfSym :: Text 327 | selfSym = "s" 328 | 329 | {- $noteFix 330 | 331 | @ 332 | self: super: 333 | @ 334 | 335 | follows the fixpoint scheme first introduced 336 | by the @haskellPackage@ set in @nixpkgs@. 337 | See the @Overlays@ documentation in the @nixpkgs@ 338 | manual for explanations of how this works. 339 | 340 | Note: originally, this was a shallow fix like 341 | 342 | @ 343 | let attrs = self: { 344 | "foo bar" = 1; 345 | bar = self."foo bar" + 2; 346 | }; 347 | in fix attrs 348 | @ 349 | 350 | which was just in place to work around referencing 351 | attrset attributes through string names. 352 | The new method is a lot more general and allows deep 353 | overrides of arbitrary packages in the dependency set. 354 | -} 355 | --------------------------------------------------------------------------------