├── README.md ├── de-generate.py ├── nix-de-generate └── template.nix /README.md: -------------------------------------------------------------------------------- 1 | # De-generate 2 | 3 | Dependency generator for prebuilt binaries and shared libraries in NixOS. 4 | 5 | ## How? 6 | # Install 7 | nix-env -iA nixos.nix-index nixos.git 8 | nix-index 9 | git clone https://github.com/lexleogryfon/de-generate.git 10 | cd ./de-generate 11 | # Use 12 | nix-shell template.nix 13 | ./nix-de-generate /home/usr/path/to/folder_with_executables 14 | # a file newenv.nix should appear in current directory 15 | # with list of packages to satisfy dependency requirements. 16 | 17 | 18 | ## Why? 19 | 20 | Due current state of the art NixOS design, you can't just run downloaded dynamically linked portable application outside of nixpkgs repo and expect it to work. When you attempt to run such app inside pkgs.buildFHSUserEnv, you may see that it couldn't find some libraries. 21 | Previously you might be forced to find package for each lib manually with nix-locate, fortunately now you could just execute nix-de-generate against target directory and it will generate newenv.nix with possible dependencies. Theoretically project could be scaled to resolve dependecies in other distros too, such as Arch or Fedora. 22 | 23 | 24 | https://nixos.wiki/wiki/FAQ#I.27ve_downloaded_a_binary.2C_but_I_can.27t_run_it.2C_what_can_I_do.3F 25 | 26 | https://nixos.wiki/wiki/Packaging_Binaries#Extra_Dynamic_Libraries 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /de-generate.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | import os, sys 3 | import subprocess as sp 4 | from pathlib import Path 5 | from string import Template 6 | 7 | """ 8 | De-generate, nix-de-generate: Dependency generator for prebuilt binaries and shared libraries in NixOS. 9 | 10 | for usage example see https://github.com/lexleogryfon/de-generate/blob/master/README.md 11 | made with emacs, elpy, color-theme-ld-dark, pypy3.5.3 12 | """ 13 | 14 | # DONE check if file is executable before calling ldd 15 | # DONE get initial list of libs that can't be found by ldd 16 | # DONE use subprocess https://docs.python.org/3/library/subprocess.html#replacing-shell-pipeline ; 17 | # DONE check for files that _do not exits in directory_, because otherwise those who exists could be dynamically linked during runtime by overriding LD LIB PATH 18 | # DONE install nix-index to locate dependecies based on files 19 | # DONE call nix-locate --top-level libgstinterfaces-0.10.so.0 | grep " x " 20 | # option nix-locate is case sensitive , so call for both cases? 21 | # option handle multi string output from nix-locate 22 | # DONE guess proper package names 23 | # DONE generate text output file 24 | # DONE refactor 25 | # DONE try use raw package prefixes from nix-locate 26 | # option add reStructuredText docstrings 27 | # BUG what if file.so does not have +x permission? 28 | 29 | # DONE make template.nix look a like FHSsmall.nix 30 | # DONE write readme.md 31 | # DONE clear out input variable 32 | # DONE comment nixOs.wiki 33 | 34 | def scan(path_string, libs=[], all_files_param=set()): 35 | """ 36 | return set of libraries that can't be linked by ldd AND set of all_files 37 | """ 38 | all_files = all_files_param 39 | notfound_libraries = libs 40 | p = Path(path_string) 41 | for child in p.iterdir(): 42 | if child.is_dir(): 43 | scan(child, libs=notfound_libraries) 44 | else: 45 | all_files.add(child.name) 46 | if os.access(str(child), os.X_OK): 47 | try: 48 | output = sp.check_output('ldd {} 2>&1 | grep "not found"'.format(child), shell=True, stderr=sp.STDOUT, universal_newlines=True) 49 | output = output.splitlines() 50 | notfound_libraries.extend(output) 51 | except OSError as e: 52 | # trying to catch unclosed pipe > ulimit 53 | print(e) 54 | except sp.CalledProcessError: 55 | # catches exitcode that not equal 0, common in our case 56 | # do nothing because ldd usually spits non zero exit code when applied to non ELF\.so , and grep spits non zero exit code when did not found mathces (e.g. no output) 57 | pass 58 | 59 | # removing dublicates and split strings to leave only library filename 60 | notfound_libraries = {i.split()[0] for i in notfound_libraries} 61 | 62 | return notfound_libraries, all_files 63 | 64 | 65 | def remove_existing_libs(libs, files): 66 | non_existing_libs = set() 67 | for lib in libs: 68 | if lib in files: 69 | # print(lib + " is in folder") 70 | pass 71 | else: 72 | # print(lib + " NOT in folder") 73 | non_existing_libs.add(lib) 74 | return non_existing_libs 75 | 76 | 77 | def guess_pkgs(libs, output_prefixes=False): 78 | pkgs = set() 79 | 80 | if output_prefixes: 81 | for lib in libs: 82 | # pkg = sp.check_output('nix-locate --top-level {} | grep " x "'.format(lib), shell=True, stderr=sp.STDOUT, universal_newlines=True).split()[0] 83 | output = sp.check_output('nix-locate --top-level {}'.format(lib), shell=True, stderr=sp.STDOUT, universal_newlines=True).splitlines() 84 | for line in output: 85 | pkg = line.split()[0] 86 | # print(pkg) 87 | pkgs.add(pkg) 88 | else: 89 | for lib in libs: 90 | pkg = sp.check_output('nix-locate --top-level {} | grep " x "'.format(lib), shell=True, stderr=sp.STDOUT, universal_newlines=True).split()[0].split(".")[-2] 91 | # print(pkg) 92 | pkgs.add(pkg) 93 | return pkgs 94 | 95 | 96 | 97 | def generate_nix(pkgs): 98 | """ 99 | pkgs : accept raw string_of_guessed_pkgs (with \n separators) 100 | substitute $output_of_dee_generate in template.nix and produces newenv.nix 101 | """ 102 | 103 | string_of_guessed_pkgs = pkgs 104 | with open('template.nix', 'r') as file1, open('newenv.nix', 'w') as file2: 105 | tmplt = Template(file1.read()) 106 | d = {'output_of_dee_generate': string_of_guessed_pkgs} 107 | result = tmplt.substitute(d) 108 | # print(result) 109 | file2.write(result) 110 | 111 | 112 | def main(argv): 113 | 114 | input = argv[1] 115 | notfound_libraries, files = scan(input) 116 | print('libraries that cannot be found by ldd') 117 | for i in notfound_libraries: print(i) 118 | print(len(notfound_libraries), type(notfound_libraries)) 119 | 120 | notfound_libraries = remove_existing_libs(notfound_libraries, files) 121 | print('\n\nlibraries that are not in the folder') 122 | for i in notfound_libraries: print(i) 123 | print(len(notfound_libraries)) 124 | 125 | guessed_pkgs = guess_pkgs(notfound_libraries, output_prefixes=True) 126 | print('\npackages that could possibly satisfy dependecies\n', guessed_pkgs) 127 | string_of_guessed_pkgs = " \n".join(guessed_pkgs) 128 | string_of_guessed_pkgs = 'for path ' + argv[1] + '\n' + string_of_guessed_pkgs 129 | 130 | generate_nix(string_of_guessed_pkgs) 131 | exit(0) 132 | 133 | 134 | if __name__ == "__main__": 135 | main(sys.argv) 136 | -------------------------------------------------------------------------------- /nix-de-generate: -------------------------------------------------------------------------------- 1 | de-generate.py -------------------------------------------------------------------------------- /template.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} }: # 2 | 3 | (pkgs.buildFHSUserEnv { 4 | 5 | name = "template-env"; 6 | targetPkgs = pkgs: with pkgs; [ 7 | coreutils git python35Full nix-index 8 | ]; 9 | multiPkgs = pkgs: with pkgs; [ 10 | 11 | #output of de-generate 12 | #$output_of_dee_generate 13 | 14 | ]; 15 | runScript = "bash"; 16 | }).env 17 | --------------------------------------------------------------------------------