├── .envrc ├── .github ├── dependabot.yml └── workflows │ ├── build.yml │ └── scrape.yml ├── .gitignore ├── LICENSE ├── README.md ├── flake.lock ├── flake.nix ├── latest.json ├── module.nix ├── programs-sqlite.nix ├── sources.json ├── src ├── .gitignore ├── tests │ └── helpers │ │ ├── test_getchannel.nim │ │ └── test_getmeta.nim ├── updater.nimble └── updater │ └── updater.nim ├── test.nix └── updater.nix /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | - package-ecosystem: github-actions 5 | directory: "/" 6 | schedule: 7 | interval: daily 8 | time: '00:00' 9 | timezone: UTC 10 | target-branch: "tooling" 11 | open-pull-requests-limit: 10 12 | commit-message: 13 | prefix: "chore" 14 | include: "scope" -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: "Build scraper" 2 | on: 3 | pull_request: 4 | push: 5 | workflow_dispatch: 6 | jobs: 7 | build: 8 | runs-on: ubuntu-22.04 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v4 12 | - name: Install Nix 13 | uses: cachix/install-nix-action@v31 14 | with: 15 | nix_path: nixpkgs=channel:nixos-unstable 16 | extra_nix_config: "system-features = nixos-test benchmark big-parallel kvm" 17 | - name: Cache on Cachix 18 | uses: cachix/cachix-action@v16 19 | with: 20 | name: flake-programs-sqlite 21 | authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' 22 | - name: Cache build result on GitHub 23 | uses: actions/cache@v4 24 | with: 25 | path: updater.bundle 26 | key: ${{ runner.os }}-bundle-${{ hashFiles('**/flake.lock') }}-${{ hashFiles('**/*.nix') }}-${{ hashFiles('**/*.nim*') }} 27 | 28 | - name: Build and Bundle 29 | run: | 30 | nix build .#updater 31 | nix bundle -o updater.bundle.lnk .#packages.x86_64-linux.updater && cp $(readlink updater.bundle.lnk) updater.bundle && chmod u+w updater.bundle 32 | 33 | - name: Smoke test 34 | run: | 35 | echo {} > ${{ runner.temp }}/sources.json 36 | echo {} > ${{ runner.temp }}/latest.json 37 | nix run .#updater -- --dir:${{ runner.temp }} --channel:https://releases.nixos.org/nixos/20.03/nixos-20.03.2400.ff1b66eaea4 38 | cat ${{ runner.temp }}/sources.json 39 | [ `nix eval --impure nixpkgs#lib.importJSON --apply "x : (x ${{ runner.temp }}/sources.json).ff1b66eaea4399d297abda7419a330239842d715.programs_sqlite_hash"` = '"6097c544f012fc21f8cff9a6305ebab335d148e6385d7288a612e10c3cc82df0"' ] 40 | 41 | - name: Integration test 42 | run: nix build --override-input nixpkgs "github:NixOS/nixpkgs?rev=04f574a1c0fde90b51bf68198e2297ca4e7cccf4" .#checks.x86_64-linux.vmtest 43 | -------------------------------------------------------------------------------- /.github/workflows/scrape.yml: -------------------------------------------------------------------------------- 1 | name: "Update Channel Info" 2 | 3 | on: 4 | schedule: 5 | - cron: '*/10 * * * *' 6 | workflow_dispatch: 7 | 8 | jobs: 9 | update: 10 | runs-on: ubuntu-22.04 11 | steps: 12 | - name: Check out repository 13 | uses: actions/checkout@v4 14 | 15 | - name: Configure Git 16 | run: | 17 | git config user.name github-actions 18 | git config user.email github-actions@github.com 19 | 20 | - name: fetch from cache 21 | uses: actions/cache@v4 22 | with: 23 | path: updater.bundle 24 | key: ${{ runner.os }}-bundle-${{ hashFiles('**/flake.lock') }}-${{ hashFiles('**/*.nix') }}-${{ hashFiles('**/*.nim*') }} 25 | 26 | - name: update JSON 27 | run: ./updater.bundle -d:$PWD 28 | 29 | - name: create commit 30 | run: git commit -a -m "update sources.json" || true 31 | 32 | - name: Push commit with updated inputs 33 | run: | 34 | git pull --rebase --autostash 35 | git push 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .direnv 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Markus Wamser 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # programs.sqlite for Nix Flake based systems 2 | 3 | [![Update Channel Info](https://github.com/wamserma/flake-programs-sqlite/actions/workflows/scrape.yml/badge.svg?branch=main)](https://github.com/wamserma/flake-programs-sqlite/actions/workflows/scrape.yml) 4 | 5 | ## TL;DR 6 | 7 | (assuming a flake similar to ) 8 | 9 | Add to `inputs` in `flake.nix`: 10 | 11 | ```nix 12 | flake-programs-sqlite.url = "github:wamserma/flake-programs-sqlite"; 13 | flake-programs-sqlite.inputs.nixpkgs.follows = "nixpkgs"; 14 | ``` 15 | 16 | Then just add the module. 17 | 18 | ### NixOS module 19 | 20 | Usage with a minimal system flake: 21 | 22 | ```nix 23 | { 24 | inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-22.11; 25 | inputs.flake-programs-sqlite.url = "github:wamserma/flake-programs-sqlite"; 26 | inputs.flake-programs-sqlite.inputs.nixpkgs.follows = "nixpkgs"; 27 | 28 | outputs = inputs@{ self, nixpkgs, ... }: { 29 | 30 | nixosConfigurations.mymachine = nixpkgs.lib.nixosSystem { 31 | system = "x86_64-linux"; 32 | modules = 33 | [ 34 | (import configuration.nix) 35 | inputs.flake-programs-sqlite.nixosModules.programs-sqlite 36 | ]; 37 | }; 38 | }; 39 | } 40 | ``` 41 | 42 | The module's functionality is enabled as soon as the module is imported. 43 | 44 | ### alternative: without using a module 45 | 46 | Add `flake-programs-sqlite` to the arguments of the flake's `outputs` function. 47 | 48 | Add `programs-sqlite-db = flake-programs-sqlite.packages.${system}.programs-sqlite` 49 | to the `specialArgs` argument of `lib.nixosSystem`. 50 | 51 | Add `programs-sqlite-db`to the inputs of your system configuration (`configuration.nix`) 52 | and to the configuration itself add: 53 | 54 | ```nix 55 | programs.command-not-found.dbPath = programs-sqlite-db; 56 | ``` 57 | 58 | ## Why? 59 | 60 | NixOS systems configured with flakes and thus lacking channels usually have a broken 61 | `command-not-found`. The reason is that the backing database `programs.sqlite` is only 62 | available on channels. The problem is that the channel URL can not be determined from 63 | the `nixpkgs` revision alone, as it also contains a build number. 64 | 65 |
66 | 67 | command-not-what? 68 | 69 | 70 | Bash and other shells have a special handler that is invoked when an unknown command 71 | is issued to the shell. For bash this is `command_not_found_handler`. 72 | 73 | `command-not-found` is [a Perl script](https://github.com/NixOS/nixpkgs/blob/7c44c865ee736afba33ee8788b59e4a123800437/nixos/modules/programs/command-not-found/command-not-found.pl) 74 | that is hooked into this handler when 75 | the option [`programs.command-not-found.enable`](https://search.nixos.org/options?show=programs.command-not-found.enable) 76 | is set to `true`. This perl script evaluates a pre-made database to suggest 77 | packages that might be able to provide the command. 78 | 79 | The pre-made database is generated as part of a channel, hence pure-flake-systems 80 | do not have access to it. This flake extracts the database from the channels 81 | and passes it to the `command-not-found` script to restore functionality that 82 | was previously only available when using channels. 83 |
84 | 85 | This is an attempt to provide a usable solution, motivated by 86 | 87 | ## How? 88 | 89 | The channel page is regularly scraped for the revision and file hashes, then a 90 | [lookup table](./sources.json) from revisions to URL and hashes is amended with any 91 | new information. 92 | The lookup table is used to create a fixed-output-derivation (FOD) for `programs.sqlite` 93 | based on the revision of `nixpkgs` passed as input of this flake. 94 | 95 | ## Usage 96 | 97 | see TL:DR above 98 | 99 | ## Development 100 | 101 | The flake provides a minimal devshell, but hacking on the code with a editor and 102 | running `nix run .#updater` is valid, too. 103 | 104 | Development happens on the `tooling` branch, which is then merged into the `main` 105 | branch. Updates to the JSON file go directly to `main`. Releases of the tooling are 106 | also cut from the `tooling` branch. There are no releases for the JSON files. 107 | 108 | ## Fetching selected channel revisions 109 | 110 | e.g. to fetch info for older revisions/releases from before this project was started 111 | 112 | ```sh 113 | [ -f sources.json ] || echo {} > sources.json 114 | nix run github:wamserma/flake-programs-sqlite#updater -- --dir:. --channel:https://releases.nixos.org/nixos/20.03/nixos-20.03.2400.ff1b66eaea4 115 | ``` 116 | 117 | Multiple channels/revisions may be passed for a single run. 118 | If no channel is given, the current channels are guessed and their latest revisions are fetched. 119 | 120 | ## Alternatives 121 | 122 | - [nix-index](https://github.com/bennofs/nix-index#usage-as-a-command-not-found-replacement) 123 | 124 | ## Licensing 125 | 126 | The Nim code to scrape the metadata is released under MIT License. 127 | The Nix code to provide the FODs is released under MIT License. 128 | The database itself (JSON) is public domain. 129 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixpkgs": { 4 | "locked": { 5 | "lastModified": 1680487167, 6 | "narHash": "sha256-9FNIqrxDZgSliGGN2XJJSvcDYmQbgOANaZA4UWnTdg4=", 7 | "owner": "NixOS", 8 | "repo": "nixpkgs", 9 | "rev": "53dad94e874c9586e71decf82d972dfb640ef044", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "NixOS", 14 | "ref": "nixpkgs-unstable", 15 | "repo": "nixpkgs", 16 | "type": "github" 17 | } 18 | }, 19 | "root": { 20 | "inputs": { 21 | "nixpkgs": "nixpkgs", 22 | "utils": "utils" 23 | } 24 | }, 25 | "utils": { 26 | "locked": { 27 | "lastModified": 1678901627, 28 | "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=", 29 | "owner": "numtide", 30 | "repo": "flake-utils", 31 | "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6", 32 | "type": "github" 33 | }, 34 | "original": { 35 | "owner": "numtide", 36 | "repo": "flake-utils", 37 | "type": "github" 38 | } 39 | } 40 | }, 41 | "root": "root", 42 | "version": 7 43 | } 44 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | inputs = { 3 | nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; 4 | utils.url = "github:numtide/flake-utils"; 5 | }; 6 | 7 | outputs = { self, nixpkgs, utils }: 8 | let 9 | # use tools from given pkgs to extract the db from the download 10 | getDB = pkgs: pkgs.callPackage ./programs-sqlite.nix { inherit (nixpkgs) rev; }; 11 | in 12 | 13 | # provide db-package for all archs, but scaper/test only for most common 14 | (utils.lib.eachSystem utils.lib.allSystems (system: 15 | let 16 | pkgs = nixpkgs.legacyPackages.${system}; 17 | in 18 | nixpkgs.lib.recursiveUpdate 19 | (nixpkgs.lib.optionalAttrs (builtins.elem system utils.lib.defaultSystems) rec { 20 | packages.updater = pkgs.callPackage ./updater.nix {}; 21 | apps.updater = { type = "app"; program = "${packages.updater}/bin/updater";}; 22 | devShells.default = with pkgs; mkShell { 23 | buildInputs = [ nim nimble-unwrapped nimlsp ]; 24 | }; 25 | checks.vmtest = import ./test.nix { inherit pkgs; flake = self; }; # nixpkgs must be set to a revision present in the JSON file 26 | }) 27 | { 28 | packages.programs-sqlite = getDB pkgs; 29 | }) 30 | ) // 31 | 32 | # NixOS module 33 | { 34 | nixosModules.programs-sqlite = pass@{ pkgs, ... }: (import ./module.nix { programs-sqlite = (getDB pkgs); }) pass; 35 | }; 36 | } 37 | -------------------------------------------------------------------------------- /latest.json: -------------------------------------------------------------------------------- 1 | { 2 | "24.05": { 3 | "name": "nixos-24.05.7389.0da3c44a9460", 4 | "url": "/nixos/24.05-small/nixos-24.05.7389.0da3c44a9460/nixexprs.tar.xz", 5 | "nixexprs_hash": "2c0987935239b20e7f3293d7f39cc3b03502abd5ac184eedc90d75680c77794a", 6 | "programs_sqlite_hash": "32b539abd5f6e2caeddeb7f42011f3c8d048287a0e58e0a1c715621e75086c25" 7 | }, 8 | "22.11": { 9 | "name": "nixos-22.11.4773.ea4c80b39be4", 10 | "url": "/nixos/22.11/nixos-22.11.4773.ea4c80b39be4/nixexprs.tar.xz", 11 | "nixexprs_hash": "c4828378407ebd3255ff9d51d31e942c4a9acd088540c6b83dbd8dc7d24283dd", 12 | "programs_sqlite_hash": "fbb9c05caae92e51e1c4d14018656aeddb2d6222009f6d483dd38ddcbc1f2076" 13 | }, 14 | "23.05": { 15 | "name": "nixos-23.05.5541.a1982c92d898", 16 | "url": "/nixos/23.05-small/nixos-23.05.5541.a1982c92d898/nixexprs.tar.xz", 17 | "nixexprs_hash": "ba1902d5919de07869b0cb03c238a8ffa0fb321794530b3ce8da6820019cc144", 18 | "programs_sqlite_hash": "e6f48b8e44a12663831c869a3edd2b9370e0d8e592ef9b4027fd44cad837c9af" 19 | }, 20 | "23.11": { 21 | "name": "nixos-23.11.7870.205fd4226592", 22 | "url": "/nixos/23.11-small/nixos-23.11.7870.205fd4226592/nixexprs.tar.xz", 23 | "nixexprs_hash": "394d8e173403bb780bd85bbcafe335ed4d52b27dab4170a39e4439e5bcf0d531", 24 | "programs_sqlite_hash": "565f9b26a01db9cd18f9cc38580b56423ee6fe084e15a01deffa55c9a824720f" 25 | }, 26 | "24.11": { 27 | "name": "nixos-24.11.718735.5e70f6a10315", 28 | "url": "/nixos/24.11-small/nixos-24.11.718735.5e70f6a10315/nixexprs.tar.xz", 29 | "nixexprs_hash": "634b2ea66a5b4668608685d29459a768c8da0ada6c59e650fc78b8a1e16c7815", 30 | "programs_sqlite_hash": "8cbe59e316935aa29c7e6601a77b0ec3a06f65aa2968f3f388c559e505d3b8c0" 31 | }, 32 | "25.05": { 33 | "name": "nixos-25.05.803921.0cd4cf33252d", 34 | "url": "/nixos/25.05-small/nixos-25.05.803921.0cd4cf33252d/nixexprs.tar.xz", 35 | "nixexprs_hash": "62a45fca95aca2b190bdc64f5650bf1c43f60d128a0b210e41833b296444af76", 36 | "programs_sqlite_hash": "badebced6cdd83bceb0b2caa937b12310f6d26a5de432dec1f7333b61b93649a" 37 | }, 38 | "25.11": { 39 | "name": "nixos-25.11pre813869.0e5743dd17f7", 40 | "url": "/nixos/unstable-small/nixos-25.11pre813869.0e5743dd17f7/nixexprs.tar.xz", 41 | "nixexprs_hash": "e0bd002450e6429b83f2e219d532fdcf95dbb3340949efe5de117a0891c87dc0", 42 | "programs_sqlite_hash": "5c92abd8307e40bcccfffb07671b5d32f52e003022e4276ea2cf14ea87b6b429" 43 | } 44 | } -------------------------------------------------------------------------------- /module.nix: -------------------------------------------------------------------------------- 1 | { programs-sqlite }: ({ config, lib, ... }: 2 | with lib; 3 | let cfg = config.programs-sqlite; 4 | in { 5 | options.programs-sqlite = { 6 | enable = mkEnableOption "fetching a `programs.sqlite` for `command-not-found`" // 7 | { 8 | default = true; 9 | description = '' 10 | fetch a `programs.sqlite` file matching the current nixpks revision and use it for the 11 | `command-not-found` hook. 12 | ''; 13 | }; 14 | }; 15 | 16 | config = mkIf cfg.enable { 17 | assertions = [ 18 | { 19 | assertion = config.programs.command-not-found.enable; 20 | message = "Using programs.sqlite was requested but command-not-found itself is not enabled."; 21 | } 22 | ]; 23 | 24 | programs.command-not-found.dbPath = programs-sqlite; 25 | }; 26 | }) 27 | -------------------------------------------------------------------------------- /programs-sqlite.nix: -------------------------------------------------------------------------------- 1 | { lib, fetchurl, pkgs, rev }: 2 | let 3 | meta = (lib.importJSON ./sources.json)."${rev}" or (lib.importJSON ./latest.json).${lib.trivial.release}; 4 | in 5 | pkgs.stdenvNoCC.mkDerivation { 6 | pname = "programs-sqlite"; 7 | version = meta.name; 8 | 9 | src = fetchurl { 10 | url = "https://releases.nixos.org${meta.url}"; 11 | sha256 = meta.nixexprs_hash; 12 | }; 13 | dontConfigure = true; 14 | dontBuild = true; 15 | installPhase = '' 16 | cp programs.sqlite $out 17 | ''; 18 | outputHashAlgo = "sha256"; 19 | outputHashMode = "flat"; 20 | outputHash = meta.programs_sqlite_hash; 21 | } 22 | -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | tests/megatest* 2 | testresults/* 3 | nimcache/* 4 | outputExpected.txt 5 | outputGotten.txt 6 | -------------------------------------------------------------------------------- /src/tests/helpers/test_getchannel.nim: -------------------------------------------------------------------------------- 1 | import 2 | std/[times], 3 | ../../updater/updater 4 | 5 | discard """ """ 6 | 7 | assert getChannels(parse("2100-04-01", "yyyy-MM-dd")) == @[ 8 | "https://channels.nixos.org/nixos-unstable", 9 | "https://channels.nixos.org/nixos-99.05", 10 | "https://channels.nixos.org/nixos-99.11", 11 | "https://channels.nixos.org/nixos-unstable-small", 12 | "https://channels.nixos.org/nixos-99.05-small", 13 | "https://channels.nixos.org/nixos-99.11-small" 14 | ] 15 | assert getChannels(parse("2100-05-01", "yyyy-MM-dd")) == @[ 16 | "https://channels.nixos.org/nixos-unstable", 17 | "https://channels.nixos.org/nixos-99.05", 18 | "https://channels.nixos.org/nixos-99.11", 19 | "https://channels.nixos.org/nixos-unstable-small", 20 | "https://channels.nixos.org/nixos-99.05-small", 21 | "https://channels.nixos.org/nixos-99.11-small" 22 | ] 23 | assert getChannels(parse("2100-05-25", "yyyy-MM-dd")) == @[ 24 | "https://channels.nixos.org/nixos-unstable", 25 | "https://channels.nixos.org/nixos-99.05", 26 | "https://channels.nixos.org/nixos-99.11", 27 | "https://channels.nixos.org/nixos-00.05", 28 | "https://channels.nixos.org/nixos-unstable-small", 29 | "https://channels.nixos.org/nixos-99.05-small", 30 | "https://channels.nixos.org/nixos-99.11-small", 31 | "https://channels.nixos.org/nixos-00.05-small" 32 | ] 33 | assert getChannels(parse("2100-06-01", "yyyy-MM-dd")) == @[ 34 | "https://channels.nixos.org/nixos-unstable", 35 | "https://channels.nixos.org/nixos-99.11", 36 | "https://channels.nixos.org/nixos-00.05", 37 | "https://channels.nixos.org/nixos-unstable-small", 38 | "https://channels.nixos.org/nixos-99.11-small", 39 | "https://channels.nixos.org/nixos-00.05-small" 40 | ] 41 | assert getChannels(parse("2100-10-01", "yyyy-MM-dd")) == @[ 42 | "https://channels.nixos.org/nixos-unstable", 43 | "https://channels.nixos.org/nixos-99.11", 44 | "https://channels.nixos.org/nixos-00.05", 45 | "https://channels.nixos.org/nixos-unstable-small", 46 | "https://channels.nixos.org/nixos-99.11-small", 47 | "https://channels.nixos.org/nixos-00.05-small" 48 | ] 49 | assert getChannels(parse("2100-11-01", "yyyy-MM-dd")) == @[ 50 | "https://channels.nixos.org/nixos-unstable", 51 | "https://channels.nixos.org/nixos-99.11", 52 | "https://channels.nixos.org/nixos-00.05", 53 | "https://channels.nixos.org/nixos-unstable-small", 54 | "https://channels.nixos.org/nixos-99.11-small", 55 | "https://channels.nixos.org/nixos-00.05-small" 56 | ] 57 | assert getChannels(parse("2100-11-25", "yyyy-MM-dd")) == @[ 58 | "https://channels.nixos.org/nixos-unstable", 59 | "https://channels.nixos.org/nixos-99.11", 60 | "https://channels.nixos.org/nixos-00.05", 61 | "https://channels.nixos.org/nixos-00.11", 62 | "https://channels.nixos.org/nixos-unstable-small", 63 | "https://channels.nixos.org/nixos-99.11-small", 64 | "https://channels.nixos.org/nixos-00.05-small", 65 | "https://channels.nixos.org/nixos-00.11-small" 66 | ] 67 | assert getChannels(parse("2100-12-01", "yyyy-MM-dd")) == @[ 68 | "https://channels.nixos.org/nixos-unstable", 69 | "https://channels.nixos.org/nixos-00.05", 70 | "https://channels.nixos.org/nixos-00.11", 71 | "https://channels.nixos.org/nixos-unstable-small", 72 | "https://channels.nixos.org/nixos-00.05-small", 73 | "https://channels.nixos.org/nixos-00.11-small" 74 | ] 75 | -------------------------------------------------------------------------------- /src/tests/helpers/test_getmeta.nim: -------------------------------------------------------------------------------- 1 | import 2 | ../../updater/updater 3 | 4 | discard """ """ 5 | let 6 | html = """nixos-22.11-small release nixos-22.11.740.7a6a010c3a1

nixos-22.11-small release nixos-22.11.740.7a6a010c3a1

Released on 2022-12-09 11:35:15 from Git commit 7a6a010c3a1d00f8470a5ca888f2f927f1860a19 via Hydra evaluation 1787172.

File nameSizeSHA-256 hash
binary-cache-url23eb8d2a51d85521c07092ed8126df19bb6f9d7c16a5275395a0e110868a4ac6c0
git-revision40a1a32fbf326a730ce581e797a8ce78a013876b65104b69d8904ec31f37c6f105
nixexprs.tar.xz234746681a4947287bce87a5e6fc1e681673e93d2ca22eaeed9672da85038926305e37cf
nixos-minimal-22.11.740.7a6a010c3a1-aarch64-linux.iso88289075258bf5ebaabb2349fd0058ffa551340255fe9875604d472fd7d9fd75b2f21ddba
nixos-minimal-22.11.740.7a6a010c3a1-x86_64-linux.iso8671723522c5abbfeb0cb5700aa55efab1f62e7b37c4ff17893477f32b191a5d98677ae32
options.json.br6954467b1fee6540ef896e3a6ea7f77ccf559b1c562c5e95f58cf58c56b96fccc7928a
packages.json.br4538523950030eb2c2f83eae8fb380f35c4b706524d65bdb9bcd5dd9f04a7d186a53477
src-url361d75588d569a0e9191dfb2e66c11ef75b3a0e7bd4b0114dbffdba5e79ffa5a67
store-paths.xz4008bf75b4fbd88a6054fc50b66da23d51c7ba224b41fba346c967bd4d4ecd22a4c8
""" 7 | parsed = getMetadata(html) 8 | assert parsed of SqliteInfo 9 | assert parsed.name == "nixos-22.11.740.7a6a010c3a1" 10 | assert parsed.url == "https://releases.nixos.org/nixos/22.11-small/nixos-22.11.740.7a6a010c3a1/nixexprs.tar.xz" 11 | assert parsed.rev == "7a6a010c3a1d00f8470a5ca888f2f927f1860a19" 12 | assert parsed.nixexprs_hash == "1a4947287bce87a5e6fc1e681673e93d2ca22eaeed9672da85038926305e37cf" 13 | assert parsed.hash == "" 14 | -------------------------------------------------------------------------------- /src/updater.nimble: -------------------------------------------------------------------------------- 1 | # Package 2 | 3 | version = "0.3.0" 4 | author = "Markus S. Wamser" 5 | description = "A small tool to build lookup-tables for programs.sqlite from nixpkgs revisions" 6 | license = "MIT" 7 | srcDir = "updater" 8 | bin = @["updater"] 9 | 10 | 11 | # Dependencies 12 | 13 | requires "nim >= 1.6.10" 14 | requires "q >= 0.0.7" 15 | -------------------------------------------------------------------------------- /src/updater/updater.nim: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Markus S. Wamser 2 | # SPDX: MIT 3 | 4 | import std/[json, logging, options, os, parseopt, re, sequtils, strformat, strutils, tempfiles, times, xmltree] 5 | 6 | import osproc, streams 7 | 8 | import httpClient 9 | import q 10 | 11 | const 12 | channelBaseUrl = "https://channels.nixos.org/nixos-" 13 | releaseBaseUrl = "https://releases.nixos.org" 14 | jsonFile = "sources.json" 15 | jsonFileLatest = "latest.json" 16 | systemXZ = "xz" 17 | systemTar = "tar" 18 | systemSha256 = "sha256sum" 19 | systemShell = "/bin/sh" 20 | printDebugInfo = false 21 | 22 | type 23 | SqliteInfo* = object 24 | rev*: string # git revision 25 | name*: string # name, e.g. nixos-22.11.740.7a6a010c3a1 26 | url*: string # URL of nixexprs.tar.xz, e.g. /nixos/22.11-small/nixos-22.11.740.7a6a010c3a1/nixexprs.tar.xz 27 | nixexprs_hash*: string # SHA256 of nixexprs.tar.xz as given on webpage 28 | hash* : string # SHA256 of programs.sqlite 29 | 30 | var 31 | logger = newConsoleLogger() 32 | 33 | proc getChannels*(now: DateTime): seq[string] = 34 | # get the current and previous release plus unstable 35 | # also take care of the time window between branch-off and relase 36 | var 37 | chans: seq[string] = @["unstable"] 38 | chanURLs: seq[string] = @[] 39 | 40 | let 41 | y = int(now.year) %% 100 42 | ly = (int(now.year)-1) %% 100 43 | m = int(now.month) 44 | d = int(now.monthday) 45 | 46 | if m <= 5: # both releases from previous year 47 | chans.add(@[fmt"{ly:02}.05", fmt"{ly:02}.11"]) 48 | if m == 5 and d > 21: # .05 branch off, but not yet released 49 | chans.add(@[fmt"{y:02}.05"]) 50 | if 5 < m and m <= 11: # .05 from current year, .11 from previous year 51 | chans.add(@[fmt"{ly:02}.11", fmt"{y:02}.05"]) 52 | if m == 11 and d > 21: # .11 branch off, but not yet released 53 | chans.add(@[fmt"{y:02}.11"]) 54 | if m > 11: # both releases from current year 55 | chans.add(@[fmt"{y:02}.05", fmt"{y:02}.11"]) 56 | for suffix in ["", "-small"]: 57 | for chan in chans: 58 | chanURLs.add(channelBaseUrl & chan & suffix) 59 | return chanURLs 60 | 61 | proc stripTags(d: XmlNode): string = 62 | return ($d).multiReplace([(re"<[^>]*>", "")]) 63 | 64 | proc extractName(d: XmlNode): string = 65 | return d.stripTags().split()[2] 66 | 67 | proc extractNixexpr(xml: seq[XmlNode]) : array[2, string] = 68 | let 69 | nixexprsRow = xml.filterIt("nixexprs.tar.xz" in $it)[0] 70 | url = nixexprsRow.select("a")[0].attr("href") 71 | hash = nixexprsRow.select("tt")[0].stripTags() 72 | return [url, hash] 73 | 74 | proc fetchFile(channelUrl: string): Option[string] = 75 | var 76 | client = newHttpClient() 77 | try: 78 | return some(client.getContent(channelUrl)) 79 | except HttpRequestError as hce: 80 | logger.log(lvlWarn, "Unable to fetch " & channelUrl & " (" & hce.msg & ")") 81 | finally: 82 | client.close() 83 | 84 | proc getMetadata*(htmlRaw: string): SqliteInfo = 85 | # scrape the data from the website and build an SqliteInfo object 86 | var 87 | info: SqliteInfo 88 | 89 | let 90 | qHtmlRaw = q(htmlRaw) 91 | uh = (qHtmlRaw.select("tr+td+a")).extractNixexpr() 92 | 93 | info.name = (qHtmlRaw.select("h1")[0]).extractName() 94 | info.rev = (qHtmlRaw.select("p+a+tt")[0]).select("tt")[0].stripTags() 95 | info.url = uh[0] 96 | info.nixexprs_hash = uh[1] 97 | info.hash = "" # getting this is expensive, fill on demand later 98 | return info 99 | 100 | proc extractProgramsSqliteHash(tarball: string): string = 101 | let (cfile, path) = createTempFile("nixexpr_tgz_", "_end.tmp") 102 | cfile.write(tarball) 103 | close cfile 104 | let cmdLine = systemXZ & " --decompress --stdout " & path & " | " & systemTar & " -xf - " & " --wildcards */programs.sqlite -O | " & systemSha256 105 | let pout = execProcess(command = systemShell, args = ["-c", cmdLine], options = {poUsePath, poStdErrToStdOut}) 106 | if printDebugInfo: echo pout 107 | let hash = pout.split()[0] 108 | removeFile(path) 109 | 110 | return hash 111 | 112 | proc getHashfromNixexprs(info: SqliteInfo): Option[SqliteInfo] = 113 | let nixexprs = fetchFile(releaseBaseUrl & info.url) 114 | if nixexprs.isNone: 115 | return none(SqliteInfo) 116 | var updatedInfo = info 117 | if printDebugInfo: echo "Fetching " & releaseBaseUrl & info.url 118 | updatedInfo.hash = extractProgramsSqliteHash(nixexprs.get()) 119 | if printDebugInfo: echo "\\-> hash of programs.sqlite:" & updatedInfo.hash 120 | return some(updatedInfo) 121 | 122 | proc writeHelp() = 123 | echo "Run as: updater --dir:path-to-json\n or as: updater --dir:path-to-json --channel:channel-revision" 124 | quit(QuitSuccess) 125 | proc writeVersion() = 126 | echo "updater, v0.2.1" 127 | quit(QuitSuccess) 128 | 129 | when isMainModule: 130 | var 131 | positionalArgs = newSeq[string]() 132 | directories = newSeq[string]() 133 | requestedChannels = newSeq[string]() 134 | optparser = initOptParser(quoteShellCommand(commandLineParams())) 135 | 136 | for kind, key, val in optparser.getopt(): 137 | case kind 138 | of cmdArgument: 139 | discard 140 | of cmdLongOption, cmdShortOption: 141 | case key 142 | of "help", "h": writeHelp() 143 | of "version", "v": writeVersion() 144 | of "dir", "d": 145 | directories.add(val) 146 | of "channel", "c": 147 | requestedChannels.add(val) 148 | of cmdEnd: assert(false) # cannot happen 149 | 150 | if len(directories) > 1: 151 | echo "Only one dir argument allowed, ignoring all but first." 152 | if len(directories) < 1: 153 | directories.add(".") 154 | var jsonPath = directories[0] 155 | normalizePathEnd(jsonPath, trailingSep = true) 156 | 157 | if len(requestedChannels) < 1: 158 | requestedChannels.add(getChannels(now().utc)) 159 | 160 | let 161 | channels = requestedChannels.mapIt(fetchFile(it)).filterIt(it.isSome).mapIt(getMetadata(it.get())) 162 | sourcesJson = parseFile(jsonPath & jsonFile) 163 | sourcesLatestJson = parseFile(jsonPath & jsonFileLatest) 164 | 165 | var 166 | queuedInfos: seq[SqliteInfo] = @[] 167 | queuedRevs: seq[string] = @[] 168 | 169 | for c in channels: 170 | if sourcesJson{c.rev} == nil and c.rev notin queuedRevs: 171 | queuedInfos.add(c) 172 | queuedRevs.add(c.rev) 173 | 174 | let newInfos = queuedInfos.mapIt(getHashfromNixexprs(it)).filterIt(it.isSome).mapIt(it.get()).filterIt(len(it.hash) == 64 and match(it.hash, re"^[A-Fa-f\d]{64}$")) 175 | for c in newInfos: 176 | sourcesJson[c.rev] = %* {"name": c.name, "url": c.url, "nixexprs_hash": c.nixexprs_hash, "programs_sqlite_hash": c.hash} 177 | let release = c.name[6..10] # extract release number from "nixos-YY.mmSUFFIX" 178 | sourcesLatestJson[release] = %* {"name": c.name, "url": c.url, "nixexprs_hash": c.nixexprs_hash, "programs_sqlite_hash": c.hash} 179 | 180 | writeFile(jsonPath & jsonFile, pretty(sourcesJson)) 181 | writeFile(jsonPath & jsonFileLatest, pretty(sourcesLatestJson)) 182 | -------------------------------------------------------------------------------- /test.nix: -------------------------------------------------------------------------------- 1 | { pkgs, flake } : 2 | let 3 | 4 | # Single source of truth for all VM constants 5 | system = "x86_64-linux"; 6 | 7 | # NixOS module shared between server and client 8 | sharedModule = { 9 | virtualisation = { 10 | graphics = false; 11 | # jnsgruk suggests these limits 12 | cores = 2; 13 | memorySize = 5120; 14 | diskSize = 10240; 15 | }; 16 | programs.command-not-found.enable = true; 17 | environment.systemPackages = [ pkgs.gnugrep pkgs.coreutils ]; 18 | }; 19 | 20 | programs-sqlite-db = flake.packages.${system}.programs-sqlite; 21 | rev = flake.inputs.nixpkgs.rev; 22 | 23 | programs-sqlite-db-for-fallback-test = pkgs.callPackage ./programs-sqlite.nix { 24 | rev = "0000000000000000000000000000000000000000"; 25 | }; 26 | 27 | in 28 | # for the fallback test, we need a rev not in sources.json 29 | assert (false == (pkgs.lib.importJSON ./sources.json) ? revForFallbackTest); 30 | 31 | pkgs.nixosTest { 32 | name = "packages-sqlite-test"; 33 | nodes = { 34 | directConfig = { config, pkgs, ... }: { 35 | imports = [ sharedModule ]; 36 | users = { 37 | mutableUsers = false; 38 | users = { 39 | root.password = ""; 40 | }; 41 | }; 42 | programs.command-not-found.dbPath = programs-sqlite-db; 43 | }; 44 | 45 | moduleConfig = { config, pkgs, ... }: { 46 | imports = [ sharedModule flake.nixosModules.programs-sqlite ]; 47 | users = { 48 | mutableUsers = false; 49 | users = { 50 | root.password = ""; 51 | }; 52 | }; 53 | }; 54 | 55 | directConfigFallback = { config, pkgs, ... }: { 56 | imports = [ sharedModule ]; 57 | users = { 58 | mutableUsers = false; 59 | users = { 60 | root.password = ""; 61 | }; 62 | }; 63 | programs.command-not-found.dbPath = programs-sqlite-db-for-fallback-test; 64 | }; 65 | }; 66 | 67 | testScript = '' 68 | import json; 69 | 70 | def check(machine, expected_rev, db): 71 | with open(f"${flake.outPath}/{db}", "r") as f: 72 | hashes = json.load(f) 73 | 74 | machine.start() 75 | machine.wait_for_unit("multi-user.target") 76 | 77 | with subtest("check functionality"): 78 | err, response = machine.execute("command-not-found ponysay 2>&1") 79 | print(response) 80 | expected1 = "The program 'ponysay' is not in your PATH. You can make it available in an" 81 | expected2 = "ephemeral shell by typing:" 82 | expected3 = "nix-shell -p ponysay" 83 | assert expected1 in response and expected2 in response and expected3 in response, "command-not-found does nor work" 84 | 85 | with subtest("check database"): 86 | cnfsrc = machine.succeed("readlink $(which command-not-found)").strip() 87 | cnfdb = machine.succeed("grep -m 1 dbPath '" + cnfsrc + "' | cut -d '" + '"' + "' -f 2").strip() 88 | cnfdbhash = machine.succeed("sha256sum " + cnfdb + " | cut -d ' ' -f 1").strip() 89 | assert hashes.get(expected_rev).get("programs_sqlite_hash") == cnfdbhash, "incorrect programs.sqlite is used" 90 | 91 | machine.shutdown() 92 | 93 | check(directConfig, "${rev}", "sources.json") 94 | check(moduleConfig, "${rev}", "sources.json") 95 | check(directConfigFallback, "${pkgs.lib.trivial.release}", "latest.json") 96 | ''; 97 | } 98 | -------------------------------------------------------------------------------- /updater.nix: -------------------------------------------------------------------------------- 1 | { lib, pkgs, nimPackages, fetchurl, fetchFromGitHub, busybox, gnutar, xz }: 2 | 3 | let 4 | q = fetchFromGitHub { 5 | owner = "OpenSystemsLab"; 6 | repo = "q.nim"; 7 | rev = "0.0.8"; 8 | sha256 = "sha256-juYoPW1pIizSNeEf203gs/3zm64iHxzV41fKFeSuqaY="; 9 | }; 10 | in 11 | nimPackages.buildNimPackage rec { 12 | pname = "updater"; 13 | version = "0.3.0"; 14 | 15 | nimBinOnly = true; 16 | nimRelease = true; 17 | 18 | nimDefines = [ "ssl" ]; 19 | 20 | src = ./src; 21 | # set relative paths for dependencies, so they can be discovered in a nix bundle 22 | postPatch = '' 23 | substituteInPlace updater/updater.nim --replace 'systemXZ = "xz"' 'systemXZ = "${xz}/bin/xz"' 24 | substituteInPlace updater/updater.nim --replace 'systemTar = "tar"' 'systemTar = "${gnutar}/bin/tar"' 25 | substituteInPlace updater/updater.nim --replace 'systemSha256 = "sha256sum"' 'systemSha256 = "${busybox}/bin/sha256sum"' 26 | substituteInPlace updater/updater.nim --replace 'systemShell = "/bin/sh"' 'systemShell = "${busybox}/bin/sh"' 27 | ''; 28 | 29 | doCheck = true; 30 | checkPhase = ''testament all''; 31 | 32 | nativeBuildInputs = [ pkgs.removeReferencesTo ]; 33 | buildInputs = (with nimPackages; [ 34 | q 35 | ]) ++ 36 | [ pkgs.nim-unwrapped ]; # needs to be declared als buildInput, so the path is known for the postFixup 37 | 38 | propagatedBuildInputs = [ busybox gnutar xz ]; 39 | postFixup = '' 40 | remove-references-to -t ${pkgs.nim-unwrapped} $out/bin/updater 41 | ''; 42 | } 43 | --------------------------------------------------------------------------------