├── .direnv ├── .envrc ├── .github └── workflows │ ├── action-nix.yml │ ├── example.yml │ ├── tests-cache.yml │ ├── tests-empty.yml │ ├── tests-impure.yml │ ├── tests-stages.yml │ └── tests-tasks.yml ├── .gitignore ├── .gitmodules ├── README.md ├── actions ├── core.js └── nix │ ├── build │ ├── action.yml │ └── main.js │ ├── install │ ├── action.yml │ └── main.js │ └── run │ ├── action.yml │ └── main.js ├── default.nix ├── examples ├── .gitignore ├── ci.nix ├── deploy.nix ├── docs.nix └── example.sh ├── nix ├── actions-ci.nix ├── actions.nix ├── compat.nix ├── config.nix ├── default.nix ├── doc │ ├── man-pages.xml │ └── manual.xml ├── env.nix ├── exec.nix ├── global.nix ├── lib.nix ├── lib │ ├── bootstrap.nix │ ├── build │ │ ├── build.sh │ │ ├── cache.sh │ │ ├── default.nix │ │ ├── dirty.sh │ │ ├── realise.sh │ │ └── summarise.sh │ ├── channels.nix │ ├── cipkgs.nix │ ├── data.nix │ ├── env-builder.nix │ ├── env.nix │ ├── exec-ssh.nix │ ├── impure.nix │ ├── lib.nix │ ├── overlay.nix │ ├── scope.nix │ └── setup.nix ├── modules.nix ├── overlay.nix ├── pkgs.nix ├── project.nix ├── tasks.nix └── tools │ ├── default.nix │ ├── dirty.sh │ ├── install.sh │ └── query.sh ├── shell.nix ├── src └── tests ├── cache.nix ├── empty.nix ├── example.nix ├── impure.nix ├── stages.nix └── tasks.nix /.direnv: -------------------------------------------------------------------------------- 1 | if ! use nix; then 2 | export CI_ROOT=$PWD 3 | export CI_CONFIG_ROOT=$PWD 4 | export NIX_PATH=ci=$CI_ROOT:$NIX_PATH 5 | fi 6 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | export CI_ROOT=$PWD 2 | export CI_CONFIG_ROOT=$PWD 3 | 4 | use nix 5 | -------------------------------------------------------------------------------- /.github/workflows/action-nix.yml: -------------------------------------------------------------------------------- 1 | name: actions/nix 2 | on: 3 | pull_request: 4 | paths: 5 | - actions/*.js 6 | - actions/nix/** 7 | - .github/workflows/action-nix.yml 8 | - nix/tools/install.sh 9 | - nix/compat.nix 10 | - nix/lib/cipkgs.nix 11 | push: 12 | paths: 13 | - actions/*.js 14 | - actions/nix/** 15 | - .github/workflows/action-nix.yml 16 | - nix/tools/install.sh 17 | - nix/compat.nix 18 | - nix/lib/cipkgs.nix 19 | 20 | jobs: 21 | install: 22 | strategy: 23 | matrix: 24 | os: 25 | - ubuntu-latest 26 | - macos-latest 27 | nix-version: 28 | - latest 29 | - '2.3' 30 | - 2.13.6 31 | exclude: 32 | - os: macos-latest 33 | nix-version: '2.3' 34 | - os: macos-latest 35 | nix-version: 2.13.6 36 | runs-on: ${{ matrix.os }} 37 | steps: 38 | - uses: actions/checkout@v3 39 | - id: nix-install 40 | uses: ./actions/nix/install 41 | with: 42 | version: ${{ matrix.nix-version }} 43 | - run: nix --version 44 | - run: | 45 | if [[ $(nix --version | cut -d ' ' -f 3) != $NIX_VERSION ]]; then 46 | echo ::error::Installed nix version did not match expected $NIX_VERSION 47 | false 48 | fi 49 | env: 50 | NIX_VERSION: ${{ steps.nix-install.outputs.version }} 51 | install-nix-path: 52 | strategy: 53 | matrix: 54 | os: 55 | - ubuntu-latest 56 | - macos-latest 57 | runs-on: ${{ matrix.os }} 58 | steps: 59 | - uses: actions/checkout@v1 60 | - uses: ./actions/nix/install 61 | with: 62 | nix-path: nixpkgs=https://nixos.org/channels/nixos-24.11/nixexprs.tar.xz 63 | - uses: ./actions/nix/run 64 | with: 65 | attrs: nixpkgs.hello 66 | command: hello 67 | build: 68 | strategy: 69 | matrix: 70 | os: 71 | - ubuntu-latest 72 | - macos-latest 73 | nix2: 74 | - true 75 | - false 76 | runs-on: ${{ matrix.os }} 77 | steps: 78 | - uses: actions/checkout@v1 79 | - uses: ./actions/nix/install 80 | - uses: ./actions/nix/run 81 | with: 82 | attrs: nixpkgs.hello 83 | command: hello 84 | quiet: false 85 | - run: '! hello' 86 | - uses: ./actions/nix/build 87 | with: 88 | attrs: nixpkgs.hello 89 | add-path: true 90 | nix2: ${{ matrix.nix2 }} 91 | - run: hello 92 | -------------------------------------------------------------------------------- /.github/workflows/example.yml: -------------------------------------------------------------------------------- 1 | env: 2 | CI_ALLOW_ROOT: '1' 3 | CI_CONFIG: ./tests/example.nix 4 | CI_PLATFORM: gh-actions 5 | jobs: 6 | ci: 7 | name: example 8 | runs-on: ubuntu-latest 9 | steps: 10 | - id: checkout 11 | name: git clone 12 | uses: actions/checkout@v4 13 | with: 14 | submodules: true 15 | - id: nix-install 16 | name: nix install 17 | uses: ./actions/nix/install 18 | - id: ci-setup 19 | name: nix setup 20 | uses: ./actions/nix/run 21 | with: 22 | attrs: ci.run.bootstrap 23 | quiet: false 24 | - id: crex 25 | run: crex --help | lolcat --force 26 | - id: ci-dirty 27 | name: nix test dirty 28 | uses: ./actions/nix/run 29 | with: 30 | attrs: ci.run.test 31 | command: ci-build-dirty 32 | quiet: false 33 | stdout: ${{ runner.temp }}/ci.build.dirty 34 | - id: ci-test 35 | name: nix test build 36 | uses: ./actions/nix/run 37 | with: 38 | attrs: ci.run.test 39 | command: ci-build-realise 40 | ignore-exit-code: true 41 | quiet: false 42 | stdin: ${{ runner.temp }}/ci.build.dirty 43 | - env: 44 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 45 | id: ci-summary 46 | name: nix test results 47 | uses: ./actions/nix/run 48 | with: 49 | attrs: ci.run.test 50 | command: ci-build-summarise 51 | quiet: false 52 | stdin: ${{ runner.temp }}/ci.build.dirty 53 | stdout: ${{ runner.temp }}/ci.build.cache 54 | - env: 55 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 56 | id: ci-cache 57 | if: always() 58 | name: nix test cache 59 | uses: ./actions/nix/run 60 | with: 61 | attrs: ci.run.test 62 | command: ci-build-cache 63 | quiet: false 64 | stdin: ${{ runner.temp }}/ci.build.cache 65 | ci-check: 66 | name: example check 67 | runs-on: ubuntu-latest 68 | steps: 69 | - id: checkout 70 | name: git clone 71 | uses: actions/checkout@v4 72 | with: 73 | submodules: true 74 | - id: nix-install 75 | name: nix install 76 | uses: ./actions/nix/install 77 | - id: ci-action-build 78 | name: nix build ci.gh-actions.configFile 79 | uses: ./actions/nix/build 80 | with: 81 | attrs: ci.gh-actions.configFile 82 | out-link: .ci/workflow.yml 83 | - id: ci-action-compare 84 | name: gh-actions compare 85 | uses: ./actions/nix/run 86 | with: 87 | args: -u .github/workflows/example.yml .ci/workflow.yml 88 | attrs: nixpkgs.diffutils 89 | command: diff 90 | deploy: 91 | name: deploy 92 | runs-on: ubuntu-latest 93 | steps: 94 | - id: checkout 95 | name: git clone 96 | uses: actions/checkout@v4 97 | with: 98 | fetch-depth: 0 99 | submodules: true 100 | - id: nix-install 101 | name: nix install 102 | uses: ./actions/nix/install 103 | - id: ci-dirty 104 | name: nix test dirty 105 | uses: ./actions/nix/run 106 | with: 107 | attrs: ci.stage.deploy.run.test 108 | command: ci-build-dirty 109 | quiet: false 110 | stdout: ${{ runner.temp }}/ci.build.dirty 111 | - id: ci-test 112 | name: nix test build 113 | uses: ./actions/nix/run 114 | with: 115 | attrs: ci.stage.deploy.run.test 116 | command: ci-build-realise 117 | ignore-exit-code: true 118 | quiet: false 119 | stdin: ${{ runner.temp }}/ci.build.dirty 120 | - env: 121 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 122 | id: ci-summary 123 | name: nix test results 124 | uses: ./actions/nix/run 125 | with: 126 | attrs: ci.stage.deploy.run.test 127 | command: ci-build-summarise 128 | quiet: false 129 | stdin: ${{ runner.temp }}/ci.build.dirty 130 | stdout: ${{ runner.temp }}/ci.build.cache 131 | - env: 132 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 133 | id: ci-cache 134 | if: always() 135 | name: nix test cache 136 | uses: ./actions/nix/run 137 | with: 138 | attrs: ci.stage.deploy.run.test 139 | command: ci-build-cache 140 | quiet: false 141 | stdin: ${{ runner.temp }}/ci.build.cache 142 | docs: 143 | name: docs 144 | runs-on: ubuntu-latest 145 | steps: 146 | - id: checkout 147 | name: git clone 148 | uses: actions/checkout@v4 149 | with: 150 | fetch-depth: 0 151 | submodules: true 152 | - id: nix-install 153 | name: nix install 154 | uses: ./actions/nix/install 155 | - id: ci-dirty 156 | name: nix test dirty 157 | uses: ./actions/nix/run 158 | with: 159 | attrs: ci.stage.docs.run.test 160 | command: ci-build-dirty 161 | quiet: false 162 | stdout: ${{ runner.temp }}/ci.build.dirty 163 | - id: ci-test 164 | name: nix test build 165 | uses: ./actions/nix/run 166 | with: 167 | attrs: ci.stage.docs.run.test 168 | command: ci-build-realise 169 | ignore-exit-code: true 170 | quiet: false 171 | stdin: ${{ runner.temp }}/ci.build.dirty 172 | - env: 173 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 174 | id: ci-summary 175 | name: nix test results 176 | uses: ./actions/nix/run 177 | with: 178 | attrs: ci.stage.docs.run.test 179 | command: ci-build-summarise 180 | quiet: false 181 | stdin: ${{ runner.temp }}/ci.build.dirty 182 | stdout: ${{ runner.temp }}/ci.build.cache 183 | - env: 184 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 185 | id: ci-cache 186 | if: always() 187 | name: nix test cache 188 | uses: ./actions/nix/run 189 | with: 190 | attrs: ci.stage.docs.run.test 191 | command: ci-build-cache 192 | quiet: false 193 | stdin: ${{ runner.temp }}/ci.build.cache 194 | mac: 195 | name: example-mac 196 | runs-on: macos-13 197 | steps: 198 | - id: checkout 199 | name: git clone 200 | uses: actions/checkout@v4 201 | with: 202 | submodules: true 203 | - id: nix-install 204 | name: nix install 205 | uses: ./actions/nix/install 206 | - id: ci-setup 207 | name: nix setup 208 | uses: ./actions/nix/run 209 | with: 210 | attrs: ci.job.mac.run.bootstrap 211 | quiet: false 212 | - id: ci-dirty 213 | name: nix test dirty 214 | uses: ./actions/nix/run 215 | with: 216 | attrs: ci.job.mac.run.test 217 | command: ci-build-dirty 218 | quiet: false 219 | stdout: ${{ runner.temp }}/ci.build.dirty 220 | - id: ci-test 221 | name: nix test build 222 | uses: ./actions/nix/run 223 | with: 224 | attrs: ci.job.mac.run.test 225 | command: ci-build-realise 226 | ignore-exit-code: true 227 | quiet: false 228 | stdin: ${{ runner.temp }}/ci.build.dirty 229 | - env: 230 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 231 | id: ci-summary 232 | name: nix test results 233 | uses: ./actions/nix/run 234 | with: 235 | attrs: ci.job.mac.run.test 236 | command: ci-build-summarise 237 | quiet: false 238 | stdin: ${{ runner.temp }}/ci.build.dirty 239 | stdout: ${{ runner.temp }}/ci.build.cache 240 | - env: 241 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 242 | id: ci-cache 243 | if: always() 244 | name: nix test cache 245 | uses: ./actions/nix/run 246 | with: 247 | attrs: ci.job.mac.run.test 248 | command: ci-build-cache 249 | quiet: false 250 | stdin: ${{ runner.temp }}/ci.build.cache 251 | old: 252 | name: example-old 253 | runs-on: ubuntu-latest 254 | steps: 255 | - id: checkout 256 | name: git clone 257 | uses: actions/checkout@v4 258 | with: 259 | submodules: true 260 | - id: nix-install 261 | name: nix install 262 | uses: ./actions/nix/install 263 | - id: ci-setup 264 | name: nix setup 265 | uses: ./actions/nix/run 266 | with: 267 | attrs: ci.job.old.run.bootstrap 268 | quiet: false 269 | - id: ci-dirty 270 | name: nix test dirty 271 | uses: ./actions/nix/run 272 | with: 273 | attrs: ci.job.old.run.test 274 | command: ci-build-dirty 275 | quiet: false 276 | stdout: ${{ runner.temp }}/ci.build.dirty 277 | - id: ci-test 278 | name: nix test build 279 | uses: ./actions/nix/run 280 | with: 281 | attrs: ci.job.old.run.test 282 | command: ci-build-realise 283 | ignore-exit-code: true 284 | quiet: false 285 | stdin: ${{ runner.temp }}/ci.build.dirty 286 | - env: 287 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 288 | id: ci-summary 289 | name: nix test results 290 | uses: ./actions/nix/run 291 | with: 292 | attrs: ci.job.old.run.test 293 | command: ci-build-summarise 294 | quiet: false 295 | stdin: ${{ runner.temp }}/ci.build.dirty 296 | stdout: ${{ runner.temp }}/ci.build.cache 297 | - env: 298 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 299 | id: ci-cache 300 | if: always() 301 | name: nix test cache 302 | uses: ./actions/nix/run 303 | with: 304 | attrs: ci.job.old.run.test 305 | command: ci-build-cache 306 | quiet: false 307 | stdin: ${{ runner.temp }}/ci.build.cache 308 | script: 309 | name: example script 310 | runs-on: ubuntu-latest 311 | steps: 312 | - id: checkout 313 | name: git clone 314 | uses: actions/checkout@v4 315 | with: 316 | submodules: true 317 | - id: nix-install 318 | name: nix install 319 | uses: ./actions/nix/install 320 | - uses: actions/checkout@v3 321 | - name: example.sh 322 | run: ./example.sh 323 | working-directory: examples 324 | name: example 325 | 'on': 326 | - push 327 | - pull_request 328 | -------------------------------------------------------------------------------- /.github/workflows/tests-cache.yml: -------------------------------------------------------------------------------- 1 | env: 2 | CI_ALLOW_ROOT: '1' 3 | CI_CONFIG: ./tests/cache.nix 4 | CI_PLATFORM: gh-actions 5 | jobs: 6 | ci: 7 | name: tests-cache 8 | runs-on: ubuntu-latest 9 | steps: 10 | - id: checkout 11 | name: git clone 12 | uses: actions/checkout@v4 13 | with: 14 | submodules: true 15 | - id: nix-install 16 | name: nix install 17 | uses: ./actions/nix/install 18 | - id: ci-setup 19 | name: nix setup 20 | uses: ./actions/nix/run 21 | with: 22 | attrs: ci.run.setup 23 | quiet: false 24 | - id: ci-dirty 25 | name: nix test dirty 26 | uses: ./actions/nix/run 27 | with: 28 | attrs: ci.run.test 29 | command: ci-build-dirty 30 | quiet: false 31 | stdout: ${{ runner.temp }}/ci.build.dirty 32 | - id: ci-test 33 | name: nix test build 34 | uses: ./actions/nix/run 35 | with: 36 | attrs: ci.run.test 37 | command: ci-build-realise 38 | ignore-exit-code: true 39 | quiet: false 40 | stdin: ${{ runner.temp }}/ci.build.dirty 41 | - env: 42 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 43 | id: ci-summary 44 | name: nix test results 45 | uses: ./actions/nix/run 46 | with: 47 | attrs: ci.run.test 48 | command: ci-build-summarise 49 | quiet: false 50 | stdin: ${{ runner.temp }}/ci.build.dirty 51 | stdout: ${{ runner.temp }}/ci.build.cache 52 | - env: 53 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 54 | id: ci-cache 55 | if: always() 56 | name: nix test cache 57 | uses: ./actions/nix/run 58 | with: 59 | attrs: ci.run.test 60 | command: ci-build-cache 61 | quiet: false 62 | stdin: ${{ runner.temp }}/ci.build.cache 63 | ci-check: 64 | name: tests-cache check 65 | runs-on: ubuntu-latest 66 | steps: 67 | - id: checkout 68 | name: git clone 69 | uses: actions/checkout@v4 70 | with: 71 | submodules: true 72 | - id: nix-install 73 | name: nix install 74 | uses: ./actions/nix/install 75 | - id: ci-action-build 76 | name: nix build ci.gh-actions.configFile 77 | uses: ./actions/nix/build 78 | with: 79 | attrs: ci.gh-actions.configFile 80 | out-link: .ci/workflow.yml 81 | - id: ci-action-compare 82 | name: gh-actions compare 83 | uses: ./actions/nix/run 84 | with: 85 | args: -u .github/workflows/tests-cache.yml .ci/workflow.yml 86 | attrs: nixpkgs.diffutils 87 | command: diff 88 | name: tests-cache 89 | 'on': 90 | - push 91 | - pull_request 92 | -------------------------------------------------------------------------------- /.github/workflows/tests-empty.yml: -------------------------------------------------------------------------------- 1 | env: 2 | CI_ALLOW_ROOT: '1' 3 | CI_CONFIG: ./tests/empty.nix 4 | CI_PLATFORM: gh-actions 5 | jobs: 6 | ci: 7 | name: tests-empty 8 | runs-on: ubuntu-latest 9 | steps: 10 | - id: checkout 11 | name: git clone 12 | uses: actions/checkout@v4 13 | with: 14 | submodules: true 15 | - id: nix-install 16 | name: nix install 17 | uses: ./actions/nix/install 18 | - id: ci-dirty 19 | name: nix test dirty 20 | uses: ./actions/nix/run 21 | with: 22 | attrs: ci.run.test 23 | command: ci-build-dirty 24 | quiet: false 25 | stdout: ${{ runner.temp }}/ci.build.dirty 26 | - id: ci-test 27 | name: nix test build 28 | uses: ./actions/nix/run 29 | with: 30 | attrs: ci.run.test 31 | command: ci-build-realise 32 | ignore-exit-code: true 33 | quiet: false 34 | stdin: ${{ runner.temp }}/ci.build.dirty 35 | - env: 36 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 37 | id: ci-summary 38 | name: nix test results 39 | uses: ./actions/nix/run 40 | with: 41 | attrs: ci.run.test 42 | command: ci-build-summarise 43 | quiet: false 44 | stdin: ${{ runner.temp }}/ci.build.dirty 45 | stdout: ${{ runner.temp }}/ci.build.cache 46 | - env: 47 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 48 | id: ci-cache 49 | if: always() 50 | name: nix test cache 51 | uses: ./actions/nix/run 52 | with: 53 | attrs: ci.run.test 54 | command: ci-build-cache 55 | quiet: false 56 | stdin: ${{ runner.temp }}/ci.build.cache 57 | ci-check: 58 | name: tests-empty check 59 | runs-on: ubuntu-latest 60 | steps: 61 | - id: checkout 62 | name: git clone 63 | uses: actions/checkout@v4 64 | with: 65 | submodules: true 66 | - id: nix-install 67 | name: nix install 68 | uses: ./actions/nix/install 69 | - id: ci-action-build 70 | name: nix build ci.gh-actions.configFile 71 | uses: ./actions/nix/build 72 | with: 73 | attrs: ci.gh-actions.configFile 74 | out-link: .ci/workflow.yml 75 | - id: ci-action-compare 76 | name: gh-actions compare 77 | uses: ./actions/nix/run 78 | with: 79 | args: -u .github/workflows/tests-empty.yml .ci/workflow.yml 80 | attrs: nixpkgs.diffutils 81 | command: diff 82 | name: tests-empty 83 | 'on': 84 | - push 85 | - pull_request 86 | -------------------------------------------------------------------------------- /.github/workflows/tests-impure.yml: -------------------------------------------------------------------------------- 1 | env: 2 | CI_ALLOW_ROOT: '1' 3 | CI_CONFIG: ./tests/impure.nix 4 | CI_PLATFORM: gh-actions 5 | jobs: 6 | ci: 7 | name: tests-impure 8 | runs-on: ubuntu-latest 9 | steps: 10 | - id: checkout 11 | name: git clone 12 | uses: actions/checkout@v4 13 | with: 14 | submodules: true 15 | - id: nix-install 16 | name: nix install 17 | uses: ./actions/nix/install 18 | - id: ci-dirty 19 | name: nix test dirty 20 | uses: ./actions/nix/run 21 | with: 22 | attrs: ci.run.test 23 | command: ci-build-dirty 24 | quiet: false 25 | stdout: ${{ runner.temp }}/ci.build.dirty 26 | - id: ci-test 27 | name: nix test build 28 | uses: ./actions/nix/run 29 | with: 30 | attrs: ci.run.test 31 | command: ci-build-realise 32 | ignore-exit-code: true 33 | quiet: false 34 | stdin: ${{ runner.temp }}/ci.build.dirty 35 | - env: 36 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 37 | id: ci-summary 38 | name: nix test results 39 | uses: ./actions/nix/run 40 | with: 41 | attrs: ci.run.test 42 | command: ci-build-summarise 43 | quiet: false 44 | stdin: ${{ runner.temp }}/ci.build.dirty 45 | stdout: ${{ runner.temp }}/ci.build.cache 46 | - env: 47 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 48 | id: ci-cache 49 | if: always() 50 | name: nix test cache 51 | uses: ./actions/nix/run 52 | with: 53 | attrs: ci.run.test 54 | command: ci-build-cache 55 | quiet: false 56 | stdin: ${{ runner.temp }}/ci.build.cache 57 | ci-check: 58 | name: tests-impure check 59 | runs-on: ubuntu-latest 60 | steps: 61 | - id: checkout 62 | name: git clone 63 | uses: actions/checkout@v4 64 | with: 65 | submodules: true 66 | - id: nix-install 67 | name: nix install 68 | uses: ./actions/nix/install 69 | - id: ci-action-build 70 | name: nix build ci.gh-actions.configFile 71 | uses: ./actions/nix/build 72 | with: 73 | attrs: ci.gh-actions.configFile 74 | out-link: .ci/workflow.yml 75 | - id: ci-action-compare 76 | name: gh-actions compare 77 | uses: ./actions/nix/run 78 | with: 79 | args: -u .github/workflows/tests-impure.yml .ci/workflow.yml 80 | attrs: nixpkgs.diffutils 81 | command: diff 82 | name: tests-impure 83 | 'on': 84 | - push 85 | - pull_request 86 | -------------------------------------------------------------------------------- /.github/workflows/tests-stages.yml: -------------------------------------------------------------------------------- 1 | env: 2 | CI_ALLOW_ROOT: '1' 3 | CI_CONFIG: ./tests/stages.nix 4 | CI_PLATFORM: gh-actions 5 | jobs: 6 | another: 7 | name: another 8 | runs-on: ubuntu-latest 9 | steps: 10 | - id: checkout 11 | name: git clone 12 | uses: actions/checkout@v4 13 | with: 14 | submodules: true 15 | - id: nix-install 16 | name: nix install 17 | uses: ./actions/nix/install 18 | - id: ci-dirty 19 | name: nix test dirty 20 | uses: ./actions/nix/run 21 | with: 22 | attrs: ci.stage.another.run.test 23 | command: ci-build-dirty 24 | quiet: false 25 | stdout: ${{ runner.temp }}/ci.build.dirty 26 | - id: ci-test 27 | name: nix test build 28 | uses: ./actions/nix/run 29 | with: 30 | attrs: ci.stage.another.run.test 31 | command: ci-build-realise 32 | ignore-exit-code: true 33 | quiet: false 34 | stdin: ${{ runner.temp }}/ci.build.dirty 35 | - env: 36 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 37 | id: ci-summary 38 | name: nix test results 39 | uses: ./actions/nix/run 40 | with: 41 | attrs: ci.stage.another.run.test 42 | command: ci-build-summarise 43 | quiet: false 44 | stdin: ${{ runner.temp }}/ci.build.dirty 45 | stdout: ${{ runner.temp }}/ci.build.cache 46 | - env: 47 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 48 | id: ci-cache 49 | if: always() 50 | name: nix test cache 51 | uses: ./actions/nix/run 52 | with: 53 | attrs: ci.stage.another.run.test 54 | command: ci-build-cache 55 | quiet: false 56 | stdin: ${{ runner.temp }}/ci.build.cache 57 | ci: 58 | name: tests-stages 59 | runs-on: ubuntu-latest 60 | steps: 61 | - id: checkout 62 | name: git clone 63 | uses: actions/checkout@v4 64 | with: 65 | submodules: true 66 | - id: nix-install 67 | name: nix install 68 | uses: ./actions/nix/install 69 | - id: ci-dirty 70 | name: nix test dirty 71 | uses: ./actions/nix/run 72 | with: 73 | attrs: ci.run.test 74 | command: ci-build-dirty 75 | quiet: false 76 | stdout: ${{ runner.temp }}/ci.build.dirty 77 | - id: ci-test 78 | name: nix test build 79 | uses: ./actions/nix/run 80 | with: 81 | attrs: ci.run.test 82 | command: ci-build-realise 83 | ignore-exit-code: true 84 | quiet: false 85 | stdin: ${{ runner.temp }}/ci.build.dirty 86 | - env: 87 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 88 | id: ci-summary 89 | name: nix test results 90 | uses: ./actions/nix/run 91 | with: 92 | attrs: ci.run.test 93 | command: ci-build-summarise 94 | quiet: false 95 | stdin: ${{ runner.temp }}/ci.build.dirty 96 | stdout: ${{ runner.temp }}/ci.build.cache 97 | - env: 98 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 99 | id: ci-cache 100 | if: always() 101 | name: nix test cache 102 | uses: ./actions/nix/run 103 | with: 104 | attrs: ci.run.test 105 | command: ci-build-cache 106 | quiet: false 107 | stdin: ${{ runner.temp }}/ci.build.cache 108 | ci-check: 109 | name: tests-stages check 110 | runs-on: ubuntu-latest 111 | steps: 112 | - id: checkout 113 | name: git clone 114 | uses: actions/checkout@v4 115 | with: 116 | submodules: true 117 | - id: nix-install 118 | name: nix install 119 | uses: ./actions/nix/install 120 | - id: ci-action-build 121 | name: nix build ci.gh-actions.configFile 122 | uses: ./actions/nix/build 123 | with: 124 | attrs: ci.gh-actions.configFile 125 | out-link: .ci/workflow.yml 126 | - id: ci-action-compare 127 | name: gh-actions compare 128 | uses: ./actions/nix/run 129 | with: 130 | args: -u .github/workflows/tests-stages.yml .ci/workflow.yml 131 | attrs: nixpkgs.diffutils 132 | command: diff 133 | name: tests-stages 134 | 'on': 135 | - push 136 | - pull_request 137 | -------------------------------------------------------------------------------- /.github/workflows/tests-tasks.yml: -------------------------------------------------------------------------------- 1 | env: 2 | CI_ALLOW_ROOT: '1' 3 | CI_CONFIG: ./tests/tasks.nix 4 | CI_PLATFORM: gh-actions 5 | jobs: 6 | ci-check: 7 | name: tests-tasks check 8 | runs-on: ubuntu-latest 9 | steps: 10 | - id: checkout 11 | name: git clone 12 | uses: actions/checkout@v4 13 | with: 14 | submodules: true 15 | - id: nix-install 16 | name: nix install 17 | uses: ./actions/nix/install 18 | - id: ci-action-build 19 | name: nix build ci.gh-actions.configFile 20 | uses: ./actions/nix/build 21 | with: 22 | attrs: ci.gh-actions.configFile 23 | out-link: .ci/workflow.yml 24 | - id: ci-action-compare 25 | name: gh-actions compare 26 | uses: ./actions/nix/run 27 | with: 28 | args: -u .github/workflows/tests-tasks.yml .ci/workflow.yml 29 | attrs: nixpkgs.diffutils 30 | command: diff 31 | linux: 32 | name: tests-tasks-linux 33 | runs-on: ubuntu-latest 34 | steps: 35 | - id: checkout 36 | name: git clone 37 | uses: actions/checkout@v4 38 | with: 39 | submodules: true 40 | - id: nix-install 41 | name: nix install 42 | uses: ./actions/nix/install 43 | - id: ci-dirty 44 | name: nix test dirty 45 | uses: ./actions/nix/run 46 | with: 47 | attrs: ci.job.linux.run.test 48 | command: ci-build-dirty 49 | quiet: false 50 | stdout: ${{ runner.temp }}/ci.build.dirty 51 | - id: ci-test 52 | name: nix test build 53 | uses: ./actions/nix/run 54 | with: 55 | attrs: ci.job.linux.run.test 56 | command: ci-build-realise 57 | ignore-exit-code: true 58 | quiet: false 59 | stdin: ${{ runner.temp }}/ci.build.dirty 60 | - env: 61 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 62 | id: ci-summary 63 | name: nix test results 64 | uses: ./actions/nix/run 65 | with: 66 | attrs: ci.job.linux.run.test 67 | command: ci-build-summarise 68 | quiet: false 69 | stdin: ${{ runner.temp }}/ci.build.dirty 70 | stdout: ${{ runner.temp }}/ci.build.cache 71 | - env: 72 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 73 | id: ci-cache 74 | if: always() 75 | name: nix test cache 76 | uses: ./actions/nix/run 77 | with: 78 | attrs: ci.job.linux.run.test 79 | command: ci-build-cache 80 | quiet: false 81 | stdin: ${{ runner.temp }}/ci.build.cache 82 | mac: 83 | name: tests-tasks-mac 84 | runs-on: macos-latest 85 | steps: 86 | - id: checkout 87 | name: git clone 88 | uses: actions/checkout@v4 89 | with: 90 | submodules: true 91 | - id: nix-install 92 | name: nix install 93 | uses: ./actions/nix/install 94 | - id: ci-dirty 95 | name: nix test dirty 96 | uses: ./actions/nix/run 97 | with: 98 | attrs: ci.job.mac.run.test 99 | command: ci-build-dirty 100 | quiet: false 101 | stdout: ${{ runner.temp }}/ci.build.dirty 102 | - id: ci-test 103 | name: nix test build 104 | uses: ./actions/nix/run 105 | with: 106 | attrs: ci.job.mac.run.test 107 | command: ci-build-realise 108 | ignore-exit-code: true 109 | quiet: false 110 | stdin: ${{ runner.temp }}/ci.build.dirty 111 | - env: 112 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 113 | id: ci-summary 114 | name: nix test results 115 | uses: ./actions/nix/run 116 | with: 117 | attrs: ci.job.mac.run.test 118 | command: ci-build-summarise 119 | quiet: false 120 | stdin: ${{ runner.temp }}/ci.build.dirty 121 | stdout: ${{ runner.temp }}/ci.build.cache 122 | - env: 123 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 124 | id: ci-cache 125 | if: always() 126 | name: nix test cache 127 | uses: ./actions/nix/run 128 | with: 129 | attrs: ci.job.mac.run.test 130 | command: ci-build-cache 131 | quiet: false 132 | stdin: ${{ runner.temp }}/ci.build.cache 133 | mac-x86: 134 | name: tests-tasks-mac-x86 135 | runs-on: macos-13 136 | steps: 137 | - id: checkout 138 | name: git clone 139 | uses: actions/checkout@v4 140 | with: 141 | submodules: true 142 | - id: nix-install 143 | name: nix install 144 | uses: ./actions/nix/install 145 | - id: ci-dirty 146 | name: nix test dirty 147 | uses: ./actions/nix/run 148 | with: 149 | attrs: ci.job.mac-x86.run.test 150 | command: ci-build-dirty 151 | quiet: false 152 | stdout: ${{ runner.temp }}/ci.build.dirty 153 | - id: ci-test 154 | name: nix test build 155 | uses: ./actions/nix/run 156 | with: 157 | attrs: ci.job.mac-x86.run.test 158 | command: ci-build-realise 159 | ignore-exit-code: true 160 | quiet: false 161 | stdin: ${{ runner.temp }}/ci.build.dirty 162 | - env: 163 | CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }} 164 | id: ci-summary 165 | name: nix test results 166 | uses: ./actions/nix/run 167 | with: 168 | attrs: ci.job.mac-x86.run.test 169 | command: ci-build-summarise 170 | quiet: false 171 | stdin: ${{ runner.temp }}/ci.build.dirty 172 | stdout: ${{ runner.temp }}/ci.build.cache 173 | - env: 174 | CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} 175 | id: ci-cache 176 | if: always() 177 | name: nix test cache 178 | uses: ./actions/nix/run 179 | with: 180 | attrs: ci.job.mac-x86.run.test 181 | command: ci-build-cache 182 | quiet: false 183 | stdin: ${{ runner.temp }}/ci.build.cache 184 | name: tests-tasks 185 | 'on': 186 | - push 187 | - pull_request 188 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | result 2 | result-* 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "nix/lib/lib"] 2 | path = nix/lib/lib 3 | url = https://github.com/arcnmx/nixpkgs-lib.git 4 | branch = master 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ci 2 | 3 | [![ci-badge][]][ci] [![docs-badge][]][docs] 4 | 5 | A configurable continuous integration and testing system built on top of nix and 6 | the NixOS module system. 7 | 8 | 9 | ## Getting Started 10 | 11 | See the proper [documentation page][docs] for a full description. 12 | 13 | 14 | ### Quick Sample 15 | 16 | With [nix](https://nixos.org/nix/) installed... 17 | 18 | ```bash 19 | export NIX_PATH=ci=https://github.com/arcnmx/ci/archive/v0.7.tar.gz 20 | nix run --arg config '' -f '' test 21 | ``` 22 | 23 | 24 | ### Provider Support 25 | 26 | Though a simple command like the above can be run on any machine or CI service, 27 | automated configuration generators and full support for job descriptions and 28 | integrated features such as matrix builds are currently supported for: 29 | 30 | - [GitHub Actions](https://github.com/features/actions) 31 | 32 | 33 | [ci-badge]: https://github.com/arcnmx/ci/workflows/tests-tasks/badge.svg 34 | [ci]: https://github.com/arcnmx/ci/actions 35 | [docs-badge]: https://img.shields.io/badge/API-docs-blue.svg?style=flat-square 36 | [docs]: https://arcnmx.github.io/ci 37 | -------------------------------------------------------------------------------- /actions/core.js: -------------------------------------------------------------------------------- 1 | const process = require('process'); 2 | const os = require('os'); 3 | const fs = require('fs'); 4 | const crypto = require("crypto"); 5 | const { spawnSync } = require('child_process'); 6 | 7 | const envfile = process.env['GITHUB_ENV']; 8 | const pathfile = process.env['GITHUB_PATH']; 9 | const outputfile = process.env['GITHUB_OUTPUT']; 10 | const statefile = process.env['GITHUB_STATE']; 11 | const delim = crypto.randomBytes(32).toString('hex'); 12 | 13 | process.env['PWD'] = process.cwd(); 14 | 15 | function writeCommand(cmd) { 16 | // https://github.com/actions/runner/blob/6bec1e3bb832aad26f4ad5b64759a8e4d468df24/src/Runner.Common/ActionCommand.cs 17 | process.stdout.write(`${cmd}${os.EOL}`) 18 | } 19 | 20 | // Available actions: 21 | // https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions 22 | // https://github.com/actions/runner/blob/6bec1e3bb832aad26f4ad5b64759a8e4d468df24/src/Runner.Worker/ActionCommandManager.cs 23 | 24 | exports.error = function(msg) { 25 | writeCommand(`::error::${msg}`); 26 | }; 27 | 28 | exports.warning = function(msg) { 29 | writeCommand(`::warning::${msg}`); 30 | }; 31 | 32 | exports.setOutput = function(name, value) { 33 | if (outputfile) { 34 | fs.appendFileSync(outputfile, `${name}=${value}${os.EOL}`); 35 | } else { 36 | writeCommand(`::set-output name=${name}::${value}`); 37 | } 38 | }; 39 | 40 | exports.saveState = function(name, value) { 41 | fs.appendFileSync(statefile, `${name}=${value}${os.EOL}`); 42 | }; 43 | 44 | exports.addPath = function(path) { 45 | if (pathfile) { 46 | fs.appendFileSync(pathfile, `${path}${os.EOL}`); 47 | } else { 48 | writeCommand(`::add-path::${path}`); 49 | } 50 | // TODO: modify process.env like @actions/core does? 51 | }; 52 | 53 | exports.exportVariable = function(name, value) { 54 | if (envfile) { 55 | fs.appendFileSync(envfile, `${name}<<${delim}${os.EOL}${value}${os.EOL}${delim}`); 56 | } else { 57 | writeCommand(`::set-env name=${name}::${value}`); 58 | } 59 | // TODO: modify process.env like @actions/core does? 60 | }; 61 | 62 | exports.getInput = function(name) { 63 | return (process.env[`INPUT_${name.toUpperCase()}`] || '').trim(); 64 | }; 65 | 66 | exports.nix = { }; 67 | exports.nix.version = function() { 68 | const env_ver = process.env['NIX_VERSION']; 69 | if (env_ver) { 70 | return env_ver; 71 | } else { 72 | const res = spawnSync('nix', ['--version'], { 73 | windowsHide: true, 74 | stdio: [ 75 | 'ignore', 76 | 'pipe', 77 | 'inherit', 78 | ], 79 | }); 80 | if (res.error) { 81 | throw res.error; 82 | } else { 83 | const stdout = res.stdout.split(' '); 84 | if (stdout[0] === 'nix') { 85 | return stdout[2].trimRight(); 86 | } else { 87 | exports.error(`Unexpected nix --version output: ${res.stdout}`); 88 | throw 'unexpected'; 89 | } 90 | } 91 | } 92 | }; 93 | 94 | exports.nix.versionIs24 = function(version) { 95 | return version.localeCompare("2.4", undefined, { numeric: true, sensitivity: 'base' }) != -1 96 | } 97 | 98 | exports.nix.adjustFileAttrs = function(version, file, attrs) { 99 | if (file === '' && exports.nix.versionIs24(version)) { 100 | // compatibility from nix <2.4 101 | attrs.forEach(function(attr, index) { 102 | if (attr.includes('#')) { 103 | return; // assume flake reference 104 | } 105 | 106 | const attr_split = attr.split('.'); 107 | let [ first ] = attr_split.splice(0, 1); 108 | first = `<${first}>`; 109 | if (file === '') { 110 | file = first; 111 | } else if (file !== first) { 112 | exports.error(`cannot find common base with ${file} in ${attr}`); 113 | return; // let nix deal with it 114 | } 115 | this[index] = attr_split.join('.'); 116 | }, attrs); 117 | } 118 | 119 | return file; 120 | } 121 | -------------------------------------------------------------------------------- /actions/nix/build/action.yml: -------------------------------------------------------------------------------- 1 | name: nix build 2 | runs: 3 | using: node20 4 | main: main.js 5 | description: Build a nix derivation 6 | inputs: 7 | attrs: 8 | description: Attributes to build 9 | default: "" 10 | file: 11 | description: Path of nix expression to build, or a channel such as 12 | default: "" 13 | out-link: 14 | description: Output link to write 15 | default: "" 16 | add-path: 17 | description: Add the resulting build output to PATH 18 | default: "false" 19 | options: 20 | description: Extra options to pass to nix 21 | default: "" 22 | quiet: 23 | description: Silence logging 24 | default: "false" 25 | nix-path: 26 | description: Extra nix path arguments, colon-separated 27 | default: "" 28 | nix2: 29 | description: Use the nix 2.0 interface 30 | default: "true" 31 | outputs: 32 | out-link: 33 | description: Link created by the build 34 | branding: 35 | icon: hexagon 36 | color: blue 37 | -------------------------------------------------------------------------------- /actions/nix/build/main.js: -------------------------------------------------------------------------------- 1 | const core = require('../../core.js'); 2 | const process = require('process'); 3 | const { spawn } = require('child_process'); 4 | const path = require('path') 5 | const fs = require('fs') 6 | 7 | const compat = path.resolve(path.join(__dirname, '../../../nix/compat.nix')); 8 | 9 | const add_path = core.getInput('add-path') !== ''; 10 | const nix2 = core.getInput('nix2') !== 'false'; 11 | const quiet = core.getInput('quiet') !== 'false'; 12 | const nix_path = core.getInput('nix-path').split(':').filter(a => a !== ''); 13 | const nix_version = core.nix.version(); 14 | let file = core.getInput('file'); 15 | let attrs = core.getInput('attrs').split(' '); 16 | let options = core.getInput('options').split(' '); 17 | let out_link = core.getInput('out-link'); 18 | 19 | if (add_path && out_link === '') { 20 | do { 21 | out_link = `.ci-nix-bin/result-_${Math.random()}`; // make up a path :( 22 | } while (fs.existsSync(out_link)); 23 | } 24 | 25 | if (options.length === 1) { 26 | options = options.filter(o => o !== ''); 27 | } 28 | 29 | if (attrs.length === 1) { 30 | attrs = attrs.filter(a => a !== ''); 31 | } 32 | 33 | file = core.nix.adjustFileAttrs(nix_version, file, attrs); 34 | 35 | let args; 36 | let no_link; 37 | let arg0; 38 | if (nix2) { 39 | arg0 = 'nix'; 40 | no_link = '--no-link'; 41 | args = [ 42 | 'build', 43 | ].concat(file !== '' ? ['-f', file] : []) 44 | .concat(quiet ? [] : ['-L', '--show-trace']) 45 | .concat(attrs); 46 | } else { 47 | arg0 = 'nix-build'; 48 | no_link = '--no-out-link'; 49 | args = [ 50 | file !== '' ? file : compat 51 | ].concat(attrs.map(attr => ['-A', attr]).flat()) 52 | .concat(quiet ? ['-Q'] : []); 53 | } 54 | 55 | const builder = spawn(arg0, args 56 | .concat(nix_path.map(p => ['-I', p]).flat()) 57 | .concat(out_link === '' ? [no_link] : ['-o', out_link]) 58 | .concat(options), { 59 | env: Object.assign({}, process.env, { 60 | CI_PLATFORM: 'gh-actions', 61 | }), 62 | windowsHide: true, 63 | stdio: 'inherit', 64 | }); 65 | 66 | builder.on('close', (code) => { 67 | if (code === 0 && add_path) { 68 | core.addPath(path.join(path.resolve(out_link), 'bin')); 69 | core.setOutput('out-link', out_link); 70 | } 71 | process.exit(code); 72 | }); 73 | -------------------------------------------------------------------------------- /actions/nix/install/action.yml: -------------------------------------------------------------------------------- 1 | name: nix install 2 | runs: 3 | using: node20 4 | main: main.js 5 | description: Install nix in the GitHub Actions virtual environment. 6 | inputs: 7 | version: 8 | description: nix version to install 9 | default: latest # examples: latest, 2.3, 2.15.3 10 | timeout: 11 | description: minutes to wait before failing 12 | default: "5" 13 | daemon: 14 | description: Install nix in multi-user mode 15 | default: "false" 16 | github-access-token: 17 | description: The token nix uses when fetching from GitHub 18 | default: ${{ github.token }} 19 | nix-path: 20 | description: NIX_PATH channels to make available to the job 21 | default: "" # example: nixpkgs=https://nixos.org/channels/nixos-19.09/nixexprs.tar.xz 22 | outputs: 23 | version: 24 | description: The nix version installed 25 | nix-path: 26 | description: The NIX_PATH search directories 27 | branding: 28 | icon: hexagon 29 | color: blue 30 | -------------------------------------------------------------------------------- /actions/nix/install/main.js: -------------------------------------------------------------------------------- 1 | const core = require('../../core.js'); 2 | const process = require('process'); 3 | const { spawn } = require('child_process'); 4 | const path = require('path') 5 | 6 | const ci_root = path.resolve(path.join(__dirname, '../../..')); 7 | const installer_script = path.join(ci_root, 'nix/tools/install.sh'); 8 | 9 | // provide and a fallback nixpkgs matching the version of nix installed 10 | // TODO: option to turn this off? 11 | let nix_path = core.getInput('nix-path').split(':').filter(p => p !== ''); 12 | nix_path = nix_path.concat([`ci=${ci_root}`]); 13 | process.env['CI_NIX_PATH_NIXPKGS'] = '1'; // instruct script to add nixpkgs to NIX_PATH 14 | // TODO: if (nix_path.filter(p => p.startsWith('ci=') || !p.includes('=')).length === 0) ? 15 | process.env['NIX_INSTALLER'] = core.getInput('daemon') !== 'false' ? '--daemon' : ''; 16 | 17 | const gh_token = core.getInput('github-access-token'); 18 | if (gh_token !== '') { 19 | process.env['NIX_GITHUB_TOKEN'] = gh_token 20 | } 21 | 22 | const installer = spawn('bash', [installer_script], { 23 | env: Object.assign({}, process.env, { 24 | CI_PLATFORM: 'gh-actions', 25 | CI_ROOT: ci_root, 26 | NIX_VERSION: core.getInput('version'), 27 | NIX_PATH: nix_path.join(':'), 28 | }), 29 | windowsHide: true, 30 | stdio: 'inherit', 31 | }); 32 | 33 | installer.on('close', (code) => { 34 | if (code === 0) { 35 | } 36 | process.exit(code); 37 | }); 38 | 39 | const timeout = core.getInput('timeout'); 40 | if (timeout !== '') { 41 | setTimeout(() => { 42 | core.error('Nix installer timed out'); 43 | process.exit(1); 44 | }, timeout * 60 * 1000); 45 | } 46 | -------------------------------------------------------------------------------- /actions/nix/run/action.yml: -------------------------------------------------------------------------------- 1 | name: nix run 2 | runs: 3 | using: node20 4 | main: main.js 5 | description: Run a command with nix packages in the environment 6 | inputs: 7 | attrs: 8 | description: Attributes to build 9 | default: "" 10 | file: 11 | description: Path of nix expression to build, or a channel such as 12 | default: "" 13 | options: 14 | description: Extra options to pass to nix 15 | default: "" 16 | command: 17 | description: Command to run 18 | default: "" 19 | args: 20 | description: Command arguments 21 | default: "" 22 | stdout: 23 | description: Save stdout to the specified path 24 | default: "" 25 | stdin: 26 | description: Read stdin from the specified path 27 | default: "" 28 | ignore-exit-code: 29 | description: Ignore command failures 30 | default: "false" 31 | quiet: 32 | description: Silence logging 33 | default: "true" 34 | nix-path: 35 | description: Extra nix path arguments, colon-separated 36 | default: "" 37 | # TODO: option to tee stdout? 38 | outputs: 39 | exit-code: 40 | description: The code the process exited with 41 | branding: 42 | icon: hexagon 43 | color: blue 44 | -------------------------------------------------------------------------------- /actions/nix/run/main.js: -------------------------------------------------------------------------------- 1 | const core = require('../../core.js'); 2 | const process = require('process'); 3 | const { spawn } = require('child_process'); 4 | const path = require('path') 5 | const fs = require('fs') 6 | 7 | const quiet = core.getInput('quiet') !== 'false'; 8 | const nix_path = core.getInput('nix-path').split(':').filter(a => a !== ''); 9 | const ignore_exit = core.getInput('ignore-exit-code') !== 'false'; 10 | const stdout_path = core.getInput('stdout'); 11 | const stdin_path = core.getInput('stdin'); 12 | const nix_version = core.nix.version(); 13 | const nix2_4 = core.nix.versionIs24(nix_version); 14 | let file = core.getInput('file'); 15 | let command = core.getInput('command'); 16 | let cargs = core.getInput('args').split(' '); 17 | let attrs = core.getInput('attrs').split(' '); 18 | let options = core.getInput('options').split(' '); 19 | 20 | if (options.length === 1) { 21 | options = options.filter(o => o !== ''); 22 | } 23 | 24 | if (attrs.length === 1) { 25 | attrs = attrs.filter(a => a !== ''); 26 | } 27 | 28 | if (cargs.length === 1) { 29 | cargs = cargs.filter(a => a !== ''); 30 | } 31 | 32 | file = core.nix.adjustFileAttrs(nix_version, file, attrs); 33 | 34 | let stdout; 35 | if (stdout_path === '') { 36 | stdout = 'inherit'; 37 | } else { 38 | const fd = fs.openSync(stdout_path, 'w', 0o666); // TODO: append+mode options? 39 | stdout = fs.createWriteStream(stdout_path, { 40 | encoding: 'binary', 41 | fd: fd, 42 | }); 43 | } 44 | 45 | let stdin; 46 | if (stdin_path === '') { 47 | stdin = 'inherit'; 48 | } else { 49 | const fd = fs.openSync(stdin_path, 'r'); 50 | stdin = fs.createReadStream(stdin_path, { 51 | encoding: 'binary', 52 | fd: fd, 53 | }); 54 | } 55 | 56 | let cmd = 'run'; 57 | if (nix2_4 && command !== '') { 58 | cmd = 'shell'; 59 | } 60 | 61 | let args = [ 62 | cmd, 63 | ].concat(quiet ? [] : ['-L', '--show-trace']) 64 | .concat(file !== '' ? ['-f', file] : []) 65 | .concat(attrs) 66 | .concat(nix_path.map(p => ['-I', p]).flat()) 67 | .concat(options); 68 | 69 | if (cmd === 'shell' || !nix2_4) { 70 | if (command !== '' || cargs.length > 0) { 71 | if (command === '') { 72 | command = 'bash'; 73 | } 74 | args = args.concat(['-c', command]).concat(cargs); 75 | } 76 | } else if (cargs.length > 0) { 77 | args = args.concat(['--']).concat(cargs); 78 | } 79 | 80 | const builder = spawn('nix', args, { 81 | env: Object.assign({}, process.env, { 82 | CI_PLATFORM: 'gh-actions', 83 | }), 84 | windowsHide: true, 85 | stdio: [ 86 | stdin, 87 | stdout, 88 | 'inherit', 89 | ], 90 | }); 91 | 92 | builder.on('close', (code) => { 93 | core.setOutput('exit-code', code); 94 | process.exit(ignore_exit ? 0 : code); 95 | }); 96 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? if builtins.getEnv "CI_PLATFORM" == "impure" then true else null 2 | , config ? let env = builtins.getEnv "CI_CONFIG"; in if env != "" then env else null 3 | }@args: import ./nix { 4 | inherit pkgs; 5 | configuration = config; 6 | } 7 | -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | /result 2 | -------------------------------------------------------------------------------- /examples/ci.nix: -------------------------------------------------------------------------------- 1 | { pkgs, lib, config, channels, ... }: with lib; { 2 | name = "example"; 3 | ci.version = "v0.7"; 4 | 5 | # https://github.com/arcnmx/ci/actions?workflow=example 6 | ci.gh-actions.enable = true; 7 | 8 | channels = { 9 | # shorthands are available for common channels 10 | nixpkgs = "19.09"; 11 | # custom NIX_PATH, pinned channels, etc. 12 | nur.url = "https://github.com/nix-community/NUR/archive/master.tar.gz"; 13 | }; 14 | 15 | # Use a cache to remember which of our tests passed 16 | cache.cachix.ci.enable = true; 17 | 18 | # We can refer to a special pinned nixpkgs to make packages part of the base environment 19 | # These are meant to be readily available so that they can be used before caches are set up. 20 | environment.bootstrap = with channels.cipkgs; { 21 | inherit hello; 22 | }; 23 | 24 | # dependencies that can use custom caches and channels 25 | environment.test = let 26 | nur = channels.nur; 27 | in { 28 | inherit (pkgs) lolcat ncurses; 29 | inherit (nur.repos.dtz.pkgs) crex; 30 | }; 31 | 32 | tasks.hello = { 33 | # a task is a group of tests to run or packages to build 34 | name = "hello, world"; 35 | inputs = pkgs.ci.command { 36 | # commands run tests without necessarily generating any output, they either succeed or fail 37 | name = "hello"; 38 | displayName = "hihi"; 39 | command = '' 40 | hello | lolcat 41 | ''; 42 | }; 43 | }; 44 | 45 | jobs = { 46 | # additional jobs are submodules that can contain overrides to augment our config 47 | old = { 48 | channels.nixpkgs = mkForce "18.09"; 49 | }; 50 | mac = { 51 | system = "x86_64-darwin"; 52 | }; 53 | }; 54 | 55 | stages = { 56 | # stages on the other hand are fresh new sub-configs 57 | script = { 58 | gh-actions.jobs.script = { 59 | # just making sure the provided ./example.sh script works 60 | name = "example script"; 61 | steps = mkForce [ { 62 | uses = { 63 | owner = "actions"; 64 | repo = "checkout"; 65 | version = "v3"; 66 | }; 67 | } { 68 | name = "example.sh"; 69 | run = "./example.sh"; 70 | working-directory = "examples"; 71 | } ]; 72 | }; 73 | }; 74 | docs = ./docs.nix; 75 | deploy = ./deploy.nix; 76 | }; 77 | 78 | # May be necessary to include depending on what you're testing or building... 79 | environment.glibcLocales = [ channels.cipkgs.glibcLocales pkgs.glibcLocales ]; 80 | 81 | # we're adding a github-exclusive step here 82 | ci.gh-actions = { 83 | emit = true; # normally the existence of other jobs disables the default/implicit job 84 | export = true; # want our dependencies to be available in $PATH 85 | }; 86 | gh-actions.jobs = { 87 | ci.step.crex = { 88 | # using ci.gh-actions.export, we can also access the environment implicitly 89 | run = "crex --help | lolcat --force"; 90 | }; 91 | }; 92 | } 93 | -------------------------------------------------------------------------------- /examples/deploy.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, env, ... }: with lib; { 2 | ci.gh-actions.checkoutOptions.fetch-depth = 0; 3 | tasks = { 4 | smoke.inputs = [ 5 | (pkgs.ci.command { 6 | name = "pure"; 7 | command = '' 8 | echo hello from inside the $NIX_BUILD_TOP sandbox 9 | ''; 10 | }) 11 | ]; 12 | deploy = let 13 | inherit (import ../nix/lib/data.nix { }) ciRepoInfo; 14 | deploy-tag = pkgs.ci.command { 15 | name = "deploy-tag"; 16 | displayName = "deploy tag"; 17 | impure = true; 18 | skip = 19 | if env.platform != "gh-actions" || env.gh-event-name != "push" then env.gh-event-name or env.platform 20 | else if env.git-branch != ciRepoInfo.devBranch then "branch" 21 | else false; 22 | gitCommit = env.git-commit; 23 | gitTag = removePrefix "refs/tags/" ciRepoInfo.releaseRef; 24 | releaseTag = optionalString (ciRepoInfo.releaseRef != "refs/tags/${ciRepoInfo.releaseName}") ciRepoInfo.releaseName; 25 | smokeTest = config.tasks.smoke.drv; 26 | command = '' 27 | git tag -f $gitTag $gitCommit 28 | if [[ -n $releaseTag ]]; then 29 | git tag -f $releaseTag $gitTag 30 | fi 31 | git push -fq origin $releaseTag $gitTag 32 | ''; 33 | }; 34 | in { 35 | inputs = 36 | optional (hasPrefix "refs/tags/" ciRepoInfo.releaseRef) deploy-tag 37 | ++ optional (hasPrefix "refs/heads/" ciRepoInfo.releaseRef) (throw "TODO: release branch unsupported"); 38 | }; 39 | }; 40 | } 41 | -------------------------------------------------------------------------------- /examples/docs.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, env, ... }: with lib; { 2 | ci.gh-actions.checkoutOptions.fetch-depth = 0; 3 | tasks = { 4 | build.inputs = [ config.doc.manual ]; 5 | deploy = let 6 | inherit (import ../nix/lib/data.nix { }) ciRepoInfo; 7 | deploy-docs = pkgs.ci.command { 8 | name = "deploy-docs"; 9 | displayName = "deploy docs"; 10 | impure = true; 11 | skip = 12 | if env.platform != "gh-actions" || env.gh-event-name != "push" then env.gh-event-name or env.platform 13 | else if env.git-branch != ciRepoInfo.devBranch then "branch" 14 | else if !ciRepoInfo.latestVersion then "outdated" 15 | else false; 16 | gitCommit = env.git-commit; 17 | docsBranch = "gh-pages"; 18 | command = '' 19 | DOCDIR=$(mktemp -d) 20 | git fetch origin $docsBranch 21 | git worktree add $DOCDIR $docsBranch 22 | cd $DOCDIR 23 | 24 | cp -a ${config.doc.manual}/share/doc/ci/* ./ 25 | 26 | if [[ -n $(git status --porcelain) ]]; then 27 | git add -A . 28 | git config user.name ghost 29 | git config user.email ghost@konpa.ku 30 | git commit -m "manual of $gitCommit" 31 | 32 | git push -q origin HEAD:$docsBranch 33 | fi 34 | ''; 35 | }; 36 | in { 37 | inputs = [ deploy-docs ]; 38 | }; 39 | }; 40 | } 41 | -------------------------------------------------------------------------------- /examples/example.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -xeu 3 | unset NIX_PATH 4 | 5 | if ! command -v nix > /dev/null; then 6 | # install nix! 7 | NIX_VERSION=${NIX_VERSION-latest} 8 | export _NIX_INSTALLER_TEST=1 9 | sh <(curl https://nixos.org/releases/nix/$NIX_VERSION/install) --no-daemon 10 | fi 11 | 12 | # set up a known path where our environment goes 13 | export CI_ENV=$PWD/result 14 | export NIX_PATH="ci=${CI_ROOT-$PWD/../}" 15 | export CI_CONFIG_ROOT=$PWD 16 | export CI_CONFIG=./ci.nix 17 | 18 | # build the base/bootstrap environment and replace CI_ENV with final environment 19 | # this step installs dependencies from channels, can use additional caches, etc. 20 | bash -lc "nix run -L -f '' run.bootstrap" 21 | 22 | # environment ready to go at this point 23 | export BASH_ENV=$CI_ENV/ci/source 24 | bash -c "crex --help | lolcat --force" 25 | 26 | set +x 27 | # or from a single script 28 | source $BASH_ENV 29 | crex --help | lolcat --force 30 | -------------------------------------------------------------------------------- /nix/actions-ci.nix: -------------------------------------------------------------------------------- 1 | { config, lib, channels, configPath, ... }: with lib; let 2 | cfg = config.ci.gh-actions; 3 | action = name: if ! hasPrefix "http" config.ci.url 4 | then { 5 | path = "${config.ci.url}/actions/${name}"; 6 | } else { 7 | owner = "arcnmx"; 8 | repo = "ci"; 9 | version = config.ci.version; 10 | path = "actions/${name}"; 11 | }; 12 | subjobs = configs: (filterAttrs (_: v: v != null) (mapAttrs 13 | (k: configs: let 14 | job = configs.gh-actions.jobs.${configs.id} or null; 15 | job' = job // { 16 | env = filterAttrs (k: v: config.gh-actions.env.${k} or null != v) ( 17 | configs.gh-actions.env // job.env or {} 18 | ); 19 | }; 20 | in if job == null then null else job') configs 21 | )); 22 | ciJob = { 23 | id 24 | , name ? null 25 | , platform ? systems.elaborate config.channels.nixpkgs.args.system 26 | , step ? { } 27 | , steps ? [ ] 28 | , env ? { } 29 | }: { ${id} = { 30 | /* TODO: additional/manual setting overrides 31 | - Like say runs-on for testing different OS versions 32 | - Also ability to matrix without actually creating a whole new jobId? because... 33 | - it can be useful to test on old and new macOS versions for ex 34 | - ... but when running locally or on platforms that can't differentiate, there's no point in building the jobId multiple times! 35 | */ 36 | # TODO: needs (jobId deps) 37 | # TODO: if conditionals 38 | 39 | runs-on = mkDefault (if platform.isLinux then "ubuntu-latest" 40 | else if platform.isDarwin && platform.isAarch64 then "macos-latest" 41 | else if platform.isDarwin then "macos-13" 42 | else throw "unknown GitHub Actions platform for ${platform.system}"); 43 | inherit env; 44 | step = step // { 45 | checkout = { 46 | order = 10; 47 | name = "git clone"; 48 | uses = { 49 | owner = "actions"; 50 | repo = "checkout"; 51 | version = cfg.checkoutVersion; 52 | }; 53 | "with" = cfg.checkoutOptions; 54 | } // step.checkout or {}; 55 | nix-install = { 56 | order = 100; 57 | name = "nix install"; 58 | uses = action "nix/install"; 59 | } // step.nix-install or {}; 60 | }; 61 | inherit steps; 62 | } // optionalAttrs (name != null) { 63 | inherit name; 64 | }; }; 65 | in { 66 | options.ci.gh-actions = { 67 | enable = mkEnableOption "GitHub Actions CI"; 68 | checkoutOptions = mkOption { 69 | type = types.attrsOf types.unspecified; 70 | defaultText = ''{ submodules = true; }''; 71 | }; 72 | checkoutVersion = mkOption { 73 | type = types.str; 74 | default = "v4"; 75 | }; 76 | name = mkOption { 77 | type = types.str; 78 | default = foldl' (s: i: 79 | if i == null || i == s then s 80 | else if s == null then i 81 | else "${s}-${i}") null [ config.name config.stageId config.jobId ]; 82 | }; 83 | path = mkOption { 84 | type = types.nullOr types.str; 85 | default = ".github/workflows/${config.name}.yml"; 86 | }; 87 | export = mkOption { 88 | # export runtime test environment to the host 89 | type = types.bool; 90 | default = false; 91 | }; 92 | emit = mkOption { 93 | type = types.bool; 94 | default = config.jobs == { }; 95 | }; 96 | }; 97 | options.export.gh-actions = { 98 | configFile = mkOption { 99 | type = types.package; 100 | }; 101 | }; 102 | config.ci.gh-actions = { 103 | checkoutOptions = { 104 | submodules = mkOptionDefault true; 105 | }; 106 | }; 107 | config.export.gh-actions = { 108 | inherit (config.gh-actions) configFile; 109 | }; 110 | config.gh-actions = mkIf config.ci.gh-actions.enable { 111 | enable = true; 112 | env = mapAttrs (_: mkDefault) { 113 | CI_ALLOW_ROOT = "1"; 114 | #CI_CLOSE_STDIN = "1"; # TODO: is this necessary on actions or just azure pipelines? 115 | CI_PLATFORM = "gh-actions"; 116 | CI_CONFIG = config.ci.configPath; 117 | }; 118 | # TODO: on push/pull or on check? what is check? 119 | name = config.name; 120 | jobs = mkMerge [ 121 | (mkIf cfg.emit (ciJob { 122 | inherit (config) id; 123 | name = mkDefault cfg.name; 124 | step = { 125 | ci-setup = mkIf (cfg.export || any (c: c.enable && c.publicKey == null) (attrValues config.cache.cachix)) { 126 | order = 200; 127 | name = "nix setup"; 128 | uses = action "nix/run"; 129 | "with" = { 130 | attrs = "ci.${config.exportAttrDot}run.${if cfg.export then "bootstrap" else "setup"}"; 131 | quiet = false; 132 | }; 133 | }; 134 | }; 135 | steps = [ { # TODO: nix/build with export-path instead to avoid repeating evaluation 136 | id = "ci-dirty"; 137 | name = "nix test dirty"; 138 | uses = action "nix/run"; 139 | "with" = { 140 | attrs = "ci.${config.exportAttrDot}run.test"; 141 | command = "ci-build-dirty"; 142 | stdout = "\${{ runner.temp }}/ci.build.dirty"; 143 | quiet = false; 144 | }; 145 | } { 146 | id = "ci-test"; 147 | name = "nix test build"; 148 | uses = action "nix/run"; 149 | "with" = { 150 | attrs = "ci.${config.exportAttrDot}run.test"; 151 | command = "ci-build-realise"; 152 | stdin = "\${{ runner.temp }}/ci.build.dirty"; 153 | quiet = false; 154 | ignore-exit-code = true; 155 | }; 156 | } { 157 | id = "ci-summary"; 158 | name = "nix test results"; 159 | uses = action "nix/run"; 160 | "with" = { 161 | attrs = "ci.${config.exportAttrDot}run.test"; 162 | command = "ci-build-summarise"; 163 | stdin = "\${{ runner.temp }}/ci.build.dirty"; 164 | stdout = "\${{ runner.temp }}/ci.build.cache"; 165 | quiet = false; 166 | }; 167 | env.CI_EXIT_CODE = "\${{ steps.ci-test.outputs.exit-code }}"; 168 | } { 169 | id = "ci-cache"; 170 | name = "nix test cache"; 171 | uses = action "nix/run"; 172 | "with" = { 173 | attrs = "ci.${config.exportAttrDot}run.test"; 174 | command = "ci-build-cache"; 175 | stdin = "\${{ runner.temp }}/ci.build.cache"; 176 | quiet = false; 177 | }; 178 | "if" = "always()"; 179 | env.CACHIX_SIGNING_KEY = "\${{ secrets.CACHIX_SIGNING_KEY }}"; 180 | } ]; 181 | })) 182 | (subjobs config.jobs) 183 | (subjobs config.stages) 184 | (mkIf (config.jobId == null && config.ci.gh-actions.path != null) (ciJob { 185 | id = "${config.id}-check"; 186 | name = mkDefault "${cfg.name} check"; 187 | step = { 188 | # alternatively, run gh-actions-generate and check for unclean git repo instead? 189 | ci-action-build = { 190 | name = "nix build ci.gh-actions.configFile"; 191 | uses = action "nix/build"; 192 | "with" = { 193 | attrs = "ci.gh-actions.configFile"; 194 | out-link = ".ci/workflow.yml"; 195 | }; 196 | }; 197 | ci-action-compare = { 198 | name = "gh-actions compare"; 199 | uses = action "nix/run"; 200 | "with" = { 201 | attrs = "nixpkgs.diffutils"; 202 | command = "diff"; 203 | args = "-u ${config.ci.gh-actions.path} .ci/workflow.yml"; 204 | }; 205 | }; 206 | }; 207 | })) 208 | ]; 209 | }; 210 | config.export.run.gh-actions-generate = let 211 | gen = (channels.cipkgs.writeShellScriptBin "gh-actions-generate" '' 212 | install -Dm0644 ${config.export.gh-actions.configFile} ${cfg.path} 213 | '').overrideAttrs (old: { 214 | meta = old.meta or {} // { 215 | description = "generate or update the GitHub Actions workflow file"; 216 | }; 217 | }); 218 | in mkIf (cfg.enable && cfg.path != null) (config.lib.ci.nixRunWrapper "gh-actions-generate" gen); 219 | } 220 | -------------------------------------------------------------------------------- /nix/actions.nix: -------------------------------------------------------------------------------- 1 | { channels, config, lib, ... }: with lib; let 2 | filterEmpty = filterAttrs (_: v: v != null && v != { } && v != [ ]); 3 | cfg = config.gh-actions; 4 | containerType = types.submodule ({ name, ... }: { 5 | options = { 6 | image = mkOption { 7 | # TODO: submodule for version spec? 8 | type = types.str; 9 | default = name; 10 | }; 11 | env = mkOption { 12 | type = types.attrsOf types.str; 13 | default = { }; 14 | }; 15 | ports = mkOption { 16 | # TODO: submodule? can be ints, or "1234/tcp" 17 | type = types.listOf types.str; 18 | default = [ ]; 19 | }; 20 | volumes = mkOption { 21 | # TODO: submodule for this? can be /src:/dst or src_name:/dst or /sameSrcDst 22 | type = types.listOf types.str; 23 | default = [ ]; 24 | }; 25 | options = mkOption { 26 | type = types.nullOr types.str; 27 | default = null; 28 | }; 29 | }; 30 | }); 31 | actionType = types.submodule ({ config, ... }: { 32 | options = { 33 | owner = mkOption { 34 | type = types.nullOr types.str; 35 | default = null; 36 | }; 37 | repo = mkOption { 38 | type = types.nullOr types.str; 39 | default = null; 40 | }; 41 | slug = mkOption { 42 | type = types.nullOr types.str; 43 | }; 44 | path = mkOption { 45 | type = types.nullOr types.str; 46 | default = null; 47 | }; 48 | version = mkOption { 49 | type = types.str; 50 | default = "v1"; 51 | }; 52 | spec = mkOption { 53 | type = types.str; 54 | }; 55 | docker = mkOption { 56 | type = types.nullOr types.str; 57 | default = null; 58 | }; 59 | }; 60 | 61 | config = { 62 | slug = mkOptionDefault ( 63 | if config.repo != null then "${config.owner}/${config.repo}" else null); 64 | spec = mkOptionDefault ( 65 | if config.docker != null then "docker://${config.docker}" 66 | else if config.repo == null then config.path 67 | else "${config.slug}${optionalString (config.path != null) "/${config.path}"}@${config.version}"); 68 | }; 69 | }); 70 | stepType = { isList }: types.submodule ({ name, config, ... }: { 71 | options = { 72 | id = mkOption { 73 | type = types.nullOr types.str; 74 | default = if isList then null else name; 75 | }; 76 | name = mkOption { 77 | type = types.nullOr types.str; 78 | default = null; 79 | }; 80 | order = mkOption { 81 | type = (if isList then types.nullOr else id) types.int; 82 | default = if isList then null else 1000; 83 | }; 84 | "if" = mkOption { 85 | type = types.nullOr types.str; 86 | default = null; 87 | }; 88 | uses = mkOption { 89 | type = types.nullOr actionType; 90 | default = null; 91 | }; 92 | run = mkOption { 93 | type = types.nullOr types.str; 94 | default = null; 95 | }; 96 | shell = mkOption { 97 | type = types.nullOr (types.enum [ "bash" "pwsh" "python" "sh" "cmd" "powershell" ]); 98 | default = null; 99 | example = "bash"; 100 | }; 101 | "with" = let 102 | # mostly just anything that coerces to string 103 | valueType = foldl' types.either (types.nullOr types.bool) [ types.str types.int types.float ]; 104 | in mkOption { 105 | # TODO: abstract this away into an action option type 106 | # TODO: with.entrypoint and with.args are special? 107 | type = types.attrsOf valueType; 108 | default = { }; 109 | }; 110 | env = mkOption { 111 | type = types.attrsOf types.str; 112 | default = { }; 113 | }; 114 | working-directory = mkOption { 115 | type = types.nullOr types.str; 116 | default = null; 117 | }; 118 | continue-on-error = mkOption { 119 | type = types.bool; 120 | default = false; 121 | }; 122 | timeout-minutes = mkOption { 123 | type = types.nullOr types.ints.positive; 124 | default = null; 125 | }; 126 | shellTemplate = mkOption { 127 | type = types.nullOr types.str; 128 | default = null; 129 | example = "-xeu {0} scriptArg"; 130 | }; 131 | }; 132 | }); 133 | jobType = types.submodule ({ name, config, ... }: let 134 | sortedSteps = steps: foldl' (list: step: let 135 | prev = if list == [] then { order = 1000; } else last list; 136 | order = /*if prev.id or null != null 137 | then config.step.${prev.id}.order 138 | else*/ prev.order; 139 | in list ++ singleton (step // { 140 | order = (if step.order != null then step.order else (order + 10)); 141 | })) [] steps; 142 | in { 143 | options = { 144 | id = mkOption { 145 | type = types.str; 146 | default = name; 147 | }; 148 | name = mkOption { 149 | type = types.str; 150 | default = config.id; 151 | }; 152 | needs = let 153 | type = types.listOf types.str; 154 | fudge = types.coercedTo types.str singleton type; 155 | in mkOption { 156 | type = fudge; 157 | default = [ ]; 158 | }; 159 | runs-on = let 160 | githubHostedRunners = [ 161 | "ubuntu-latest" "ubuntu-22.04" "ubuntu-20.04" 162 | "windows-latest" "windows-2022" "windows-2019" 163 | "macos-latest" "macos-14" "macos-13" "macos-12" "macos-11" 164 | ]; 165 | in mkOption { 166 | type = types.oneOf [ 167 | (types.enum githubHostedRunners) 168 | (types.listOf types.str) 169 | types.str 170 | types.attrs 171 | ]; 172 | default = "ubuntu-latest"; 173 | }; 174 | permissions = mkOption { 175 | # TODO: proper types here 176 | type = types.unspecified; 177 | default = null; 178 | }; 179 | env = mkOption { 180 | type = types.attrsOf types.str; 181 | default = { }; 182 | }; 183 | "if" = mkOption { 184 | type = types.nullOr types.str; 185 | default = null; 186 | }; 187 | steps = mkOption { 188 | type = types.listOf (stepType { isList = true; }); 189 | default = [ ]; 190 | }; 191 | step = mkOption { 192 | type = types.attrsOf (stepType { isList = false; }); 193 | default = { }; 194 | }; 195 | timeout-minutes = mkOption { 196 | type = types.ints.positive; 197 | default = 360; 198 | }; 199 | strategy = { 200 | # TODO: complicated! 201 | matrix = mkOption { 202 | type = types.attrsOf types.unspecified; 203 | default = { }; 204 | }; 205 | fail-fast = mkOption { 206 | type = types.bool; 207 | default = true; 208 | }; 209 | max-parallel = mkOption { 210 | type = types.nullOr types.ints.positive; 211 | default = null; 212 | }; 213 | }; 214 | container = mkOption { 215 | type = types.nullOr containerType; 216 | default = null; 217 | }; 218 | services = mkOption { 219 | type = types.attrsOf containerType; 220 | default = { }; 221 | }; 222 | }; 223 | 224 | config.step = let 225 | steps = sortedSteps config.steps; 226 | in foldl' (steps: s: steps // { 227 | ${if s.id or null != null then s.id else "step${toString (length (attrNames steps))}"} = s; 228 | }) { } steps; 229 | }); 230 | in { 231 | options.gh-actions = { 232 | enable = mkEnableOption "GitHub Actions"; 233 | name = mkOption { 234 | type = types.str; 235 | }; 236 | on = mkOption { 237 | # TODO: proper types here 238 | type = types.unspecified; 239 | default = [ "push" "pull_request" ]; 240 | }; 241 | permissions = mkOption { 242 | # TODO: proper types here 243 | type = types.unspecified; 244 | default = null; 245 | }; 246 | env = mkOption { 247 | type = types.attrsOf types.str; 248 | default = { }; 249 | }; 250 | jobs = mkOption { 251 | type = types.attrsOf jobType; 252 | default = { }; 253 | }; 254 | configFile = mkOption { 255 | type = types.package; 256 | internal = true; 257 | }; 258 | }; 259 | config.gh-actions = mkIf config.gh-actions.enable { 260 | configFile = channels.cipkgs.stdenvNoCC.mkDerivation { 261 | name = "gh-actions.yml"; 262 | preferLocalBuild = true; 263 | allowSubstitutes = false; 264 | nativeBuildInputs = with channels.cipkgs; [ yq ]; 265 | data = builtins.toJSON (filterEmpty { 266 | inherit (cfg) name on permissions env; 267 | jobs = mapAttrs' (_: j: nameValuePair j.id (filterEmpty { 268 | inherit (j) name needs runs-on permissions env "if"; 269 | ${if j.timeout-minutes != 360 then "timeout-minutes" else null} = j.timeout-minutes; 270 | steps = map (s: filterEmpty { 271 | inherit (s) id name "if" run shell "with" env working-directory timeout-minutes shellTemplate; 272 | ${if s.uses != null then "uses" else null} = s.uses.spec; 273 | ${if s.continue-on-error then "continue-on-error" else null} = s.continue-on-error; 274 | }) (sort (l: r: l.order < r.order) (attrValues j.step)); 275 | } // optionalAttrs (j.strategy.matrix != { }) { 276 | matrix = filterEmpty { 277 | inherit (j.strategy) matrix max-parallel; 278 | ${if !s.fail-fast then "fail-fast" else null} = j.strategy.fail-fast; 279 | }; 280 | } // optionalAttrs (j.container != null) { 281 | container = filterEmpty { 282 | inherit (j.container) image env ports volumes options; 283 | }; 284 | } // optionalAttrs (j.services != { }) { 285 | services = mapAttrs (_: s: filterEmpty { 286 | inherit (s) image env ports volumes options; 287 | }) j.services; 288 | })) cfg.jobs; 289 | }); 290 | passAsFile = [ "data" "buildCommand" ]; 291 | buildCommand = '' 292 | yq ${optionalString (versionAtLeast (channels.cipkgs.yq.version or "2") "2.8.0") "--indentless-lists "}--yaml-output -c . $dataPath > $out 293 | ''; 294 | }; 295 | }; 296 | } 297 | -------------------------------------------------------------------------------- /nix/compat.nix: -------------------------------------------------------------------------------- 1 | with builtins; let 2 | # TODO: prefixless paths can be supported using readDir, or use the logic nix does: 3 | # 1. generate keys from the explicit prefixes 4 | # 2. search prefixless paths first before actually importing the explicit key 5 | toPath = path: if builtins.match "http.*" path != null 6 | then builtins.fetchTarball path # TODO: include hash if we know it for nixpkgs? 7 | else path; 8 | updateFlipped = a: b: b // a; # TODO: overrides should only happen if the path is invalid, store a list and try in order! 9 | remap = { prefix, path }: { ${prefix} = toPath path; }; 10 | prefixed = filter ({ prefix, ... }: prefix != "") nixPath; 11 | paths = foldl' updateFlipped {} (map remap prefixed); 12 | in mapAttrs (_: import) paths 13 | -------------------------------------------------------------------------------- /nix/config.nix: -------------------------------------------------------------------------------- 1 | { config, configPath, lib, ... }: with lib; let 2 | cfg = config.ci; 3 | inherit (import ./lib/data.nix { }) ciRepoInfo; 4 | in { 5 | options.ci = { 6 | version = mkOption { 7 | type = types.str; 8 | default = ciRepoInfo.releaseName; 9 | }; 10 | url = mkOption { 11 | type = types.str; 12 | default = "https://github.com/arcnmx/ci/archive/${cfg.version}.tar.gz"; 13 | }; 14 | configPath = mkOption { 15 | type = types.str; 16 | # TODO: check for config.ci.env.impure first? 17 | default = let 18 | root = builtins.getEnv "CI_CONFIG_ROOT"; 19 | pwd = builtins.getEnv "PWD"; 20 | configRoot = if root != "" then root else pwd; 21 | path = toString configPath; 22 | in if configRoot != "" && hasPrefix configRoot path 23 | then "." + removePrefix configRoot path 24 | else (import ./global.nix).defaultConfigPath; 25 | }; 26 | }; 27 | 28 | options.doc = { 29 | json = mkOption { 30 | type = types.unspecified; 31 | }; 32 | manPages = mkOption { 33 | type = types.unspecified; 34 | }; 35 | manual = mkOption { 36 | type = types.unspecified; 37 | }; 38 | open = mkOption { 39 | type = types.unspecified; 40 | }; 41 | }; 42 | 43 | options.export.doc = mkOption { 44 | type = types.unspecified; 45 | }; 46 | 47 | config.doc = { 48 | inherit (config.bootstrap.pkgs.ci.doc) manPages manual open json; 49 | }; 50 | config.export = { 51 | inherit (config) doc; 52 | }; 53 | } 54 | -------------------------------------------------------------------------------- /nix/default.nix: -------------------------------------------------------------------------------- 1 | { configuration 2 | , pkgs ? null 3 | , check ? true 4 | }@args: let 5 | inherit (builtins.import ./lib/cipkgs.nix) nixpkgsPath; 6 | #import = (builtins.import ./lib/scope.nix { }).nixPathImport { 7 | # nixpkgs = nixpkgsPath; 8 | #}; 9 | libPath = builtins.import ./lib/lib.nix + "/lib"; 10 | lib = builtins.import libPath; 11 | in with lib; let 12 | pkgs'args = { }; 13 | pkgs'path = builtins.tryEval (import ); 14 | pkgs = 15 | if args.pkgs or null == true && pkgs'path.success then pkgs'path.value pkgs'args 16 | else if args.pkgs or null == null || args.pkgs or null == true then builtins.import nixpkgsPath pkgs'args 17 | else args.pkgs; 18 | 19 | impureConfig = if configuration != null then configuration else (import ./global.nix).defaultConfigPath; 20 | impureConfigRoot = findFirst (v: v != "") null [ (builtins.getEnv "CI_CONFIG_ROOT") (builtins.getEnv "PWD") ]; 21 | relativePath = /. + (if impureConfigRoot != null && builtins.match "/.*" impureConfig == null 22 | then "${impureConfigRoot}/${impureConfig}" 23 | else impureConfig); 24 | configPath = 25 | if configuration == null && ! builtins.pathExists relativePath then warn "no CI configuration provided" ../tests/empty.nix 26 | else if configuration == null then relativePath 27 | else if builtins.typeOf configuration != "string" then configuration 28 | else if hasPrefix "/" configuration || impureConfigRoot != null then relativePath 29 | else throw "could not find configuration ${toString configuration}"; 30 | 31 | collectFailed = cfg: 32 | map (x: x.message) (filter (x: !x.assertion) cfg.assertions); 33 | showWarnings = res: let 34 | f = w: x: warn w x; 35 | in fold f res res.config.warnings; 36 | #nixosModulesPath = pkgs.path + "/nixos/modules"; 37 | 38 | rawModule = evalModules { 39 | modules = [ configPath ] ++ (builtins.import ./modules.nix { 40 | inherit check pkgs; 41 | }); 42 | specialArgs = { 43 | inherit /*nixosModulesPath*/ libPath; 44 | modulesPath = builtins.toString ./.; 45 | configPath = toString configPath; 46 | rootConfigPath = toString configPath; 47 | }; 48 | }; 49 | 50 | module = showWarnings (let 51 | failed = collectFailed rawModule.config; 52 | failedStr = concatStringsSep "\n" (map (x: "- ${x}") failed); 53 | in if failed == [] 54 | then rawModule 55 | else throw "\nFailed assertions:\n${failedStr}" 56 | ); 57 | in module.config.export // { 58 | inherit (module) options config; 59 | } 60 | -------------------------------------------------------------------------------- /nix/doc/man-pages.xml: -------------------------------------------------------------------------------- 1 | 4 | Reference Pages 5 | 6 | 7 | 8 | ci.nix 9 | CI configuration 10 | 11 | 12 | Description 13 | TBD 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /nix/doc/manual.xml: -------------------------------------------------------------------------------- 1 | 6 | 7 | CI/nix Manual 8 | 9 | 10 | CI/nix 11 | 12 | CI is a declarative continuous 13 | integration and testing system built on top of Nix and the NixOS module system. Its aim is to 14 | be flexible in order to meet the many needs of the various technologies and cloud providers in 15 | use with CI today, while being simple and self-contained so that all actions can be performed 16 | on a local machine. Build results should be reproducible so if it works on your machine then 17 | it will work on the CI server as well! 18 | 19 | 20 | Some of the project's goals and features are... 21 | 22 | 23 | 24 | The same reproducibility guarantees provided by the Nix and NixOS ecosystem, offering consistent builds that will produce the same results whether they run on your laptop or in the cloud. 25 | 26 | 27 | Matrix builds that run locally as well as on remote CI servers. 28 | 29 | 30 | Integration with the Nix store to cache and avoid repeating unnecessary work. 31 | 32 | 33 | Included escape hatches for tests, commands, and deployments that require the network or other resources that would normally be restricted by using Nix. 34 | 35 | 36 | Integration with existing CI providers while being agnostic to the underlying service, allowing the same configuration to work with multiple providers while still providing dashboards and results that reflect your configuration. 37 | 38 | 39 | Ability to publish results and perform follow-up actions and notifications without being constrained to any particular provider's capabilities. 40 | 41 | 42 | 43 | (this is currently all very much WIP right now, but please do file issues if you actually use it and have questions or concerns!) 44 | 45 | 46 | 47 | Setup 48 | 49 | To start, it's covenient to have the ci channel available to nix on your 50 | system: 51 | 52 | 53 | $ export NIX_PATH=$NIX_PATH:ci=https://github.com/arcnmx/ci/archive/v0.6.tar.gz 54 | 55 | 56 | This allows the channel to be easily referred to while testing locally, like so: 57 | 58 | 59 | $ nix run -f '<ci>' help 60 | $ nix run -f '<ci>' --arg config '<ci/examples/ci.nix>' test 61 | ✔️ hello, world ok 62 | 63 | 64 | You can also manage it with nix-channel, though due to a nix limitation you 65 | are still required to include the channel name in NIX_PATH in some way: 66 | 67 | 68 | $ nix-channel --add https://github.com/arcnmx/ci/archive/v0.6.tar.gz ci 69 | $ nix-channel --update 70 | $ export NIX_PATH=$NIX_PATH:ci=/xxx 71 | 72 | 73 | 74 | Commands 75 | 76 | Global and CI commands can be found with the list command: 77 | 78 | 79 | $ nix run -f '<ci>' list 80 | list 81 | run.gh-actions-generate 82 | run.test.example 83 | test 84 | 85 | 86 | Commands under a job or stage sub-configuration can be accessed by name via 87 | ci.job.jobName.run and ci.stage.name.run respectively. 88 | 89 | 90 | 91 | Examples 92 | 93 | Documentation is a bit sparse right now, but a few examples are currently available: 94 | 95 | 96 | 97 | Example and associated dashboard 98 | 99 | 100 | CI Tests 101 | 102 | 103 | rust channel tests 104 | 105 | 106 | 107 | 108 | Configuration 109 |
110 | Options 111 | 112 | lists all the options available for customizing your CI 113 | process! 114 | 115 |
116 |
117 | Environment 118 | 119 | documents the environment variables provided by and used for 120 | configuring the CI. It may be convenient to set some of these for a project in the relevant 121 | shell.nix or .direnv files. 122 | 123 |
124 | 125 | 126 | Configuration Options 127 | 128 | 129 | 130 | Environment Variables 131 |
132 | Input 133 | 134 | The following environment variables can be used to tweak CI behaviour and configuration. Some 135 | of these may be ignored when config.environment.impure = false. 136 | 137 | 138 | 139 | CI_CONFIG points to your CI configuration, if not provided via --arg config. 140 | 141 | 142 | CI_CONFIG_ROOT points to your project root. This is used to resolve relative CI_CONFIG paths when generating cloud service configurations. 143 | 144 | 145 | CI_PLATFORM indicates what environment the tests are currently running under. This should generally only be set when configuring a CI server, but the special impure value can be used to instruct the tests to use nixpkgs from your NIX_PATH environment. This may be useful when testing local changes or to avoid nix store bloat from using pinned channels. 146 | 147 | 148 | 149 | TBD: CI_ALLOW_ROOT, CI_CLOSE_STDIN 150 | 151 | 152 |
153 |
154 | Output 155 | 156 | Environment variables are also used to relay information about the build environment to the configuration, and can be accessed via builtins.getEnv. 157 | 158 | 159 | 160 | CI_ROOT points to the path of the CI channel currently being used for evaluation. 161 | 162 | 163 | TBD: CI_ENV, CI_PATH 164 | 165 | 166 |
167 | 168 | 169 | -------------------------------------------------------------------------------- /nix/env.nix: -------------------------------------------------------------------------------- 1 | { pkgs, lib, config, modulesPath, libPath, configPath, rootConfigPath, ... }@args: with lib; let 2 | channels = import ./lib/channels.nix lib; 3 | channelArgs = { 4 | inherit (config.lib) channelUrls; 5 | inherit pkgs; 6 | bootpkgs = config.nixpkgs.import; 7 | }; 8 | storePath = p: if builtins.getEnv "NIX_IGNORE_SYMLINK_STORE" != "1" 9 | then builtins.storePath p 10 | else if builtins.isPath p || builtins.hasContext p then p else /. + p; # there are some alternatives but... 11 | channelType = channels.channelTypeCoerced (channels.channelType (channelArgs // { 12 | inherit (config) channels; 13 | specialImport = config._module.args.import; 14 | ciOverlayArgs = args; 15 | defaultConfig = { 16 | nixpkgs = { 17 | args = with { mkDefault = config.lib.ci.mkOptionDefault1; }; { 18 | localSystem = mkDefault config.nixpkgs.args.localSystem; 19 | crossSystem = mkDefault config.nixpkgs.args.crossSystem; 20 | system = mkDefault config.nixpkgs.args.system; 21 | config = mapAttrs (_: mkDefault) config.nixpkgs.args.config; 22 | # TODO: overlays? 23 | crossOverlays = mkDefault config.nixpkgs.args.crossOverlays; 24 | stdenvStages = mkDefault config.nixpkgs.args.stdenvStages; 25 | }; 26 | path = config.lib.ci.mkOptionDefault1 config.nixpkgs.path; 27 | }; 28 | ci = { 29 | version = config.ci.version; 30 | path = toString ../.; 31 | }; 32 | nmd = { 33 | version = config.lib.ci.mkOptionDefault1 "4db11ab82c8a9fdecbc19d290d998c635afe225a"; 34 | sha256 = config.lib.ci.mkOptionDefault1 "1yn3i25bcpg0zs1yfw0az3d81hkj9gv3ygp9raic06702m59ba2r"; 35 | args = { 36 | pkgs = mkOptionDefault config.bootstrap.pkgs; 37 | inherit lib; 38 | }; 39 | }; 40 | } // mapAttrs (_: v: { version = mkDefault v; }) (optionalAttrs config.environment.impure (channelsFromEnv screamingSnakeCase "NIX_CHANNELS_")); 41 | })); 42 | nixpkgsType = channels.channelTypeCoerced (channels.channelType (channelArgs // { 43 | channels = { 44 | ci = { 45 | inherit (config.channels.ci) enable overlays; 46 | }; 47 | inherit (config.bootstrap) nixpkgs; 48 | }; 49 | specialImport = throw "nixPathImport unsupported"; 50 | isNixpkgs = true; 51 | ciOverlayArgs = args; 52 | defaultConfig.nixpkgs = { 53 | args.system = mkIf (config.system != null) (config.lib.ci.mkOptionDefault2 config.system); 54 | path = config.lib.ci.mkOptionDefault1 (config.lib.nixpkgsPathFor.${builtins.nixVersion} or config.lib.nixpkgsPathFor."24.11"); 55 | # TODO: defaults in url + sha256 instead? doesn't really matter... 56 | }; 57 | })); 58 | inherit (import ./lib/env.nix { inherit lib config; }) env envIsSet; 59 | channelsFromEnv = trans: prefix: filterAttrs (_: v: v != null) ( 60 | listToAttrs (map (ch: nameValuePair ch (env.get "${prefix}${trans ch}")) (attrNames config.lib.channelUrls)) 61 | ); 62 | screamingSnakeCase = s: builtins.replaceStrings [ "-" ] [ "_" ] (toUpper s); 63 | nixosCache = "https://cache.nixos.org/"; 64 | nixosKey = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY"; 65 | filteredSource = path: config.bootstrap.pkgs.nix-gitignore.gitignoreSourcePure [ 66 | "/.git" 67 | ] path; # TODO: name = "source"? 68 | bootstrapStorePath = v: storePath (/. + v + "/../.."); 69 | envBuilder = config.bootstrap.pkgs.buildPackages.callPackage (import ./lib/env-builder.nix) { inherit config; }; 70 | needsCache = any (c: c.url != nixosCache) (attrValues config.cache.substituters); 71 | needsCachix = any (c: c.enable && (c.publicKey == null || c.signingKey != null)) (attrValues config.cache.cachix); 72 | #envBuilder = { pname, packages, command ? "", ... }: throw "aaa"; 73 | in { 74 | options = { 75 | nix = let 76 | configValueType = with types; oneOf [ str bool int (listOf str) ]; 77 | in { 78 | corepkgs = { 79 | config = mkOption { 80 | type = types.nullOr (types.attrsOf types.unspecified); 81 | default = if versionOlder builtins.nixVersion "2.4" 82 | then import 83 | else null; # removed in nix 2.4 84 | defaultText = "import "; 85 | internal = true; 86 | }; 87 | }; 88 | experimental-features = mkOption { 89 | type = types.listOf types.str; 90 | }; 91 | config = mkOption { 92 | type = types.attrsOf configValueType; 93 | }; 94 | extraConfig = mkOption { 95 | type = types.lines; 96 | default = ""; 97 | }; 98 | configText = mkOption { 99 | type = types.lines; 100 | internal = true; 101 | }; 102 | configFile = mkOption { 103 | type = types.path; 104 | description = "/etc/nix/nix.conf"; 105 | }; 106 | settings = mkOption { 107 | type = types.attrsOf configValueType; 108 | }; 109 | extraSettings = mkOption { 110 | type = types.lines; 111 | default = ""; 112 | }; 113 | settingsText = mkOption { 114 | type = types.lines; 115 | internal = true; 116 | }; 117 | settingsFile = mkOption { 118 | type = types.path; 119 | description = "appended to $NIX_USER_CONF_FILES"; 120 | }; 121 | }; 122 | nixpkgs = mkOption { 123 | type = nixpkgsType; 124 | default = { }; 125 | }; 126 | system = mkOption { 127 | type = types.nullOr channels.systemType; 128 | default = null; 129 | }; 130 | bootstrap = { 131 | pkgs = mkOption { 132 | type = types.unspecified; 133 | default = config.nixpkgs.import; 134 | internal = true; 135 | }; 136 | runtimeShell = mkOption { 137 | type = types.path; 138 | default = if config.nix.corepkgs.config != null 139 | then storePath (/. + config.nix.corepkgs.config.shell) 140 | else config.bootstrap.pkgs.runtimeShell; 141 | defaultText = "corepkgs.shell"; 142 | internal = true; 143 | }; 144 | packages = { 145 | # nix appears to expect these to be available in PATH 146 | tar = mkOption { 147 | type = types.package; 148 | default = if config.nix.corepkgs.config != null 149 | then bootstrapStorePath config.nix.corepkgs.config.tar 150 | else config.bootstrap.pkgs.tar; 151 | defaultText = "corepkgs.tar"; 152 | visible = false; 153 | }; 154 | gzip = mkOption { 155 | type = types.package; 156 | default = if config.nix.corepkgs.config != null 157 | then bootstrapStorePath config.nix.corepkgs.config.gzip 158 | else config.bootstrap.pkgs.gzip; 159 | defaultText = "corepkgs.gzip"; 160 | visible = false; 161 | }; 162 | xz = mkOption { 163 | type = types.package; 164 | default = if config.nix.corepkgs.config != null 165 | then bootstrapStorePath config.nix.corepkgs.config.xz 166 | else config.bootstrap.pkgs.xz; 167 | defaultText = "corepkgs.xz"; 168 | visible = false; 169 | }; 170 | bzip2 = mkOption { 171 | type = types.package; 172 | default = if config.nix.corepkgs.config != null 173 | then bootstrapStorePath config.nix.corepkgs.config.bzip2 174 | else config.bootstrap.pkgs.bzip2; 175 | defaultText = "corepkgs.bzip2"; 176 | visible = false; 177 | }; 178 | shell = mkOption { 179 | type = types.package; 180 | default = if config.nix.corepkgs.config != null 181 | then bootstrapStorePath config.nix.corepkgs.config.shell 182 | else config.bootstrap.pkgs.bash; 183 | defaultText = "corepkgs.shell"; 184 | visible = false; 185 | }; 186 | coreutils = mkOption { 187 | type = types.package; 188 | default = if config.nix.corepkgs.config != null 189 | then storePath (/. + config.nix.corepkgs.config.coreutils + "/..") 190 | else config.bootstrap.pkgs.coreutils; 191 | defaultText = "corepkgs.coreutils"; 192 | visible = false; 193 | }; 194 | nix = mkOption { 195 | type = types.nullOr types.package; 196 | default = if config.nix.corepkgs.config != null 197 | then storePath (/. + config.nix.corepkgs.config.nixPrefix) 198 | else if env.isSet "NIX_BIN_DIR" then 199 | storePath (/. + env.get "NIX_BIN_DIR" + "/..") 200 | else null; 201 | defaultText = "corepkgs.nix"; 202 | }; 203 | cachix = mkOption { 204 | type = types.package; 205 | default = getBin config.bootstrap.pkgs.cachix; 206 | defaultText = "channels.cipkgs.cachix"; 207 | }; 208 | ci-dirty = mkOption { 209 | type = types.package; 210 | default = (import ./tools { inherit (config.bootstrap) pkgs; }).ci-dirty.override { 211 | inherit (config.bootstrap) runtimeShell; 212 | }; 213 | defaultText = "channels.cipkgs.ci-dirty"; 214 | visible = false; 215 | }; 216 | ci-query = mkOption { 217 | type = types.package; 218 | default = (import ./tools { inherit (config.bootstrap) pkgs; }).ci-query.override { 219 | inherit (config.bootstrap) runtimeShell; 220 | inherit (config.bootstrap.packages) nix; 221 | }; 222 | defaultText = "channels.cipkgs.ci-query"; 223 | visible = false; 224 | }; 225 | ci-build = mkOption { 226 | type = types.package; 227 | default = config.bootstrap.pkgs.runCommand "ci-build.sh" ({ 228 | scriptBuild = ./lib/build/build.sh; 229 | scriptDirty = ./lib/build/dirty.sh; 230 | scriptRealise = ./lib/build/realise.sh; 231 | scriptSummarise = ./lib/build/summarise.sh; 232 | scriptCache = ./lib/build/cache.sh; 233 | inherit (config.bootstrap.packages) nix; 234 | inherit (config.bootstrap.pkgs) gnugrep gnused; 235 | inherit (config.bootstrap) runtimeShell; 236 | inherit (config.lib.ci.op) sourceOps; 237 | cachix = optionalString needsCachix config.bootstrap.pkgs.cachix; 238 | } // config.lib.ci.colours) '' 239 | mkdir -p $out/bin 240 | substituteAll $scriptBuild $out/bin/ci-build 241 | substituteAll $scriptDirty $out/bin/ci-build-dirty 242 | substituteAll $scriptRealise $out/bin/ci-build-realise 243 | substituteAll $scriptSummarise $out/bin/ci-build-summarise 244 | substituteAll $scriptCache $out/bin/ci-build-cache 245 | chmod +x $out/bin/* 246 | ''; 247 | defaultText = "channels.cipkgs.ci-build"; 248 | visible = false; 249 | }; 250 | }; 251 | }; 252 | channels = mkOption { 253 | type = types.attrsOf channelType; 254 | }; 255 | nixPath = mkOption { 256 | type = types.attrsOf types.path; 257 | }; 258 | environment = { 259 | impure = mkOption { 260 | type = types.bool; 261 | default = true; 262 | }; 263 | allowRoot = mkOption { 264 | type = types.bool; 265 | default = envIsSet "CI_ALLOW_ROOT"; 266 | defaultText = ''getEnv "CI_ALLOW_ROOT" != ""''; 267 | }; 268 | closeStdin = mkOption { 269 | type = types.bool; 270 | default = envIsSet "CI_CLOSE_STDIN"; 271 | defaultText = ''getEnv "CI_CLOSE_STDIN" != ""''; 272 | }; 273 | workingDirectory = mkOption { 274 | type = types.path; 275 | default = env.getOr "/" "PWD"; 276 | defaultText = ''getEnv "PWD"''; 277 | }; 278 | glibcLocales = mkOption { 279 | type = types.listOf types.package; 280 | default = [ ]; 281 | }; 282 | bootstrap = mkOption { 283 | type = types.attrsOf types.package; 284 | defaultText = ''with pkgs; { inherit nix coreutils gzip tar xz bzip2 shell; }''; 285 | }; 286 | shell = mkOption { 287 | type = types.attrsOf types.package; 288 | defaultText = ''{ inherit (pkgs) less; }''; 289 | }; 290 | test = mkOption { 291 | type = types.attrsOf types.package; 292 | defaultText = ''config.environment.bootstrap''; 293 | }; 294 | }; 295 | export.env = { 296 | setup = mkOption { 297 | type = types.package; 298 | readOnly = true; 299 | }; 300 | bootstrap = mkOption { 301 | type = types.package; 302 | readOnly = true; 303 | }; 304 | test = mkOption { 305 | type = types.package; 306 | readOnly = true; 307 | }; 308 | shell = mkOption { 309 | type = types.package; 310 | readOnly = true; 311 | }; 312 | }; 313 | cache = let 314 | substituterType = types.submodule ({ ... }: { 315 | options = { 316 | url = mkOption { 317 | type = types.str; 318 | }; 319 | publicKeys = mkOption { 320 | type = types.listOf types.str; 321 | }; 322 | }; 323 | }); 324 | cachixType = types.submodule ({ name, ... }: { 325 | options = { 326 | enable = mkEnableOption "cachix cache" // { 327 | default = true; 328 | }; 329 | name = mkOption { 330 | type = types.str; 331 | default = name; 332 | }; 333 | publicKey = mkOption { 334 | type = types.nullOr types.str; 335 | default = null; 336 | }; 337 | signingKey = mkOption { 338 | type = types.nullOr types.str; 339 | default = if length (attrNames config.cache.cachix) == 1 then env.get "CACHIX_SIGNING_KEY" else null; 340 | }; 341 | }; 342 | }); 343 | in { 344 | substituters = mkOption { 345 | type = types.attrsOf substituterType; 346 | defaultText = nixosCache; 347 | }; 348 | cachix = mkOption { 349 | type = types.attrsOf cachixType; 350 | default = { }; 351 | }; 352 | }; 353 | }; 354 | config = { 355 | nix = let 356 | nixListSep = k: { 357 | builders = ":"; 358 | extra-builders = ":"; 359 | }.${k} or " "; 360 | toNixValue = k: v: 361 | if v == true then "true" 362 | else if v == false then "false" 363 | else if isList v then concatMapStringsSep (nixListSep k) (toNixValue k) v 364 | else toString v; 365 | toNixConf = settings: extra: mkMerge ( 366 | (mapAttrsToList (k: v: "${k} = ${toNixValue k v}") settings) 367 | ++ singleton extra 368 | ); 369 | substituters = mapAttrsToList (_: s: s.url) config.cache.substituters; 370 | publicKeys = concatLists (mapAttrsToList (_: s: s.publicKeys) config.cache.substituters); 371 | hasPublicKeys = any (s: s.publicKeys != []) (attrValues config.cache.substituters); 372 | in { 373 | config = mapAttrs (_: mkOptionDefault) { 374 | cores = 0; 375 | max-jobs = 8; 376 | http2 = false; 377 | fsync-metadata = false; 378 | max-silent-time = 60 * 30; 379 | substituters = [ nixosCache ] ++ optionals (config.cache.substituters != { }) substituters; 380 | trusted-public-keys = [ nixosKey ] ++ optionals hasPublicKeys publicKeys; 381 | trusted-users = let 382 | user = env.get "USER"; 383 | in [ "root" "@wheel" ] ++ optional (user != null) user; 384 | } // { 385 | ssl-cert-file = let 386 | sslCert = env.get "NIX_SSL_CERT_FILE"; 387 | in mkIf (sslCert != null) (mkOptionDefault sslCert); 388 | }; 389 | settings = { 390 | max-silent-time = mkOptionDefault config.nix.config.max-silent-time; 391 | accept-flake-config = mkIf (elem "flakes" config.nix.experimental-features) (mkOptionDefault true); 392 | extra-substituters = mkIf (config.cache.substituters != { }) substituters; 393 | extra-trusted-public-keys = mkIf hasPublicKeys publicKeys; 394 | extra-experimental-features = mkIf (config.nix.experimental-features != []) config.nix.experimental-features; 395 | extra-access-tokens = let 396 | ghToken = env.get "NIX_GITHUB_TOKEN"; 397 | in mkIf (ghToken != null) [ "github.com=${ghToken}" ]; 398 | }; 399 | experimental-features = optionals (versionAtLeast builtins.nixVersion "2.4") [ "nix-command" "flakes" "recursive-nix" ] 400 | ++ optional false "ca-derivations"; 401 | configText = toNixConf config.nix.config config.nix.extraConfig; 402 | configFile = mkOptionDefault (builtins.toFile "nix.conf" config.nix.configText); 403 | settingsText = toNixConf config.nix.settings config.nix.extraSettings; 404 | settingsFile = mkOptionDefault (builtins.toFile "nix.user.conf" config.nix.settingsText); 405 | }; 406 | environment = { 407 | bootstrap = mapAttrs (_: mkOptionDefault) (optionalAttrs (config.bootstrap.packages.nix != null) { 408 | inherit (config.bootstrap.packages) nix; 409 | } // optionalAttrs (config.nix.corepkgs.config != null) { 410 | inherit (config.bootstrap.packages) coreutils gzip xz bzip2 tar shell; 411 | } // optionalAttrs (needsCache) { 412 | inherit (config.bootstrap.packages) ci-query ci-dirty; 413 | } // optionalAttrs (needsCachix) { 414 | inherit (config.bootstrap.packages) cachix; 415 | }); 416 | shell = mapAttrs (_: mkOptionDefault) { 417 | inherit (config.bootstrap.pkgs) less; 418 | }; 419 | test = mapAttrs (_: mkOptionDefault) config.environment.bootstrap; 420 | }; 421 | export.env = { 422 | setup = envBuilder (import ./lib/setup.nix { inherit lib config; }); 423 | bootstrap = envBuilder (import ./lib/bootstrap.nix { inherit lib config; }); 424 | test = envBuilder { 425 | pname = "ci-env"; 426 | packages = attrValues config.environment.test; 427 | }; 428 | shell = config.export.env.test.override (old: { 429 | pname = "ci-shell"; 430 | packages = old.packages ++ builtins.attrValues config.environment.shell; 431 | }); 432 | }; 433 | cache.cachix = optionalAttrs (envIsSet "CACHIX_CACHE") { 434 | ${cachixCache} = { 435 | signingKey = env.get "CACHIX_SIGNING_KEY"; 436 | }; 437 | }; 438 | channels = { 439 | nixpkgs = mkOptionDefault { }; 440 | ci = mkOptionDefault { }; 441 | nmd = mkOptionDefault { }; 442 | }; 443 | # TODO: cipkgs? dunno, this ends up in the environment... but maybe that's fine? you can't get away with building an env without using cipkgs! 444 | nixPath = mapAttrs (_: c: config.lib.ci.storePathFor c.path) config.channels; 445 | cache.substituters = { 446 | nixos = { 447 | url = nixosCache; 448 | publicKeys = [ nixosKey ]; 449 | }; 450 | } // mapAttrs' (k: v: nameValuePair "${k}.cachix" { 451 | url = "https://${k}.cachix.org"; 452 | publicKeys = optional (v.publicKey != null) v.publicKey; 453 | }) (filterAttrs (_: c: c.enable) config.cache.cachix); 454 | 455 | lib = { 456 | channelUrls = channels.channelUrls { 457 | inherit (config.lib) nixpkgsChannels; 458 | inherit (config.lib.ci) githubChannel gitlabChannel; 459 | inherit (systems.elaborate config.nixpkgs.args.system) isDarwin; 460 | }; 461 | nixpkgsChannels = channels.nixpkgsChannels { 462 | inherit (config.lib) nixpkgsChannels; 463 | inherit (systems.elaborate config.nixpkgs.args.system) isLinux; 464 | }; 465 | nixpkgsPathFor = mapAttrs (_: builtins.fetchTarball) (import ./lib/cipkgs.nix).nixpkgsFor; 466 | ci = { 467 | inherit env; 468 | inherit (channels) githubChannel gitlabChannel; 469 | import = config.lib.ci.nixPathImport config.nixPath; 470 | mkOverrideAdj = mkOverride: adj: content: let 471 | res = mkOverride content; 472 | in res // { 473 | priority = res.priority + adj; 474 | }; 475 | mkOptionDefault1 = config.lib.ci.mkOverrideAdj mkOptionDefault (-100); 476 | mkOptionDefault2 = config.lib.ci.mkOverrideAdj config.lib.ci.mkOptionDefault1 (-100); 477 | storePathFor = path: if hasPrefix builtins.storeDir (toString path) || hasPrefix "/usr/local/nix" (toString path) 478 | then storePath path 479 | else if builtins.getEnv "CI_PLATFORM" == "impure" then toString path 480 | else filteredSource path; 481 | }; 482 | }; 483 | 484 | _module.args = { 485 | inherit (config.lib.ci) import env; 486 | channels = mapAttrs (_: c: c.import) config.channels // { 487 | cipkgs = config.nixpkgs.import; 488 | }; 489 | pkgs = mkOptionDefault config.channels.nixpkgs.import; 490 | }; 491 | }; 492 | } 493 | -------------------------------------------------------------------------------- /nix/exec.nix: -------------------------------------------------------------------------------- 1 | { pkgs, channels, lib, config, configPath, ... }: with lib; with config.lib.ci; let 2 | inherit (config.bootstrap.packages) ci-query ci-dirty; 3 | nixExe = exe: if config.bootstrap.packages.nix != null 4 | then "${config.bootstrap.packages.nix}/bin/${exe}" 5 | else "${exe}"; 6 | cfg = config.exec; 7 | tasks = mapAttrs (_: { drv, ... }: drv) config.tasks; 8 | nixRequiresDrvPath = versionAtLeast builtins.nixVersion "2.15.0"; 9 | in { 10 | options.exec = { 11 | useNix2 = mkOption { 12 | # 2.3 introduced --print-build-logs, and was unsuitable for CI prior to that 13 | type = types.bool; 14 | # TODO: check bootstrap.packages.nix version, which may differ? 15 | default = versionAtLeast builtins.nixVersion "2.3"; 16 | }; 17 | nixRealise = mkOption { 18 | type = types.enum [ "nix build" "nix-build" "nix-store" ]; 19 | default = if cfg.useNix2 then "nix build" else "nix-store"; 20 | }; 21 | verbosity = mkOption { 22 | # TODO: make quiet delay logs until after build completes? 23 | type = types.enum [ "build" "quiet" "silent" ]; 24 | default = "build"; 25 | }; 26 | list = { 27 | run = mkOption { 28 | type = types.attrsOf (types.nullOr types.str); 29 | default = { }; 30 | internal = true; 31 | }; 32 | }; 33 | }; 34 | options.export = { 35 | environment = mkOption { 36 | type = types.package; 37 | }; 38 | source = mkOption { 39 | type = types.lines; 40 | }; 41 | shell = mkOption { 42 | type = types.package; 43 | }; 44 | test = mkOption { 45 | type = types.unspecified; 46 | }; 47 | help = mkOption { 48 | type = types.unspecified; 49 | }; 50 | exec = mkOption { 51 | type = types.attrsOf types.unspecified; 52 | }; 53 | run = mkOption { 54 | type = types.attrsOf types.unspecified; 55 | }; 56 | list = mkOption { 57 | type = types.package; 58 | }; 59 | }; 60 | 61 | config.lib.ci = { 62 | colours = { 63 | red = ''$'\e[31m''\'''; 64 | green = ''$'\e[32m''\'''; 65 | yellow = ''$'\e[33m''\'''; 66 | blue = ''$'\e[34m''\'''; 67 | magenta = ''$'\e[35m''\'''; 68 | cyan = ''$'\e[36m''\'''; 69 | clear = ''$'\e[0m''\'''; 70 | }; 71 | op = { 72 | # TODO: add a --substituters flag for any caches mentioned in config? 73 | nixRealise = { 74 | nix-store = "${nixExe "nix-store"} ${optionalString (cfg.verbosity != "build") "-Q"} -r"; 75 | nix-build = "${nixExe "nix-build"} ${optionalString (cfg.verbosity != "build") "-Q"} --no-out-link"; 76 | "nix build" = "${nixExe "nix"} build ${optionalString (cfg.verbosity == "build") "-L"} --no-link"; 77 | }.${cfg.nixRealise}; 78 | query = drvImports: "${ci-query}/bin/ci-query -f ${drvImports}"; 79 | dirty = "${ci-dirty}/bin/ci-dirty"; 80 | realise = drvs: "${config.lib.ci.op.nixRealise} ${drvs}"; # --keep-going 81 | filter = drvImports: 82 | config.lib.ci.op.query drvImports 83 | + " | ${config.lib.ci.op.dirty}"; 84 | filterNoisy = drvImports: 85 | config.lib.ci.op.query drvImports 86 | + logpipe "${config.lib.ci.colours.magenta}::::: Dirty Derivations ::::: ${config.lib.ci.colours.green}" 87 | + " | ${config.lib.ci.op.dirty} -v" 88 | + logpipe "${config.lib.ci.colours.magenta}::::::::::::::::::::::::::::: ${config.lib.ci.colours.clear}"; 89 | buildDirty = drvImports: config.lib.ci.op.realise "$(cat <(${config.lib.ci.op.filter drvImports}))"; 90 | build = drvs: config.lib.ci.op.realise (drvPathsOf drvs); 91 | sourceOps = let 92 | transformDrv = { 93 | "nix build" = '' 94 | op_sep='^' 95 | op_outputs=('*') 96 | ''; 97 | nix-build = '' 98 | op_outputs=$(${nixExe "nix-instantiate"} --eval --strict --expr "toString (map (d: d.outputName) (import \"$op_in\").all)") 99 | op_outputs=''${op_outputs#\"} 100 | op_outputs=''${op_outputs%\"} 101 | op_outputs=($op_outputs) 102 | ''; 103 | nix-store = '' 104 | op_outputs=("") 105 | ''; 106 | }.${cfg.nixRealise}; 107 | transformDrvs = if nixRequiresDrvPath then '' 108 | local op_in op_sep='!' op_outputs op_output OP_IN=() 109 | for op_in in "$@"; do 110 | if [[ $op_in = *.drv ]]; then 111 | ${transformDrv} 112 | for op_output in "''${op_outputs[@]}"; do 113 | OP_IN+=("$op_in$op_sep$op_output") 114 | done 115 | else 116 | OP_IN+=("$op_in") 117 | fi 118 | done 119 | '' else '' 120 | local OP_IN=("$@") 121 | ''; 122 | realiseIn = config.lib.ci.op.realise "\${OP_IN[@]+\"\${OP_IN[@]}\"}" 123 | # we don't want it to spit out paths 124 | + optionalString (cfg.nixRealise != "nix build") " >/dev/null"; 125 | in '' 126 | function opFilter { 127 | ${config.lib.ci.op.filter "$1"} 128 | } 129 | function opFilterNoisy { 130 | ${config.lib.ci.op.filterNoisy "$1"} 131 | } 132 | 133 | function opRealise { 134 | ${transformDrvs} 135 | ${realiseIn} 136 | } 137 | ''; 138 | }; 139 | nixRunner = binName: channels.cipkgs.stdenvNoCC.mkDerivation { 140 | preferLocalBuild = true; 141 | allowSubstitutes = false; 142 | name = "nix-run-wrapper-${binName}"; 143 | meta.mainProgram = "run"; 144 | defaultCommand = "bash"; # `nix run` execvp's bash by default 145 | inherit binName; 146 | inherit (config.bootstrap) runtimeShell; 147 | passAsFile = [ "buildCommand" "script" ]; 148 | buildCommand = '' 149 | mkdir -p $out/bin 150 | substituteAll $scriptPath $out/bin/$defaultCommand 151 | chmod +x $out/bin/$defaultCommand 152 | ln -s $out/bin/$defaultCommand $out/bin/run 153 | ''; 154 | script = '' 155 | #!@runtimeShell@ 156 | set -eu 157 | 158 | if [[ -n ''${CI_NO_RUN-} ]]; then 159 | # escape hatch 160 | exec bash "$@" 161 | fi 162 | 163 | # also bail out if we're not called via `nix run` 164 | #PPID=($(@ps@/bin/ps -o ppid= $$)) 165 | #if [[ $(readlink /proc/$PPID/exe) = */nix ]]; then 166 | # exec bash "$@" 167 | #fi 168 | 169 | IFS=: PATHS=($PATH) 170 | join_path() { 171 | local IFS=: 172 | echo "$*" 173 | } 174 | 175 | # remove us from PATH 176 | OPATH=() 177 | for p in "''${PATHS[@]}"; do 178 | if [[ $p != @out@/bin ]]; then 179 | OPATH+=("$p") 180 | fi 181 | done 182 | export PATH=$(join_path "''${OPATH[@]}") 183 | 184 | exec @binName@ "$@" ''${CI_RUN_ARGS-} 185 | ''; 186 | }; 187 | nixRunWrapper = binName: package: channels.cipkgs.stdenvNoCC.mkDerivation { 188 | name = "nix-run-${binName}"; 189 | preferLocalBuild = true; 190 | allowSubstitutes = false; 191 | wrapper = config.lib.ci.nixRunner binName; 192 | inherit package; 193 | buildCommand = '' 194 | mkdir -p $out/nix-support 195 | echo $package $wrapper > $out/nix-support/propagated-user-env-packages 196 | if [[ -e $package/bin ]]; then 197 | ln -s $package/bin $out/bin 198 | fi 199 | ''; 200 | meta = package.meta or {} // { 201 | mainProgram = binName; 202 | }; 203 | passthru = package.passthru or {}; 204 | }; 205 | drvOf = drv: builtins.unsafeDiscardStringContext drv.drvPath; 206 | drvOutputs = drv: drv.outputs or (map (drv: drv.outputName) drv.all or (import (drvOf drv)).all); 207 | drvPathOutputs = drv: let 208 | drvPath = drvOf drv; 209 | in if !nixRequiresDrvPath then [ drvPath ] else { 210 | "nix build" = [ "${drvPath}^*" ]; 211 | nix-build = map (outputName: "${drvPath}!${outputName}") (drvOutputs drv); 212 | nix-store = [ "${drvPath}!" ]; 213 | }.${cfg.nixRealise}; 214 | drvPathsOf = drvs: escapeShellArgs (concatMap drvPathOutputs drvs); 215 | buildDrvs = drvs: "${config.lib.ci.op.nixRealise} ${drvPathsOf drvs}"; 216 | buildDrv = drv: buildDrvs [drv]; 217 | logpipe = msg: " | (cat && echo ${msg} >&2)"; 218 | #logpipe = msg: " | (${config.bootstrap.packages.coreutils}/bin/tee >(cat >&2) && echo ${msg} >&2)"; 219 | drvImports = drvs: builtins.toFile "drvs.nix" ''[ 220 | ${builtins.concatStringsSep "\n" (map (d: "(import ${drvOf d})") drvs)} 221 | ]''; 222 | buildAnd = drvs: run: 223 | ''${buildDrvs drvs} > /dev/null && ${run}''; 224 | buildAndRun = drvs: 225 | buildAnd drvs "${nixExe "nix"} run ${toString drvs}"; 226 | taskDrvs = builtins.attrValues tasks; 227 | taskDrvImports = drvImports taskDrvs; 228 | buildTask = task: let 229 | drv = if builtins.isString task then config.tasks.${task} else task; 230 | in config.lib.ci.op.build [drv.drv]; 231 | toNix = val: with builtins; # honestly why not just use from/to json? 232 | if isString val then ''"${val}"'' 233 | else if builtins ? isPath && builtins.isPath val then ''${toString val}'' 234 | else if typeOf val == "path" then ''${toString val}'' 235 | else if isList val then ''[ ${concatStringsSep " " (map toNix val)} ]'' 236 | else if isAttrs val then ''{ 237 | ${concatStringsSep "\n" (map (key: ''"${key}" = ${toNix val.${key}};'') (attrNames val))} 238 | }'' 239 | else if isBool val && val then "true" else if isBool then "false" 240 | else if isInt val || isFloat val then toString val 241 | else if val == null then "null" 242 | else throw "unknown nix value ${toString val}"; 243 | cacheInputsOf = drv: let 244 | buildInputs = throw "cache.buildInputs unimplemented"; 245 | default = ! isFunction drv.ci.cache or null; 246 | # TODO: do this recursively over all inputs? 247 | in optional (drv.ci.cache.enable or default && drv.allowSubstitutes or true) drv 248 | ++ optionals (drv.ci.cache.buildInputs or false) buildInputs # TODO: make this true by default? 249 | ++ optionals (isFunction drv.ci.cache or null) (drv.ci.cache drv) 250 | ++ concatMap cacheInputsOf (drv.ci.cache.inputs or []); 251 | commandExecutor = { 252 | stdenv ? channels.cipkgs.stdenvNoCC 253 | , drv 254 | , executor 255 | }: let 256 | exDrv = stdenv.mkDerivation (executor.attrs // { 257 | name = "${executor.name}-${builtins.unsafeDiscardStringContext (builtins.baseNameOf drv.outPath)}"; 258 | outputHashMode = "flat"; 259 | outputHashAlgo = "sha256"; 260 | outputHash = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; 261 | 262 | commandDrv = drvOf drv; 263 | nativeBuildInputs = executor.attrs.nativeBuildInputs or [] ++ [ drv ]; 264 | commandExec = drv.ci.exec; # also ensures the command is built before running the executor 265 | 266 | inherit (drv) meta; 267 | passthru = drv.passthru or {} // { 268 | ci = builtins.removeAttrs drv.passthru.ci or {} [ "exec" "cache" ] // { 269 | cache = { 270 | enable = false; 271 | inputs = [ drv ]; 272 | }; 273 | }; 274 | }; 275 | buildCommand = '' 276 | ${executor.buildCommand} 277 | touch $out 278 | ''; 279 | }); 280 | in exDrv; 281 | inherit (import ./lib/exec-ssh.nix { inherit lib config; }) execSsh; 282 | inherit (import ./lib/build { inherit lib config; }) buildScriptFor; 283 | }; 284 | 285 | config.exec.list = { 286 | run = let 287 | stageList = key: stage: mapAttrsToList (k: v: nameValuePair "${key}.${k}" v) stage.exec.list.run; 288 | in { 289 | # TODO: mention sub-test attrs, also list jobs/stages on their own... 290 | test = config.export.test.meta.description or null; 291 | list = config.export.list.meta.description or null; 292 | help = config.export.help.meta.description or null; 293 | } // mapAttrs' (k: v: nameValuePair "run.${k}" v.meta.description or null) config.export.run 294 | // listToAttrs (concatLists (mapAttrsToList (k: stageList "job.${k}") config.jobs)) 295 | // listToAttrs (concatLists (mapAttrsToList (k: stageList "stage.${k}") config.stages)); 296 | }; 297 | 298 | config.export = { 299 | environment = config.export.env.bootstrap; 300 | source = '' 301 | CI_ENV=${config.export.env.test} 302 | 303 | function ci_refresh() { 304 | local CONFIG_ARGS=(--arg configuration '${toNix configPath}') 305 | if [[ $# -gt 0 ]]; then 306 | CONFIG_ARGS=(--argstr configuration "$1") 307 | fi 308 | 309 | eval "$(${nixExe "nix"} eval --show-trace --raw source -f ${toString ./.} "''${CONFIG_ARGS[@]}")" 310 | } 311 | 312 | ${builtins.concatStringsSep "\n" (mapAttrsToList (name: eval: '' 313 | function ci_${name} { 314 | ${eval} "$@" 315 | } 316 | '') config.export.exec)} 317 | ''; 318 | shell = pkgs.mkShell { 319 | nativeBuildInputs = [ config.export.env.test ] ++ 320 | attrValues config.export.env.shell; 321 | 322 | shellHook = '' 323 | eval "${config.export.source}" 324 | source ${config.export.env.test}/${global.prefix}/env 325 | ci_env_impure 326 | ''; 327 | }; 328 | inherit (config.export.run) test list help; 329 | run = { 330 | setup = (config.lib.ci.nixRunWrapper "ci-setup" config.export.env.setup).overrideAttrs (old: { 331 | meta = old.meta or {} // { 332 | description = "build and setup the bootstrap environment"; 333 | }; 334 | }); 335 | bootstrap = (config.lib.ci.nixRunWrapper "ci-setup" config.export.env.bootstrap).overrideAttrs (old: { 336 | meta = old.meta or {} // { 337 | description = "build and setup the test environment"; 338 | }; 339 | }); 340 | run = (config.lib.ci.nixRunWrapper "ci-run" config.export.env.bootstrap).overrideAttrs (old: { 341 | meta = old.meta or {} // { 342 | description = "run a command in the test environment"; 343 | }; 344 | }); 345 | help = (config.lib.ci.nixRunWrapper "ci-help" config.export.doc.open).overrideAttrs (old: { 346 | meta = old.meta or {} // { 347 | description = "open the manual"; 348 | }; 349 | }); 350 | list = let 351 | list = (channels.cipkgs.writeShellScriptBin "ci-list" '' 352 | for item in ${toString (mapAttrsToList (k: v: "\"${k}=${toString v}\"") config.exec.list.run)}; do 353 | key=''${item%%=*} 354 | desc=''${item#*=} 355 | if [[ -n $desc ]]; then 356 | echo "$key: $desc" 357 | else 358 | echo "$key" 359 | fi 360 | done 361 | '').overrideAttrs (old: { 362 | meta = old.meta or {} // { 363 | description = "lists all commands that are available"; 364 | }; 365 | }); 366 | in config.lib.ci.nixRunWrapper "ci-list" list; 367 | } // { 368 | test = config.lib.ci.nixRunWrapper "ci-build" (buildScriptFor config.tasks) // { 369 | # TODO: turn this into a megatask using host-exec so it can all run in parallel? sshd ports though :( 370 | all = config.lib.ci.nixRunWrapper "ci-build" (channels.cipkgs.writeShellScriptBin "ci-build" '' 371 | set -eu 372 | ${config.export.test}/bin/ci-build 373 | ${concatStringsSep "\n" (mapAttrsToList (k: v: "echo testing ${k} ... >&2 && ${v.test}/bin/ci-build") config.export.job)} 374 | ${concatStringsSep "\n" (mapAttrsToList (k: v: "echo testing ${k} ... >&2 && ${v.test}/bin/ci-build") config.export.stage)} 375 | ''); 376 | } // mapAttrs (name: task: config.lib.ci.nixRunWrapper "ci-build" (buildScriptFor { ${name} = task; })) config.tasks; 377 | }; 378 | exec = { 379 | shell = ''${buildAndRun [config.export.env.shell]} -c ci-shell''; 380 | dirty = buildAnd [ci-query ci-dirty] (config.lib.ci.op.filterNoisy taskDrvImports); 381 | build = buildAnd [ci-query ci-dirty] (config.lib.ci.op.buildDirty taskDrvImports); 382 | buildAll = config.lib.ci.op.build taskDrvs; 383 | } // mapAttrs' (name: value: nameValuePair "task_${name}" (buildTask value)) config.tasks 384 | // config.project.exec; 385 | }; 386 | } 387 | -------------------------------------------------------------------------------- /nix/global.nix: -------------------------------------------------------------------------------- 1 | { 2 | prefix = "ci"; 3 | defaultConfigPath = "./ci.nix"; 4 | } 5 | -------------------------------------------------------------------------------- /nix/lib.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: with lib; { 2 | options = { 3 | lib = mkOption { 4 | type = types.attrsOf types.attrs; 5 | default = { }; 6 | }; 7 | }; 8 | } 9 | -------------------------------------------------------------------------------- /nix/lib/bootstrap.nix: -------------------------------------------------------------------------------- 1 | { lib, config }: with lib; { 2 | pname = "ci-env-bootstrap"; 3 | packages = builtins.attrValues config.environment.bootstrap; 4 | 5 | passAsFile = [ "setup" "build" "run" ]; 6 | 7 | setup = '' 8 | #!@runtimeShell@ 9 | set -eu 10 | 11 | @setupEnv@/bin/ci-setup 12 | 13 | @out@/bin/ci-build 14 | ''; 15 | 16 | build = '' 17 | #!@runtimeShell@ 18 | set -eu 19 | 20 | source @out@/@prefix@/env 21 | ci_env_impure 22 | 23 | BUILD_ARGS=(-E "import @runtimeDrv@") 24 | if [[ -n ''${CI_ENV-} ]]; then 25 | BUILD_ARGS+=(-o $CI_ENV) 26 | fi 27 | @nix@/bin/nix-build "''${BUILD_ARGS[@]}" 28 | 29 | case "''${CI_PLATFORM-}" in 30 | gh-actions) 31 | SOURCE=@runtimeOut@/@prefix@/source 32 | if [[ -n "''${GITHUB_ENV-}" ]]; then 33 | echo "BASH_ENV=$SOURCE" >> $GITHUB_ENV 34 | else 35 | echo "::set-env name=BASH_ENV::$SOURCE" >&2 36 | fi 37 | ;; 38 | esac 39 | ''; 40 | 41 | run = '' 42 | #!@runtimeShell@ 43 | set -eu 44 | 45 | exec @nix@/bin/nix run $(@nix@/bin/nix-build --no-out-link @runtimeDrv@\!out) "$@" 46 | ''; 47 | 48 | runtimeDrv = builtins.unsafeDiscardStringContext config.export.env.test.drvPath; 49 | runtimeOut = builtins.unsafeDiscardStringContext config.export.env.test.outPath; 50 | setupEnv = config.export.env.setup; 51 | passthru = { 52 | ciEnv = config.export.env.test; 53 | }; 54 | 55 | command = '' 56 | substituteBin $buildPath ci-build 57 | substituteBin $setupPath ci-setup 58 | substituteBin $runPath ci-run 59 | ''; 60 | } 61 | -------------------------------------------------------------------------------- /nix/lib/build/build.sh: -------------------------------------------------------------------------------- 1 | #!@runtimeShell@ 2 | set -eu 3 | 4 | CI_DRV_DIRTY=$(ci-build-dirty) 5 | 6 | CI_EXIT_CODE=0 7 | if [[ -z ${CI_DRY_RUN-} ]]; then 8 | echo $CI_DRV_DIRTY | ci-build-realise || CI_EXIT_CODE=$? 9 | fi 10 | export CI_EXIT_CODE 11 | 12 | EXIT_CODE=0 13 | CI_CACHE_LIST=$(echo $CI_DRV_DIRTY | ci-build-summarise) || EXIT_CODE=$? 14 | 15 | echo $CI_CACHE_LIST | ci-build-cache 16 | 17 | exit $EXIT_CODE 18 | -------------------------------------------------------------------------------- /nix/lib/build/cache.sh: -------------------------------------------------------------------------------- 1 | #!@runtimeShell@ 2 | set -eu 3 | 4 | CI_CACHE_LIST=($(cat)) 5 | 6 | source $CI_BUILD_ATTRS 7 | 8 | if [[ ${#CI_CACHE_LIST[@]} -gt 0 && -n ${CACHIX_SIGNING_KEY-} && -n ${CACHIX_CACHE-} ]]; then 9 | FILTERED=() 10 | for path in "${CI_CACHE_LIST[@]}"; do 11 | if [[ -e $path ]]; then 12 | FILTERED+=("$path") 13 | fi 14 | done 15 | echo ${FILTERED[*]} | @cachix@/bin/cachix push "$CACHIX_CACHE" || true 16 | fi 17 | -------------------------------------------------------------------------------- /nix/lib/build/default.nix: -------------------------------------------------------------------------------- 1 | { lib, config }: with lib; with config.lib.ci; let 2 | scriptData = tasks: let 3 | executor = config.project.executor.drv; 4 | drvOf = drv: let 5 | ph = "${builtins.placeholder drv.name}-${drv.name}"; 6 | drvPath = builtins.tryEval (config.lib.ci.drvOf drv); 7 | in 8 | if drv.ci.omit or false != false || ! drvPath.success then ph 9 | else drvPath.value; 10 | tasks'partition = partition (t: t.skip != false) (attrValues tasks); 11 | tasks'skipped = tasks'partition.right; 12 | tasks'build = tasks'partition.wrong; 13 | drvsSkipped = map (t: t.drv) tasks'skipped 14 | ++ concatMap (t: t.internal.inputs.skipped) tasks'build 15 | ++ concatMap (t: t.internal.inputs.all) tasks'skipped; 16 | drvs = map (t: t.drv) tasks'build ++ concatMap (task: (with task.internal.inputs; wrappedImpure ++ pure)) (attrValues tasks); 17 | drvAttrs = fn: drvs: listToAttrs (map (drv: nameValuePair (drvOf drv) (fn drv)) drvs); 18 | drvCachePaths = drv: let 19 | inputs = cacheInputsOf drv; 20 | cachePaths = input: builtins.unsafeDiscardStringContext (concatMapStringsSep " " (out: toString input.${out}) input.outputs); 21 | in concatMapStringsSep " " cachePaths inputs; 22 | cachixCaches = attrValues config.cache.cachix; 23 | writableCaches = filter (c: c.enable && c.signingKey != null) cachixCaches; 24 | in { 25 | # TODO: support multiple caches 26 | ${if writableCaches != [ ] then "CACHIX_CACHE" else null} = (head writableCaches).name; 27 | # structured input data for buildScript 28 | drvs = map drvOf drvs; 29 | drvExecutor = if executor == null then "" else executor.exec; 30 | drvTasks = mapAttrsToList (_: t: drvOf t.drv) tasks; 31 | drvSkipped = drvAttrs (drv: 32 | if isString (drv.ci.skip or null) then drv.ci.skip 33 | else if drv.meta.broken or false then "broken" 34 | else if drv.meta.available or true == false then "unavailable" 35 | else drv.ci.skip or true) drvsSkipped; 36 | drvImports = drvImports drvs; 37 | drvWarn = drvAttrs 38 | (drv: true) 39 | (filter (drv: drv.ci.warn or false) 40 | drvs); 41 | drvCache = drvAttrs drvCachePaths drvs; 42 | drvInputs = mapAttrs' (_: t: nameValuePair (drvOf t.drv) (concatStringsSep " " (map drvOf (with t.internal.inputs; wrappedImpure ++ pure ++ skipped)))) tasks; 43 | drvName = drvAttrs (drv: drv.meta.name or drv.name) (drvs ++ drvsSkipped); 44 | preBuild = concatMapStringsSep "\n" (t: t.preBuild) tasks'build; 45 | }; 46 | # TODO: manual derivation rather than using stdenv? 47 | buildScriptFor = tasks: config.bootstrap.pkgs.buildPackages.runCommand config.name ({ 48 | __structuredAttrs = true; 49 | attrsPath = "${placeholder "out"}/${(import ../../global.nix).prefix}/attrs.sh"; 50 | preferLocalBuild = true; 51 | allowSubstitutes = false; 52 | builder = config.bootstrap.pkgs.writeScript "ci-build.sh" '' 53 | #!${config.bootstrap.runtimeShell} 54 | source .attrs.sh 55 | 56 | ${config.bootstrap.packages.coreutils}/bin/install -D .attrs.sh $attrsPath 57 | for bin in ${config.bootstrap.packages.ci-build}/bin/ci-build*; do 58 | ${config.bootstrap.packages.coreutils}/bin/cat > ci-build <> $GITHUB_ENV 13 | else 14 | echo "::set-env name=CI_BUILD_ATTRS::$CI_BUILD_ATTRS" >&2 15 | fi 16 | ;; 17 | azure-pipelines) 18 | echo "##vso[task.setvariable variable=CI_BUILD_ATTRS]$CI_BUILD_ATTRS" >&2 19 | ;; 20 | esac 21 | 22 | opFilterNoisy $drvImports 23 | -------------------------------------------------------------------------------- /nix/lib/build/realise.sh: -------------------------------------------------------------------------------- 1 | #!@runtimeShell@ 2 | set -eu 3 | 4 | CI_DRV_DIRTY=($(cat)) 5 | 6 | source $CI_BUILD_ATTRS 7 | 8 | @sourceOps@ 9 | 10 | eval "$preBuild" 11 | 12 | if [[ -n $drvExecutor ]]; then 13 | # TODO: just make this part of preBuild? 14 | export EX_PIDFILE=$(mktemp) 15 | $drvExecutor 16 | trap 'kill -QUIT $(cat $EX_PIDFILE)' EXIT 17 | fi 18 | 19 | # TODO: use --add-root with --indirect in a ci cache dir? 20 | (( ${#CI_DRV_DIRTY[@]} == 0 )) || opRealise "${CI_DRV_DIRTY[@]}" --show-trace --keep-going "$@" 21 | -------------------------------------------------------------------------------- /nix/lib/build/summarise.sh: -------------------------------------------------------------------------------- 1 | #!@runtimeShell@ 2 | set -eu 3 | if [[ -n "@nix@" ]]; then 4 | export PATH="@nix@/bin:$PATH" 5 | fi 6 | 7 | CI_DRV_DIRTY=($(cat)) 8 | 9 | source $CI_BUILD_ATTRS 10 | 11 | @sourceOps@ 12 | 13 | drv_dirty() { 14 | drv_skipped $1 || [[ " ${CI_DRV_DIRTY[@]} " =~ " $1 " ]] 15 | } 16 | 17 | drv_valid() { 18 | if drv_skipped $1; then 19 | return 1 20 | else 21 | nix-store -u -q --hash "$1" > /dev/null 2>&1 22 | fi 23 | } 24 | 25 | drv_warn() { 26 | [[ -n ${drvWarn[$1]-} ]] 27 | } 28 | 29 | drv_skipped() { 30 | [[ -n ${drvSkipped[$1]-} ]] 31 | } 32 | 33 | drv_report() { 34 | # drv, status = ok|fail|cache, nested=1 35 | local REPORT_MSG REPORT_ICON REPORT_COLOUR 36 | 37 | if [[ $2 = ok ]] && ! drv_dirty $1; then 38 | REPORT_MSG=cache 39 | elif [[ $2 = fail && -n ${CI_DRY_RUN-} ]]; then 40 | REPORT_MSG=dry 41 | else 42 | REPORT_MSG=$2 43 | fi 44 | 45 | case $REPORT_MSG in 46 | fail) 47 | if drv_warn $1; then 48 | REPORT_COLOUR=@yellow@ 49 | REPORT_MSG="failed (allowed, ignored)" 50 | #REPORT_ICON="⚠️" 51 | REPORT_ICON="!" 52 | if [[ ${CI_PLATFORM-} = gh-actions ]]; then 53 | echo "::warning::${drvName[$1]} failed to build" >&2 54 | fi 55 | else 56 | REPORT_COLOUR=@red@ 57 | REPORT_MSG=failed 58 | REPORT_ICON=❌ 59 | if [[ -z ${3-} ]]; then 60 | EXIT_CODE=1 61 | fi 62 | if [[ ${CI_PLATFORM-} = gh-actions ]]; then 63 | echo "::error::${drvName[$1]} failed to build" >&2 64 | fi 65 | fi 66 | ;; 67 | ok) 68 | REPORT_COLOUR=@blue@ 69 | REPORT_MSG=ok 70 | REPORT_ICON="✔️" 71 | CI_CACHE_LIST+=(${drvCache[$1]-}) 72 | ;; 73 | cache) 74 | REPORT_COLOUR=@magenta@ 75 | REPORT_MSG="ok (cached)" 76 | REPORT_ICON="✔️" 77 | ;; 78 | skip|dry) 79 | REPORT_COLOUR=@yellow@ 80 | REPORT_ICON="•" 81 | if [[ $REPORT_MSG = dry ]]; then 82 | REPORT_MSG="skipped (dry run)" 83 | elif [[ -z ${drvSkipped[$1]-} || ${drvSkipped[$1]} = 1 ]]; then 84 | REPORT_MSG="skipped" 85 | else 86 | REPORT_MSG="skipped (${drvSkipped[$1]})" 87 | fi 88 | ;; 89 | esac 90 | echo "$REPORT_COLOUR${3+"  "}$REPORT_ICON ${drvName[$1]} $REPORT_MSG" >&2 91 | } 92 | 93 | # TODO: verbose option for opFilter vs opFilterNoisy? 94 | CI_CACHE_LIST=() 95 | EXIT_CODE=0 96 | 97 | if (( ${#CI_DRV_DIRTY[@]} > 0 && $CI_EXIT_CODE != 0 )) || [[ -n ${CI_DRY_RUN-} ]]; then 98 | for drv in "${drvTasks[@]}"; do 99 | if drv_skipped $drv; then 100 | drv_report $drv skip 101 | for input in ${drvInputs[$drv]}; do 102 | drv_report $input skip 1 103 | done 104 | elif ! drv_dirty $drv || drv_valid $drv; then 105 | # TODO: maybe use path-info -Sh and record the size to show with the results? 106 | drv_report $drv ok 107 | for input in ${drvInputs[$drv]}; do 108 | if drv_skipped $input; then 109 | drv_report $input skip 1 110 | else 111 | drv_report $input ok 1 112 | fi 113 | done 114 | else 115 | drv_report $drv fail 116 | for input in ${drvInputs[$drv]}; do 117 | if drv_skipped $input; then 118 | drv_report $input skip 1 119 | elif ! drv_dirty $input || drv_valid $input; then 120 | drv_report $input ok 1 121 | else 122 | drv_report $input fail 1 123 | if [[ -z ${CI_DRY_RUN-} ]]; then 124 | nix-store -r "$input" --dry-run 2>&1 | (@gnugrep@/bin/grep -vFe 'these derivations will be built' -e "$input" | @gnused@/bin/sed -n '/^these paths will be fetched/q;p' >&2 || true) 125 | # TODO: parse the above list and show more info via nix-store query or something? 126 | fi 127 | fi 128 | done 129 | # TODO: print out part of failure log? 130 | fi 131 | done 132 | else 133 | for drv in "${drvTasks[@]}"; do 134 | if drv_skipped $drv; then 135 | drv_report $drv skip 136 | else 137 | drv_report $drv ok 138 | for input in ${drvInputs[$drv]}; do 139 | if drv_skipped $input; then 140 | drv_report $input skip 1 141 | else 142 | drv_report $input ok 1 143 | fi 144 | done 145 | fi 146 | done 147 | fi 148 | 149 | printf %s @clear@ >&2 150 | 151 | # TODO: consider listing tasks that aren't cached (local) but don't get rebuilt because dirty filter includes local? 152 | 153 | echo ${CI_CACHE_LIST[*]} 154 | 155 | exit $EXIT_CODE 156 | -------------------------------------------------------------------------------- /nix/lib/channels.nix: -------------------------------------------------------------------------------- 1 | lib: with lib; rec { 2 | channelType = { channelUrls, channels, /*system,*/ bootpkgs, ciOverlayArgs, defaultConfig, isNixpkgs ? false, pkgs ? channels.nixpkgs.import, specialImport }: types.submodule ({ name, config, ... }: { 3 | options = { 4 | enable = mkEnableOption "channel" // { default = true; }; 5 | name = mkOption { 6 | type = types.str; 7 | default = name; 8 | }; 9 | version = mkOption { 10 | type = types.nullOr types.str; 11 | }; 12 | url = mkOption { 13 | type = types.nullOr types.str; 14 | }; 15 | sha256 = mkOption { 16 | type = types.nullOr types.str; 17 | }; 18 | path = mkOption { 19 | type = types.either types.str types.path; 20 | }; 21 | file = mkOption { 22 | type = types.nullOr types.str; 23 | }; 24 | args = mkOption { 25 | type = if isNixpkgs || name == "nixpkgs" 26 | then nixpkgsType { inherit channels; } 27 | else types.attrsOf types.unspecified; 28 | }; 29 | overlays = mkOption { 30 | type = types.listOf types.unspecified; 31 | }; 32 | import = mkOption { 33 | type = types.unspecified; 34 | internal = true; 35 | }; 36 | nixPathImport = mkOption { 37 | type = types.bool; 38 | default = false; 39 | }; 40 | }; 41 | config = { 42 | version = defaultConfig.${config.name}.version or (mkOptionDefault null); 43 | sha256 = defaultConfig.${config.name}.sha256 or (mkOptionDefault null); 44 | 45 | args = defaultConfig.${config.name}.args or ({ 46 | nur = { 47 | nurpkgs = mkOptionDefault bootpkgs; 48 | }; 49 | ci = { pkgs = mkOptionDefault bootpkgs; }; 50 | }.${config.name} or (mkOptionDefault { })); 51 | 52 | file = defaultConfig.${config.name}.file or (mkOptionDefault { 53 | mozilla = "package-set.nix"; 54 | }.${config.name} or null); 55 | 56 | overlays = defaultConfig.${config.name}.overlays or (mkOptionDefault { 57 | rust = [ (config.path + "/overlay.nix") ]; 58 | arc = [ (config.path + "/overlay.nix") ]; 59 | home-manager = [ (config.path + "/overlay.nix") ]; 60 | mozilla = import (config.path + "/overlays.nix"); 61 | nur = [ (self: super: let 62 | args = optionalAttrs (config.args ? repoOverrides) { 63 | inherit (config.args) repoOverrides; 64 | }; 65 | in { 66 | nur = import config.path (args // { 67 | nurpkgs = bootpkgs; # can self work here? 68 | pkgs = self; 69 | }); 70 | nur-no-pkgs = import config.path (args // { 71 | nurpkgs = bootpkgs; 72 | }); 73 | }) ]; 74 | ci = [ 75 | (config.path + "/nix/overlay.nix") 76 | (import (config.path + "/nix/lib/overlay.nix") ciOverlayArgs) 77 | ]; 78 | }.${config.name} or []); 79 | 80 | path = mkMerge ([ 81 | (mkIf (config.url != null) (mkDefault ( 82 | if hasPrefix builtins.storeDir (toString config.url) then /. + builtins.storePath config.url 83 | else if hasPrefix "/" (toString config.url) then toString config.url 84 | else builtins.fetchTarball ({ 85 | name = "source"; # or config.name? 86 | inherit (config) url; 87 | } // optionalAttrs (config.sha256 != null) { 88 | inherit (config) sha256; 89 | }) 90 | ))) 91 | ] ++ optional (defaultConfig ? ${config.name}.path) defaultConfig.${config.name}.path); 92 | 93 | url = defaultConfig.${config.name}.url or (mkOptionDefault ( 94 | if config.version != null 95 | then channelUrls.${config.name} config.version 96 | else null 97 | )); 98 | 99 | import = mkOptionDefault (defaultConfig.${config.name}.import or (let 100 | args = optionalAttrs (isFunction channel && ((functionArgs channel) ? pkgs)) { inherit pkgs; } 101 | // config.args.ciChannelArgs or config.args; 102 | file = if config.file != null 103 | then config.path + "/${config.file}" 104 | else config.path; 105 | channel = (if config.nixPathImport then specialImport else import) file; 106 | imported = if isFunction channel then channel args else channel; 107 | in imported)); 108 | }; 109 | }); 110 | channelTypeCoerced = channelType: types.coercedTo types.str (version: { 111 | inherit version; 112 | }) channelType; 113 | systemType = types.coercedTo types.str (system: { inherit system; }) (types.attrsOf types.unspecified); 114 | nixpkgsType = { channels }: types.submodule ({ config, ... }: { 115 | options = { 116 | system = mkOption { 117 | type = systemType; 118 | }; 119 | localSystem = mkOption { 120 | type = systemType; 121 | }; 122 | crossSystem = mkOption { 123 | type = types.nullOr systemType; 124 | }; 125 | config = mkOption { 126 | type = types.attrsOf types.unspecified; 127 | default = { }; 128 | }; 129 | overlays = mkOption { 130 | type = types.listOf types.unspecified; 131 | default = [ ]; 132 | }; 133 | crossOverlays = mkOption { 134 | type = types.listOf types.unspecified; 135 | default = [ ]; 136 | }; 137 | stdenvStages = mkOption { 138 | type = types.nullOr types.unspecified; 139 | default = null; 140 | }; 141 | ciChannelArgs = mkOption { 142 | type = types.unspecified; 143 | internal = true; 144 | }; 145 | }; 146 | 147 | config = { 148 | localSystem = { 149 | system = mkOptionDefault builtins.currentSystem; 150 | }; 151 | crossSystem = mkOptionDefault null; 152 | system = mkOptionDefault ( 153 | if config.crossSystem != null then config.crossSystem 154 | else config.localSystem 155 | ); 156 | config = { 157 | checkMetaRecursively = mkOptionDefault true; 158 | checkMeta = mkOptionDefault false; 159 | }; 160 | overlays = let 161 | overlayChannels = filterAttrs (k: c: k != "nixpkgs" && c.enable) channels; 162 | overlays = concatMap (c: c.overlays) (attrValues overlayChannels); 163 | in map (o: if isFunction o then o else import o) overlays; 164 | ciChannelArgs = removeAttrs config [ 165 | "_module" "ciChannelArgs" "system" "stdenvStages" 166 | ] // optionalAttrs (config.stdenvStages != null) { 167 | inherit (config) stdenvStages; 168 | } // optionalAttrs (config.crossSystem == null && config.system != config.localSystem) { 169 | crossSystem = config.system; 170 | }; 171 | }; 172 | }); 173 | nixpkgsChannels = { nixpkgsChannels, isLinux }: { 174 | stable = "24.11"; 175 | stable-small = "${config.lib.nixpkgsChannels.stable}-small"; 176 | unstable = if isLinux 177 | then "nixos-unstable" 178 | else "nixpkgs-unstable"; 179 | unstable-small = if isLinux 180 | then "nixos-unstable-small" 181 | else nixpkgsChannels.unstable; 182 | "25.05" = nixpkgsChannels.unstable; 183 | "25.05-small" = nixpkgsChannels.unstable-small; 184 | }; 185 | # TODO: think about how this will work with flakes. want to expand this to include overlays! 186 | githubChannel = slug: c: "https://github.com/${slug}/archive/${c}.tar.gz"; 187 | gitlabChannel = slug: c: "https://gitlab.com/${slug}/-/archive/${c}/${baseNameOf slug}-${c}.tar.gz"; 188 | channelUrls = { nixpkgsChannels, githubChannel, gitlabChannel, isDarwin }: { 189 | # TODO: if nixpkgs is a git ref use githubChannel instead 190 | nixpkgs = c: let 191 | c' = nixpkgsChannels.${c} or c; 192 | stable = builtins.match "([0-9][0-9]\\.[0-9][0-9]).*" c'; 193 | channel = if stable != null then 194 | (if isDarwin 195 | then "nixpkgs-${builtins.elemAt stable 0}-darwin" 196 | else "nixos-${c'}") 197 | else if builtins.match ".*-.*" c' != null then c' 198 | else null; 199 | in if channel != null 200 | then "https://nixos.org/channels/${channel}/nixexprs.tar.xz" 201 | else githubChannel "nixos/nixpkgs" c'; 202 | home-manager = githubChannel "rycee/home-manager"; 203 | mozilla = githubChannel "mozilla/nixpkgs-mozilla"; 204 | rust = githubChannel "arcnmx/nixexprs-rust"; 205 | nur = githubChannel "nix-community/NUR"; 206 | arc = githubChannel "arcnmx/nixexprs"; 207 | ci = githubChannel "arcnmx/ci"; 208 | #nmd = gitlabChannel "rycee/nmd"; # gitlab seems to have rate limits (429 Too Many Requests)? 209 | nmd = githubChannel "arcnmx/nmd"; 210 | }; 211 | } 212 | -------------------------------------------------------------------------------- /nix/lib/cipkgs.nix: -------------------------------------------------------------------------------- 1 | rec { 2 | getNixpkgsHashFor = { version }: let 3 | # nix eval -f nix/lib/cipkgs.nix getNixpkgsHashFor.sourceInfo --argstr version 2.10.3 4 | nix = builtins.getFlake "github:NixOS/nix/${version}"; 5 | in { 6 | sourceInfo = { 7 | inherit (nix.inputs.nixpkgs.sourceInfo) rev narHash; 8 | }; 9 | }; 10 | nixpkgsSource = { rev, sha256 }: { 11 | name = "source"; 12 | url = "https://github.com/nixos/nixpkgs/archive/${rev}.tar.gz"; 13 | inherit sha256; 14 | }; 15 | nixpkgsFor = { 16 | # pinned nixpkgs evaluations bundled with nix binary releases (https://github.com/NixOS/nix/blob/master/flake.lock) 17 | "2.26." = nixpkgsFor."2.26.1"; 18 | "2.26.1" = nixpkgsFor."2.26.0"; 19 | "2.26.0" = nixpkgsSource { 20 | rev = "48d12d5e70ee91fe8481378e540433a7303dbf6a"; 21 | sha256 = "sha256-1Noao/H+N8nFB4Beoy8fgwrcOQLVm9o4zKW1ODaqK9E="; 22 | }; 23 | "2.25." = nixpkgsFor."2.25.5"; 24 | "2.25.5" = nixpkgsFor."2.25.4"; 25 | "2.25.4" = nixpkgsFor."2.25.3"; 26 | "2.25.3" = nixpkgsFor."2.25.2"; 27 | "2.25.2" = nixpkgsFor."2.25.1"; 28 | "2.25.1" = nixpkgsFor."2.25.0"; 29 | "2.25.0" = nixpkgsFor."2.24.3"; 30 | "2.24." = nixpkgsFor."2.24.11"; 31 | "2.24.11" = nixpkgsFor."2.24.10"; 32 | "2.24.10" = nixpkgsFor."2.24.9"; 33 | "2.24.9" = nixpkgsFor."2.24.8"; 34 | "2.24.8" = nixpkgsFor."2.24.7"; 35 | "2.24.7" = nixpkgsFor."2.24.6"; 36 | "2.24.6" = nixpkgsFor."2.24.5"; 37 | "2.24.5" = nixpkgsFor."2.24.4"; 38 | "2.24.4" = nixpkgsFor."2.24.3"; 39 | "2.24.3" = nixpkgsSource { 40 | rev = "c3d4ac725177c030b1e289015989da2ad9d56af0"; 41 | sha256 = "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg="; 42 | }; 43 | "2.24.2" = nixpkgsFor."2.24.1"; 44 | "2.24.1" = nixpkgsFor."2.24.0"; 45 | "2.24.0" = nixpkgsSource { 46 | rev = "63d37ccd2d178d54e7fb691d7ec76000740ea24a"; 47 | sha256 = "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o="; 48 | }; 49 | "2.23." = nixpkgsFor."2.23.4"; 50 | "2.23.4" = nixpkgsFor."2.23.3"; 51 | "2.23.3" = nixpkgsFor."2.23.2"; 52 | "2.23.2" = nixpkgsFor."2.23.1"; 53 | "2.23.1" = nixpkgsFor."2.23.0"; 54 | "2.23.0" = nixpkgsFor."2.22.0"; 55 | "2.22." = nixpkgsFor."2.22.4"; 56 | "2.22.4" = nixpkgsFor."2.22.3"; 57 | "2.22.3" = nixpkgsFor."2.22.2"; 58 | "2.22.2" = nixpkgsFor."2.22.1"; 59 | "2.22.1" = nixpkgsFor."2.22.0"; 60 | "2.22.0" = nixpkgsFor."2.21.0"; 61 | "2.21." = nixpkgsFor."2.21.5"; 62 | "2.21.5" = nixpkgsFor."2.21.3"; 63 | "2.21.4" = nixpkgsFor."2.21.3"; 64 | "2.21.3" = nixpkgsFor."2.21.2"; 65 | "2.21.2" = nixpkgsFor."2.21.1"; 66 | "2.21.1" = nixpkgsFor."2.21.0"; 67 | "2.21.0" = nixpkgsSource { 68 | rev = "b550fe4b4776908ac2a861124307045f8e717c8e"; 69 | sha256 = "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo="; 70 | }; 71 | "2.20." = nixpkgsFor."2.20.9"; 72 | "2.20.9" = nixpkgsFor."2.20.8"; 73 | "2.20.8" = nixpkgsFor."2.20.7"; 74 | "2.20.7" = nixpkgsFor."2.20.6"; 75 | "2.20.6" = nixpkgsFor."2.20.5"; 76 | "2.20.5" = nixpkgsFor."2.20.4"; 77 | "2.20.4" = nixpkgsFor."2.20.3"; 78 | "2.20.3" = nixpkgsFor."2.20.2"; 79 | "2.20.2" = nixpkgsFor."2.20.1"; 80 | "2.20.1" = nixpkgsFor."2.20.0"; 81 | "2.20.0" = nixpkgsSource { 82 | rev = "a1982c92d8980a0114372973cbdfe0a307f1bdea"; 83 | sha256 = "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ="; 84 | }; 85 | "2.19." = nixpkgsFor."2.19.7"; 86 | "2.19.7" = nixpkgsFor."2.19.6"; 87 | "2.19.6" = nixpkgsFor."2.19.5"; 88 | "2.19.5" = nixpkgsFor."2.19.4"; 89 | "2.19.4" = nixpkgsFor."2.19.3"; 90 | "2.19.3" = nixpkgsSource { 91 | rev = "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0"; 92 | sha256 = "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds="; 93 | }; 94 | "2.19.2" = nixpkgsFor."2.19.1"; 95 | "2.19.1" = nixpkgsFor."2.19.0"; 96 | "2.19.0" = nixpkgsSource { 97 | rev = "9eb24edd6a0027fed010ccfe300a9734d029983c"; 98 | sha256 = "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo="; 99 | }; 100 | "2.18." = nixpkgsFor."2.18.9"; 101 | "2.18.9" = nixpkgsFor."2.18.8"; 102 | "2.18.8" = nixpkgsFor."2.18.7"; 103 | "2.18.7" = nixpkgsFor."2.18.6"; 104 | "2.18.6" = nixpkgsFor."2.18.5"; 105 | "2.18.5" = nixpkgsFor."2.18.4"; 106 | "2.18.4" = nixpkgsFor."2.18.3"; 107 | "2.18.3" = nixpkgsFor."2.18.2"; 108 | "2.18.2" = nixpkgsSource { 109 | rev = "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0"; 110 | sha256 = "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds="; 111 | }; 112 | "2.18.1" = nixpkgsSource { 113 | rev = "31ed632c692e6a36cfc18083b88ece892f863ed4"; 114 | sha256 = "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg="; 115 | }; 116 | "2.18.0" = nixpkgsSource { 117 | rev = "a3d30b525535e3158221abc1a957ce798ab159fe"; 118 | sha256 = "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE="; 119 | }; 120 | "2.17." = nixpkgsFor."2.17.2"; 121 | "2.17.2" = nixpkgsFor."2.17.1"; 122 | "2.17.1" = nixpkgsFor."2.17.0"; 123 | "2.17.0" = nixpkgsFor."2.16.1"; 124 | "2.16." = nixpkgsFor."2.16.3"; 125 | "2.16.3" = nixpkgsFor."2.16.2"; 126 | "2.16.2" = nixpkgsFor."2.16.1"; 127 | "2.16.1" = nixpkgsFor."2.16.0"; 128 | "2.16.0" = nixpkgsFor."2.15.1"; 129 | "2.15." = nixpkgsFor."2.15.3"; 130 | "2.15.3" = nixpkgsFor."2.15.2"; 131 | "2.15.2" = nixpkgsFor."2.15.1"; 132 | "2.15.1" = nixpkgsFor."2.15.0"; 133 | "2.15.0" = nixpkgsFor."2.14.1"; 134 | "2.14." = nixpkgsFor."2.14.1"; 135 | "2.14.1" = nixpkgsFor."2.14.0"; 136 | "2.14.0" = nixpkgsFor."2.13.3"; 137 | "2.13." = nixpkgsFor."2.13.6"; 138 | "2.13.6" = nixpkgsFor."2.13.5"; 139 | "2.13.5" = nixpkgsFor."2.13.4"; 140 | "2.13.4" = nixpkgsFor."2.13.3"; 141 | "2.13.3" = nixpkgsFor."2.13.2"; 142 | "2.13.2" = nixpkgsFor."2.13.1"; 143 | "2.13.1" = nixpkgsFor."2.13.0"; 144 | "2.13.0" = nixpkgsSource { 145 | rev = "04a75b2eecc0acf6239acf9dd04485ff8d14f425"; 146 | sha256 = "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8="; 147 | }; 148 | "2.12." = nixpkgsFor."2.12.1"; 149 | "2.12.1" = nixpkgsFor."2.12.0"; 150 | "2.12.0" = nixpkgsFor."2.11.1"; 151 | "2.11." = nixpkgsFor."2.11.1"; 152 | "2.11.1" = nixpkgsFor."2.11.0"; 153 | "2.11.0" = nixpkgsFor."2.10.3"; 154 | "2.10." = nixpkgsFor."2.10.3"; 155 | "2.10.3" = nixpkgsFor."2.10.2"; 156 | "2.10.2" = nixpkgsSource { 157 | rev = "365e1b3a859281cf11b94f87231adeabbdd878a2"; 158 | sha256 = "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8="; 159 | }; 160 | "2.10.1" = nixpkgsFor."2.10.0"; 161 | "2.10.0" = nixpkgsSource { 162 | rev = "2fa57ed190fd6c7c746319444f34b5917666e5c1"; 163 | sha256 = "sha256-ZaqFFsSDipZ6KVqriwM34T739+KLYJvNmCWzErjAg7c="; 164 | }; 165 | "2.9.2" = nixpkgsFor."2.9.0"; 166 | "2.9.1" = nixpkgsFor."2.9.0"; 167 | "2.9.0" = nixpkgsFor."2.8.1"; 168 | "2.8.1" = nixpkgsFor."2.8.0"; 169 | "2.8.0" = nixpkgsSource { 170 | rev = "530a53dcbc9437363471167a5e4762c5fcfa34a1"; 171 | sha256 = "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU="; 172 | }; 173 | "2.7.0" = nixpkgsFor."2.6.1"; 174 | "2.6.1" = nixpkgsFor."2.6.0"; 175 | "2.6.0" = nixpkgsFor."2.5.1"; 176 | "2.5.1" = nixpkgsFor."2.5.0"; 177 | "2.5.0" = nixpkgsFor."2.4"; 178 | "2.4" = nixpkgsSource { 179 | rev = "82891b5e2c2359d7e58d08849e4c89511ab94234"; 180 | sha256 = "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM="; 181 | }; 182 | "2.3" = nixpkgsSource { 183 | rev = "56b84277cc8c52318a99802878b0725b2e34648e"; 184 | sha256 = "0rhpkfcfvszvcga7lcy4zqzchglsnvrzphkz59ifp9ihvmxrq14y"; 185 | }; 186 | "2.3.1" = nixpkgsSource { 187 | rev = "df7e351af91e6cbf4434e281d35fec39348a5d91"; 188 | sha256 = "19fpg1pya2iziwk10wja69ma65r985zdcd9blkplyg0l1lnn8haq"; 189 | }; 190 | "2.3.2" = nixpkgsSource { 191 | rev = "87c698a5ca655fe108958eb4bc6ad7a9b8bfcd82"; 192 | sha256 = "1ayipwmgbnf2vggr7jbq5l0vg0ly3g2wmcyajd3a7355g9hys3q3"; 193 | }; 194 | "2.3.3" = nixpkgsSource { 195 | rev = "4dc0c1761c8dd15e9ddfff793f22dba2a0828986"; 196 | sha256 = "1pw9xfsqfaig1vdmm6a4cgbqdw5bc0by84g7rip53m68p8fa33c5"; 197 | }; 198 | "2.3.7" = nixpkgsSource { 199 | rev = "0cfa467f8771ca585b3e122806c8337f5025bd92"; 200 | sha256 = "09fp8fvhc1jhaxjabf9nspqxn47lkknkfxp66a09y4bxf9120q18"; 201 | }; 202 | "2.3.8" = nixpkgsSource { 203 | rev = "afcf35320d8cdce3f569f611819c302c1b724609"; 204 | sha256 = "1ls88988mpyapgr5plky1bavr1ibi3qpl1m2jdx39r35sjlwb83q"; 205 | }; 206 | "2.3.9" = nixpkgsSource { 207 | rev = "d488daf8504ef1838c6b89f7add9bf370757afe4"; 208 | sha256 = "04rpvdn4s81d7dqcrv2v1qd7yx2n65l9fgc2m1258dcmjqbzv9ah"; 209 | }; 210 | "2.3.10" = nixpkgsSource { 211 | rev = "929768261a3ede470eafb58d5b819e1a848aa8bf"; 212 | sha256 = "0zi54vbfi6i6i5hdd4v0l144y1c8rg6hq6818jjbbcnm182ygyfa"; 213 | }; 214 | "2.3.11" = nixpkgsSource { 215 | rev = "1db42b7fe3878f3f5f7a4f2dc210772fd080e205"; 216 | sha256 = "05k9y9ki6jhaqdhycnidnk5zrdzsdammbk5lsmsbz249hjhhgcgh"; 217 | }; 218 | "2.3.12" = nixpkgsSource { 219 | rev = "1db42b7fe3878f3f5f7a4f2dc210772fd080e205"; 220 | sha256 = "05k9y9ki6jhaqdhycnidnk5zrdzsdammbk5lsmsbz249hjhhgcgh"; 221 | }; 222 | "2.3.13" = nixpkgsSource { 223 | rev = "0ccd0d91361dc42dd32ffcfafed1a4fc23d1c8b4"; 224 | sha256 = "0dmwi3r1hsv3f11pzf0qmaw5b7w4bncrjypiwfc8sym5bvxb2lcz"; 225 | }; 226 | "2.3.14" = nixpkgsSource { 227 | rev = "806c01c9f9945dcd63f6daea8f12a787fbb54dd2"; 228 | sha256 = "0d7sr35yb7z13c6940xpk0ngjs1avpvwjhgbhfwxy6zsmhwi2m2z"; 229 | }; 230 | "2.3.15" = nixpkgsSource { 231 | rev = "ccd782596c1fdd82ec79ed16707bb117cd9d5d11"; 232 | sha256 = "0sxf09v04ldddx2n87zhxw4lvc2hw068pqwad7c5c0xl6jc99dx1"; 233 | }; 234 | "2.3.16" = nixpkgsSource { 235 | rev = "49017a1c5ac37461144d3b2a6efab02b87bdf066"; 236 | sha256 = "0pm93801rh70cy43gpi3mamdk19cxf9fdnl98v8hf8bvn0f2vz6j"; 237 | }; 238 | "2.3.17" = nixpkgsSource { 239 | rev = "022caabb5f2265ad4006c1fa5b1ebe69fb0c3faf"; 240 | sha256 = "sha256-lkA5X3VNMKirvA+SUzvEhfA7XquWLci+CGi505YFAIs="; 241 | }; 242 | "2.3.18" = nixpkgsSource { 243 | rev = "9b19f5e77dd906cb52dade0b7bd280339d2a1f3d"; 244 | sha256 = "sha256-rCIsyE80jgiOU78gCWN3A0wE0tR2GI5nH6MlS+HaaSQ="; 245 | }; 246 | "2.2.1" = nixpkgsSource { 247 | rev = "d26f11d38903768bf10036ce70d67e981056424b"; 248 | sha256 = "16d986r76ps7542mbm63dxiavxw9af08l4ffpjp38lpam2cd9zpp"; 249 | }; 250 | "2.2.2" = nixpkgsSource { 251 | rev = "2296f0fc9559d0b6e08a7c07b25bd0a5f03eebe5"; 252 | sha256 = "197f6glm69717a5pj7bwm57vf1wrgh8nb13sa9qpjkz4803xpzdf"; 253 | }; 254 | "21.11" = nixpkgsSource { 255 | # 21.11 release 256 | rev = "a7ecde854aee5c4c7cd6177f54a99d2c1ff28a31"; 257 | sha256 = "162dywda2dvfj1248afxc45kcrg83appjd0nmdb541hl7rnncf02"; 258 | }; 259 | "22.05" = nixpkgsSource { 260 | # 22.05 release 261 | rev = "ce6aa13369b667ac2542593170993504932eb836"; 262 | sha256 = "0d643wp3l77hv2pmg2fi7vyxn4rwy0iyr8djcw1h5x72315ck9ik"; 263 | }; 264 | "22.11" = nixpkgsSource { 265 | # 22.11 release 266 | rev = "4d2b37a84fad1091b9de401eb450aae66f1a741e"; 267 | sha256 = "11w3wn2yjhaa5pv20gbfbirvjq6i3m7pqrq2msf0g7cv44vijwgw"; 268 | }; 269 | "23.05" = nixpkgsSource { 270 | # 23.05 release 271 | rev = "4ecab3273592f27479a583fb6d975d4aba3486fe"; 272 | sha256 = "10wn0l08j9lgqcw8177nh2ljrnxdrpri7bp0g7nvrsn9rkawvlbf"; 273 | }; 274 | "23.11" = nixpkgsSource { 275 | # 23.11 release 276 | rev = "057f9aecfb71c4437d2b27d3323df7f93c010b7e"; 277 | sha256 = "1ndiv385w1qyb3b18vw13991fzb9wg4cl21wglk89grsfsnra41k"; 278 | }; 279 | "24.05" = nixpkgsSource { 280 | # 24.05 release 281 | rev = "63dacb46bf939521bdc93981b4cbb7ecb58427a0"; 282 | sha256 = "1lr1h35prqkd1mkmzriwlpvxcb34kmhc9dnr48gkm8hh089hifmx"; 283 | }; 284 | "24.11" = nixpkgsSource { 285 | # 24.11 release (ish, missing tag but deployed same-day) 286 | rev = "62c435d93bf046a5396f3016472e8f7c8e2aed65"; 287 | sha256 = "sha256-F7thesZPvAMSwjRu0K8uFshTk3ZZSNAsXTIFvXBT+34="; 288 | }; 289 | }; 290 | nixpkgsUrl = nixpkgsFor.${builtins.nixVersion} or nixpkgsFor.${builtins.substring 0 5 builtins.nixVersion} or nixpkgsFor."24.11"; 291 | nixpkgsPath = builtins.fetchTarball nixpkgsUrl; 292 | nixpkgs = args: import nixpkgsPath args; 293 | } 294 | -------------------------------------------------------------------------------- /nix/lib/data.nix: -------------------------------------------------------------------------------- 1 | { ... }@args: { 2 | ciRepoInfo = rec { 3 | latestVersion = true; 4 | version = "0.7"; 5 | releaseName = "v${version}"; 6 | releaseRef = "refs/tags/${releaseName}"; 7 | devBranch = "main"; 8 | }; 9 | } 10 | -------------------------------------------------------------------------------- /nix/lib/env-builder.nix: -------------------------------------------------------------------------------- 1 | { lib, runCommand, hostPlatform, cacert, config }: with lib; let 2 | glibcLocales = listToAttrs (map (glibc: 3 | lib.nameValuePair (replaceStrings [ "." ] [ "_" ] glibc.version) "${glibc}/lib/locale/locale-archive" 4 | ) (config.environment.glibcLocales or [ ])); 5 | in makeOverridable ({ pname, packages ? [], command ? "", passAsFile ? [], ... }@args: runCommand pname ({ 6 | inherit cacert; 7 | inherit (import ../global.nix) prefix; 8 | inherit (config.bootstrap) runtimeShell; 9 | inherit (config.bootstrap.packages) nix; 10 | 11 | passAsFile = [ "source" "env" "rc" "shellBin" ] ++ passAsFile; 12 | 13 | packages = map getBin packages; 14 | ciRoot = config.lib.ci.storePathFor ../..; 15 | nixPathStr = builtins.concatStringsSep ":" (builtins.attrValues (builtins.mapAttrs (k: v: "${k}=${v}") config.nixPath)); 16 | glibcLocaleVars = optionals hostPlatform.isLinux (mapAttrsToList (name: path: 17 | "LOCALE_ARCHIVE_${name}=${path}" 18 | ) glibcLocales); 19 | env = '' 20 | ci_env_host() { 21 | export PATH=$HOST_PATH 22 | } 23 | 24 | ci_env_nix() { 25 | export PATH=$CI_PATH 26 | } 27 | 28 | ci_env_impure() { 29 | export PATH=$CI_PATH:$HOST_PATH 30 | } 31 | 32 | if [[ -n ''${CI_PATH-} ]]; then return; fi 33 | 34 | if [[ $- != *i* ]]; then 35 | # non-interactive shells should bail on any error 36 | set -euo pipefail 37 | fi 38 | ${optionalString config.environment.closeStdin "exec 0<&-"} 39 | 40 | export NIX_PATH=@nixPathStr@ 41 | if [[ -n "@nix@" ]]; then 42 | export NIX_PREFIX=@nix@ 43 | fi 44 | export HOST_PATH=$PATH 45 | export CI_PATH=@out@/bin 46 | export CI_ROOT=@ciRoot@ 47 | export NIX_SSL_CERT_FILE=@cacert@/etc/ssl/certs/ca-bundle.crt 48 | export TERMINFO_DIRS=''${TERMINFO_DIRS-}:/usr/share/terminfo:@out@/share/terminal 49 | for locale in @glibcLocaleVars@; do 50 | export $locale 51 | done 52 | ''; 53 | 54 | source = '' 55 | source @out@/@prefix@/env 56 | 57 | ci_env_impure 58 | ''; 59 | 60 | rc = '' 61 | ci_rc_env() { 62 | local CI_RCFILE 63 | if [[ -n ''${BASH_VERSION-} ]]; then 64 | CI_RCFILE=''${HOME-/homeless}/.bashrc 65 | fi 66 | if [[ -e ''${CI_RCFILE-} && -n ''${CI_IMPURE-} ]]; then 67 | source $CI_RCFILE 68 | fi 69 | 70 | source @out@/@prefix@/source 71 | if [[ -n ''${CI_IMPURE-} ]]; then 72 | ci_env_impure 73 | fi 74 | } 75 | 76 | ci_rc_env 77 | ''; 78 | 79 | shellBin = '' 80 | #!@runtimeShell@ 81 | 82 | # TODO: check for zsh or other shells 83 | # TODO: assumption here that $runtimeShell is bash (accepts --rcfile) 84 | 85 | exec @runtimeShell@ --rcfile @out@/@prefix@/rc "$@" 86 | ''; 87 | } // builtins.removeAttrs args ["pname" "command" "passAsFile" "packages"]) '' 88 | install -d $out/$prefix $out/bin 89 | 90 | for pkg in $packages; do 91 | if ! cp --no-preserve=mode -rsf $pkg/* $out/; then 92 | cp --no-preserve=mode -Lrsf $pkg/* $out/ 93 | fi 94 | done 95 | 96 | substituteBin() { 97 | substituteAll $1 $out/bin/$2 98 | chmod +x $out/bin/$2 99 | } 100 | 101 | substituteAll $sourcePath $out/$prefix/source 102 | substituteAll $envPath $out/$prefix/env 103 | substituteAll $rcPath $out/$prefix/rc 104 | substituteBin $shellBinPath ci-shell 105 | 106 | ${command} 107 | '') 108 | -------------------------------------------------------------------------------- /nix/lib/env.nix: -------------------------------------------------------------------------------- 1 | { lib, config, ... }: with lib; rec { 2 | envOrNull = envOr null; 3 | envOr = fallback: key: let 4 | value = builtins.getEnv key; 5 | in if value == "" then fallback else value; 6 | envIsSet = key: if config.environment.impure 7 | then envOrNull key != null 8 | else false; 9 | envMappings = { 10 | global = { 11 | platform = 12 | if envOrNull "GITHUB_ACTIONS" == "true" then "gh-actions" 13 | else "none"; 14 | git-commit = null; # TODO: fall back to IFD pulling this info from .git? 15 | git-ref = null; # TODO: ditto, just read .git/HEAD? 16 | git-tag = let 17 | tag = builtins.match "refs/tags/(.*)" (toString config.lib.ci.env.git-ref); 18 | in if tag != null then head tag else null; 19 | git-branch = let 20 | branch = builtins.match "refs/heads/(.*)" (toString config.lib.ci.env.git-ref); 21 | in if branch != null then head branch else null; 22 | slug = null; # TODO: fall back to getting this from the git remote url..? 23 | tmpdir = null; 24 | build-dir = null; # TODO: fall back to config root? 25 | work-dir = null; 26 | pr-head = null; 27 | pr-base = null; 28 | }; 29 | gh-actions = { 30 | git-commit = envOrNull "GITHUB_SHA"; 31 | git-ref = envOrNull "GITHUB_REF"; 32 | slug = envOrNull "GITHUB_REPOSITORY"; 33 | gh-slug = envOrNull "GITHUB_REPOSITORY"; 34 | gh-actor = envOrNull "GITHUB_ACTOR"; 35 | tmpdir = envOrNull "RUNNER_TEMP"; 36 | build-dir = envOrNull "GITHUB_WORKSPACE"; # git repo checkout goes here 37 | work-dir = envOrNull "RUNNER_WORKSPACE"; # this is um, a workspace? it's actually the parent of the build-dir... 38 | gh-event-name = envOrNull "GITHUB_EVENT_NAME"; # "push", "pull_request", etc 39 | gh-event = importJSON (envOrNull "GITHUB_EVENT_PATH"); 40 | gh-workflow = envOrNull "GITHUB_WORKFLOW"; # workflow name/id 41 | gh-action = envOrNull "GITHUB_ACTION"; # an id of sorts? 42 | pr-head = envOrNull "GITHUB_HEAD_REF"; 43 | pr-base = envOrNull "GITHUB_BASE_REF"; 44 | # TODO: option for making sure nix does builds in a tmpfs? disks are slow! 45 | }; 46 | }; 47 | envKey = k: "CI_${replaceStrings [ "-" ] [ "_" ] (toUpper k)}"; 48 | env = let 49 | filtered = filterAttrs (_: v: v != null); 50 | globalEnv = mapAttrs (k: v: envOrNull (envKey k)) envMappings.global; 51 | global = envMappings.global; 52 | in global // filtered envMappings.${global.platform} or { } // filtered globalEnv // { 53 | get = envOrNull; 54 | getOr = envOr; 55 | isSet = envIsSet; 56 | }; 57 | } 58 | -------------------------------------------------------------------------------- /nix/lib/exec-ssh.nix: -------------------------------------------------------------------------------- 1 | { lib, config }: with lib; let 2 | inherit (config.bootstrap.pkgs.buildPackages) pkgs; 3 | sshdkey = pkgs.stdenvNoCC.mkDerivation { 4 | name = "ci-ssh-serverkey"; 5 | 6 | nativeBuildInputs = [ pkgs.openssh ]; 7 | 8 | outputs = [ "out" "priv" ]; 9 | buildCommand = '' 10 | ssh-keygen -q -N "" -t ed25519 -f sshd_key 11 | mv sshd_key.pub $out 12 | mv sshd_key $priv 13 | ''; 14 | }; 15 | commandKeyFor = command: pkgs.stdenvNoCC.mkDerivation { 16 | name = "ci-ssh-commandkey"; 17 | 18 | nativeBuildInputs = [ pkgs.openssh ]; 19 | 20 | commandName = command.name; 21 | 22 | outputs = [ "out" "priv" ]; 23 | buildCommand = '' 24 | ssh-keygen -q -N "" -t ed25519 -f ssh_key 25 | mv ssh_key.pub $out 26 | mv ssh_key $priv 27 | ''; 28 | }; 29 | sshExecutor = { 30 | drv 31 | , executor 32 | }: let 33 | tty = true; 34 | in { 35 | attrs = { 36 | nativeBuildInputs = [ pkgs.openssh ]; 37 | inherit sshdkey; 38 | sshkey = (commandKeyFor drv).priv; 39 | inherit (executor.ci.connectionDetails) port address user; 40 | }; 41 | 42 | name = "ci-ssh"; 43 | buildCommand = '' 44 | echo "[$address]:$port $(cat $sshdkey)" > known_hosts 45 | CLIENT_KEY=$(mktemp) 46 | install -m600 $sshkey $CLIENT_KEY 47 | ssh ${optionalString tty "-t -t"} -q -F none -i $CLIENT_KEY -o UserKnownHostsFile=$PWD/known_hosts -o GlobalKnownHostsFile=/dev/null -p $port $user@$address false 48 | ''; 49 | }; 50 | in { 51 | execSsh = { 52 | commands 53 | , connectionDetails 54 | }: let 55 | executorFor = executor: drv: config.lib.ci.commandExecutor { 56 | inherit drv; 57 | executor = sshExecutor { 58 | inherit drv; 59 | inherit executor; 60 | }; 61 | }; 62 | executors = executor: map (executorFor executor) commands; 63 | commandsExec = listToAttrs (map (drv: 64 | nameValuePair (config.lib.ci.drvOf drv) { 65 | exec = map builtins.unsafeDiscardStringContext drv.ci.exec; 66 | key = commandKeyFor drv; 67 | } 68 | ) commands); 69 | drv = pkgs.stdenvNoCC.mkDerivation { 70 | inherit (import ../global.nix) prefix; 71 | 72 | passAsFile = [ "authorizedKeys" "sshdScript" "sshdConfig" "commandsExec" ]; 73 | 74 | name = "ci-ssh-executor"; 75 | passthru = { 76 | ci = { 77 | inherit connectionDetails; 78 | }; 79 | }; 80 | 81 | sshdkey = sshdkey.priv; 82 | inherit (connectionDetails) address port user; 83 | sshdConfig = '' 84 | ListenAddress @address@ 85 | Port @port@ 86 | #UsePrivilegeSeparation no 87 | PasswordAuthentication no 88 | AuthorizedKeysFile @out@/@prefix@/authorized_keys 89 | StrictModes no 90 | ChallengeResponseAuthentication no 91 | ${optionalString pkgs.hostPlatform.isLinux "UsePAM no"} 92 | ''; 93 | 94 | inherit (pkgs) openssh; 95 | inherit (config.bootstrap.packages) coreutils; 96 | inherit (config.bootstrap) runtimeShell; 97 | sshdScript = '' 98 | #!@runtimeShell@ 99 | HOST_KEY=$(@coreutils@/bin/mktemp) 100 | @coreutils@/bin/install -m600 @sshdkey@ $HOST_KEY # TODO: remove this after sshd exit! 101 | @openssh@/bin/sshd -e -f @out@/@prefix@/sshd_config -h $HOST_KEY -o "PidFile $EX_PIDFILE" 102 | ''; 103 | 104 | nativeBuildInputs = with pkgs; [ openssh jq ]; 105 | commands = map config.lib.ci.drvOf commands; 106 | commandsExec = builtins.toJSON commandsExec; 107 | buildCommand = '' 108 | mkdir -p $out/$prefix $out/bin 109 | for command in $commands; do 110 | IFS=$'\n' commandExec=($(jq -er ".\"$command\".exec | .[]" $commandsExecPath)) # TODO: support quoting these? 111 | commandKey=$(jq -er ".\"$command\".key" $commandsExecPath) 112 | echo "command=\"''${commandExec[*]}\" $(cat $commandKey)" 113 | done > $out/$prefix/authorized_keys 114 | substituteAll $sshdConfigPath $out/$prefix/sshd_config 115 | substituteAll $sshdScriptPath $out/bin/ci-sshd 116 | chmod +x $out/bin/ci-sshd 117 | # TODO: could we make it a systemd user service? can you spawn one-off services with systemctl?? I don't really like just randomly forking something in the background of the test script but eh, it can work fine with exit traps or pidfiles to be fair... 118 | ''; 119 | }; 120 | in drv.overrideAttrs (old: { 121 | passthru = old.passthru or {} // { 122 | exec = "${drv}/bin/ci-sshd"; 123 | ci = old.passthru.ci or {} // { 124 | executor = { 125 | executors = executors drv; 126 | for = executorFor drv; # TODO: validate command passed is in commands array? otherwise it will break! 127 | }; 128 | }; 129 | }; 130 | }); 131 | } 132 | -------------------------------------------------------------------------------- /nix/lib/impure.nix: -------------------------------------------------------------------------------- 1 | { lib, config, ... }: with lib; rec { 2 | hostPath = let 3 | paths' = splitString ":" (builtins.getEnv "PATH"); 4 | paths = builtins.filter (p: p != "") (filter builtins.pathExists paths'); 5 | in map (path: { inherit path; prefix = ""; }) paths; 6 | hostDep = name: bins: let 7 | binTry = map (bin: builtins.tryEval (builtins.findFile hostPath bin)) (toList bins); 8 | success = all (bin: bin.success) binTry; 9 | binPaths = map (bin: bin.value) binTry; 10 | drv = config.bootstrap.pkgs.linkFarm "${name}-host-impure" (map (bin: { 11 | name = "bin/${builtins.baseNameOf bin}"; 12 | path = toString bin; 13 | }) binPaths); 14 | in if success then drv else null; 15 | } 16 | -------------------------------------------------------------------------------- /nix/lib/lib.nix: -------------------------------------------------------------------------------- 1 | if builtins.pathExists ./lib/default.nix then ./lib else builtins.fetchTarball { 2 | url = "https://github.com/arcnmx/nixpkgs-lib/archive/d21353de66de858fb2998ed76da67a62ccc252dd.tar.gz"; 3 | sha256 = "00kx3agiv5d61mjhw5lc3lq1s0frvy2pz9cs4cs3ghwvz68r057j"; 4 | } 5 | -------------------------------------------------------------------------------- /nix/lib/overlay.nix: -------------------------------------------------------------------------------- 1 | { config, modulesPath, libPath, configPath, rootConfigPath, ... }: self: super: with super.lib; { 2 | ci = super.ci.extend (cself: csuper: { 3 | inherit config; 4 | 5 | doc = let 6 | inherit (config._module.args.channels) nmd; 7 | inherit (config.channels.ci) name; 8 | scrubPkgs = { 9 | channels = mapAttrs (k: c: { 10 | import = mkForce (nmd.scrubDerivations (if k == "nixpkgs" then "pkgs" else k) config.channels.${k}.import); 11 | }) config.channels; 12 | nixpkgs.import = mkForce (nmd.scrubDerivations "pkgs" config.nixpkgs.import); 13 | lib.ci = { 14 | import = mkForce builtins.import; 15 | nixPathImport = mkForce (path: builtins.import); 16 | }; 17 | _module.args = { 18 | inherit modulesPath libPath configPath rootConfigPath; 19 | }; 20 | }; 21 | options = nmd.buildModulesDocs { 22 | modules = import ../modules.nix { } ++ [ scrubPkgs ]; 23 | moduleRootPaths = [ modulesPath ]; 24 | mkModuleUrl = path: "https://github.com/arcnmx/ci/blob/${config.ci.version}/nix/${path}#blob-path"; 25 | channelName = name; 26 | docBook.id = "${name}-options"; 27 | }; 28 | docs = nmd.buildDocBookDocs { 29 | pathName = name; 30 | modulesDocs = [ options ]; 31 | documentsDirectory = ../doc; 32 | chunkToc = '' 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | ''; 41 | }; 42 | in { 43 | json = docs.json.override { 44 | path = "share/doc/${name}/options.json"; 45 | }; 46 | 47 | manPages = docs.manPages; 48 | 49 | manual = docs.html; 50 | 51 | open = docs.htmlOpenTool; 52 | }; 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /nix/lib/scope.nix: -------------------------------------------------------------------------------- 1 | { ... }: let 2 | # convert { nixpkgs = ./path; } attrsets to [ { path = ./path; prefix = "nixpkgs" } ] format 3 | nixPathList = nixPathAttrs: let 4 | nixPath = { 5 | # never really makes sense to omit ? 6 | nix = ; 7 | } // nixPathAttrs; 8 | in builtins.map (prefix: { 9 | inherit prefix; 10 | path = toString nixPath.${prefix}; 11 | }) (builtins.attrNames nixPath); 12 | 13 | # import a file with a new nixPath 14 | nixPathImport = nixPath: nixPathScopedImport nixPath { }; 15 | 16 | # import a file with a new nixPath and scope 17 | nixPathScopedImport = nixPath': newScope: let 18 | nixPath = if builtins.isAttrs nixPath' then nixPathList nixPath' else nixPath'; 19 | import = builtins.scopedImport scope; 20 | scopedImport = newScope: builtins.scopedImport (scope // newScope); 21 | scope = newScope // { 22 | inherit import scopedImport; 23 | __nixPath = nixPath; 24 | builtins = builtins // (newScope.builtins or { }) // { 25 | inherit nixPath import scopedImport; 26 | }; 27 | }; 28 | in import; 29 | in { 30 | inherit nixPathImport nixPathScopedImport nixPathList; 31 | } 32 | -------------------------------------------------------------------------------- /nix/lib/setup.nix: -------------------------------------------------------------------------------- 1 | { lib, config }: with lib; { 2 | pname = "ci-env-setup"; 3 | packages = builtins.attrValues config.environment.bootstrap; 4 | 5 | passAsFile = [ "setup" ]; 6 | 7 | cachixUse = attrNames (filterAttrs (_: c: c.enable && c.publicKey == null) config.cache.cachix); 8 | inherit (config.bootstrap.packages) cachix coreutils; 9 | inherit (config.environment) allowRoot; 10 | nixSysconfDir = "${config.nix.corepkgs.config.nixSysconfDir or "/etc"}/nix"; 11 | #inherit (config.nix) configFile; 12 | setup = '' 13 | #!@runtimeShell@ 14 | set -eu 15 | 16 | source @out@/@prefix@/env 17 | ci_env_impure 18 | 19 | asroot() { 20 | local CI_ROOT_ENV=( 21 | PATH="$PATH" 22 | ) 23 | if [[ -n ''${NIX_SSL_CERT_FILE-} ]]; then 24 | CI_ROOT_ENV+=( 25 | NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" 26 | ) 27 | fi 28 | if [[ ! -w @nixSysconfDir@ && -n "@allowRoot@" ]]; then 29 | sudo @coreutils@/bin/env "''${CI_ROOT_ENV[@]}" "$@" 30 | else 31 | "$@" 32 | fi 33 | } 34 | 35 | #asroot @coreutils@/bin/mkdir -p @nixSysconfDir@ && 36 | #asroot @coreutils@/bin/tee -a @nixSysconfDir@/nix.conf < @configFile@ || 37 | # echo failed to configure @nixSysconfDir@/nix.conf >&2 38 | 39 | for cachixCache in @cachixUse@; do 40 | echo Setting up cache $cachixCache... >&2 41 | asroot @cachix@/bin/cachix use $cachixCache || 42 | echo failed to add cache >&2 43 | done 44 | ''; 45 | 46 | command = '' 47 | substituteBin $setupPath ci-setup 48 | ''; 49 | } 50 | -------------------------------------------------------------------------------- /nix/modules.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? null, check ? true }: let 2 | libPath = import ./lib/lib.nix; 3 | module = { config, lib, ... }: with lib; { 4 | imports = [ 5 | (libPath + "/nixos/modules/misc/assertions.nix") 6 | (libPath + "/nixos/modules/misc/meta.nix") 7 | ]; 8 | 9 | config._module = { 10 | inherit check; 11 | }; 12 | #config.lib = import ./lib { inherit lib; }; 13 | config.nixpkgs = mkIf (pkgs != null) { 14 | args = { 15 | localSystem = config.lib.ci.mkOptionDefault1 pkgs.buildPlatform.system; 16 | crossSystem = mkIf (pkgs.buildPlatform != pkgs.hostPlatform) (config.lib.ci.mkOptionDefault1 pkgs.hostPlatform.system); 17 | config = mapAttrs (_: config.lib.ci.mkOptionDefault1) pkgs.config or {}; 18 | overlays = pkgs.overlays or []; 19 | crossOverlays = pkgs.crossOverlays or []; 20 | }; 21 | path = config.lib.ci.mkOptionDefault2 (toString pkgs.path); 22 | }; 23 | }; 24 | in [ 25 | ./env.nix 26 | ./lib.nix 27 | ./exec.nix 28 | ./config.nix 29 | ./project.nix 30 | ./tasks.nix 31 | ./actions.nix 32 | ./actions-ci.nix 33 | module 34 | ] 35 | -------------------------------------------------------------------------------- /nix/overlay.nix: -------------------------------------------------------------------------------- 1 | self: super: let 2 | inherit (import ./global.nix) prefix; 3 | ciWrapper = { lib, stdenvNoCC }: input: with lib; let 4 | wrapped = stdenvNoCC.mkDerivation ({ 5 | # a wrapper prevents the input itself from being a build-time dependency for a task 6 | name = if hasPrefix "ci-" input.name then input.name else "ci-${input.name}"; 7 | preferLocalBuild = true; 8 | allowSubstitutes = true; 9 | 10 | inherit input; 11 | passthru = input.passthru or {} // { 12 | ci = input.passthru.ci or {} // { 13 | wrapped = input; 14 | }; 15 | }; 16 | 17 | buildCommand = '' 18 | # no-op marker for $input 19 | mkdir -p $out/nix-support 20 | ''; 21 | } // optionalAttrs (input ? meta) { 22 | inherit (input) meta; 23 | }); 24 | in input.ci.wrapped or wrapped; 25 | ciCommand = { lib, stdenvNoCC, config }: with lib; makeOverridable ({ 26 | name 27 | , command 28 | , stdenv ? stdenvNoCC 29 | , warn ? false 30 | , skip ? null 31 | , cache ? null 32 | , displayName ? null 33 | , timeout ? null 34 | , tests ? null 35 | , impure ? false 36 | , cwd ? config.environment.workingDirectory 37 | , environment ? [] 38 | , ciEnv ? true 39 | , sha256 ? null 40 | , ... 41 | }@args: let 42 | args' = removeAttrs args [ 43 | "name" "command" "meta" "passthru" "warn" "skip" "cache" "displayName" "timeout" "tests" "impure" "sha256" "ciEnv" "passAsFile" "cwd" "environment" 44 | ] // optionalAttrs (builtins.getEnv "NIX_IGNORE_SYMLINK_STORE" == "1") { 45 | NIX_IGNORE_SYMLINK_STORE = "1"; 46 | }; 47 | argVars = attrNames args' ++ environment; 48 | commandPath = "${prefix}/run-test"; 49 | # TODO: nativeBuildInputs should work with impure commands! 50 | command' = if impure == true then '' 51 | mkdir -p $out/${prefix} 52 | { 53 | cat $commandHeaderPath 54 | echo cd ${cwd} 55 | ${optionalString (argVars != []) "declare -p $argVars"} 56 | cat $commandPath 57 | } > $out/${commandPath} 58 | chmod +x $out/${commandPath} 59 | '' else '' 60 | source $commandHeaderPath 61 | mkdir -p $out 62 | source $commandPath 63 | ''; 64 | hostExec = [ "${drv}/${commandPath}" ]; 65 | drv = stdenv.mkDerivation ({ 66 | inherit name; 67 | preferLocalBuild = true; 68 | allowSubstitutes = true; 69 | 70 | buildCommand = command'; 71 | 72 | inherit argVars; 73 | commandHeader = optionalString ciEnv '' 74 | #!${self.buildPackages.runtimeShell} 75 | source ${config.export.env.test}/${prefix}/source 76 | ci_env_impure 77 | ''; 78 | passAsFile = [ "buildCommand" "command" "commandHeader" ] ++ args.passAsFile or []; 79 | inherit command; 80 | 81 | meta = { 82 | ${mapNullable (_: "name") displayName} = displayName; 83 | ${mapNullable (_: "timeout") timeout} = timeout; 84 | } // args.meta or {}; 85 | 86 | passthru = args.passthru or {} // { 87 | ci = { 88 | inherit warn; 89 | ${mapNullable (_: "skip") skip} = skip; 90 | ${mapNullable (_: "cache") cache} = if isBool cache then { enable = cache; } else cache; 91 | ${mapNullable (_: "tests") tests} = toList tests; 92 | ${if impure == true then "exec" else null} = hostExec; 93 | } // args.passthru.ci or {}; 94 | }; 95 | } // optionalAttrs (sha256 != null) { 96 | outputHashAlgo = "sha256"; 97 | outputHash = sha256; 98 | } // genAttrs environment builtins.getEnv // args'); 99 | in drv); 100 | in { 101 | ci = super.lib.makeExtensible (cself: { 102 | wrapper = self.callPackage ciWrapper { }; 103 | command = self.callPackage ciCommand { inherit (cself) config; }; 104 | commandCC = cself.command.override { stdenvNoCC = self.stdenv; }; 105 | }); 106 | # passthru.ci.skip = true; # do not test 107 | # passthru.ci.omit = true; # do not evaluate 108 | # passthru.ci.cache = drv: [ drv ]; # inputs to cache for a given drv 109 | # passthru.ci.cache.enable = false; # always re-run, true is default 110 | # passthru.ci.cache.buildInputs = true; # cache build inputs 111 | # passthru.ci.cache.inputs = [...]; # other items to cache with it 112 | # - consider how this should differ from making a build non-deterministic (input with currentTime or CI build counter env var) 113 | # passthru.ci.exec = ["script" "and" "args"]; # a test that runs in the host environment (with the associated derivation in scope/PATH): 114 | # passthru.ci.eval = drv: assert something; true; # a test that checks whether the given expression evaluates to `true` # TODO: implement this 115 | # - can be impure and use network, caches, etc 116 | # passthru.ci.inputs = actual derivation to build/test (use to avoid recursing into unsupported attrs, or to build mkShells, etc) 117 | # passthru.ci.tests = []; # related test derivations, expects a function with a { drv }: argument. 118 | # passthru.ci.max-silent-time # seconds 119 | # meta.timeout = seconds; # see https://nixos.org/nixpkgs/manual/#sec-standard-meta-attributes 120 | # passthru.tests = []; # related test derivations for hydra, idk, ignore? 121 | # meta.broken, meta.platforms, etc. are obeyed as expected and considered the same as "ci.skip" 122 | } 123 | -------------------------------------------------------------------------------- /nix/pkgs.nix: -------------------------------------------------------------------------------- 1 | import (import ./lib/cipkgs.nix).nixpkgsPath 2 | -------------------------------------------------------------------------------- /nix/project.nix: -------------------------------------------------------------------------------- 1 | { config, lib, modulesPath, configPath, rootConfigPath, libPath, ... }: with lib; let 2 | # NOTE: perhaps submodules are the wrong way to go about this, just use evalModules again 3 | # (mostly saying this because as far as I can tell, there's no way to pass specialArgs on to submodules? I imagine that could be hacked into lib/ though?) 4 | submodule = imports: types.submodule { 5 | imports = imports ++ import ./modules.nix { 6 | inherit (config._module) check; 7 | }; 8 | 9 | config = { 10 | _module.args = { 11 | inherit modulesPath rootConfigPath libPath; 12 | configPath = mkOptionDefault configPath; 13 | parentConfigPath = configPath; 14 | }; 15 | }; 16 | } // { 17 | getSubOptions = _: {}; 18 | getSubModules = null; 19 | }; 20 | jobModule = { name, ...}: { 21 | config = { 22 | parentConfig = config; 23 | inherit (config) stageId; 24 | jobId = name; 25 | jobs = mkForce { }; # jobs only go one level deep! 26 | stages = mkForce { }; # jobs cannot contain stages! 27 | }; 28 | }; 29 | mkOptionDefaultAlmost = mkOverride 1499; 30 | stageModule = { name, ...}: { 31 | config = { 32 | parentConfig = config; 33 | stageId = name; 34 | name = mkOptionDefault name; 35 | ci = { 36 | gh-actions.enable = mkIf config.ci.gh-actions.enable (mkOptionDefaultAlmost true); 37 | url = mkOptionDefaultAlmost config.ci.url; 38 | }; 39 | }; 40 | }; 41 | in { 42 | options = { 43 | jobId = mkOption { 44 | type = types.nullOr types.str; 45 | default = null; 46 | internal = true; 47 | }; 48 | stageId = mkOption { 49 | type = types.nullOr types.str; 50 | default = null; 51 | internal = true; 52 | }; 53 | exportAttr = mkOption { 54 | type = types.nullOr types.str; 55 | internal = true; 56 | default = let 57 | prefix = if config.parentConfig == null then "" else config.parentConfig.exportAttrDot; 58 | in if config.jobId != null then "${prefix}job.${config.jobId}" 59 | else if config.stageId != null then "${prefix}stage.${config.stageId}" 60 | else null; 61 | }; 62 | exportAttrDot = mkOption { 63 | type = types.str; 64 | internal = true; 65 | default = if config.exportAttr == null then "" else "${config.exportAttr}."; 66 | }; 67 | id = mkOption { 68 | type = types.str; 69 | default = findFirst (i: i != null) "ci" [ config.jobId config.stageId ]; 70 | internal = true; 71 | }; 72 | name = mkOption { 73 | type = types.str; 74 | default = findFirst (i: i != null) "ci" [ config.jobId config.stageId ]; 75 | }; 76 | parentConfig = mkOption { 77 | type = types.nullOr types.unspecified; 78 | default = null; 79 | internal = true; 80 | }; 81 | warn = mkOption { 82 | # TODO: ability to trigger some sort of action/notification 83 | type = types.bool; 84 | default = false; 85 | }; 86 | jobs = let 87 | type = submodule [ configPath jobModule ]; 88 | in mkOption { 89 | type = types.attrsOf type; 90 | default = { }; 91 | visible = config.jobId != null; 92 | }; 93 | stages = let 94 | type = types.coercedTo types.path (configPath: { ... }: { 95 | imports = [ configPath ]; 96 | config._module.args = { 97 | #inherit configPath; 98 | stageConfigPath = configPath; 99 | }; 100 | }) (submodule [ stageModule ]); 101 | in mkOption { 102 | type = types.attrsOf type; 103 | # TODO: coercedTo types.path 104 | default = { }; 105 | visible = config.jobId != null; 106 | }; 107 | project = { 108 | exec = mkOption { 109 | type = types.attrsOf types.str; # TODO: or lines? 110 | default = { }; 111 | }; 112 | run = mkOption { 113 | type = types.attrsOf types.package; 114 | default = { }; 115 | }; 116 | }; 117 | export.job = mkOption { 118 | type = types.attrsOf types.unspecified; 119 | }; 120 | export.stage = mkOption { 121 | type = types.attrsOf types.unspecified; 122 | }; 123 | }; 124 | config = { 125 | export.job = mapAttrs (_: s: s.export) config.jobs; 126 | export.stage = mapAttrs (_: s: s.export) config.stages; 127 | lib.ci = { 128 | inherit (import ./lib/scope.nix { }) nixPathImport; 129 | }; 130 | }; 131 | } 132 | -------------------------------------------------------------------------------- /nix/tasks.nix: -------------------------------------------------------------------------------- 1 | { pkgs, config, lib, ... }: with lib; let 2 | inherit (config.bootstrap) pkgs; 3 | executor = config.project.executor.drv; 4 | warn = config.warn; 5 | flattenInputs = inputs: 6 | if inputs ? ci.inputs then flattenInputs inputs.ci.inputs 7 | #else if isDerivation inputs && inputs.ci.omit or false != false then [ ] 8 | else if isDerivation inputs then [ inputs ] 9 | else if isAttrs inputs then concatMap flattenInputs (filter (a: a.recurseForDerivations or true) (attrValues inputs)) 10 | else if isList inputs then concatMap flattenInputs inputs 11 | else [ ]; 12 | isValid = drv: assert isDerivation drv; # TODO: support lists or attrsets of derivations? 13 | !(drv.meta.broken or false) && (drv.ci.skip or false) == false && (drv.ci.omit or false) == false && drv.meta.available or true; 14 | mapInput = cache: input: if ! cache.enable then input.overrideAttrs (old: { 15 | passthru = old.passthru or {} // { 16 | allowSubstitutes = false; # this needs to be part of the derivation doesn't it :( 17 | ci = old.passthru.ci or {} // { 18 | cache.enable = false; 19 | }; 20 | }; 21 | }) else if cache.wrap || input.ci.cache.wrap or false == true || input.allowSubstitutes or true == false then (pkgs.ci.wrapper input).overrideAttrs (old: { 22 | passthru = old.passthru or {} // { 23 | ci = old.passthru.ci or {} // { 24 | cache.enable = input.allowSubstitutes or true; 25 | }; 26 | }; 27 | }) else input; 28 | taskType = types.submodule ({ name, config, ... }: { 29 | options = { 30 | id = mkOption { 31 | type = types.str; 32 | default = "ci-task-${name}"; 33 | }; 34 | name = mkOption { 35 | type = types.nullOr types.str; 36 | default = name; 37 | }; 38 | args = mkOption { 39 | type = types.attrsOf types.unspecified; 40 | default = { }; 41 | }; 42 | inputs = let 43 | #inputType = types.package; 44 | inputType = types.unspecified; # have you seen flattenInputs?? it doesn't require derivations wow 45 | type = types.listOf inputType; 46 | fudge = types.coercedTo inputType singleton type; 47 | in mkOption { 48 | type = fudge; 49 | default = [ ]; 50 | }; 51 | preBuild = mkOption { 52 | type = types.lines; 53 | default = ""; 54 | }; 55 | buildCommand = mkOption { 56 | type = types.lines; 57 | default = ""; 58 | }; 59 | warn = mkOption { 60 | type = types.bool; 61 | default = warn || any (i: i.ci.warn or false) config.inputs; 62 | }; 63 | skip = mkOption { 64 | type = types.either types.bool types.str; 65 | default = false; 66 | }; 67 | timeoutSeconds = mkOption { 68 | type = types.nullOr types.ints.positive; 69 | default = null; 70 | }; 71 | cache = { 72 | enable = mkEnableOption "cache build results" // { default = true; }; 73 | wrap = mkEnableOption "cache whether a build succeeds and not the output"; 74 | inputs = mkOption { 75 | type = types.listOf types.package; 76 | default = [ ]; 77 | }; 78 | # TODO: other attrs that are valid here? 79 | }; 80 | drv = mkOption { 81 | type = types.package; 82 | internal = true; 83 | }; 84 | internal.inputs = { 85 | all = mkOption { 86 | type = types.listOf types.package; 87 | internal = true; 88 | }; 89 | valid = mkOption { 90 | type = types.listOf types.package; 91 | internal = true; 92 | }; 93 | tests = mkOption { 94 | type = types.listOf types.package; 95 | internal = true; 96 | }; 97 | skipped = mkOption { 98 | type = types.listOf types.package; 99 | internal = true; 100 | }; 101 | impure = mkOption { 102 | type = types.listOf types.package; 103 | internal = true; 104 | }; 105 | pure = mkOption { 106 | type = types.listOf types.package; 107 | internal = true; 108 | }; 109 | wrapped = mkOption { 110 | type = types.listOf types.package; 111 | internal = true; 112 | }; 113 | wrappedImpure = mkOption { 114 | type = types.listOf types.package; 115 | internal = true; 116 | }; 117 | }; 118 | }; 119 | config = { 120 | internal.inputs = let 121 | partitioned = partition isValid config.internal.inputs.all; 122 | inputs = map (mapInput config.cache) (config.internal.inputs.valid ++ config.internal.inputs.tests); 123 | partitioned'impure = partition (d: d ? ci.exec) inputs; 124 | mapTest = drv: test: if isFunction test 125 | then test drv 126 | else test; 127 | in { 128 | all = flattenInputs config.inputs; 129 | skipped = partitioned.wrong; 130 | valid = partitioned.right; 131 | tests = concatMap (d: map (mapTest d) d.ci.tests or []) config.internal.inputs.valid; 132 | impure = partitioned'impure.right; 133 | pure = partitioned'impure.wrong; 134 | wrapped = map pkgs.ci.wrapper partitioned'impure.wrong; 135 | wrappedImpure = if executor != null 136 | then map executor.ci.executor.for config.internal.inputs.impure 137 | else [ ]; 138 | # TODO: possibly want to be able to filter out warn'd inputs so task can still run when they fail? 139 | }; 140 | drv = pkgs.stdenvNoCC.mkDerivation { 141 | name = config.id; 142 | 143 | inherit (config.internal.inputs) wrapped; 144 | inherit (config.internal.inputs) wrappedImpure; 145 | 146 | preferLocalBuild = true; 147 | allowSubstitutes = true; 148 | passAsFile = [ "buildCommand" ] ++ config.args.passAsFile or []; 149 | buildCommand = '' 150 | ${config.buildCommand} 151 | mkdir -p $out/nix-support 152 | echo $wrapped $wrappedImpure > $out/nix-support/ci-task 153 | ''; 154 | 155 | meta = { 156 | ${mapNullable (_: "timeout") config.timeoutSeconds} = config.timeoutSeconds; 157 | } // optionalAttrs (config.name != null) { 158 | inherit (config) name; 159 | }; 160 | passthru = config.args.passthru or {} // { 161 | inputs = config.internal.inputs.valid; 162 | inputsAll = config.internal.inputs.all; 163 | inputsSkipped = config.internal.inputs.skipped; 164 | ci = { 165 | tests = config.internal.inputs.valid; 166 | inherit (config) warn skip cache; 167 | } // config.args.passthru.ci or {}; 168 | }; 169 | }; 170 | }; 171 | }); 172 | in { 173 | options = { 174 | project = { 175 | executor = { 176 | connectionDetails = mkOption { 177 | type = types.attrsOf types.unspecified; 178 | default = { 179 | }; 180 | }; 181 | drv = mkOption { 182 | type = types.nullOr types.package; 183 | internal = true; 184 | }; 185 | }; 186 | }; 187 | tasks = mkOption { 188 | type = types.attrsOf taskType; 189 | default = { }; 190 | }; 191 | }; 192 | config.project.executor = { 193 | drv = let 194 | commands = concatLists (mapAttrsToList (_: t: optionals (t.skip == false) t.internal.inputs.impure) config.tasks); 195 | in mkOptionDefault (if commands == [] then null else config.lib.ci.execSsh { 196 | inherit (config.project.executor) connectionDetails; 197 | inherit commands; 198 | }); 199 | connectionDetails = with config.lib.ci; mapAttrs (_: mkOptionDefault) { 200 | address = env.getOr 201 | (if (systems.elaborate config.nixpkgs.args.system).isLinux then "127.42.0.2" else "127.0.0.1") 202 | "CI_EXEC_ADDRESS"; 203 | user = env.getOr (env.get "USER") "CI_EXEC_USER"; 204 | port = env.getOr 50650 "CI_EXEC_PORT"; # u-umm 205 | }; 206 | }; 207 | config.lib.ci = { 208 | inherit (import ./lib/impure.nix { inherit config lib; }) hostPath hostDep; 209 | }; 210 | } 211 | -------------------------------------------------------------------------------- /nix/tools/default.nix: -------------------------------------------------------------------------------- 1 | { 2 | pkgs ? import { } 3 | }: let 4 | derivations = { 5 | ci-query = { substituteAll, runtimeShell, nix }: substituteAll { 6 | name = "ci-query"; 7 | dir = "bin"; 8 | src = ./query.sh; 9 | isExecutable = true; 10 | inherit runtimeShell nix; 11 | }; 12 | ci-dirty = { substituteAll, runtimeShell, coreutils, nix }: substituteAll { 13 | name = "ci-dirty"; 14 | dir = "bin"; 15 | src = ./dirty.sh; 16 | isExecutable = true; 17 | inherit runtimeShell coreutils nix; 18 | }; 19 | }; 20 | in with derivations; { 21 | ci-query = pkgs.callPackage ci-query { }; 22 | ci-dirty = pkgs.callPackage ci-dirty { }; 23 | } 24 | -------------------------------------------------------------------------------- /nix/tools/dirty.sh: -------------------------------------------------------------------------------- 1 | #!@runtimeShell@ 2 | set -euo pipefail 3 | 4 | if [[ -n "@nix@" ]]; then 5 | export PATH="@nix@/bin:$PATH" 6 | fi 7 | 8 | OPT_IGNORE_LOCAL= 9 | OPT_VERBOSE= 10 | 11 | if [[ $# -gt 0 ]]; then 12 | if [[ $1 = -i ]]; then 13 | shift 14 | # only check substituters, ignore whether the package is installed or not 15 | OPT_IGNORE_LOCAL=1 16 | fi 17 | if [[ $1 = -v ]]; then 18 | shift 19 | # give more feedback via stderr 20 | OPT_VERBOSE=1 21 | fi 22 | fi 23 | 24 | validStorePath() { 25 | if [[ -n $OPT_IGNORE_LOCAL ]]; then 26 | false 27 | else 28 | #nix path-info "$2" > /dev/null 2>&1 29 | nix-store -u -q --hash "$1" > /dev/null 2>&1 30 | fi 31 | } 32 | 33 | { 34 | CLEAN=() 35 | while read -r line; do 36 | LINE=($line) 37 | if [[ ${LINE[1]} = /* ]]; then 38 | # fixup weird issues when name is missing from the output? 39 | LINE=(${LINE[0]} ${LINE[2]#*-} ${LINE[1]} ${LINE[2]}) 40 | fi 41 | # STATUS NAME DRV_PATH OUTPATHS 42 | # STATUS FORMAT: IPS (Installed drvPresent Substitutable, with - in place if false) 43 | if [[ ${LINE[0]} = ??- ]] && ! validStorePath ${LINE[2]} ${LINE[3]}; then 44 | # filter for derivations that are neither installed nor available from a binary substitute 45 | echo ${LINE[2]} 46 | if [[ -n $OPT_VERBOSE ]]; then 47 | echo "${LINE[1]} :: ${LINE[3]}" >&2 48 | fi 49 | elif [[ -n $OPT_VERBOSE ]]; then 50 | CLEAN+=("${LINE[1]} :: ${LINE[0]} ${LINE[3]}") 51 | fi 52 | done 53 | 54 | for clean in ${CLEAN[@]+"${CLEAN[@]}"}; do 55 | echo "[CLEAN] $clean" >&2 56 | done 57 | } | @coreutils@/bin/sort -u 58 | -------------------------------------------------------------------------------- /nix/tools/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | export_env() { 6 | case "${CI_PLATFORM-}" in 7 | gh-actions) 8 | if [[ -n "${GITHUB_ENV-}" ]]; then 9 | echo "$1=$2" >> $GITHUB_ENV 10 | else 11 | echo "::set-env name=$1::$2" >&2 12 | fi 13 | ;; 14 | azure-pipelines) 15 | echo "##vso[task.setvariable variable=$1]$2" >&2 16 | ;; 17 | esac 18 | } 19 | 20 | set_output() { 21 | case "${CI_PLATFORM-}" in 22 | gh-actions) 23 | if [[ -n "${GITHUB_OUTPUT-}" ]]; then 24 | echo "$1=$2" >> $GITHUB_OUTPUT 25 | else 26 | echo "::set-output name=$1::$2" >&2 27 | fi 28 | ;; 29 | esac 30 | } 31 | 32 | add_path() { 33 | case "${CI_PLATFORM-}" in 34 | gh-actions) 35 | if [[ -n "${GITHUB_PATH-}" ]]; then 36 | echo "$1" >> $GITHUB_PATH 37 | else 38 | echo "::add-path::$1" >&2 39 | fi 40 | ;; 41 | azure-pipelines) 42 | sudo chown 0:0 / || true 43 | cat >> ~/.bash_profile <' config.nix.settingsText --argstr config "${CI_CONFIG-$CI_ROOT/tests/empty.nix}") 74 | NIX_USER_CONF_FILE=$(maketemp ci.nix.user.conf) 75 | printf "%s" "$NIX_USER_CONF" > "$NIX_USER_CONF_FILE" 76 | export NIX_USER_CONF_FILES="${NIX_USER_CONF_FILES-${XDG_CONFIG_HOME-$HOME/.config}/nix/nix.conf}:$NIX_USER_CONF_FILE" 77 | export_env NIX_USER_CONF_FILES "$NIX_USER_CONF_FILES" 78 | 79 | if [[ -n ${CI_NIX_PATH_NIXPKGS-} ]]; then 80 | export NIX_PATH="${NIX_PATH-}${NIX_PATH+:}nixpkgs=$(nix_eval "$CI_ROOT/nix/lib/cipkgs.nix" nixpkgsUrl.url)" 81 | fi 82 | if [[ -n ${NIX_PATH-} ]]; then 83 | export_env NIX_PATH "$NIX_PATH" 84 | fi 85 | set_output nix-path "${NIX_PATH-}" 86 | } 87 | 88 | if type -P nix > /dev/null; then 89 | export NIX_PATH_DIR=$(dirname "$(readlink -f "$(type -P nix)")") 90 | 91 | NIX_VERSION=$(nix --version) 92 | if [[ $NIX_VERSION = "nix "* ]]; then 93 | NIX_VERSION=${NIX_VERSION##* } 94 | export_env NIX_VERSION "$NIX_VERSION" 95 | set_output version "$NIX_VERSION" 96 | fi 97 | 98 | setup_nix_path 99 | exit 100 | fi 101 | 102 | NIX_VERSION=${NIX_VERSION-latest} 103 | if [[ $NIX_VERSION != latest && $NIX_VERSION != nix-* ]]; then 104 | NIX_VERSION=nix-$NIX_VERSION 105 | fi 106 | 107 | case "$(uname -s).$(uname -m)" in 108 | Linux.x86_64) NIX_SYSTEM=x86_64-linux;; 109 | Linux.i?86) NIX_SYSTEM=i686-linux;; 110 | Linux.aarch64) NIX_SYSTEM=aarch64-linux;; 111 | Darwin.x86_64) NIX_SYSTEM=x86_64-darwin;; 112 | Darwin.arm64|Darwin.aarch64) NIX_SYSTEM=aarch64-darwin;; 113 | esac 114 | 115 | if [[ $NIX_VERSION = latest ]]; then 116 | NIX_VERSION=$(curl -fsSL https://nixos.org/nix/install | grep -o 'nix-[0-9.]*' | tail -n1) 117 | fi 118 | NIX_URL=https://nixos.org/releases/nix/$NIX_VERSION 119 | NIX_VERSION=${NIX_VERSION#nix-} 120 | 121 | NIX_BASE=nix-$NIX_VERSION-$NIX_SYSTEM 122 | NIX_URL=$NIX_URL/$NIX_BASE.tar 123 | 124 | echo "Downloading $NIX_BASE..." >&2 125 | 126 | makedir() { 127 | if ! mkdir -pm 0755 "$1" 2>/dev/null; then 128 | sudo mkdir -pm 0755 "$1" && sudo chown $(id -u) "$1" 129 | fi 130 | } 131 | 132 | installer_fallback() { 133 | NIX_INSTALLER=$1 134 | NIX_STORE_DIR=$HOME/nix-install 135 | mkdir $NIX_STORE_DIR 136 | } 137 | 138 | NIX_INSTALLER=${NIX_INSTALLER-} 139 | NIX_STORE_DIR=/nix 140 | if [[ -n $NIX_INSTALLER ]]; then 141 | installer_fallback "$NIX_INSTALLER" 142 | elif ! makedir $NIX_STORE_DIR; then 143 | if [[ $NIX_SYSTEM = *-darwin ]]; then 144 | # macos catalina mounts root readonly 145 | if sudo mount -uw /; then 146 | # if SIP is disabled this will still work... 147 | makedir $NIX_STORE_DIR 148 | elif /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B >/dev/null 2>&1; then 149 | # otherwise best we can do is tell macos to make us a symlink 150 | # see also: https://github.com/NixOS/nix/pull/3212 151 | NIX_STORE_CANON=/opt/nix 152 | makedir $NIX_STORE_CANON 153 | echo -e "nix\\t$NIX_STORE_CANON" | sudo tee -a /etc/synthetic.conf > /dev/null 154 | /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B 155 | export NIX_IGNORE_SYMLINK_STORE=1 156 | else 157 | installer_fallback --daemon 158 | export NIX_IGNORE_SYMLINK_STORE=1 159 | fi 160 | else 161 | installer_fallback --no-daemon 162 | fi 163 | fi 164 | makedir $NIX_STORE_DIR/var 165 | makedir $NIX_STORE_DIR/var/nix 166 | if curl -fsSLI $NIX_URL.xz > /dev/null; then 167 | tar -C $NIX_STORE_DIR --strip-components=1 -xJf <(curl -fSL $NIX_URL.xz) 168 | else 169 | tar -C $NIX_STORE_DIR --strip-components=1 -xjf <(curl -fSL $NIX_URL.bz2) 170 | fi 171 | 172 | nixvars() { 173 | NIX_STORE_NIX=$(cd $NIX_STORE_DIR/store && echo *-nix-2*) 174 | NIX_STORE_CACERT=$(cd $NIX_STORE_DIR/store && echo *-nss-cacert-*) 175 | NIX_PROFILE="$NIX_STORE_DIR/store/$NIX_STORE_NIX/etc/profile.d/nix.sh" 176 | 177 | # silence "warning: Nix search path entry '/home/runner/.nix-defexpr/channels' does not exist, ignoring" 178 | if [[ ! -e $HOME/.nix-defexpr/channels ]]; then 179 | mkdir $HOME/.nix-defexpr || true 180 | mkdir $HOME/.nix-defexpr/channels || true 181 | fi 182 | 183 | export NIX_PATH_DIR="$NIX_STORE_DIR/store/$NIX_STORE_NIX/bin" 184 | if [[ $NIX_INSTALLER = --daemon ]]; then 185 | set +eu 186 | source $NIX_STORE_DIR/var/nix/profiles/default/etc/profile.d/nix-daemon.sh 187 | set -eu 188 | if [[ ${NIX_SSL_CERT_FILE-} = $NIX_STORE_DIR/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt ]]; then 189 | unset NIX_SSL_CERT_FILE 190 | fi 191 | elif [[ ! -r /etc/ssl/certs/ca-certificates.crt ]]; then 192 | export NIX_SSL_CERT_FILE="$NIX_STORE_DIR/store/$NIX_STORE_CACERT/etc/ssl/certs/ca-bundle.crt" 193 | fi 194 | } 195 | 196 | if [[ -n $NIX_INSTALLER ]]; then 197 | INVOKED_FROM_INSTALL_IN=1 $NIX_STORE_DIR/install \ 198 | --no-channel-add --no-modify-profile \ 199 | --daemon-user-count ${NIX_USER_COUNT-8} \ 200 | $NIX_INSTALLER 201 | rm -rf $NIX_STORE_DIR 202 | NIX_STORE_DIR=/nix 203 | nixvars 204 | else 205 | nixvars 206 | 207 | $NIX_PATH_DIR/nix-store --init 208 | $NIX_PATH_DIR/nix-store --load-db < $NIX_STORE_DIR/.reginfo 209 | fi 210 | rm -f $NIX_STORE_DIR/*.sh $NIX_STORE_DIR/.reginfo 211 | 212 | setup_nix_path 213 | 214 | # set up a default config 215 | if [[ -n $NIX_INSTALLER || ! -e /etc/nix/nix.conf ]] && [[ -z ${NIX_CONF_DIR-} ]]; then 216 | NIX_CONF=$(nix_eval '' config.nix.configText --argstr config "${CI_CONFIG-$CI_ROOT/tests/empty.nix}") 217 | 218 | if [[ $NIX_INSTALLER = --daemon ]]; then 219 | makedir /etc/nix 220 | NIX_CONF_DIR=/etc/nix 221 | else 222 | export NIX_CONF_DIR=$(maketemp ci.nix.conf -d) 223 | export_env NIX_CONF_DIR "$NIX_CONF_DIR" 224 | fi 225 | 226 | if [[ -w $NIX_CONF_DIR ]]; then 227 | printf "%s" "$NIX_CONF" >> "$NIX_CONF_DIR/nix.conf" 228 | else 229 | printf "%s" "$NIX_CONF" | sudo bash -c "cat >> $NIX_CONF_DIR/nix.conf" 230 | fi 231 | 232 | if [[ $NIX_INSTALLER = --daemon ]]; then 233 | if [[ $NIX_SYSTEM = *-darwin ]]; then 234 | launchctl kickstart -k system/org.nixos.nix-daemon || 235 | sudo launchctl kickstart -k system/org.nixos.nix-daemon || 236 | true 237 | elif [[ $NIX_SYSTEM = *-linux ]]; then 238 | systemctl restart nix-daemon.service || 239 | sudo systemctl restart nix-daemon.service || 240 | true 241 | fi 242 | fi 243 | fi 244 | 245 | export_env NIX_VERSION "$NIX_VERSION" 246 | #export_env NIX_SSL_CERT_FILE "$NIX_SSL_CERT_FILE" 247 | if [[ -n ${NIX_IGNORE_SYMLINK_STORE-} ]]; then 248 | export_env NIX_IGNORE_SYMLINK_STORE "$NIX_IGNORE_SYMLINK_STORE" 249 | fi 250 | 251 | set_output version "$NIX_VERSION" 252 | add_path "$NIX_PATH_DIR" 253 | case "${CI_PLATFORM-}" in 254 | gh-actions) 255 | sudo chown 0:0 / || true 256 | ;; 257 | azure-pipelines) 258 | sudo chown 0:0 / || true 259 | cat >> ~/.bash_profile < { } }: with pkgs; let 2 | CI_CONFIG_ROOT = "\${CI_CONFIG_ROOT-${toString ./.}}"; 3 | CI_CONFIG_FILES = "${CI_CONFIG_ROOT}/tests/*"; 4 | gh-actions-generate = writeShellScriptBin "gh-actions-generate" '' 5 | for f in ${CI_CONFIG_FILES}; do 6 | nix run --arg config $f -f ${CI_CONFIG_ROOT} run.gh-actions-generate 7 | done 8 | ''; 9 | test-all = writeShellScriptBin "test-all" '' 10 | if [[ $# -gt 0 ]]; then 11 | CI_CONFIG_FILES=("$@") 12 | else 13 | CI_CONFIG_FILES=(${CI_CONFIG_FILES}) 14 | fi 15 | for f in "''${CI_CONFIG_FILES[@]}"; do 16 | if ! nix run --arg config "$(realpath $f || echo $f)" -f ${CI_CONFIG_ROOT} test; then 17 | echo failed test $f >&2 18 | exit 1 19 | fi 20 | done 21 | ''; 22 | in mkShell { 23 | #CI_CONFIG = toString ./example/ci.nix 24 | CI_PLATFORM = "impure"; # use host's nixpkgs for more convenient testing 25 | 26 | nativeBuildInputs = [ 27 | gh-actions-generate 28 | test-all 29 | ]; 30 | shellHook = '' 31 | export CI_ROOT=''${CI_ROOT-${toString ./.}} 32 | export CI_CONFIG_ROOT=${CI_CONFIG_ROOT} 33 | ''; 34 | } 35 | -------------------------------------------------------------------------------- /src: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export CI_ROOT=`dirname "${BASH_SOURCE[0]}"` 4 | 5 | if [ "$TRAVIS" = true ]; then 6 | export CI_GIT_TAG=${TRAVIS_TAG} 7 | export CI_GIT_COMMIT=${TRAVIS_COMMIT} 8 | export CI_GH_SLUG=${TRAVIS_REPO_SLUG} 9 | 10 | export CI_BUILD_DIR=${TRAVIS_BUILD_DIR} 11 | 12 | if [ -n "${TRAVIS_RUST_VERSION:-}" ]; then 13 | export CI_LANG=rust 14 | export CI_RUST_VERSION="${TRAVIS_RUST_VERSION}" 15 | fi 16 | fi 17 | 18 | export PATH="$CI_ROOT:$CI_ROOT/$CI_LANG:$PATH" 19 | 20 | if [ -s "$CI_ROOT/$CI_LANG" ]; then 21 | source "$CI_ROOT/$CI_LANG/src" 22 | fi 23 | 24 | sudo() { 25 | `which sudo` -E "$@" 26 | } 27 | export -f sudo 28 | -------------------------------------------------------------------------------- /tests/cache.nix: -------------------------------------------------------------------------------- 1 | { pkgs, lib, ... }: with lib; { 2 | ci = { 3 | url = "."; 4 | gh-actions = { 5 | enable = true; 6 | }; 7 | }; 8 | name = "tests-cache"; 9 | cache.cachix.ci.enable = true; 10 | tasks.touch.inputs = pkgs.runCommand "touch" { 11 | inherit system; 12 | } '' 13 | echo $system > $out 14 | ''; 15 | tasks.nonsubstitutable.inputs = pkgs.runCommand "nonsub" { 16 | allowSubstitutes = false; 17 | } '' 18 | touch $out 19 | ''; 20 | } 21 | -------------------------------------------------------------------------------- /tests/empty.nix: -------------------------------------------------------------------------------- 1 | { 2 | # not so empty now are you! 3 | name = "tests-empty"; 4 | ci = { 5 | url = "."; 6 | gh-actions.enable = true; 7 | }; 8 | } 9 | -------------------------------------------------------------------------------- /tests/example.nix: -------------------------------------------------------------------------------- 1 | { ... }: { 2 | imports = [ ../examples/ci.nix ]; 3 | ci.url = "."; 4 | } 5 | -------------------------------------------------------------------------------- /tests/impure.nix: -------------------------------------------------------------------------------- 1 | { pkgs, config, ... }: { 2 | ci = { 3 | url = "."; 4 | gh-actions.enable = true; 5 | }; 6 | name = "tests-impure"; 7 | environment.test = rec { 8 | jq = config.lib.ci.hostDep "jq" [ "jq" ]; 9 | jqhello = pkgs.writeShellScriptBin "jqhello" '' 10 | echo '{ "hello": "world" }' | ${jq}/bin/jq -er .hello - 11 | ''; 12 | }; 13 | tasks.impure.inputs = let 14 | jq = pkgs.ci.command { 15 | name = "impure-jq"; 16 | command = '' 17 | [[ -e /home ]] || (echo "oh no we appear to be in the nix sandbox" >&2; exit 1) 18 | jqhello 19 | ''; 20 | displayName = "impure jq host dependency"; 21 | impure = true; 22 | }; 23 | env = pkgs.ci.command { 24 | name = "impure-env"; 25 | someVar = "hello"; 26 | command = '' 27 | [[ $someVar = hello ]] 28 | ''; 29 | displayName = "impure environment variable"; 30 | impure = true; 31 | }; 32 | pure = pkgs.ci.command { 33 | name = "pure-jq"; 34 | command = '' 35 | type jqhello 36 | if [[ ! -e /home ]]; then 37 | # skip this test if nix builder isn't sandboxed 38 | ! jqhello 39 | fi 40 | ''; 41 | displayName = "jq host dependency should fail inside sandbox"; 42 | }; 43 | in [ jq env pure ]; 44 | } 45 | -------------------------------------------------------------------------------- /tests/stages.nix: -------------------------------------------------------------------------------- 1 | { pkgs, ... }: { 2 | name = "tests-stages"; 3 | ci = { 4 | url = "."; 5 | gh-actions.enable = true; 6 | }; 7 | tasks.dummy.inputs = pkgs.ci.command { 8 | name = "dummy"; 9 | command = "true"; 10 | }; 11 | stages.another = { pkgs, ... }: { 12 | tasks.something.inputs = pkgs.ci.command { 13 | name = "dummy2"; 14 | command = "true"; 15 | }; 16 | }; 17 | } 18 | -------------------------------------------------------------------------------- /tests/tasks.nix: -------------------------------------------------------------------------------- 1 | { pkgs, config, ... }: { 2 | ci = { 3 | url = "."; 4 | gh-actions.enable = true; 5 | }; 6 | name = "tests-tasks"; 7 | environment.test = { 8 | inherit (pkgs) hello; 9 | }; 10 | cache.cachix.ci = { 11 | # including the public key makes `cachix use` unnecessary 12 | publicKey = "ci.cachix.org-1:PNnkaD7orCQhpX698ERHZ5MrtdGK/DacprP+7Ye/ens="; 13 | signingKey = config.lib.ci.env.get "CACHIX_SIGNING_KEY"; 14 | }; 15 | jobs = { 16 | linux = { 17 | system = "x86_64-linux"; 18 | }; 19 | mac-x86 = { 20 | system = "x86_64-darwin"; 21 | }; 22 | mac = { 23 | system = "aarch64-darwin"; 24 | }; 25 | }; 26 | tasks = { 27 | build.inputs = let 28 | magic = "compassion"; 29 | drv = pkgs.runCommand "build-task" { 30 | inherit magic; 31 | 32 | passthru.ci = { 33 | tests = [ checkMagic ]; 34 | }; 35 | } '' 36 | echo $magic > $out; 37 | ''; 38 | nond = pkgs.runCommand "non-deterministic" { 39 | allowSubstitutes = false; 40 | passthru.ci.cache.enable = false; 41 | } '' 42 | echo ${toString builtins.currentTime} > $out; 43 | ''; 44 | checkMagic = drv: pkgs.runCommand "check-task" { 45 | inherit drv magic; 46 | } '' 47 | if [[ $(cat $drv) != $magic ]]; then 48 | echo check test mismatch for $drv 49 | exit 1 50 | fi 51 | touch $out 52 | ''; 53 | impure = pkgs.ci.command { 54 | name = "impure"; 55 | command = '' 56 | [[ -z ''${NIX_BUILD_TOP-} ]] 57 | echo "hello from outside the sandbox" 58 | [[ -e /home ]] 59 | ''; 60 | impure = true; 61 | }; 62 | forgetme = pkgs.ci.command { 63 | name = "skip-me"; 64 | skip = true; 65 | command = "false"; 66 | }; 67 | forgetme-reason = pkgs.ci.command { 68 | name = "skip-me-also"; 69 | skip = "just because"; 70 | command = "false"; 71 | }; 72 | pure = pkgs.ci.command { 73 | name = "pure"; 74 | command = '' 75 | echo hello from inside the $NIX_BUILD_TOP sandbox 76 | ''; 77 | }; 78 | in [ drv nond impure pure forgetme forgetme-reason ]; 79 | broken = let 80 | fails = pkgs.runCommand "always-fails" { 81 | passthru.ci.warn = true; 82 | } "false"; 83 | depend-broken = pkgs.runCommand "depend-broken" { 84 | buildInputs = [ fails ]; 85 | } "touch $out"; 86 | in { 87 | name = "broken stuff"; 88 | inputs = [ fails depend-broken ]; 89 | warn = true; 90 | }; 91 | }; 92 | } 93 | --------------------------------------------------------------------------------