├── .envrc ├── .github ├── dependabot.yml └── workflows │ └── update-flake-lock.yml ├── .gitignore ├── .gitmodules ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── TODO.md ├── bors.toml ├── contrib ├── evaluation │ ├── sandboxed │ │ ├── iterate_over_pypi_dump.sh │ │ └── pack_from_pypi.sh │ └── unsandboxed │ │ ├── functions.sh │ │ ├── iterate_over_pypi_dump.sh │ │ ├── iterate_over_rubygems_dump.sh │ │ ├── pack_from_pypi.sh │ │ └── pack_from_rubygems.sh ├── iterate_over_pypi_dump.sh └── pack_from_pypi.sh ├── data ├── autotools │ └── m4.toml ├── glibc │ ├── endian.toml │ └── xlocale.toml └── python │ ├── cffi.toml │ ├── cryptography.toml │ └── cython.toml ├── default.nix ├── flake.lock ├── flake.nix ├── garnix.yaml ├── src ├── cache │ ├── database.rs │ ├── files.rs │ ├── frcode.rs │ ├── mod.rs │ └── package.rs ├── design.md ├── fs.rs ├── interactive.rs ├── main.rs ├── nix.rs ├── popcount.rs ├── resolution.rs └── runner.rs ├── tests ├── flake-module.nix ├── lib.nix └── nixos-test.nix └── treefmt └── flake-module.nix /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | - package-ecosystem: "cargo" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" 11 | -------------------------------------------------------------------------------- /.github/workflows/update-flake-lock.yml: -------------------------------------------------------------------------------- 1 | name: update-flake-lock 2 | on: 3 | workflow_dispatch: # allows manual triggering 4 | schedule: 5 | - cron: '0 0 * * 1,4' # Run twice a week 6 | 7 | jobs: 8 | lockfile: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout repository 12 | uses: actions/checkout@v3 13 | - name: Install Nix 14 | uses: cachix/install-nix-action@v20 15 | with: 16 | github_access_token: ${{ secrets.GITHUB_TOKEN }} 17 | - name: Update flake.lock 18 | uses: DeterminateSystems/update-flake-lock@v19 19 | with: 20 | pr-body: | 21 | Automated changes by the update-flake-lock 22 | ``` 23 | {{ env.GIT_COMMIT_MESSAGE }} 24 | ``` 25 | bors merge 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # These are backup files generated by rustfmt 6 | **/*.rs.bk 7 | 8 | 9 | # Added by cargo 10 | 11 | /target 12 | 13 | # nix 14 | /result 15 | .direnv 16 | test-venv 17 | popcount-graph.json 18 | nix-index-files 19 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "examples"] 2 | path = examples 3 | url = https://github.com/RaitoBezarius/buildxyz-examples 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "buildxyz" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | fuser = { version = "0.12", features = [ "serializable" ] } 10 | nix = "0.26.2" 11 | log = "0.4.17" 12 | stderrlog = "0.5.4" 13 | ctrlc = "3.2.5" 14 | clap = { version = "4.1.8", features = [ "derive" ] } 15 | crossbeam-channel = "0.5.7" 16 | xdg = "2.4.1" 17 | tui = "0.19.0" 18 | crossterm = "0.26" 19 | signal-hook = "0.3.15" 20 | # nix-index dependencies 21 | regex = "1.7.1" 22 | error-chain = "0.12.4" 23 | memchr = "2.5.0" 24 | zstd = { version = "0.12.3", features = [ "zstdmt" ] } 25 | serde_json = "1.0.94" 26 | byteorder = "1.4.3" 27 | regex-syntax = "0.7.1" 28 | grep = "0.2.11" 29 | serde = "1.0.163" 30 | num_cpus = "1.15.0" 31 | serde_bytes = "0.11.9" 32 | tempfile = "3.4.0" 33 | lazy_static = "1.4.0" 34 | toml = "0.7.3" 35 | thiserror = "1.0.40" 36 | walkdir = "2.3.3" 37 | include_dir = { version = "0.7.3", features = [ "glob" ] } 38 | 39 | [profile.release] 40 | debug = true 41 | 42 | [profile.dev] 43 | opt-level = 1 # Otherwise queries takes 10s (~500ms for opt-level=1). 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `buildxyz` 2 | 3 | Build your (Nix) package automatically. 4 | 5 | ## Introduction 6 | 7 | `buildxyz` is a Rust program running your build system invocation and injecting into many commonly used environment variables extra search directories to trap 8 | any filesystem access that cannot be provided by your existing environment. 9 | 10 | By doing so, `buildxyz` can know what your build system needs and will provide it dynamically using the Nix store and a nixpkgs index database. 11 | 12 | Finally, once your build system built, `buildxyz` remembers what it actually had to provide and can rematerialize the set of dependencies provided for any usage, e.g. providing this data to `nix-init` to automatically write a Nix derivation, fix implicit dependencies, etc. 13 | 14 | ## Design 15 | 16 | `buildxyz` relies on [the FUSE technology](https://en.wikipedia.org/wiki/Filesystem_in_Userspace) to instantiate an instrumented filesystem which does not require any higher privilege. 17 | 18 | Here is a sequence example of `buildxyz` operating: 19 | 20 | ```mermaid 21 | sequenceDiagram 22 | Build system->>Operating System: open(/usr/bin/pip) 23 | Operating System-)Build system: OK 24 | Build system->>Operating System: open(/usr/lib/cuda/...) 25 | Operating System->>Build system: ENOENT 26 | Build system->>Operating System: open(/tmp/buildxyz/lib/cuda/...) 27 | Operating System->>BuildXYZ: lookup(/tmp/buildxyz/lib/cuda/...) 28 | BuildXYZ->>Database: search this path 29 | Database-)BuildXYZ: /nix/store/eeeeeeeeee-cuda/ 30 | BuildXYZ->>User: Do you want to use this dependency or provide your own? 31 | User-)BuildXYZ: Use the preferred candidate 32 | BuildXYZ->>Build system: This is a symlink to /nix/store/eeeeeeeeee-cuda/ 33 | Build system->>Operating System: readlink(/nix/store/eeee..-cuda) 34 | Operating System->>Build system: OK 35 | ``` 36 | 37 | ## Actually implemented 38 | 39 | BuildXYZ can already provide dependencies to your build system based on a precise revision of nixpkgs, pinned in the `default.nix`. 40 | 41 | It is known to work on Python's packages (through `pip install --no-binary :all:`) and sometimes on certain autotools project depending on their complexity. 42 | 43 | ## Resolutions 44 | 45 | When BuildXYZ receives a new filesystem access, it means that the existing environment failed to provide it. 46 | 47 | If the filesystem access has a match in the nixpkgs index database, two options are possible: 48 | 49 | - provide it 50 | - do not provide it 51 | 52 | We call resolution the information composed of a filesystem access identified by a canonical path and a decision: provide it or not. 53 | 54 | Not all filesystem accesses should be provided even if we have matches for them, that's why we enable custom resolutions which can be managed through policies: user interaction, language-specific resolutions, etc. 55 | 56 | The resolution data for a project is very interesting as it is exactly the "implicit dependencies" data that is required to build a project, which is often described through instructions. 57 | 58 | ## Goals & TODO 59 | 60 | Current objective: get Nix to compile without any manually provided dependency using BuildXYZ. 61 | 62 | - Proper restart & program lifecycle (Ctrl-C, SIGTERM) 63 | - Proper discovery of existing resolutions databases 64 | - Proper flags to record new resolutions or merge them in an existing file 65 | - Human/machine-readable format for resolutions 66 | - Extend graphs of dependencies with implicit dependencies 67 | - `nix-init` integration 68 | 69 | # Usage 70 | 71 | Run the project: 72 | 73 | ``` nix 74 | nix run github:RaitoBezarius/buildxyz 75 | ``` 76 | 77 | Build the project: 78 | 79 | ``` nix 80 | nix build 81 | ``` 82 | 83 | Run all tests: 84 | 85 | ``` nix 86 | nix flake check -L 87 | ``` 88 | 89 | Run formatters: 90 | 91 | ``` nix 92 | nix fmt 93 | ``` 94 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # Long term 2 | 3 | - [ ] Analyze drvs to understand popularity of dependency graph 4 | 5 | # Mid term 6 | 7 | - [ ] Integrate nix-index computations of popcount 8 | - [ ] Investigate how to understand hooks from nixpkgs model 9 | - [ ] Benchmark popularity counts using header-only libraries as an example (collect list of header-only libraries) 10 | - [ ] Interactive sandboxing 11 | - [ ] Record resolutions 12 | 13 | # Short term 14 | 15 | - [ ] Visualization of nixpkgs graph 16 | -------------------------------------------------------------------------------- /bors.toml: -------------------------------------------------------------------------------- 1 | cut_body_after = "" # don't include text from the PR body in the merge commit message 2 | status = [ 3 | "Evaluate flake.nix", 4 | "check clippy [x86_64-linux]", 5 | "check nixos-test [x86_64-linux]", 6 | "check treefmt [x86_64-linux]", 7 | "package buildxyz [aarch64-darwin]", 8 | "package buildxyz [x86_64-linux]", 9 | "package default [aarch64-darwin]", 10 | "package default [x86_64-linux]" 11 | ] 12 | -------------------------------------------------------------------------------- /contrib/evaluation/sandboxed/iterate_over_pypi_dump.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell --pure -i bash -p jq git nix bubblewrap which cacert parallel tmux --keep BUILDXYZ_NIXPKGS --keep RUST_BACKTRACE 3 | # shellcheck shell=sh 4 | 5 | JOB="${1:-pypi-job}" 6 | 7 | generic_buildxyz() { 8 | builder="$1" 9 | package="$2" 10 | echo "buildxyz $package" 11 | # This is needed for the new tmpfs 12 | export TMPDIR="/buildxyz" 13 | # CAP_SYS_ADMIN is for the fusermount 14 | # /dev bind is for /dev/fuse 15 | # --share-net is necessary for network interactions. 16 | # share also the DNS resolver. 17 | bwrap \ 18 | --ro-bind /etc/resolv.conf /etc/resolv.conf \ 19 | --share-net \ 20 | --bind /nix /nix \ 21 | --dev-bind /dev /dev \ 22 | --ro-bind $(which git) $(which git) \ 23 | --ro-bind $(pwd)/target $(pwd)/target \ 24 | --bind $(pwd)/examples $(pwd)/examples \ 25 | --tmpfs /buildxyz \ 26 | --proc /proc \ 27 | --unshare-pid \ 28 | --cap-add CAP_SYS_ADMIN \ 29 | --new-session \ 30 | ./target/debug/buildxyz --automatic --record-to "examples/python/$package.toml" "$builder $package" 31 | } 32 | 33 | pip_install() { 34 | package="$1" 35 | pip install "$package" --prefix /tmp --no-binary :all 36 | } 37 | 38 | pypi_buildxyz() { 39 | generic_buildxyz pip_install "$@" 40 | } 41 | 42 | export -f pypi_buildxyz 43 | 44 | readarray -t PYPI_PACKAGES < <(jq -rc '.rows | .[] | .project' top-pypi.json) 45 | parallel --joblog $JOB --progress --bar --delay 2.5 --jobs 50% --tmux pypi_buildxyz ::: "${PYPI_PACKAGES[@]}" 46 | -------------------------------------------------------------------------------- /contrib/evaluation/sandboxed/pack_from_pypi.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell --pure -i bash -p gcc jq git nix bubblewrap which cacert strace --keep BUILDXYZ_NIXPKGS --keep RUST_BACKTRACE --keep MANUAL --keep ENABLE_STRACE --keep NIX_DEBUG 3 | # shellcheck shell=sh 4 | 5 | buildxyz_global_flags=() 6 | STRACE="" 7 | 8 | if [[ -v ENABLE_STRACE ]]; then 9 | STRACE="strace -yy -e file -f" 10 | fi 11 | 12 | if [[ ! -v MANUAL ]]; then 13 | buildxyz_global_flags+=(--automatic) 14 | fi 15 | 16 | pypi_buildxyz() { 17 | package="$1" 18 | echo "buildxyz $package" 19 | export TMPDIR="/buildxyz" 20 | bwrap \ 21 | --ro-bind /etc/resolv.conf /etc/resolv.conf \ 22 | --share-net \ 23 | --bind /nix /nix \ 24 | --dev-bind /dev /dev \ 25 | --ro-bind $(which git) $(which git) \ 26 | --ro-bind $(pwd)/target $(pwd)/target \ 27 | --bind $(pwd)/examples $(pwd)/examples \ 28 | --tmpfs /buildxyz \ 29 | --proc /proc \ 30 | --unshare-pid \ 31 | --cap-add CAP_SYS_ADMIN \ 32 | --new-session \ 33 | $STRACE ./target/debug/buildxyz "${buildxyz_global_flags[@]}" --record-to "examples/python/$package.toml" "pip install --verbose $package --prefix /tmp --no-binary :all:" 34 | } 35 | 36 | pypi_buildxyz "$1" 37 | -------------------------------------------------------------------------------- /contrib/evaluation/unsandboxed/functions.sh: -------------------------------------------------------------------------------- 1 | set -euxo pipefail 2 | export BUILDXYZ_RELEASE_VARIANT="release" 3 | export BUILDXYZ_BINARY="./target/$BUILDXYZ_RELEASE_VARIANT/buildxyz" 4 | # Improve the performance of the evaluation 5 | # because some packages believe it's fine to adopt nightly features 6 | # in Python releases… 7 | export RUSTC_BOOTSTRAP=1 8 | 9 | export buildxyz_global_flags=() 10 | 11 | if [[ -v AUTOMATIC ]]; then 12 | buildxyz_global_flags+=(--automatic) 13 | fi 14 | 15 | # Debugging infrastructure 16 | export STRACE="" 17 | export STRACE_EXTRA_FLAGS="" 18 | 19 | if [[ -v ENABLE_STRACE ]]; then 20 | STRACE="strace -yy -f $STRACE_EXTRA_FLAGS" 21 | fi 22 | 23 | # Manual interaction 24 | if [[ ! -v MANUAL ]]; then 25 | buildxyz_global_flags+=(--automatic) 26 | fi 27 | 28 | pypi_buildxyz() { 29 | package="$1" 30 | PREFIX_DIR=$(mktemp -d) 31 | echo "buildxyz $package in pip prefix $PREFIX_DIR" 32 | $STRACE $BUILDXYZ_BINARY "${buildxyz_global_flags[@]}" --record-to "examples/python/$package.toml" "pip install --use-feature=no-binary-enable-wheel-cache --prefix $PREFIX_DIR --no-binary :all: --no-cache-dir $package" 33 | } 34 | 35 | rubygems_buildxyz() { 36 | package="$1" 37 | PREFIX_DIR=$(mktemp -d) 38 | export GEM_HOME="$PREFIX_DIR" 39 | echo "buildxyz $package in gem prefix $PREFIX_DIR" 40 | $STRACE $BUILDXYZ_BINARY "${buildxyz_global_flags[@]}" --record-to "examples/ruby/$package.toml" "gem install --bindir $(mktemp -d) --install-dir $(mktemp -d) --no-user-install $package" 41 | } 42 | -------------------------------------------------------------------------------- /contrib/evaluation/unsandboxed/iterate_over_pypi_dump.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell --pure -i bash -p jq git nix bubblewrap which parallel tmux --keep BUILDXYZ_NIXPKGS --keep RUST_BACKTRACE --keep ENABLE_STRACE --keep NIX_DEBUG --keep MANUAL 3 | # shellcheck shell=sh 4 | 5 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | . "$SCRIPT_DIR/functions.sh" 7 | 8 | JOB="${1:-pypi-job}" 9 | export -f pypi_buildxyz 10 | 11 | readarray -t PYPI_PACKAGES < <(jq -rc '.rows | .[] | .project' top-pypi.json) 12 | mkdir -p "$TMPDIR/job-logs/$JOB" 13 | parallel --output-as-files --results "$TMPDIR/job-logs/$JOB" --resume-failed --joblog $JOB --progress --bar --delay 2.5 --jobs 25% --tmuxpane pypi_buildxyz ::: "${PYPI_PACKAGES[@]}" 14 | -------------------------------------------------------------------------------- /contrib/evaluation/unsandboxed/iterate_over_rubygems_dump.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell --pure -i bash -p jq git nix bubblewrap which parallel tmux --keep BUILDXYZ_NIXPKGS --keep RUST_BACKTRACE --keep ENABLE_STRACE --keep NIX_DEBUG --keep MANUAL 3 | # shellcheck shell=sh 4 | 5 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | . "$SCRIPT_DIR/functions.sh" 7 | 8 | JOB="${1:-ruby-job}" 9 | export -f rubygems_buildxyz 10 | 11 | while IFS=',' read -ra TOP_RUBY_PACKAGES; do 12 | RUBY_PACKAGES+=("${TOP_RUBY_PACKAGES[0]}") 13 | done < top-rubygems.csv 14 | 15 | 16 | mkdir -p "$TMPDIR/job-logs/$JOB" 17 | parallel --output-as-files --results "$TMPDIR/job-logs/$JOB" --resume-failed --joblog $JOB --progress --bar --delay 2.5 --jobs 25% --tmuxpane rubygems_buildxyz ::: "${RUBY_PACKAGES[@]}" 18 | -------------------------------------------------------------------------------- /contrib/evaluation/unsandboxed/pack_from_pypi.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell --pure -i bash -p jq git nix which cacert strace --keep BUILDXYZ_NIXPKGS --keep RUST_BACKTRACE --keep MANUAL --keep ENABLE_STRACE --keep NIX_DEBUG --keep RUSTC_BOOTSTRAP 3 | # shellcheck shell=sh 4 | 5 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | . "$SCRIPT_DIR/functions.sh" 7 | 8 | pypi_buildxyz "$1" 9 | -------------------------------------------------------------------------------- /contrib/evaluation/unsandboxed/pack_from_rubygems.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell --pure -i bash -p jq git nix which cacert strace --keep BUILDXYZ_NIXPKGS --keep RUST_BACKTRACE --keep MANUAL --keep ENABLE_STRACE --keep NIX_DEBUG --keep RUSTC_BOOTSTRAP 3 | # shellcheck shell=sh 4 | 5 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | . "$SCRIPT_DIR/functions.sh" 7 | 8 | rubygems_buildxyz "$1" 9 | -------------------------------------------------------------------------------- /contrib/iterate_over_pypi_dump.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell --pure -i bash -p jq git nix bubblewrap which cacert parallel tmux --keep BUILDXYZ_NIXPKGS --keep RUST_BACKTRACE 3 | # shellcheck shell=sh 4 | 5 | JOB="${1:-pypi-job}" 6 | 7 | generic_buildxyz() { 8 | builder="$1" 9 | package="$2" 10 | echo "buildxyz $package" 11 | # This is needed for the new tmpfs 12 | export TMPDIR="/buildxyz" 13 | # CAP_SYS_ADMIN is for the fusermount 14 | # /dev bind is for /dev/fuse 15 | # --share-net is necessary for network interactions. 16 | # share also the DNS resolver. 17 | bwrap \ 18 | --ro-bind /etc/resolv.conf /etc/resolv.conf \ 19 | --share-net \ 20 | --bind /nix /nix \ 21 | --dev-bind /dev /dev \ 22 | --ro-bind $(which git) $(which git) \ 23 | --ro-bind $(pwd)/target $(pwd)/target \ 24 | --bind $(pwd)/examples $(pwd)/examples \ 25 | --tmpfs /buildxyz \ 26 | --proc /proc \ 27 | --unshare-pid \ 28 | --cap-add CAP_SYS_ADMIN \ 29 | --new-session \ 30 | ./target/debug/buildxyz --automatic --record-to "examples/python/$package.toml" "$builder $package" 31 | } 32 | 33 | pip_install() { 34 | package="$1" 35 | pip install "$package" --prefix /tmp --no-binary :all 36 | } 37 | 38 | pypi_buildxyz() { 39 | generic_buildxyz pip_install "$@" 40 | } 41 | 42 | export -f pypi_buildxyz 43 | 44 | readarray -t PYPI_PACKAGES < <(jq -rc '.rows | .[] | .project' top-pypi.json) 45 | parallel --joblog $JOB --progress --bar --delay 2.5 --jobs 50% --tmux pypi_buildxyz ::: "${PYPI_PACKAGES[@]}" 46 | -------------------------------------------------------------------------------- /contrib/pack_from_pypi.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell --pure -i bash -p gcc jq git nix bubblewrap which cacert strace --keep BUILDXYZ_NIXPKGS --keep RUST_BACKTRACE --keep MANUAL --keep ENABLE_STRACE --keep NIX_DEBUG 3 | # shellcheck shell=sh 4 | 5 | buildxyz_global_flags=() 6 | STRACE="" 7 | 8 | if [[ -v ENABLE_STRACE ]]; then 9 | STRACE="strace -yy -e file -f" 10 | fi 11 | 12 | if [[ ! -v MANUAL ]]; then 13 | buildxyz_global_flags+=(--automatic) 14 | fi 15 | 16 | pypi_buildxyz() { 17 | package="$1" 18 | echo "buildxyz $package" 19 | export TMPDIR="/buildxyz" 20 | bwrap \ 21 | --ro-bind /etc/resolv.conf /etc/resolv.conf \ 22 | --share-net \ 23 | --bind /nix /nix \ 24 | --dev-bind /dev /dev \ 25 | --ro-bind $(which git) $(which git) \ 26 | --ro-bind $(pwd)/target $(pwd)/target \ 27 | --bind $(pwd)/examples $(pwd)/examples \ 28 | --tmpfs /buildxyz \ 29 | --proc /proc \ 30 | --unshare-pid \ 31 | --cap-add CAP_SYS_ADMIN \ 32 | --new-session \ 33 | $STRACE ./target/debug/buildxyz "${buildxyz_global_flags[@]}" --record-to "examples/python/$package.toml" "pip install --verbose $package --prefix /tmp --no-binary :all:" 34 | } 35 | 36 | pypi_buildxyz "$1" 37 | -------------------------------------------------------------------------------- /data/autotools/m4.toml: -------------------------------------------------------------------------------- 1 | ["share/aclocal/ax_zoneinfo.m4"] 2 | decision = "provide" 3 | file_entry_name = "/share/aclocal/ax_zoneinfo.m4" 4 | kind = "symlink" 5 | resolution = "constant" 6 | 7 | ["share/aclocal/ax_zoneinfo.m4".store_path] 8 | hash = "0znwn1lw0268i7pj0rylhz9dnzbwijjn" 9 | name = "autoconf-archive-2022.09.03" 10 | store_dir = "/nix/store" 11 | 12 | ["share/aclocal/ax_zoneinfo.m4".store_path.origin] 13 | attr = "autoconf-archive" 14 | output = "out" 15 | system = "x86_64-linux" 16 | toplevel = true 17 | -------------------------------------------------------------------------------- /data/glibc/endian.toml: -------------------------------------------------------------------------------- 1 | ["include/sys/endian.h"] 2 | decision = "ignore" 3 | resolution = "constant" 4 | -------------------------------------------------------------------------------- /data/glibc/xlocale.toml: -------------------------------------------------------------------------------- 1 | ["include/xlocale.h"] 2 | decision = "ignore" 3 | resolution = "constant" 4 | -------------------------------------------------------------------------------- /data/python/cffi.toml: -------------------------------------------------------------------------------- 1 | ["bin/lsb_release"] 2 | decision = "provide" 3 | file_entry_name = "/bin/lsb_release" 4 | kind = "symlink" 5 | resolution = "constant" 6 | 7 | ["bin/lsb_release".store_path] 8 | hash = "ji55v58cch94h30inpnrd5h5jra9fwd8" 9 | name = "lsb_release" 10 | store_dir = "/nix/store" 11 | 12 | ["bin/lsb_release".store_path.origin] 13 | attr = "lsb-release" 14 | output = "out" 15 | system = "x86_64-linux" 16 | toplevel = true 17 | 18 | ["bin/pip"] 19 | decision = "provide" 20 | file_entry_name = "/bin/pip" 21 | kind = "symlink" 22 | resolution = "constant" 23 | 24 | ["bin/pip".store_path] 25 | hash = "8gzj48mkr8i824a4ali025hk85gmfdzy" 26 | name = "python3.10-pip-22.3.1" 27 | store_dir = "/nix/store" 28 | 29 | ["bin/pip".store_path.origin] 30 | attr = "python310Packages.pip" 31 | output = "out" 32 | system = "x86_64-linux" 33 | toplevel = true 34 | 35 | ["bin/pkg-config"] 36 | decision = "provide" 37 | file_entry_name = "/bin/pkg-config" 38 | kind = "symlink" 39 | resolution = "constant" 40 | 41 | ["bin/pkg-config".store_path] 42 | hash = "prm4x2r6997idyjxajapk1177jczvjfj" 43 | name = "pkg-config-0.29.2" 44 | store_dir = "/nix/store" 45 | 46 | ["bin/pkg-config".store_path.origin] 47 | attr = "pkg-config-unwrapped" 48 | output = "out" 49 | system = "x86_64-linux" 50 | toplevel = true 51 | 52 | ["lib/pkgconfig/libffi.pc"] 53 | decision = "provide" 54 | file_entry_name = "/lib/pkgconfig/libffi.pc" 55 | kind = "symlink" 56 | resolution = "constant" 57 | 58 | ["lib/pkgconfig/libffi.pc".store_path] 59 | hash = "4j08mgygxhi9y3957hbwqn1bg02va18y" 60 | name = "libffi-3.4.4-dev" 61 | store_dir = "/nix/store" 62 | 63 | ["lib/pkgconfig/libffi.pc".store_path.origin] 64 | attr = "libffi" 65 | output = "dev" 66 | system = "x86_64-linux" 67 | toplevel = true 68 | -------------------------------------------------------------------------------- /data/python/cryptography.toml: -------------------------------------------------------------------------------- 1 | ["bin/cargo"] 2 | decision = "provide" 3 | file_entry_name = "/bin/cargo" 4 | kind = "symlink" 5 | resolution = "constant" 6 | 7 | ["bin/cargo".store_path] 8 | hash = "qiklc9rr5jg1zck0rb7vj4wn1n7yxx67" 9 | name = "cargo-1.67.1" 10 | store_dir = "/nix/store" 11 | 12 | ["bin/cargo".store_path.origin] 13 | attr = "cargo" 14 | output = "out" 15 | system = "x86_64-linux" 16 | toplevel = true 17 | 18 | ["bin/lsb_release"] 19 | decision = "provide" 20 | file_entry_name = "/bin/lsb_release" 21 | kind = "symlink" 22 | resolution = "constant" 23 | 24 | ["bin/lsb_release".store_path] 25 | hash = "ji55v58cch94h30inpnrd5h5jra9fwd8" 26 | name = "lsb_release" 27 | store_dir = "/nix/store" 28 | 29 | ["bin/lsb_release".store_path.origin] 30 | attr = "lsb-release" 31 | output = "out" 32 | system = "x86_64-linux" 33 | toplevel = true 34 | 35 | ["bin/pip"] 36 | decision = "provide" 37 | file_entry_name = "/bin/pip" 38 | kind = "symlink" 39 | resolution = "constant" 40 | 41 | ["bin/pip".store_path] 42 | hash = "8gzj48mkr8i824a4ali025hk85gmfdzy" 43 | name = "python3.10-pip-22.3.1" 44 | store_dir = "/nix/store" 45 | 46 | ["bin/pip".store_path.origin] 47 | attr = "python310Packages.pip" 48 | output = "out" 49 | system = "x86_64-linux" 50 | toplevel = true 51 | 52 | ["bin/pkg-config"] 53 | decision = "provide" 54 | file_entry_name = "/bin/pkg-config" 55 | kind = "symlink" 56 | resolution = "constant" 57 | 58 | ["bin/pkg-config".store_path] 59 | hash = "prm4x2r6997idyjxajapk1177jczvjfj" 60 | name = "pkg-config-0.29.2" 61 | store_dir = "/nix/store" 62 | 63 | ["bin/pkg-config".store_path.origin] 64 | attr = "pkg-config-unwrapped" 65 | output = "out" 66 | system = "x86_64-linux" 67 | toplevel = true 68 | 69 | ["bin/rustc"] 70 | decision = "provide" 71 | file_entry_name = "/bin/rustc" 72 | kind = "symlink" 73 | resolution = "constant" 74 | 75 | ["bin/rustc".store_path] 76 | hash = "bxm02qp0lnm4sin3x78qrxp0gsbby1jb" 77 | name = "rustc-1.67.1" 78 | store_dir = "/nix/store" 79 | 80 | ["bin/rustc".store_path.origin] 81 | attr = "rustc" 82 | output = "out" 83 | system = "x86_64-linux" 84 | toplevel = true 85 | 86 | ["lib/pkgconfig/libffi.pc"] 87 | decision = "provide" 88 | file_entry_name = "/lib/pkgconfig/libffi.pc" 89 | kind = "symlink" 90 | resolution = "constant" 91 | 92 | ["lib/pkgconfig/libffi.pc".store_path] 93 | hash = "4j08mgygxhi9y3957hbwqn1bg02va18y" 94 | name = "libffi-3.4.4-dev" 95 | store_dir = "/nix/store" 96 | 97 | ["lib/pkgconfig/libffi.pc".store_path.origin] 98 | attr = "libffi" 99 | output = "dev" 100 | system = "x86_64-linux" 101 | toplevel = true 102 | 103 | ["lib/pkgconfig/openssl.pc"] 104 | decision = "provide" 105 | file_entry_name = "/lib/pkgconfig/openssl.pc" 106 | kind = "symlink" 107 | resolution = "constant" 108 | 109 | ["lib/pkgconfig/openssl.pc".store_path] 110 | hash = "fyiwac98bi86nd5qzz9bhvs1bvmmwgad" 111 | name = "openssl-3.0.8-dev" 112 | store_dir = "/nix/store" 113 | 114 | ["lib/pkgconfig/openssl.pc".store_path.origin] 115 | attr = "openssl" 116 | output = "dev" 117 | system = "x86_64-linux" 118 | toplevel = true 119 | -------------------------------------------------------------------------------- /data/python/cython.toml: -------------------------------------------------------------------------------- 1 | ["bin/lsb_release"] 2 | decision = "provide" 3 | file_entry_name = "/bin/lsb_release" 4 | kind = "symlink" 5 | resolution = "constant" 6 | 7 | ["bin/lsb_release".store_path] 8 | hash = "ji55v58cch94h30inpnrd5h5jra9fwd8" 9 | name = "lsb_release" 10 | store_dir = "/nix/store" 11 | 12 | ["bin/lsb_release".store_path.origin] 13 | attr = "lsb-release" 14 | output = "out" 15 | system = "x86_64-linux" 16 | toplevel = true 17 | 18 | ["bin/pgen"] 19 | decision = "ignore" 20 | resolution = "constant" 21 | 22 | ["bin/pip"] 23 | decision = "provide" 24 | file_entry_name = "/bin/pip" 25 | kind = "symlink" 26 | resolution = "constant" 27 | 28 | ["bin/pip".store_path] 29 | hash = "8gzj48mkr8i824a4ali025hk85gmfdzy" 30 | name = "python3.10-pip-22.3.1" 31 | store_dir = "/nix/store" 32 | 33 | ["bin/pip".store_path.origin] 34 | attr = "python310Packages.pip" 35 | output = "out" 36 | system = "x86_64-linux" 37 | toplevel = true 38 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { fuse3 2 | , macfuse-stubs 3 | , stdenv 4 | , pkg-config 5 | , openssl 6 | , zstd 7 | , cargo-flamegraph 8 | , rustPlatform 9 | , lib 10 | , runCommand 11 | , fetchurl 12 | , clippy 13 | , path 14 | , enableLint ? false 15 | }: 16 | let 17 | fuse = if stdenv.isDarwin then macfuse-stubs else fuse3; 18 | popcount-graph = builtins.fetchurl { 19 | url = "https://github.com/RaitoBezarius/buildxyz/releases/download/assets-0.1.0/popcount-graph.json"; 20 | sha256 = "1xbhlcmb2laa9cp5qh9vsmmvzdifaqb7x7817ppjk1wx6gf2p02a"; 21 | }; 22 | nix-index-db = builtins.fetchurl { 23 | url = "https://github.com/RaitoBezarius/buildxyz/releases/download/assets-0.1.0/files"; 24 | sha256 = "02igi3vkqg8hqwa9p03gyr6x2h99sz1gv2w4mzfw646qlckfh32p"; 25 | }; 26 | in 27 | rustPlatform.buildRustPackage 28 | { 29 | pname = "buildxyz"; 30 | version = "0.0.1"; 31 | src = runCommand "src" { } '' 32 | install -D ${./Cargo.toml} $out/Cargo.toml 33 | install -D ${./Cargo.lock} $out/Cargo.lock 34 | cp -r ${./src} $out/src 35 | ln -sf ${popcount-graph} $out/popcount-graph.json 36 | ln -sf ${nix-index-db} $out/nix-index-files 37 | ''; 38 | # Use provided zstd rather than vendored one. 39 | ZSTD_SYS_USE_PKG_CONFIG = true; 40 | BUILDXYZ_NIXPKGS = path; 41 | BUILDXYZ_CORE_RESOLUTIONS = ./data; 42 | 43 | buildInputs = [ zstd fuse ]; 44 | nativeBuildInputs = [ openssl cargo-flamegraph pkg-config ] ++ lib.optional enableLint clippy; 45 | 46 | shellHook = '' 47 | ln -s ${popcount-graph} popcount-graph.json 48 | ln -s ${nix-index-db} nix-index-files 49 | ''; 50 | 51 | cargoLock = { 52 | lockFile = ./Cargo.lock; 53 | }; 54 | meta = with lib; { 55 | description = "Provides build shell that can automatically figure out dependencies"; 56 | homepage = "https://github.com/RaitoBezarius/buildxyz"; 57 | license = licenses.mit; 58 | }; 59 | } // lib.optionalAttrs enableLint { 60 | buildPhase = '' 61 | cargo clippy --all-targets --all-features -- -D warnings 62 | if grep -R 'dbg!' ./src; then 63 | echo "use of dbg macro found in code!" 64 | false 65 | fi 66 | ''; 67 | 68 | installPhase = '' 69 | touch $out 70 | ''; 71 | 72 | } 73 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-parts": { 4 | "inputs": { 5 | "nixpkgs-lib": [ 6 | "nixpkgs" 7 | ] 8 | }, 9 | "locked": { 10 | "lastModified": 1677714448, 11 | "narHash": "sha256-Hq8qLs8xFu28aDjytfxjdC96bZ6pds21Yy09mSC156I=", 12 | "owner": "hercules-ci", 13 | "repo": "flake-parts", 14 | "rev": "dc531e3a9ce757041e1afaff8ee932725ca60002", 15 | "type": "github" 16 | }, 17 | "original": { 18 | "owner": "hercules-ci", 19 | "repo": "flake-parts", 20 | "type": "github" 21 | } 22 | }, 23 | "nixpkgs": { 24 | "locked": { 25 | "lastModified": 1677995890, 26 | "narHash": "sha256-eOnCn0o3I6LP48fAi8xWFcn49V2rL7oX5jCtJTeN1LI=", 27 | "owner": "NixOS", 28 | "repo": "nixpkgs", 29 | "rev": "a1240f6b4a0bcc84fc48008b396a140d9f3638f6", 30 | "type": "github" 31 | }, 32 | "original": { 33 | "owner": "NixOS", 34 | "ref": "nixpkgs-unstable", 35 | "repo": "nixpkgs", 36 | "type": "github" 37 | } 38 | }, 39 | "root": { 40 | "inputs": { 41 | "flake-parts": "flake-parts", 42 | "nixpkgs": "nixpkgs", 43 | "treefmt-nix": "treefmt-nix" 44 | } 45 | }, 46 | "treefmt-nix": { 47 | "inputs": { 48 | "nixpkgs": [ 49 | "nixpkgs" 50 | ] 51 | }, 52 | "locked": { 53 | "lastModified": 1677433127, 54 | "narHash": "sha256-vafj2WbhrlnwkU20yRDqtHFTUJIEygPfxJVswB3dJ9U=", 55 | "owner": "numtide", 56 | "repo": "treefmt-nix", 57 | "rev": "f7fcf3770c6cec6fd5f995ba94e6e6376019b9ff", 58 | "type": "github" 59 | }, 60 | "original": { 61 | "owner": "numtide", 62 | "repo": "treefmt-nix", 63 | "type": "github" 64 | } 65 | } 66 | }, 67 | "root": "root", 68 | "version": 7 69 | } 70 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Development environment for this project"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; 6 | treefmt-nix.url = "github:numtide/treefmt-nix"; 7 | treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; 8 | 9 | flake-parts.url = "github:hercules-ci/flake-parts"; 10 | flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; 11 | }; 12 | 13 | outputs = inputs@{ flake-parts, ... }: 14 | flake-parts.lib.mkFlake { inherit inputs; } ({ ... }: { 15 | systems = [ 16 | "x86_64-linux" 17 | "aarch64-linux" 18 | # TODO: fix eval... 19 | #"riscv64-linux" 20 | "x86_64-darwin" 21 | "aarch64-darwin" 22 | ]; 23 | imports = [ 24 | ./treefmt/flake-module.nix 25 | ./tests/flake-module.nix 26 | ]; 27 | 28 | perSystem = { self', pkgs, ... }: { 29 | packages.buildxyz = pkgs.callPackage ./default.nix { }; 30 | packages.default = self'.packages.buildxyz; 31 | checks.clippy = self'.packages.buildxyz.override { 32 | enableLint = true; 33 | }; 34 | }; 35 | }); 36 | } 37 | -------------------------------------------------------------------------------- /garnix.yaml: -------------------------------------------------------------------------------- 1 | builds: 2 | exclude: [] 3 | include: 4 | - "*.x86_64-linux.*" 5 | - "nixosConfigurations.*" 6 | - "packages.aarch64-darwin.*" 7 | - "devShells.aarch64-darwin.*" 8 | -------------------------------------------------------------------------------- /src/cache/database.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::Cursor; 3 | /// Creating and searching file databases. 4 | /// 5 | /// This module implements an abstraction for creating an index of files with meta information 6 | /// and searching that index for paths matching a specific pattern. 7 | use std::io::{self, BufReader, BufWriter, Read, Seek, SeekFrom, Write}; 8 | use std::path::Path; 9 | 10 | use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; 11 | use error_chain::error_chain; 12 | use grep::matcher::{LineMatchKind, Match, Matcher, NoError}; 13 | use grep::{self}; 14 | use memchr::{memchr, memrchr}; 15 | use regex::bytes::Regex; 16 | use regex_syntax::ast::{ 17 | Alternation, Assertion, AssertionKind, Ast, Concat, Group, Literal, Repetition, 18 | }; 19 | use serde_json; 20 | use zstd; 21 | 22 | use crate::cache::files::{FileTree, FileTreeEntry}; 23 | use crate::cache::frcode; 24 | use crate::cache::package::StorePath; 25 | 26 | /// The version of the database format supported by this nix-index version. 27 | /// 28 | /// This should be updated whenever you make an incompatible change to the database format. 29 | const FORMAT_VERSION: u64 = 1; 30 | 31 | /// The magic for nix-index database files, used to ensure that the file we're passed is 32 | /// actually a file generated by nix-index. 33 | const FILE_MAGIC: &'static [u8] = b"NIXI"; 34 | 35 | /// A writer for creating a new file database. 36 | pub struct Writer { 37 | /// The encoder used to compress the database. Will be set to `None` when the value 38 | /// is dropped. 39 | writer: Option>>, 40 | } 41 | 42 | // We need to make sure that the encoder is `finish`ed in all cases, so we need 43 | // a custom Drop. 44 | impl Drop for Writer { 45 | fn drop(&mut self) { 46 | if self.writer.is_some() { 47 | self.finish_encoder().unwrap(); 48 | } 49 | } 50 | } 51 | 52 | impl Writer { 53 | /// Creates a new database at the given path with the specified zstd compression level 54 | /// (currently, supported values range from 0 to 22). 55 | pub fn create>(path: P, level: i32) -> io::Result { 56 | let mut file = File::create(path)?; 57 | file.write_all(FILE_MAGIC)?; 58 | file.write_u64::(FORMAT_VERSION)?; 59 | let mut encoder = zstd::Encoder::new(file, level)?; 60 | encoder.multithread(num_cpus::get() as u32)?; 61 | 62 | Ok(Writer { 63 | writer: Some(BufWriter::new(encoder)), 64 | }) 65 | } 66 | 67 | /// Add a new package to the database for the given store path with its corresponding 68 | /// file tree. Entries are only added if they match `filter_prefix`. 69 | pub fn add( 70 | &mut self, 71 | path: StorePath, 72 | files: FileTree, 73 | filter_prefix: &[u8], 74 | ) -> io::Result<()> { 75 | let writer = self.writer.as_mut().expect("not dropped yet"); 76 | let mut encoder = 77 | frcode::Encoder::new(writer, b"p".to_vec(), serde_json::to_vec(&path).unwrap()); 78 | for entry in files.to_list(filter_prefix) { 79 | entry.encode(&mut encoder)?; 80 | } 81 | Ok(()) 82 | } 83 | 84 | /// Finishes encoding. After calling this function, `add` may no longer be called, since this function 85 | /// closes the stream. 86 | /// 87 | /// The return value is the underlying File. 88 | fn finish_encoder(&mut self) -> io::Result { 89 | let writer = self.writer.take().expect("not dropped yet"); 90 | let encoder = writer.into_inner()?; 91 | encoder.finish() 92 | } 93 | 94 | /// Finish the encoding and return the size in bytes of the compressed file that was created. 95 | pub fn finish(mut self) -> io::Result { 96 | let mut file = self.finish_encoder()?; 97 | file.seek(SeekFrom::Current(0)) 98 | } 99 | } 100 | 101 | error_chain! { 102 | errors { 103 | UnsupportedFileType(found: Vec) { 104 | description("unsupported file type") 105 | display("expected file to start with nix-index file magic 'NIXI', but found '{}' (is this a valid nix-index database file?)", String::from_utf8_lossy(found)) 106 | } 107 | UnsupportedVersion(found: u64) { 108 | description("unsupported file version") 109 | display("this executable only supports the nix-index database version {}, but found a database with version {}", FORMAT_VERSION, found) 110 | } 111 | MissingPackageEntry { 112 | description("missing package entry for path") 113 | display("database corrupt, found a file entry without a matching package entry") 114 | } 115 | Frcode(err: frcode::Error) { 116 | description("frcode error") 117 | display("database corrupt, frcode error: {}", err) 118 | } 119 | EntryParse(entry: Vec) { 120 | description("entry parse failure") 121 | display("database corrupt, could not parse entry: {:?}", String::from_utf8_lossy(entry)) 122 | } 123 | StorePathParse(path: Vec) { 124 | description("store path parse failure") 125 | display("database corrupt, could not parse store path: {:?}", String::from_utf8_lossy(path)) 126 | } 127 | } 128 | 129 | foreign_links { 130 | Io(io::Error); 131 | Grep(grep::regex::Error); 132 | } 133 | } 134 | 135 | impl From for Error { 136 | fn from(err: frcode::Error) -> Error { 137 | ErrorKind::Frcode(err).into() 138 | } 139 | } 140 | 141 | /// A Reader allows fast querying of a nix-index database. 142 | pub struct Reader { 143 | decoder: frcode::Decoder>>, // BufReader>>>, 144 | } 145 | 146 | pub fn read_from_path>(path: P) -> Result> { 147 | read_raw_buffer(File::open(path)?) 148 | } 149 | 150 | pub fn read_raw_buffer(mut reader: Reader) -> Result> { 151 | let mut magic = [0u8; 4]; 152 | reader.read_exact(&mut magic)?; 153 | 154 | if magic != FILE_MAGIC { 155 | return Err(ErrorKind::UnsupportedFileType(magic.to_vec()).into()); 156 | } 157 | 158 | let version = reader.read_u64::()?; 159 | if version != FORMAT_VERSION { 160 | return Err(ErrorKind::UnsupportedVersion(version).into()); 161 | } 162 | 163 | let mut decoder = zstd::Decoder::new(reader)?; 164 | let mut buffer: Vec = Vec::new(); 165 | decoder.read_to_end(&mut buffer)?; 166 | 167 | Ok(buffer) 168 | } 169 | 170 | impl Reader { 171 | /// Opens a nix-index database located at the given path. 172 | /// 173 | /// If the path does not exist or is not a valid database, an error is returned. 174 | pub fn open>(path: P) -> Result { 175 | Reader::from_buffer(read_from_path(path)?) 176 | } 177 | 178 | pub fn from_buffer(buffer: Vec) -> Result { 179 | Ok(Reader { 180 | decoder: frcode::Decoder::new(Cursor::new(buffer)), 181 | }) 182 | } 183 | 184 | /// Builds a query to find all entries in the database that have a filename matching the given pattern. 185 | /// 186 | /// Afterwards, use `Query::into_iter` to iterate over the items. 187 | pub fn query(self, exact_regex: &Regex) -> Query { 188 | Query { 189 | reader: self, 190 | exact_regex, 191 | hash: None, 192 | package_pattern: None, 193 | } 194 | } 195 | 196 | /// Dumps the contents of the database to stdout, for debugging. 197 | #[allow(clippy::print_stdout)] 198 | pub fn dump(&mut self) -> Result<()> { 199 | loop { 200 | let block = self.decoder.decode()?; 201 | if block.is_empty() { 202 | break; 203 | } 204 | for line in block.split(|c| *c == b'\n') { 205 | println!("{:?}", String::from_utf8_lossy(line)); 206 | } 207 | println!("-- block boundary"); 208 | } 209 | Ok(()) 210 | } 211 | } 212 | 213 | /// A builder for a `ReaderIter` to iterate over entries in the database matching a given pattern. 214 | pub struct Query<'a, 'b> { 215 | /// The underlying reader from which we read input. 216 | reader: Reader, 217 | 218 | /// The pattern that file paths have to match. 219 | exact_regex: &'a Regex, 220 | 221 | /// Only include the package with the given hash. 222 | hash: Option, 223 | 224 | /// Only include packages whose name matches the given pattern. 225 | package_pattern: Option<&'b Regex>, 226 | } 227 | 228 | impl<'a, 'b> Query<'a, 'b> { 229 | /// Limit results to entries from the package with the specified hash if `Some`. 230 | pub fn hash(self, hash: Option) -> Query<'a, 'b> { 231 | Query { hash, ..self } 232 | } 233 | 234 | /// Limit results to entries from packages whose name matches the given regex if `Some`. 235 | pub fn package_pattern(self, package_pattern: Option<&'b Regex>) -> Query<'a, 'b> { 236 | Query { 237 | package_pattern, 238 | ..self 239 | } 240 | } 241 | 242 | /// Runs the query, returning an Iterator that will yield all entries matching the conditions. 243 | /// 244 | /// There is no guarantee about the order of the returned matches. 245 | pub fn run(self) -> Result> { 246 | let mut expr = regex_syntax::ast::parse::Parser::new() 247 | .parse(self.exact_regex.as_str()) 248 | .expect("regex cannot be invalid"); 249 | // replace the ^ anchor by a NUL byte, since each entry is of the form `METADATA\0PATH` 250 | // (so the NUL byte marks the start of the path). 251 | { 252 | let mut stack = vec![&mut expr]; 253 | while let Some(e) = stack.pop() { 254 | match *e { 255 | Ast::Assertion(Assertion { 256 | kind: AssertionKind::StartLine, 257 | span, 258 | }) => { 259 | *e = Ast::Literal(Literal { 260 | span, 261 | c: '\0', 262 | kind: regex_syntax::ast::LiteralKind::Verbatim, 263 | }) 264 | } 265 | Ast::Group(Group { ref mut ast, .. }) => stack.push(ast), 266 | Ast::Repetition(Repetition { ref mut ast, .. }) => stack.push(ast), 267 | Ast::Concat(Concat { ref mut asts, .. }) 268 | | Ast::Alternation(Alternation { ref mut asts, .. }) => stack.extend(asts), 269 | _ => {} 270 | } 271 | } 272 | } 273 | let mut regex_builder = grep::regex::RegexMatcherBuilder::new(); 274 | regex_builder.line_terminator(Some(b'\n')).multi_line(true); 275 | 276 | let grep = regex_builder.build(&format!("{}", expr))?; 277 | Ok(ReaderIter { 278 | reader: self.reader, 279 | found: Vec::new(), 280 | found_without_package: Vec::new(), 281 | pattern: grep, 282 | exact_pattern: self.exact_regex, 283 | package_entry_pattern: regex_builder.build("^p\0").expect("valid regex"), 284 | package_name_pattern: self.package_pattern, 285 | package_hash: self.hash, 286 | }) 287 | } 288 | } 289 | 290 | /// An iterator for entries in a database matching a given pattern. 291 | pub struct ReaderIter<'a, 'b> { 292 | /// The underlying reader from which we read input. 293 | reader: Reader, 294 | /// Entries that matched the pattern but have not been returned by `next` yet. 295 | found: Vec<(StorePath, FileTreeEntry)>, 296 | /// Entries that matched the pattern but for which we don't know yet what package they belong to. 297 | /// This may happen if the entry we matched was at the end of the search buffer, so that the entry 298 | /// for the package did not fit into the buffer anymore (since the package is stored after the entries 299 | /// of the package). In this case, we need to look for the package entry in the next iteration when 300 | /// we read the next block of input. 301 | found_without_package: Vec, 302 | /// The pattern for which to search package paths. 303 | /// 304 | /// This pattern should work on the raw bytes of file entries. In particular, the file path is not the 305 | /// first data in a file entry, so the regex `^` anchor will not work correctly. 306 | /// 307 | /// The pattern here may produce false positives (for example, if it matches inside the metadata of a file 308 | /// entry). This is not a problem, as matches are later checked against `exact_pattern`. 309 | pattern: grep::regex::RegexMatcher, 310 | /// The raw pattern, as supplied to `find_iter`. This is used to verify matches, since `pattern` itself 311 | /// may produce false positives. 312 | exact_pattern: &'a Regex, 313 | /// Pattern that matches only package entries. 314 | package_entry_pattern: grep::regex::RegexMatcher, 315 | /// Pattern that the package name should match. 316 | package_name_pattern: Option<&'b Regex>, 317 | /// Only search the package with the given hash. 318 | package_hash: Option, 319 | } 320 | 321 | fn consume_no_error(e: NoError) -> T { 322 | panic!("impossible: {}", e) 323 | } 324 | 325 | fn next_matching_line>( 326 | matcher: M, 327 | buf: &[u8], 328 | mut start: usize, 329 | ) -> Option { 330 | while let Some(candidate) = matcher 331 | .find_candidate_line(&buf[start..]) 332 | .unwrap_or_else(consume_no_error) 333 | { 334 | // the buffer may end with a newline character, so we may get a match 335 | // for an empty "line" at the end of the buffer 336 | // since this is not a line match, return None 337 | if start == buf.len() { 338 | return None; 339 | }; 340 | 341 | let (pos, confirmed) = match candidate { 342 | LineMatchKind::Confirmed(pos) => (start + pos, true), 343 | LineMatchKind::Candidate(pos) => (start + pos, false), 344 | }; 345 | 346 | let line_start = memrchr(b'\n', &buf[..pos]).map(|x| x + 1).unwrap_or(0); 347 | let line_end = memchr(b'\n', &buf[pos..]) 348 | .map(|x| x + pos + 1) 349 | .unwrap_or(buf.len()); 350 | 351 | if !confirmed 352 | && !matcher 353 | .is_match(&buf[line_start..line_end]) 354 | .unwrap_or_else(consume_no_error) 355 | { 356 | start = line_end; 357 | continue; 358 | } 359 | 360 | return Some(Match::new(line_start, line_end)); 361 | } 362 | None 363 | } 364 | 365 | impl<'a, 'b> ReaderIter<'a, 'b> { 366 | /// Reads input until `self.found` contains at least one entry or the end of the input has been reached. 367 | #[allow(unused_assignments)] // because of https://github.com/rust-lang/rust/issues/22630 368 | fn fill_buf(&mut self) -> Result<()> { 369 | // the input is processed in blocks until we've found at least a single entry 370 | while self.found.is_empty() { 371 | let &mut ReaderIter { 372 | ref mut reader, 373 | ref package_entry_pattern, 374 | ref package_name_pattern, 375 | ref package_hash, 376 | .. 377 | } = self; 378 | let block = reader.decoder.decode()?; 379 | 380 | // if the block is empty, the end of input has been reached 381 | if block.is_empty() { 382 | return Ok(()); 383 | } 384 | 385 | // when we find a match, we need to know the package that this match belongs to. 386 | // the `find_package` function will skip forward until a package entry is found 387 | // (the package entry comes after all file entries for a package). 388 | // 389 | // to be more efficient if there are many matches, we cache the current package here. 390 | // this package is valid for all positions up to the second element of the tuple 391 | // (after that, a new package begins). 392 | let mut cached_package: Option<(StorePath, usize)> = None; 393 | let mut no_more_package = false; 394 | let mut find_package = |item_end| -> Result<_> { 395 | if let Some((ref pkg, end)) = cached_package { 396 | if item_end < end { 397 | return Ok(Some((pkg.clone(), end))); 398 | } 399 | } 400 | 401 | if no_more_package { 402 | return Ok(None); 403 | } 404 | 405 | let mat = match next_matching_line(&package_entry_pattern, &block, item_end) { 406 | Some(v) => v, 407 | None => { 408 | no_more_package = true; 409 | return Ok(None); 410 | } 411 | }; 412 | 413 | let json = &block[mat.start() + 2..mat.end() - 1]; 414 | let pkg: StorePath = serde_json::from_slice(json) 415 | .chain_err(|| ErrorKind::StorePathParse(json.to_vec()))?; 416 | cached_package = Some((pkg.clone(), mat.end())); 417 | Ok(Some((pkg, mat.end()))) 418 | }; 419 | 420 | // Tests if a store path matches the `package_name_pattern` and `package_hash` constraints. 421 | let should_search_package = |pkg: &StorePath| -> bool { 422 | package_name_pattern.map_or(true, |r| r.is_match(pkg.name().as_bytes())) 423 | && package_hash.as_ref().map_or(true, |h| h == &pkg.hash()) 424 | }; 425 | 426 | let mut pos = 0; 427 | // if there are any entries without a package left over from the previous iteration, see 428 | // if this block contains the package entry. 429 | if !self.found_without_package.is_empty() { 430 | if let Some((pkg, end)) = find_package(0)? { 431 | if !should_search_package(&pkg) { 432 | // all entries before end will have the same package 433 | pos = end; 434 | self.found_without_package.truncate(0); 435 | } else { 436 | for entry in self.found_without_package.split_off(0) { 437 | self.found.push((pkg.clone(), entry)); 438 | } 439 | } 440 | } 441 | } 442 | 443 | // process all matches in this block 444 | while let Some(mat) = next_matching_line(&self.pattern, &block, pos) { 445 | pos = mat.end(); 446 | let entry = &block[mat.start()..mat.end() - 1]; 447 | // skip entries that aren't describing file paths 448 | if self 449 | .package_entry_pattern 450 | .is_match(entry) 451 | .unwrap_or_else(consume_no_error) 452 | { 453 | continue; 454 | } 455 | 456 | // skip if package name or hash doesn't match 457 | // we can only skip if we know the package 458 | if let Some((pkg, end)) = find_package(mat.end())? { 459 | if !should_search_package(&pkg) { 460 | // all entries before end will have the same package 461 | pos = end; 462 | continue; 463 | } 464 | } 465 | 466 | let entry = FileTreeEntry::decode(entry) 467 | .ok_or_else(|| Error::from(ErrorKind::EntryParse(entry.to_vec())))?; 468 | 469 | // check for false positives 470 | if !self.exact_pattern.is_match(&entry.path) { 471 | continue; 472 | } 473 | 474 | match find_package(mat.end())? { 475 | None => self.found_without_package.push(entry), 476 | Some((pkg, _)) => self.found.push((pkg, entry)), 477 | } 478 | } 479 | } 480 | Ok(()) 481 | } 482 | 483 | /// Returns the next match in the database. 484 | fn next_match(&mut self) -> Result> { 485 | self.fill_buf()?; 486 | Ok(self.found.pop()) 487 | } 488 | } 489 | 490 | impl<'a, 'b> Iterator for ReaderIter<'a, 'b> { 491 | type Item = Result<(StorePath, FileTreeEntry)>; 492 | 493 | fn next(&mut self) -> Option { 494 | match self.next_match() { 495 | Err(e) => Some(Err(e)), 496 | Ok(v) => v.map(Ok), 497 | } 498 | } 499 | } 500 | 501 | #[cfg(test)] 502 | mod tests { 503 | use super::*; 504 | 505 | #[test] 506 | fn test_next_matching_line_package() { 507 | let matcher = grep::regex::RegexMatcherBuilder::new() 508 | .line_terminator(Some(b'\n')) 509 | .multi_line(true) 510 | .build("^p") 511 | .expect("valid regex"); 512 | let buffer = br#" 513 | SOME LINE 514 | pDATA 515 | ANOTHER LINE 516 | "#; 517 | 518 | let mat = next_matching_line(matcher, buffer, 0); 519 | assert_eq!(mat, Some(Match::new(11, 17))); 520 | } 521 | } 522 | -------------------------------------------------------------------------------- /src/cache/files.rs: -------------------------------------------------------------------------------- 1 | //! Data types for working with trees of files. 2 | //! 3 | //! The main type here is `FileTree` which represents 4 | //! such as the file listing for a store path. 5 | use std::collections::HashMap; 6 | use std::io::{self, Write}; 7 | use std::str::{self, FromStr}; 8 | 9 | use clap::builder::PossibleValue; 10 | use clap::ValueEnum; 11 | use memchr::memchr; 12 | use serde::{Deserialize, Serialize}; 13 | use serde_bytes::ByteBuf; 14 | 15 | use crate::cache::frcode; 16 | 17 | /// This enum represents a single node in a file tree. 18 | /// 19 | /// The type is generic over the contents of a directory node, 20 | /// because we want to use this enum to represent both a flat 21 | /// structure where a directory only stores some meta-information about itself 22 | /// (such as the number of children) and full file trees, where a 23 | /// directory contains all the child nodes. 24 | /// 25 | /// Note that file nodes by themselves do not have names. Names are given 26 | /// to file nodes by the parent directory, which has a map of entry names to 27 | /// file nodes. 28 | #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] 29 | pub enum FileNode { 30 | /// A regular file. This is the normal kind of file which is 31 | /// neither a directory not a symlink. 32 | Regular { 33 | /// The size of this file, in bytes. 34 | size: u64, 35 | /// Whether or not this file has the `executable` bit set. 36 | executable: bool, 37 | }, 38 | /// A symbolic link that points to another file path. 39 | Symlink { 40 | /// The path that this symlink points to. 41 | target: ByteBuf, 42 | }, 43 | /// A directory. It usually has a mapping of names to child nodes (in 44 | /// the case of a fill tree), but we also support a reduced form where 45 | /// we only store the number of entries in the directory. 46 | Directory { 47 | /// The size of a directory is the number of children it contains. 48 | size: u64, 49 | 50 | /// The contents of this directory. These are generic, as explained 51 | /// in the documentation for this type. 52 | contents: T, 53 | }, 54 | } 55 | 56 | /// The type of a file. 57 | /// 58 | /// This mirrors the variants of `FileNode`, but without storing 59 | /// data in each variant. 60 | /// 61 | /// An exception to this is the `executable` field for the regular type. 62 | /// This is needed since we present `regular` and `executable` files as different 63 | /// to the user, so we need a way to represent both types. 64 | #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] 65 | pub enum FileType { 66 | Regular { executable: bool }, 67 | Directory, 68 | Symlink, 69 | } 70 | 71 | impl ValueEnum for FileType { 72 | fn value_variants<'a>() -> &'a [Self] { 73 | &[ 74 | FileType::Regular { executable: false }, 75 | FileType::Regular { executable: true }, 76 | FileType::Directory, 77 | FileType::Symlink, 78 | ] 79 | } 80 | 81 | fn to_possible_value(&self) -> Option { 82 | match self { 83 | FileType::Regular { executable: false } => Some(PossibleValue::new("r")), 84 | FileType::Regular { executable: true } => Some(PossibleValue::new("x")), 85 | FileType::Directory => Some(PossibleValue::new("d")), 86 | FileType::Symlink => Some(PossibleValue::new("s")), 87 | } 88 | } 89 | } 90 | 91 | impl FromStr for FileType { 92 | type Err = &'static str; 93 | 94 | fn from_str(s: &str) -> Result { 95 | match s { 96 | "r" => Ok(FileType::Regular { executable: false }), 97 | "x" => Ok(FileType::Regular { executable: true }), 98 | "d" => Ok(FileType::Directory), 99 | "s" => Ok(FileType::Symlink), 100 | _ => Err("invalid file type"), 101 | } 102 | } 103 | } 104 | 105 | /// This lists all file types that can currently be represented. 106 | pub const ALL_FILE_TYPES: &'static [FileType] = &[ 107 | FileType::Regular { executable: true }, 108 | FileType::Regular { executable: false }, 109 | FileType::Directory, 110 | FileType::Symlink, 111 | ]; 112 | 113 | impl FileNode { 114 | /// Split this node into a node without contents and optionally the contents themselves, 115 | /// if the node was a directory. 116 | pub fn split_contents(&self) -> (FileNode<()>, Option<&T>) { 117 | use self::FileNode::*; 118 | match *self { 119 | Regular { size, executable } => ( 120 | Regular { 121 | size: size, 122 | executable: executable, 123 | }, 124 | None, 125 | ), 126 | Symlink { ref target } => ( 127 | Symlink { 128 | target: target.clone(), 129 | }, 130 | None, 131 | ), 132 | Directory { size, ref contents } => ( 133 | Directory { 134 | size: size, 135 | contents: (), 136 | }, 137 | Some(contents), 138 | ), 139 | } 140 | } 141 | 142 | /// Return the type of this file. 143 | pub fn get_type(&self) -> FileType { 144 | match *self { 145 | FileNode::Regular { executable, .. } => FileType::Regular { 146 | executable: executable, 147 | }, 148 | FileNode::Directory { .. } => FileType::Directory, 149 | FileNode::Symlink { .. } => FileType::Symlink, 150 | } 151 | } 152 | } 153 | 154 | impl FileNode<()> { 155 | fn encode(&self, encoder: &mut frcode::Encoder) -> io::Result<()> { 156 | use self::FileNode::*; 157 | match *self { 158 | Regular { executable, size } => { 159 | let e = if executable { "x" } else { "r" }; 160 | encoder.write_meta(format!("{}{}", size, e).as_bytes())?; 161 | } 162 | Symlink { ref target } => { 163 | encoder.write_meta(target)?; 164 | encoder.write_meta(b"s")?; 165 | } 166 | Directory { size, contents: () } => { 167 | encoder.write_meta(format!("{}d", size).as_bytes())?; 168 | } 169 | } 170 | Ok(()) 171 | } 172 | 173 | pub fn decode(buf: &[u8]) -> Option { 174 | use self::FileNode::*; 175 | buf.split_last().and_then(|(kind, buf)| match *kind { 176 | b'x' | b'r' => { 177 | let executable = *kind == b'x'; 178 | str::from_utf8(buf) 179 | .ok() 180 | .and_then(|s| s.parse().ok()) 181 | .map(|size| Regular { 182 | executable: executable, 183 | size: size, 184 | }) 185 | } 186 | b's' => Some(Symlink { 187 | target: ByteBuf::from(buf), 188 | }), 189 | b'd' => str::from_utf8(buf) 190 | .ok() 191 | .and_then(|s| s.parse().ok()) 192 | .map(|size| Directory { 193 | size: size, 194 | contents: (), 195 | }), 196 | _ => None, 197 | }) 198 | } 199 | } 200 | 201 | /// This type represents a full tree of files. 202 | /// 203 | /// A *file tree* is a *file node* where each directory contains 204 | /// the tree for its children. 205 | #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] 206 | pub struct FileTree(FileNode>); 207 | 208 | /// An entry in a file tree is a path to a node paired with that node. 209 | /// 210 | /// If the entry refers to a directory, it only stores information about that 211 | /// directory itself. It does not contain the children of the directory. 212 | #[derive(Debug, Clone)] 213 | pub struct FileTreeEntry { 214 | pub path: Vec, 215 | pub node: FileNode<()>, 216 | } 217 | 218 | impl FileTreeEntry { 219 | pub fn encode(self, encoder: &mut frcode::Encoder) -> io::Result<()> { 220 | self.node.encode(encoder)?; 221 | encoder.write_path(self.path)?; 222 | Ok(()) 223 | } 224 | 225 | pub fn decode(buf: &[u8]) -> Option { 226 | memchr(b'\0', buf).and_then(|sep| { 227 | let path = &buf[(sep + 1)..]; 228 | let node = &buf[0..sep]; 229 | FileNode::decode(node).map(|node| FileTreeEntry { 230 | path: path.to_vec(), 231 | node: node, 232 | }) 233 | }) 234 | } 235 | } 236 | 237 | impl FileTree { 238 | pub fn regular(size: u64, executable: bool) -> Self { 239 | FileTree(FileNode::Regular { 240 | size: size, 241 | executable: executable, 242 | }) 243 | } 244 | 245 | pub fn symlink(target: ByteBuf) -> Self { 246 | FileTree(FileNode::Symlink { target: target }) 247 | } 248 | 249 | pub fn directory(entries: HashMap) -> Self { 250 | FileTree(FileNode::Directory { 251 | size: entries.len() as u64, 252 | contents: entries, 253 | }) 254 | } 255 | 256 | pub fn to_list(&self, filter_prefix: &[u8]) -> Vec { 257 | let mut result = Vec::new(); 258 | 259 | let mut stack = Vec::with_capacity(16); 260 | stack.push((Vec::new(), self)); 261 | 262 | while let Some(entry) = stack.pop() { 263 | let path = entry.0; 264 | let &FileTree(ref current) = entry.1; 265 | let (node, contents) = current.split_contents(); 266 | if let Some(entries) = contents { 267 | let mut entries = entries.iter().collect::>(); 268 | entries.sort_by(|a, b| Ord::cmp(a.0, b.0)); 269 | for (name, entry) in entries { 270 | let mut path = path.clone(); 271 | path.push(b'/'); 272 | path.extend_from_slice(name); 273 | stack.push((path, entry)); 274 | } 275 | } 276 | if path.starts_with(filter_prefix) { 277 | result.push(FileTreeEntry { path, node }); 278 | } 279 | } 280 | result 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /src/cache/frcode.rs: -------------------------------------------------------------------------------- 1 | //! A compact encoding for file tree entries based on sharing prefixes. 2 | //! 3 | //! This module contains a rust implementation of a variant of the `frcode` tool 4 | //! used by GNU findutils' locate. It has been extended to allow meta information 5 | //! to be attached to each entry so it is no longer compatible with the original 6 | //! frcode format. 7 | //! (See http://www.delorie.com/gnu/docs/findutils/locatedb.5.html for a description of the frcode format.) 8 | //! 9 | //! The basic building block of the encoding is a line. Each line has the following format: 10 | //! (the spaces are for readability only, they are not present in the encoding) 11 | //! 12 | //! ```text 13 | //! <\x00 byte> 14 | //! ``` 15 | //! 16 | //! Each entry holds two parts of data: metadata, which is just some arbitrary blob of NUL-terminated bytes 17 | //! and a path. Because we are storing file trees, the path will likely share a long prefix with the previous 18 | //! entry's path (we traverse directory entries in sorted order to maximize this chance), so we first store 19 | //! the length of the shared prefix. 20 | //! 21 | //! Since this length will likely be similar to the previous one (if there are many entries in `/foo/bar`, then they will 22 | //! all share a prefix of at least the length of `/foo/bar`) we only store the signed *difference* to the previous shared prefix length 23 | //! (This is why it's called a differential). For differences smaller than +/-127 we store them directly as a single byte. If the 24 | //! difference is greater than that, the first byte will by `0x80` (-128) indicating that the following two bytes represent the 25 | //! difference (with the high byte first [big endian]). 26 | //! 27 | //! As an example, consider the following non-encoded plaintext, where `:` separates the metadata from the path: 28 | //! 29 | //! ```text 30 | //! d:/ 31 | //! d:/foo 32 | //! d:/foo/bar 33 | //! f:/foo/bar/test.txt 34 | //! f:/foo/bar/text.txt 35 | //! d:/foo/baz 36 | //! ``` 37 | //! 38 | //! This text would be encoded as (using `[v]` to indicate a byte with the value of v) 39 | //! 40 | //! ```text 41 | //! d[0][0]/ 42 | //! d[0][1]foo 43 | //! d[0][3]/bar 44 | //! f[0][4]/test.txt 45 | //! f[0][3]xt.txt 46 | //! d[0][-4]z 47 | //! ``` 48 | //! 49 | //! At the beginning, there is no previous entry, so the shared prefix length must always be `0` (and so must the shared prefix differential). 50 | //! The second entry shares `1` byte with the first path so the difference is `1`. The third entry shares `4` bytes with the second one, which 51 | //! is `3` more than the shared length of the second one, so we encode a `3` followed by the non-shared bytes, and so on for the remaining entries. 52 | //! The last entry shares four bytes less than the second to last one did with its predecessor, so here the differential is negative. 53 | //! 54 | //! Through this encoding, the size of the index is typically reduces by a factor of 3 to 5. 55 | use std::cmp; 56 | use std::io::{self, BufRead, Seek, Write}; 57 | use std::ops::{Deref, DerefMut}; 58 | 59 | use error_chain::{bail, error_chain}; 60 | use memchr; 61 | 62 | error_chain! { 63 | foreign_links { 64 | Io(io::Error); 65 | } 66 | errors { 67 | SharedOutOfRange { previous_len: usize, shared_len: isize } { 68 | description("shared prefix length out of bounds") 69 | display("length of shared prefix must be >= 0 and <= {} (length of previous item), but found: {}", previous_len, shared_len) 70 | } 71 | SharedOverflow { shared_len: isize, diff: isize } { 72 | description("shared prefix length too big (overflow)") 73 | display("length of shared prefix too big: cannot add {} to {} without overflow", shared_len, diff) 74 | } 75 | MissingNul { 76 | description("missing terminating NUL byte for entry") 77 | } 78 | MissingNewline { 79 | description("missing newline separator for entry") 80 | } 81 | MissingPrefixDifferential { 82 | description("missing the shared prefix length differential for entry") 83 | } 84 | } 85 | } 86 | 87 | /// A buffer that may be resizable or not. This is used for decoding, 88 | /// where we want to make the buffer resizable as long as we haven't decoded 89 | /// a full entry yet but want to lock it as soon as we got a full entry. 90 | /// 91 | /// This is necessary because we always need to be able to decode at least 92 | /// one entry to make progress, as we never return partial entries during decoding. 93 | struct ResizableBuf { 94 | allow_resize: bool, 95 | data: Vec, 96 | } 97 | 98 | impl ResizableBuf { 99 | /// Allocates a new resizable buffer with the given initial size. 100 | /// 101 | /// The new buffer will allow resizing initially. 102 | fn new(capacity: usize) -> ResizableBuf { 103 | ResizableBuf { 104 | data: vec![0; capacity], 105 | allow_resize: true, 106 | } 107 | } 108 | 109 | /// Resizes the buffer to hold at least `new_size` elements. Returns `true` 110 | /// if resizing was successful (so that buffer can now hold at least `new_size` elements) 111 | /// or `false` if not (meaning `new_size` is greater than the current size and resizing 112 | /// was not allowed). 113 | fn resize(&mut self, new_size: usize) -> bool { 114 | if new_size <= self.data.len() { 115 | return true; 116 | } 117 | 118 | if !self.allow_resize { 119 | return false; 120 | } 121 | 122 | self.data.resize(new_size, b'\x00'); 123 | true 124 | } 125 | } 126 | 127 | impl Deref for ResizableBuf { 128 | type Target = [u8]; 129 | fn deref(&self) -> &[u8] { 130 | &self.data 131 | } 132 | } 133 | 134 | impl DerefMut for ResizableBuf { 135 | fn deref_mut(&mut self) -> &mut [u8] { 136 | &mut self.data 137 | } 138 | } 139 | 140 | /// A decoder for the frcode format. It reads data from some input source 141 | /// and returns blocks of decoded entries. 142 | /// 143 | /// It will not split the metadata/path parts of individual entries since 144 | /// the primary use case for this is searching, where it is enough to decode 145 | /// the entries that match. 146 | pub struct Decoder { 147 | /// The input source from which we decode 148 | reader: R, 149 | /// Position of the first byte of the path part of the last entry. 150 | /// We need this to copy the shared prefix. 151 | last_path: usize, 152 | /// Position of the start of the entry that didn't fully fit in the buffer in the 153 | /// last decode iteration. Since this entry was partial, it hasn't been returned to 154 | /// the user yet and we need to continue decoding this entry in this iteration. 155 | partial_entry_start: usize, 156 | /// The length of the shared prefix for the current entry. This is necessary because 157 | /// the shared length is stored as a difference, so we need the previous value to update it. 158 | shared_len: isize, 159 | /// The buffer into which we store the decoded bytes. 160 | buf: ResizableBuf, 161 | /// Current write position in buf. The next decoded byte should be written to buf[pos]. 162 | pos: usize, 163 | } 164 | 165 | impl Decoder { 166 | /// Resets the decoder at the start for further reuse. 167 | pub fn reset(&mut self) -> Result<()> { 168 | self.reader.seek(io::SeekFrom::Start(0))?; 169 | self.pos = 0; 170 | self.last_path = 0; 171 | self.shared_len = 0; 172 | self.partial_entry_start = 0; 173 | Ok(()) 174 | } 175 | } 176 | 177 | impl Decoder { 178 | /// Construct a new decoder for the given source. 179 | pub fn new(reader: R) -> Decoder { 180 | let capacity = 1_000_000; 181 | Decoder { 182 | reader, 183 | buf: ResizableBuf::new(capacity), 184 | pos: 0, 185 | last_path: 0, 186 | shared_len: 0, 187 | partial_entry_start: 0, 188 | } 189 | } 190 | 191 | /// Copies `self.shared_len` bytes from the previous entry's path into the output buffer. 192 | /// 193 | /// Returns false if the buffer was too small and could not be resized. In this case, no 194 | /// bytes will be copied. 195 | fn copy_shared(&mut self) -> Result { 196 | let shared_len = self.shared_len as usize; 197 | let new_pos = self.pos + shared_len; 198 | let new_last_path = self.pos; 199 | if !self.buf.resize(new_pos) { 200 | return Ok(false); 201 | } 202 | 203 | if self.shared_len < 0 || self.last_path + shared_len > self.pos { 204 | bail!(ErrorKind::SharedOutOfRange { 205 | previous_len: self.pos - self.last_path, 206 | shared_len: self.shared_len, 207 | }); 208 | } 209 | 210 | let (_, last) = self.buf.split_at_mut(self.last_path); 211 | let (last, new) = last.split_at_mut(self.pos - self.last_path); 212 | new[..shared_len].copy_from_slice(&last[..shared_len]); 213 | 214 | self.pos += shared_len; 215 | self.last_path = new_last_path; 216 | Ok(true) 217 | } 218 | 219 | /// Copies bytes from the input reader to the output buffer until a `\x00` byte is read. 220 | /// The NUL byte is included in the output buffer. 221 | /// 222 | /// Returns false if the output buffer was exhausted before a NUL byte could be found and 223 | /// could not be resized. All bytes that were read before this situation was detected will 224 | /// have already been copied to the output buffer in this case. 225 | /// 226 | /// It will also return false if the end of the input was reached. 227 | fn read_to_nul(&mut self) -> Result { 228 | loop { 229 | let (done, len) = { 230 | let &mut Decoder { 231 | ref mut reader, 232 | ref mut buf, 233 | ref mut pos, 234 | .. 235 | } = self; 236 | let input = match reader.fill_buf() { 237 | Ok(data) => data, 238 | Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, 239 | Err(e) => return Err(Error::from(e)), 240 | }; 241 | 242 | if input.is_empty() { 243 | return Ok(false); 244 | } 245 | 246 | let (done, len) = match memchr::memchr(b'\x00', input) { 247 | Some(i) => (true, i + 1), 248 | None => (false, input.len()), 249 | }; 250 | 251 | let new_pos = *pos + len; 252 | if buf.resize(new_pos) { 253 | buf[*pos..new_pos].copy_from_slice(&input[..len]); 254 | *pos = new_pos; 255 | (done, len) 256 | } else { 257 | return Ok(false); 258 | } 259 | }; 260 | self.reader.consume(len); 261 | if done { 262 | return Ok(true); 263 | } 264 | } 265 | } 266 | 267 | /// Read the differential from the input reader. This function will return an error 268 | /// if the end of input has been reached. 269 | fn decode_prefix_diff(&mut self) -> Result { 270 | let mut buf = [0; 1]; 271 | self.reader 272 | .read_exact(&mut buf) 273 | .chain_err(|| ErrorKind::MissingPrefixDifferential)?; 274 | 275 | if buf[0] != 0x80 { 276 | Ok((buf[0] as i8) as i16) 277 | } else { 278 | let mut buf = [0; 2]; 279 | self.reader 280 | .read_exact(&mut buf) 281 | .chain_err(|| ErrorKind::MissingPrefixDifferential)?; 282 | let high = buf[0] as i16; 283 | let low = buf[1] as i16; 284 | Ok(high << 8 | low) 285 | } 286 | } 287 | 288 | /// Decodes some entries to fill the buffer and returns a block of decoded entries. 289 | /// 290 | /// It will decode as many entries as fit into the internal buffer, but at least one. 291 | /// In the returned block of bytes, an entry's metadata and path will be separated by a NUL byte 292 | /// and entries will be terminated with a newline character. This allows for fast searching with 293 | /// a line based searcher. 294 | /// 295 | /// The function does not return partially decoded entries. Because of this, the size of returned 296 | /// slice will vary from call to call. The last entry which did not fully fit into the buffer yet 297 | /// will be returned as the first entry at the next call. 298 | pub fn decode(&mut self) -> Result<&mut [u8]> { 299 | // Save end pointer from previous iteration and reset write position 300 | let end = self.pos; 301 | self.pos = 0; 302 | 303 | // We need to preserve some data from the previous iteration, namely: 304 | // 305 | // * all data after the `self.last_path` position, for copying the shared prefix 306 | // * everything from the start of the partial entry, since this entry wasn't fully decoded 307 | // in the last iteration and we want to continue decoding it now 308 | // 309 | // If we stopped decoding the partial entry after already copying the shared prefix, then 310 | // `last_path` will already point to the partial entry so it will be greater than `partial_entry_start`. 311 | // 312 | // If we stopped decoding during copying the metadata though, which comes before we copy the shared 313 | // prefix, then `last_path` will point to the previous entry's path, so it will be smaller than 314 | // `partial_entry_start`. 315 | // 316 | // To support both these cases, we take the minimum here. 317 | let mut copy_pos = cmp::min(self.partial_entry_start, self.last_path); 318 | 319 | // Since we sometimes copy more than just the partial entry, we need to know where the partial entry 320 | // starts as that is the first position that we want to return (everything before that was already 321 | // part of an entry returned in the last iteration). 322 | let item_start = self.partial_entry_start - copy_pos; 323 | 324 | // Shift the last path, because we copy it from copy_pos.. to 0.. 325 | self.last_path -= copy_pos; 326 | 327 | // Now we can do the actual copying. We cannot use copy_from_slice here since source and target 328 | // may overlap. 329 | while copy_pos < end { 330 | self.buf[self.pos] = self.buf[copy_pos]; 331 | self.pos += 1; 332 | copy_pos += 1; 333 | } 334 | 335 | // Allow resizing the buffer, since we haven't decoded a full entry yet 336 | self.buf.allow_resize = true; 337 | 338 | // If the the last decoded byte in the buffer is a NUL byte, that means that 339 | // we are now at the start of the path part of the entry. This means that 340 | // we need to copy the shared prefix now. 341 | let mut found_nul = self.pos > 0 && self.buf[self.pos - 1] == b'\x00'; 342 | if found_nul { 343 | self.copy_shared()?; 344 | } 345 | 346 | // At this point, we are guaranteed to be in either the metadata part or the non-shared part 347 | // of an entry. In both cases, the action that we need to take is the same: copy data till 348 | // the next NUL byte. After the NUL byte, we know that we are at the end of the metadata part, 349 | // so we read a differential and copy the shared prefix, and repeat. 350 | // 351 | // Note that this loop doesn't care about where entries end. Only the path part of each entry requires 352 | // special processing, so we can jump from NUL byte to NUL byte, decode the path and then just copy 353 | // the data from the source when jumping to the next NUL byte. 354 | loop { 355 | // Read data up to the next nul byte. 356 | if !self.read_to_nul()? { 357 | break; 358 | } 359 | 360 | // If we have already found a NUL byte before this, so we've now got two NUL bytes, so 361 | // we've got at least one full entry in between. 362 | self.buf.allow_resize = !found_nul; 363 | 364 | // We found a NUL byte. Note that we need to set this *after* updating allow_resize, 365 | // since allow_resize should be set to false only after we've found two NUL bytes. 366 | found_nul = true; 367 | 368 | // Parse the next prefix length difference 369 | let diff = self.decode_prefix_diff()? as isize; 370 | 371 | // Update the shared len 372 | self.shared_len = 373 | self.shared_len 374 | .checked_add(diff) 375 | .ok_or_else(|| ErrorKind::SharedOverflow { 376 | shared_len: self.shared_len, 377 | diff, 378 | })?; 379 | 380 | // Copy the shared prefix 381 | if !self.copy_shared()? { 382 | break; 383 | } 384 | } 385 | 386 | // Since we don't want to return partially decoded items, we need to find the end of the last entry. 387 | self.partial_entry_start = memchr::memrchr(b'\n', &self.buf[..self.pos]) 388 | .ok_or_else(|| ErrorKind::MissingNewline)? 389 | + 1; 390 | 391 | Ok(&mut self.buf[item_start..self.partial_entry_start]) 392 | } 393 | } 394 | 395 | /// This struct implements an encoder for the frcode format. The encoder 396 | /// writes directly to the underlying `Write` instance. 397 | /// 398 | /// To encode an entry you should first call `write_meta` a number of times 399 | /// to fill the meta data portion. Then, call `write_path` once to finialize the entry. 400 | /// 401 | /// One important property of this encoder is that it is safe to open and close 402 | /// it multiple times on the same stream, like this: 403 | /// 404 | /// ```text 405 | /// { 406 | /// let encoder1 = Encoder::new(&mut stream); 407 | /// } // encoder1 gets dropped here 408 | /// { 409 | /// let encoder2 = Encoder::new(&mut stream); 410 | /// } 411 | /// ``` 412 | /// 413 | /// To support this, the encoder has a "footer" item that will get written when it is dropped. 414 | /// This is necessary because we need to write at least one more entry to reset the shared prefix 415 | /// length to zero, since the next encoder will expect that as initial state. 416 | pub struct Encoder { 417 | writer: W, 418 | last: Vec, 419 | shared_len: i16, 420 | footer_meta: Vec, 421 | footer_path: Vec, 422 | footer_written: bool, 423 | } 424 | 425 | impl Drop for Encoder { 426 | fn drop(&mut self) { 427 | self.write_footer().expect("failed to write footer") 428 | } 429 | } 430 | 431 | impl Encoder { 432 | /// Constructs a new encoder for the specific writer. 433 | /// 434 | /// The encoder will write the given `footer_meta` and `footer_path` as the last entry. 435 | /// 436 | /// # Panics 437 | /// 438 | /// If either `footer_meta` or `footer_path` contain NUL or newline bytes. 439 | pub fn new(writer: W, footer_meta: Vec, footer_path: Vec) -> Encoder { 440 | assert!( 441 | !footer_meta.contains(&b'\x00'), 442 | "footer meta must not contain null bytes" 443 | ); 444 | assert!( 445 | !footer_path.contains(&b'\x00'), 446 | "footer path must not contain null bytes" 447 | ); 448 | assert!( 449 | !footer_meta.contains(&b'\n'), 450 | "footer meta must not contain newlines" 451 | ); 452 | assert!( 453 | !footer_path.contains(&b'\n'), 454 | "footer path must not contain newlines" 455 | ); 456 | Encoder { 457 | writer, 458 | last: Vec::new(), 459 | shared_len: 0, 460 | footer_meta, 461 | footer_path, 462 | footer_written: false, 463 | } 464 | } 465 | 466 | /// Writes the specific shared prefix differential to the output stream. 467 | /// 468 | /// This function takes care of the variable-length encoding using for prefix differentials 469 | /// in the frcode format. 470 | fn encode_diff(&mut self, diff: i16) -> io::Result<()> { 471 | let low = (diff & 0xFF) as u8; 472 | if diff.abs() < i8::max_value() as i16 { 473 | self.writer.write_all(&[low])?; 474 | } else { 475 | let high = ((diff >> 8) & 0xFF) as u8; 476 | self.writer.write_all(&[0x80, high, low])?; 477 | } 478 | Ok(()) 479 | } 480 | 481 | /// Writes the meta data of an entry to the output stream. 482 | /// 483 | /// This function can be called multiple times to extend the current meta data part. 484 | /// Since the meta data is written as-is to the output stream, calling the function 485 | /// multiple times will concatenate the meta data of all calls. 486 | /// 487 | /// # Panics 488 | /// 489 | /// If the meta data contains NUL bytes or newlines. 490 | pub fn write_meta(&mut self, meta: &[u8]) -> io::Result<()> { 491 | assert!( 492 | !meta.contains(&b'\x00'), 493 | "entry must not contain null bytes" 494 | ); 495 | assert!(!meta.contains(&b'\n'), "entry must not contain newlines"); 496 | 497 | self.writer.write_all(meta)?; 498 | Ok(()) 499 | } 500 | 501 | /// Finalizes an entry by encoding its path to the output stream. 502 | /// 503 | /// This function should be called after you've finished writing the meta data for 504 | /// the current entry. It will terminate the meta data part by writing the NUL byte 505 | /// and then encode the path into the output stream. 506 | /// 507 | /// The entry will be terminated with a newline. 508 | /// 509 | /// # Panics 510 | /// 511 | /// If the path contains NUL bytes or newlines. 512 | pub fn write_path(&mut self, path: Vec) -> io::Result<()> { 513 | assert!( 514 | !path.contains(&b'\x00'), 515 | "entry must not contain null bytes" 516 | ); 517 | assert!(!path.contains(&b'\x00'), "entry must not contain newlines"); 518 | self.writer.write_all(&[b'\x00'])?; 519 | 520 | let mut shared: isize = 0; 521 | let max_shared = i16::max_value() as isize; 522 | for (a, b) in self.last.iter().zip(path.iter()) { 523 | if a != b || shared > max_shared { 524 | break; 525 | } 526 | shared += 1; 527 | } 528 | let shared = shared as i16; 529 | 530 | let diff = shared - self.shared_len; 531 | self.encode_diff(diff)?; 532 | 533 | self.last = path; 534 | self.shared_len = shared; 535 | 536 | let pos = shared as usize; 537 | self.writer.write_all(&self.last[pos..])?; 538 | self.writer.write_all(b"\n")?; 539 | 540 | Ok(()) 541 | } 542 | 543 | /// Writes the footer entry. 544 | /// 545 | /// The footer entry will not share any prefix with the preceding entry, 546 | /// so after this function, the shared prefix length is zero. This guarantees 547 | /// that we can start another Encoder after this item, since the Encoder expects 548 | /// the initial shared prefix length to be zero. 549 | fn write_footer(&mut self) -> io::Result<()> { 550 | if self.footer_written { 551 | return Ok(()); 552 | } 553 | 554 | let diff = -self.shared_len; 555 | self.writer.write_all(&self.footer_meta)?; 556 | self.writer.write_all(b"\x00")?; 557 | self.encode_diff(diff)?; 558 | self.writer.write_all(&self.footer_path)?; 559 | self.writer.write_all(b"\n")?; 560 | self.footer_written = true; 561 | Ok(()) 562 | } 563 | 564 | /// Finishes the encoder by writing the footer entry. 565 | /// 566 | /// This function is called by drop, but calling it explictly is recommended as 567 | /// drop has no way to report IO errors that may occur during writing the footer. 568 | pub fn finish(mut self) -> io::Result<()> { 569 | self.write_footer()?; 570 | 571 | Ok(()) 572 | } 573 | } 574 | -------------------------------------------------------------------------------- /src/cache/mod.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::OsStr; 2 | 3 | pub mod database; 4 | mod files; 5 | mod frcode; 6 | mod package; 7 | 8 | pub use files::{FileNode, FileTreeEntry}; 9 | pub use package::StorePath; 10 | 11 | pub fn cache_dir() -> &'static OsStr { 12 | let base = xdg::BaseDirectories::with_prefix("nix-index").unwrap(); 13 | 14 | Box::leak(Box::new(base.get_cache_home())).as_os_str() 15 | } 16 | -------------------------------------------------------------------------------- /src/cache/package.rs: -------------------------------------------------------------------------------- 1 | //! Data types for representing meta information about packages and store paths. 2 | //! 3 | //! The main data type in this `StorePath`, which represents a single output of 4 | //! some nix derivation. We also sometimes call a `StorePath` a package, to avoid 5 | //! confusion with file paths. 6 | use std::borrow::Cow; 7 | use std::io::{self, Write}; 8 | use std::str; 9 | 10 | use serde::{Deserialize, Serialize}; 11 | 12 | use super::FileTreeEntry; 13 | 14 | /// A type for describing how to reach a given store path. 15 | /// 16 | /// When building an index, we collect store paths from various sources, such 17 | /// as the output of nix-env -qa and the references of those store paths. 18 | /// 19 | /// To show the user how we reached a given store path, each store path tracks 20 | /// its origin. For example, for top-level store paths, we know which attribute 21 | /// of nixpkgs builds this store path. 22 | #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] 23 | pub struct PathOrigin { 24 | /// The attribute of nixpkgs that lead to this store path being discovered. 25 | /// 26 | /// If the store path is a top-level path, then the store path corresponds 27 | /// to an output of the derivation assigned to this attribute path. 28 | pub attr: String, 29 | 30 | /// The output of the derivation specified by `attr` that we want to refer to. 31 | /// 32 | /// If a derivation does not support multiple outputs, then this should just be "out", 33 | /// the default output. 34 | pub output: String, 35 | 36 | /// Indicates that this path is listed in the output of nix-env -qaP --out-name. 37 | /// 38 | /// We may index paths for which we do not know the exact attribute path. In this 39 | /// case, `attr` and `output` will be set to the values for the top-level path that 40 | /// contains the path in its closure. (This is also how we discovered the path in the 41 | /// first place: through being referenced by another, top-level path). It is unspecified 42 | /// which top-level path they will refer to though if there exist multiple ones whose closure 43 | /// contains this path. 44 | pub toplevel: bool, 45 | 46 | /// Target system 47 | pub system: Option, 48 | } 49 | 50 | impl PathOrigin { 51 | /// Encodes a path origin as a sequence of bytes, such that it can be decoed using `decode`. 52 | /// 53 | /// The encoding does not use the bytes `0x00` nor `0x01`, as long as neither `attr` nor `output` 54 | /// contain them. This is important since it allows the result to be encoded with [frcode](mod.frcode.html). 55 | /// 56 | /// # Panics 57 | /// 58 | /// The `attr` and `output` of the path origin must not contain the byte value `0x02`, otherwise 59 | /// this function panics. 60 | /// 61 | /// # Errors 62 | /// 63 | /// Returns any errors that were encountered while writing to the supplied `Writer`. 64 | pub fn encode(&self, writer: &mut W) -> io::Result<()> { 65 | assert!( 66 | !self.attr.contains('\x02'), 67 | "origin attribute path must not contain the byte value 0x02 anywhere" 68 | ); 69 | assert!( 70 | !self.output.contains('\x02'), 71 | "origin output name must not contain the byte value 0x02 aynwhere" 72 | ); 73 | write!( 74 | writer, 75 | "{}\x02{}{}", 76 | self.attr, 77 | self.output, 78 | if self.toplevel { "" } else { "\x02" } 79 | )?; 80 | Ok(()) 81 | } 82 | 83 | /// Decodes a path that was encoded by `encode` function of this trait. 84 | /// 85 | /// Returns the decoded path origin, or `None` if `buf` could not be decoded as path origin. 86 | pub fn decode(buf: &[u8]) -> Option { 87 | let mut iter = buf.splitn(2, |c| *c == b'\x02'); 88 | iter.next() 89 | .and_then(|v| String::from_utf8(v.to_vec()).ok()) 90 | .and_then(|attr| { 91 | iter.next() 92 | .and_then(|v| String::from_utf8(v.to_vec()).ok()) 93 | .and_then(|mut output| { 94 | let mut toplevel = true; 95 | if let Some(l) = output.pop() { 96 | if l == '\x02' { 97 | toplevel = false 98 | } else { 99 | output.push(l) 100 | } 101 | } 102 | Some(PathOrigin { 103 | attr: attr, 104 | output: output, 105 | toplevel: toplevel, 106 | system: None, 107 | }) 108 | }) 109 | }) 110 | } 111 | } 112 | 113 | /// Represents a store path which is something that is produced by `nix-build`. 114 | /// 115 | /// A store path represents an output in the nix store, matching the pattern 116 | /// `store_dir/hash-name` (most often, `store_dir` will be `/nix/store`). 117 | /// 118 | /// Using nix, a store path can be produced by calling `nix-build`. 119 | /// 120 | /// Note that even if a store path is a directory, the files inside that directory 121 | /// themselves are *not* store paths. For example, while the following is a store path: 122 | /// 123 | /// ```text 124 | /// /nix/store/010yd8jls8w4vcnql4zhjbnyp2yay5pl-bash-4.4-p5 125 | /// ```` 126 | /// 127 | /// while this is not: 128 | /// 129 | /// ```text 130 | /// /nix/store/010yd8jls8w4vcnql4zhjbnyp2yay5pl-bash-4.4-p5/bin/ 131 | /// ``` 132 | /// 133 | /// To avoid any confusion with file paths, we sometimes also refer to a store path as a *package*. 134 | #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] 135 | pub struct StorePath { 136 | store_dir: String, 137 | hash: String, 138 | name: String, 139 | origin: PathOrigin, 140 | } 141 | 142 | impl StorePath { 143 | /// Parse a store path from an absolute file path. 144 | /// 145 | /// Since this function does not know where that path comes from, it takes 146 | /// `origin` as an argument. 147 | /// 148 | /// This function returns `None` if the path could not be parsed as a 149 | /// store path. You should not rely on that to check whether a path is a store 150 | /// path though, since it only does minimal validation (for one example, it does 151 | /// not check the length of the hash). 152 | pub fn parse(origin: PathOrigin, path: &str) -> Option { 153 | let mut parts = path.splitn(2, '-'); 154 | parts.next().and_then(|prefix| { 155 | parts.next().and_then(|name| { 156 | let mut iter = prefix.rsplitn(2, '/'); 157 | iter.next().map(|hash| { 158 | let store_dir = iter.next().unwrap_or(""); 159 | StorePath { 160 | store_dir: store_dir.to_string(), 161 | hash: hash.to_string(), 162 | name: name.to_string(), 163 | origin: origin, 164 | } 165 | }) 166 | }) 167 | }) 168 | } 169 | 170 | /// Encodes a store path as a sequence of bytes, so that it can be decoded with `decode`. 171 | /// 172 | /// The encoding does not use the bytes `0x00` nor `0x01`, as long as none of the fields of 173 | /// this path contain those bytes (this includes `store_dir`, `hash`, `name` and `origin`). 174 | /// This is important since it allows the result to be encoded with [frcode](mod.frcode.html). 175 | /// 176 | /// # Panics 177 | /// 178 | /// The `attr` and `output` of the path origin must not contain the byte value `0x02`, otherwise 179 | /// this function panics. 180 | pub fn encode(&self) -> io::Result> { 181 | let mut result = Vec::with_capacity(self.as_str().len()); 182 | result.extend(self.as_str().bytes()); 183 | result.push(b'\n'); 184 | self.origin().encode(&mut result)?; 185 | Ok(result) 186 | } 187 | 188 | pub fn decode(buf: &[u8]) -> Option { 189 | let mut parts = buf.splitn(2, |c| *c == b'\n'); 190 | parts 191 | .next() 192 | .and_then(|v| str::from_utf8(v).ok()) 193 | .and_then(|path| { 194 | parts 195 | .next() 196 | .and_then(PathOrigin::decode) 197 | .and_then(|origin| StorePath::parse(origin, path)) 198 | }) 199 | } 200 | 201 | pub fn join_entry(&self, entry: FileTreeEntry) -> Cow { 202 | self.join(Cow::Borrowed( 203 | String::from_utf8_lossy(&entry.path) 204 | .into_owned() 205 | .strip_prefix("/") 206 | .unwrap(), 207 | )) 208 | } 209 | 210 | pub fn join(&self, entry: Cow) -> Cow { 211 | Cow::Owned(format!("{}/{}", self.as_str(), entry)) 212 | } 213 | 214 | /// Returns the name of the store path, which is the part of the file name that 215 | /// is not the hash. In the above example, it would be `bash-4.4-p5`. 216 | /// 217 | /// # Example 218 | /// 219 | /// ``` 220 | /// use nix_index::package::{PathOrigin, StorePath}; 221 | /// 222 | /// let origin = PathOrigin { attr: "dummy".to_string(), output: "out".to_string(), toplevel: true, system: None }; 223 | /// let store_path = StorePath::parse(origin, "/nix/store/010yd8jls8w4vcnql4zhjbnyp2yay5pl-bash-4.4-p5").unwrap(); 224 | /// assert_eq!(&store_path.name(), "bash-4.4-p5"); 225 | /// ``` 226 | pub fn name(&self) -> Cow { 227 | Cow::Borrowed(&self.name) 228 | } 229 | 230 | /// The hash of the store path. This is the part just before the name of 231 | /// the path. 232 | /// 233 | /// # Example 234 | /// 235 | /// ``` 236 | /// use nix_index::package::{PathOrigin, StorePath}; 237 | /// 238 | /// let origin = PathOrigin { attr: "dummy".to_string(), output: "out".to_string(), toplevel: true, system: None }; 239 | /// let store_path = StorePath::parse(origin, "/nix/store/010yd8jls8w4vcnql4zhjbnyp2yay5pl-bash-4.4-p5").unwrap(); 240 | /// assert_eq!(&store_path.name(), "bash-4.4-p5"); 241 | /// ``` 242 | pub fn hash(&self) -> Cow { 243 | Cow::Borrowed(&self.hash) 244 | } 245 | 246 | /// The store dir for which this store path was built. 247 | /// 248 | /// Currently, this will be `/nix/store` in almost all cases, but 249 | /// we include it here anyway for completeness. 250 | /// 251 | /// # Example 252 | /// 253 | /// ``` 254 | /// use nix_index::package::{PathOrigin, StorePath}; 255 | /// 256 | /// let origin = PathOrigin { attr: "dummy".to_string(), output: "out".to_string(), toplevel: true, system: None }; 257 | /// let store_path = StorePath::parse(origin, "/nix/store/010yd8jls8w4vcnql4zhjbnyp2yay5pl-bash-4.4-p5").unwrap(); 258 | /// assert_eq!(&store_path.store_dir(), "/nix/store"); 259 | /// ``` 260 | pub fn store_dir(&self) -> Cow { 261 | Cow::Borrowed(&self.store_dir) 262 | } 263 | 264 | /// Converts the store path back into an absolute path. 265 | /// 266 | /// # Example 267 | /// 268 | /// ``` 269 | /// use nix_index::package::{PathOrigin, StorePath}; 270 | /// 271 | /// let origin = PathOrigin { attr: "dummy".to_string(), output: "out".to_string(), toplevel: true, system: None }; 272 | /// let store_path = StorePath::parse(origin, "/nix/store/010yd8jls8w4vcnql4zhjbnyp2yay5pl-bash-4.4-p5").unwrap(); 273 | /// assert_eq!(&store_path.as_str(), "/nix/store/010yd8jls8w4vcnql4zhjbnyp2yay5pl-bash-4.4-p5"); 274 | /// ``` 275 | pub fn as_str(&self) -> Cow { 276 | Cow::Owned(format!("{}/{}-{}", self.store_dir, self.hash, self.name)) 277 | } 278 | 279 | /// Returns the origin that describes how we discovered this store path. 280 | /// 281 | /// See the documentation of `PathOrigin` for more information about this field. 282 | /// 283 | /// # Example 284 | /// 285 | /// ``` 286 | /// use nix_index::package::{PathOrigin, StorePath}; 287 | /// 288 | /// let origin = PathOrigin { attr: "dummy".to_string(), output: "out".to_string(), toplevel: true, system: None }; 289 | /// let store_path = StorePath::parse(origin.clone(), "/nix/store/010yd8jls8w4vcnql4zhjbnyp2yay5pl-bash-4.4-p5").unwrap(); 290 | /// assert_eq!(store_path.origin().as_ref(), &origin); 291 | /// ``` 292 | pub fn origin(&self) -> Cow { 293 | Cow::Borrowed(&self.origin) 294 | } 295 | } 296 | -------------------------------------------------------------------------------- /src/design.md: -------------------------------------------------------------------------------- 1 | # Theoretical design 2 | 3 | # Implementation design 4 | 5 | BuildXYZ runs a program with a tightly controlled `envp`, namely: 6 | 7 | TODO: insert a table of all env vars resetted and their actual paths. 8 | 9 | In priority, it will try to read the build environment in tmpfs^[TODO: this is bad for large build environment such as chromium.] and fallback to FUSE filesystem. 10 | 11 | ## FUSE filesystem 12 | 13 | ### Basic operations 14 | 15 | When the fs receives a lookup request, it is of the form `(parent inode, name)`. 16 | 17 | We can resolve the parent inode to some "non prefixed" path, e.g. `parent inode -> bin/` or `parent inode -> include/boost`. 18 | 19 | Then, we build the final path `{resolved parent inode path}/{name}`, this is the path we look for in our database of Nix paths using [nix-index](https://github.com/bennofs/nix-index)'s structures. 20 | 21 | We may have multiple candidates but we want to assert that all candidates are of the same "kind", e.g. either all symlinks or file *XOR* all symlinks or directories. 22 | 23 | Mixing directories and regular files is dangerous because answering to the lookup request with the wrong kind can make legitimate system calls fail. 24 | 25 | Now, on to the candidate selection problem. 26 | 27 | ### Candidate selection 28 | 29 | During the lookup call, it is not a big deal to return any candidate (?) as the only call that count is the follow-up readlink that actually ask for the data. 30 | 31 | Considering a request for a Boost library, there is a version problem, see: `nix-locate -r include/boost$` as an example. 32 | 33 | Also, this example illustrates two more things: 34 | 35 | - Development packages containing other development outputs: `bulletml` for example 36 | - Multi-language development package containing native dependency output: `python$VERPackages.boost$VER.dev`. 37 | 38 | Also, consider `pip`, the Python package manager, it can be found in many closures of Python environments, some of them can be very big (3GiB), it is desirable to select the candidate that minimize realizations and moving bytes around. Unfortunately, the smallest closure containing a `bin/pip` is not `python3Packages.pip` but `cope`. Therefore, relying only on minimizing the closure size is not a good idea, otherwise false positives will probably be important. 39 | For this, a popularity database can be built out of the nix-index machinery to index also a popularity score based on (reverse-)dependency relation. 40 | 41 | For BuildXYZ to operate properly, it should be possible to detect the desired version or to operate in some sort of "fuzzing" mechanism where it will record the candidate it has chose for some branching situation and it will try to find out what are the range of possibilities regarding a certain packaging situation. 42 | -------------------------------------------------------------------------------- /src/fs.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{HashMap, HashSet}; 2 | use std::io; 3 | use std::path::{Path, PathBuf}; 4 | use std::time::{Duration, Instant, SystemTime}; 5 | 6 | use std::sync::mpsc::{channel, Receiver, Sender}; 7 | 8 | // TODO: is it Linux-specific? 9 | use std::cell::RefCell; 10 | use std::ffi::{OsStr, OsString}; 11 | 12 | use std::os::unix::ffi::OsStringExt; 13 | 14 | use fuser::{FileAttr, FileType, Filesystem}; 15 | 16 | use log::{debug, info, trace, warn}; 17 | 18 | use regex::bytes::Regex; 19 | use walkdir::WalkDir; 20 | 21 | use crate::cache::database::Reader; 22 | use crate::cache::{FileNode, FileTreeEntry, StorePath}; 23 | use crate::interactive::UserRequest; 24 | use crate::nix::realize_path; 25 | use crate::popcount::Popcount; 26 | 27 | use crate::read_raw_buffer; 28 | use crate::resolution::{db_to_human_toml, Decision, ProvideData, Resolution, ResolutionDB}; 29 | 30 | const UNIX_EPOCH: SystemTime = SystemTime::UNIX_EPOCH; 31 | 32 | pub enum FsEventMessage { 33 | /// Flush all current pending filesystem access to ENOENT 34 | IgnorePendingRequests, 35 | /// A package suggestion as a reply to a user interactive search 36 | PackageSuggestion((StorePath, FileTreeEntry)), 37 | } 38 | 39 | pub struct BuildXYZ { 40 | pub index_buffer: Vec, 41 | pub popcount_buffer: Popcount, 42 | /// resolution information for this instance 43 | pub resolution_db: ResolutionDB, 44 | /// where to write this instance resolutions 45 | pub resolution_record_filepath: Option, 46 | /// recorded ENOENTs 47 | pub recorded_enoent: HashSet<(u64, String)>, 48 | pub global_dirs: HashMap, 49 | /// "global path" -> inode 50 | pub parent_prefixes: HashMap, 51 | /// inode -> "virtual paths" 52 | pub nix_paths: HashMap>, 53 | /// inode -> "virtual foreign paths" (on another filesystem) 54 | pub redirections: HashMap>, 55 | /// fast working tree for subgraph extraction 56 | pub fast_working_tree: PathBuf, 57 | /// inode -> nix store paths 58 | pub last_inode: RefCell, 59 | /// Receiver channel for commands 60 | pub recv_fs_event: Receiver, 61 | /// Sender channel for UI requests 62 | pub send_ui_event: Sender, 63 | } 64 | 65 | impl Default for BuildXYZ { 66 | fn default() -> Self { 67 | // Those are useless channels. 68 | let (_send, recv) = channel(); 69 | let (send, _recv) = channel(); 70 | 71 | BuildXYZ { 72 | popcount_buffer: serde_json::from_slice(include_bytes!("../popcount-graph.json")) 73 | .expect("Failed to deserialize the popcount graph"), 74 | index_buffer: read_raw_buffer(std::io::Cursor::new(include_bytes!( 75 | "../nix-index-files" 76 | ))) 77 | .expect("Failed to deserialize the index buffer"), 78 | resolution_db: Default::default(), 79 | resolution_record_filepath: Default::default(), 80 | recorded_enoent: HashSet::new(), 81 | global_dirs: HashMap::new(), 82 | parent_prefixes: HashMap::new(), 83 | fast_working_tree: String::new().into(), 84 | nix_paths: HashMap::new(), 85 | redirections: HashMap::new(), 86 | last_inode: 2.into(), 87 | recv_fs_event: recv, 88 | send_ui_event: send, 89 | } 90 | } 91 | } 92 | 93 | fn prompt_user(prompt: String) -> bool { 94 | loop { 95 | let mut answer = String::new(); 96 | println!("{}", prompt); 97 | io::stdin() 98 | .read_line(&mut answer) 99 | .ok() 100 | .expect("Failed to read line"); 101 | 102 | print!("{}", answer.as_str()); 103 | 104 | match answer.as_str() { 105 | "y" => return true, 106 | "n" => return false, 107 | _ => {} 108 | } 109 | } 110 | } 111 | 112 | #[inline] 113 | fn build_fake_fattr(ino: u64, kind: FileType) -> FileAttr { 114 | fuser::FileAttr { 115 | kind, 116 | ino, 117 | size: 1, 118 | blocks: 1, 119 | blksize: 1, 120 | atime: UNIX_EPOCH, 121 | mtime: UNIX_EPOCH, 122 | crtime: UNIX_EPOCH, 123 | ctime: UNIX_EPOCH, 124 | flags: 0, 125 | uid: 0, 126 | gid: 0, 127 | nlink: 1, 128 | rdev: 0, 129 | perm: 777, 130 | } 131 | } 132 | 133 | fn is_file_or_symlink(n: &FileNode) -> bool { 134 | match n { 135 | FileNode::Regular { .. } => true, 136 | FileNode::Symlink { .. } => true, 137 | FileNode::Directory { .. } => false, 138 | } 139 | } 140 | 141 | fn is_dir(n: &FileNode) -> bool { 142 | // FIXME 143 | // investigate interaction with symlinkJoin 144 | match n { 145 | FileNode::Regular { .. } => false, 146 | FileNode::Symlink { .. } => true, // /nix/store/b6ks67mjvh2hzy3k1rnvmlri6p63b4vj-python3-3.9.12-env/include/boost is a symlink for example. 147 | FileNode::Directory { .. } => true, 148 | } 149 | } 150 | 151 | // TODO: two policies — return fake information or lstat the file and return true information 152 | impl Into for FileNode { 153 | fn into(self) -> fuser::FileAttr { 154 | let kind = match self { 155 | Self::Regular { .. } => fuser::FileType::Symlink, // No matter what, we want readlink, 156 | // not read. 157 | Self::Symlink { .. } => fuser::FileType::Symlink, 158 | Self::Directory { .. } => fuser::FileType::Directory, 159 | }; 160 | 161 | build_fake_fattr(1, kind) 162 | } 163 | } 164 | 165 | /// This will go through all candidates 166 | /// according to the sort function order 167 | /// and return the best 168 | /// It will perform some debug asserts on the list. 169 | fn extract_optimal_path( 170 | candidates: &mut Vec<(StorePath, FileTreeEntry)>, 171 | sort_key_function: F, 172 | ) -> (&StorePath, &FileTreeEntry) 173 | where 174 | F: FnMut(&(StorePath, FileTreeEntry)) -> i32, 175 | { 176 | // 1. There cannot be a folder and a file at the same time in `candidates` 177 | debug_assert!( 178 | candidates 179 | .into_iter() 180 | .all(|(_, c)| is_file_or_symlink(&c.node)) 181 | || candidates.into_iter().all(|(_, c)| is_dir(&c.node)), 182 | "either candidates are all directories, either all files, not in-between." 183 | ); 184 | 185 | // FIXME: is it enough for the ranking algorithm? 186 | candidates.sort_by_cached_key(sort_key_function); 187 | 188 | let (store_path, ft_entry) = candidates.first().unwrap(); 189 | 190 | (store_path, ft_entry) 191 | /*let mut fattr: fuser::FileAttr = ft_entry.node.clone().into(); 192 | fattr.ino = offered_inode; 193 | 194 | // This dance is necessary because ft_entry.path starts with a / 195 | // and join will keep only the second arg for an absolute path. 196 | let nix_store_dir = store_path.as_str().into_owned(); 197 | let nix_store_dir_path = Path::new(&nix_store_dir); 198 | let nix_path = nix_store_dir_path.join( 199 | String::from_utf8_lossy(&ft_entry.path) 200 | .into_owned() 201 | .strip_prefix("/") 202 | .unwrap(), 203 | ); 204 | (store_path, fattr, nix_path.as_os_str().as_bytes().to_vec())*/ 205 | } 206 | 207 | /// This will create all the directories and symlink only the leaves. 208 | /// It will fail in case of incompatibility. 209 | fn shadow_symlink_leaves(src_dir: &Path, target_dir: &Path, excluded_dirs: &Vec<&str>, already_seen: &mut HashSet) -> std::io::Result<()> { 210 | // Do not follow symlinks 211 | // Otherwise, you will get an entry.path() which does not share a base prefix with src_dir 212 | // Therefore, you don't know where to send it. 213 | // Symlink compression should be done only at the end as an optimization if needed. 214 | already_seen.insert(src_dir.canonicalize().expect("Failed to canonicalize the source path for cycle detection").into()); 215 | trace!("shadow symlinking {} -> {}...", src_dir.display(), target_dir.display()); 216 | for entry in WalkDir::new(src_dir).follow_links(false).into_iter().filter_map(|e| e.ok()) { 217 | // ensure target_dir.join(entry modulo src_dir) is a directory 218 | // or a symlink. 219 | let ft = entry.file_type(); 220 | let suffix_path = entry.path().strip_prefix(src_dir).unwrap(); 221 | let target_path = target_dir.join(suffix_path); 222 | 223 | trace!("examining entry: {} -> {}", entry.path().display(), target_path.display()); 224 | 225 | // If the target path already exist, ignore this. 226 | if target_path.exists() { 227 | trace!("{} already exist, skipping...", target_path.display()); 228 | continue; 229 | } 230 | 231 | // Skip stuff like nix-support/* 232 | if excluded_dirs.iter().any(|forbidden_dir| suffix_path.starts_with(forbidden_dir)) { 233 | trace!("skipped {}", suffix_path.display()); 234 | continue; 235 | } 236 | 237 | if ft.is_dir() { 238 | trace!("mkdir -p {} based on {}", target_path.display(), entry.path().display()); 239 | std::fs::create_dir_all(target_path)?; 240 | } else if ft.is_file() { 241 | trace!("symlink {} -> {}", entry.path().display(), target_path.display()); 242 | std::os::unix::fs::symlink(entry.path(), target_path)?; 243 | } else if ft.is_symlink() { 244 | // Two things has to be done 245 | // 1. Resolve completely the entry into resolved_target 246 | // 2. Recurse on resolved_target -> target_path 247 | // 2. Symlink target_path -> resolved_target 248 | let mut resolved_target = std::fs::read_link(entry.path())?; 249 | trace!("resolve {} -> {}", entry.path().display(), resolved_target.display()); 250 | while resolved_target.is_symlink() { 251 | resolved_target = std::fs::read_link(resolved_target.as_path())?; 252 | trace!("--> {}", resolved_target.display()); 253 | } 254 | // Now, `resolved_target` is completely resolved. 255 | // Either, it's relative, either it's absolute. 256 | // If it's relative, we correct it to an absolute link, by concatenating 257 | // $src_dir/$resolved_target. 258 | // If it's absolute, we proceed to recurse into it. 259 | if resolved_target.is_relative() { 260 | resolved_target = entry.path().parent().expect("Expected a symlink parented by at least /").join(resolved_target); 261 | } 262 | trace!("encountered an internal symlink: {} -> {}, symlinking or recursing depending on file type", entry.path().display(), resolved_target.display()); 263 | // If it's a dir, recurse the symlinkage 264 | if resolved_target.is_dir() { 265 | trace!("recursing into the symlink {} -> {} for directory symlinkage", entry.path().display(), resolved_target.display()); 266 | if already_seen.contains(&resolved_target.canonicalize().expect("Failed to canonicalize the resolved target")) { 267 | trace!("… but this source path {} was already seen, skipping.", entry.path().display()); 268 | continue; 269 | } 270 | 271 | shadow_symlink_leaves( 272 | &resolved_target, 273 | &target_path, 274 | excluded_dirs, 275 | already_seen 276 | )?; 277 | } 278 | else if resolved_target.is_file() { 279 | trace!("symlink ({} ->) {} -> {}", entry.path().display(), resolved_target.display(), target_path.display()); 280 | std::os::unix::fs::symlink(entry.path(), target_path)?; 281 | } 282 | } 283 | } 284 | 285 | Ok(()) 286 | } 287 | 288 | impl BuildXYZ { 289 | fn allocate_inode(&self) -> u64 { 290 | *self.last_inode.borrow_mut() += 1; 291 | *self.last_inode.borrow() - 1 292 | } 293 | 294 | fn build_in_construction_path(&self, parent: u64, name: &OsStr) -> PathBuf { 295 | let prefix = Path::new( 296 | self.parent_prefixes 297 | .get(&parent) 298 | .expect("Unknown parent inode!"), 299 | ); 300 | 301 | prefix.join(name) 302 | } 303 | 304 | fn record_resolution(&mut self, parent: u64, name: &OsStr, decision: Decision) { 305 | let current_path = self 306 | .build_in_construction_path(parent, name) 307 | .to_string_lossy() 308 | .to_string(); 309 | trace!("Recording {} for {:?}", current_path, decision); 310 | self.resolution_db.insert( 311 | current_path.clone(), 312 | Resolution::ConstantResolution(crate::resolution::ResolutionData { 313 | requested_path: current_path, 314 | decision, 315 | }), 316 | ); 317 | } 318 | 319 | fn get_resolution(&self, parent: u64, name: &OsStr) -> Option<&Resolution> { 320 | let current_path = self 321 | .build_in_construction_path(parent, name) 322 | .to_string_lossy() 323 | .to_string(); 324 | self.resolution_db.get(¤t_path) 325 | } 326 | 327 | fn get_decision(&self, parent: u64, name: &OsStr) -> Option<&Decision> { 328 | match self.get_resolution(parent, name) { 329 | Some(Resolution::ConstantResolution(data)) => Some(&data.decision), 330 | _ => None, 331 | } 332 | } 333 | 334 | // Shadow symlink in the fast working tree 335 | // this Nix path 336 | fn extend_fast_working_tree( 337 | &mut self, 338 | store_path: &StorePath 339 | ) { 340 | let npath: PathBuf = OsString::from_vec(store_path.as_str().as_bytes().to_vec()).into(); 341 | debug!("Shadow symlinking all the leaves {} -> {}", npath.display(), self.fast_working_tree.display()); 342 | // We do not want to symlink nix-support 343 | shadow_symlink_leaves(&npath, &self.fast_working_tree, &vec![ 344 | "nix-support" 345 | ], &mut HashSet::new()) 346 | .expect("Failed to shadow symlink the Nix path inside the fast working tree, potential incompatibility"); 347 | } 348 | 349 | /// Serve the path as an answer to the filesystem 350 | /// It realizes the Nix path if it's not already. 351 | fn serve_path( 352 | &mut self, 353 | nix_path: Vec, 354 | requested_path: PathBuf, 355 | attribute: fuser::FileAttr, 356 | reply: fuser::ReplyEntry, 357 | ) { 358 | let nix_path_as_str = String::from_utf8_lossy(&nix_path); 359 | trace!("{}: {:?}", nix_path_as_str, attribute); 360 | self.parent_prefixes 361 | .insert(attribute.ino, requested_path.to_string_lossy().to_string()); 362 | 363 | realize_path(nix_path_as_str.into()) 364 | .expect("Nix path should be realized, database seems incoherent with Nix store."); 365 | 366 | self.nix_paths.insert(attribute.ino, nix_path); 367 | 368 | reply.entry(&Duration::from_secs(60 * 20), &attribute, attribute.ino); 369 | } 370 | 371 | /// Redirect to a filesystem file 372 | /// via symlink 373 | fn redirect_to_fs( 374 | &mut self, 375 | reply: fuser::ReplyEntry, 376 | onfs_path: PathBuf 377 | ) { 378 | trace!("redirecting to {} on another filesystem", onfs_path.display()); 379 | 380 | let ft_attribute = build_fake_fattr(self.allocate_inode(), 381 | fuser::FileType::Symlink); 382 | self.redirections.insert(ft_attribute.ino, onfs_path.to_string_lossy().as_bytes().to_vec()); 383 | reply.entry(&Duration::from_secs(60 * 20), &ft_attribute, ft_attribute.ino); 384 | } 385 | 386 | /// Runs a query using our index 387 | fn search_in_index(&self, requested_path: &PathBuf) -> Vec<(StorePath, FileTreeEntry)> { 388 | let escaped_path = regex::escape(&requested_path.to_string_lossy()); 389 | debug!( 390 | "looking for: `{}$` in Nix database", 391 | requested_path.to_string_lossy(), 392 | ); 393 | let now = Instant::now(); 394 | // TODO: put me behind Arc 395 | let db = Reader::from_buffer(self.index_buffer.clone()).expect("Failed to open database"); 396 | 397 | let candidates: Vec<(StorePath, FileTreeEntry)> = db 398 | .query(&Regex::new(format!(r"^/{}$", escaped_path).as_str()).unwrap()) 399 | .run() 400 | .expect("Failed to query the database") 401 | .into_iter() 402 | .map(|result| result.expect("Failed to obtain candidate")) 403 | .filter(|(spath, _)| spath.origin().toplevel) // It must be a top-level path, otherwise 404 | // it is propagated, so not to consider. 405 | .collect(); 406 | trace!("{:?}", candidates); 407 | debug!("search took {:.2?}", now.elapsed()); 408 | 409 | candidates 410 | } 411 | 412 | /// Register known "FHS" structure 413 | /// Assume parents are already created. 414 | fn mkdir_fhs_directory(&mut self, path: &str) { 415 | let inode = self.allocate_inode(); 416 | self.parent_prefixes.insert(inode, path.to_string()); 417 | self.global_dirs.insert(path.to_string(), inode); 418 | } 419 | } 420 | 421 | // Allow parallel calls to lookup() as it should be fine. 422 | const FUSE_CAP_PARALLEL_DIROPS: u32 = 1 << 18; 423 | // Cache the symlinks we provide in the page cache. 424 | const FUSE_CAP_CACHE_SYMLINKS: u32 = 1 << 23; 425 | 426 | impl Filesystem for BuildXYZ { 427 | fn init( 428 | &mut self, 429 | _req: &fuser::Request<'_>, 430 | config: &mut fuser::KernelConfig, 431 | ) -> Result<(), i32> { 432 | // https://www.kernel.org/doc/html/latest/filesystems/fuse.html 433 | // https://libfuse.github.io/doxygen/fuse__common_8h.html 434 | config 435 | .add_capabilities(FUSE_CAP_PARALLEL_DIROPS) 436 | .map_err(|err| -(err as i32))?; 437 | self.parent_prefixes.insert(1, "".to_string()); 438 | // Create bin, lib, include, pkg-config inodes 439 | // TODO: Keep this list synchronized with created search paths in runner.rs? 440 | [ 441 | "bin", 442 | "include", 443 | "perl", 444 | "aclocal", 445 | "cmake", 446 | "lib", 447 | "lib/pkgconfig", 448 | ] 449 | .into_iter() 450 | .for_each(|c| self.mkdir_fhs_directory(c)); 451 | 452 | info!( 453 | "Loaded {} resolutions from the database.", 454 | self.resolution_db.len() 455 | ); 456 | 457 | let store_paths = self.resolution_db 458 | .values() 459 | .filter_map(|resolution| { 460 | debug!("store path: {:?}", resolution); 461 | match resolution { 462 | Resolution::ConstantResolution(data) => { 463 | if let Decision::Provide(provide_data) = &data.decision { 464 | return Some(provide_data.store_path.clone()); 465 | } 466 | } 467 | } 468 | 469 | None 470 | }) 471 | .collect::>(); 472 | 473 | info!( 474 | "Will fast extend {} store paths.", 475 | store_paths.len() 476 | ); 477 | 478 | for spath in store_paths { 479 | debug!("{} being extended in the working tree", spath.as_str()); 480 | self.extend_fast_working_tree(&spath); 481 | } 482 | 483 | info!( 484 | "Fast working tree ready based on the resolutions." 485 | ); 486 | 487 | Ok(()) 488 | } 489 | 490 | fn destroy(&mut self) { 491 | if let Some(filepath) = &self.resolution_record_filepath { 492 | debug!( 493 | "Writing {} resolutions on disk...", 494 | self.resolution_db.len() 495 | ); 496 | // Write this resolution on disk. 497 | std::fs::write( 498 | filepath, 499 | toml::to_string_pretty(&db_to_human_toml(&self.resolution_db)) 500 | .expect("Failed to serialize in a human-way the resolution database"), 501 | ) 502 | .expect("Failed to write resolution data"); 503 | } 504 | } 505 | 506 | fn lookup( 507 | &mut self, 508 | _req: &fuser::Request<'_>, 509 | parent: u64, 510 | name: &OsStr, 511 | reply: fuser::ReplyEntry, 512 | ) { 513 | let target_path = self.build_in_construction_path(parent, name); 514 | 515 | // global directory 516 | if let Some(inode) = self 517 | .global_dirs 518 | .get(&target_path.to_string_lossy().to_string()) 519 | { 520 | trace!( 521 | "global directory hit: {}", 522 | &target_path.to_string_lossy().to_string() 523 | ); 524 | reply.entry( 525 | &Duration::from_secs(60 * 60), 526 | &build_fake_fattr(*inode, FileType::Directory), 527 | *inode, 528 | ); 529 | return; 530 | } 531 | 532 | // No other global directories. 533 | if parent == 1 { 534 | return reply.error(nix::errno::Errno::ENOENT as i32); 535 | } 536 | 537 | // Fast path: ignore temporarily recorded ENOENTs. 538 | if self 539 | .recorded_enoent 540 | .contains(&(parent, name.to_string_lossy().to_string())) 541 | { 542 | return reply.error(nix::errno::Errno::ENOENT as i32); 543 | } 544 | 545 | // Fast path: fast working tree 546 | // Rebase the target path based on the working tree structure 547 | if self.fast_working_tree.join(&target_path).exists() { 548 | trace!("FAST PATH — Path already exist in the fast working tree"); 549 | return self.redirect_to_fs(reply, self.fast_working_tree.join(target_path)); 550 | } 551 | 552 | // Fast path: general resolutions 553 | let path_provide_data: Option<&ProvideData> = match self.get_decision(parent, name) { 554 | Some(Decision::Provide(data)) => Some(data), 555 | Some(Decision::Ignore) => return reply.error(nix::errno::Errno::ENOENT as i32), 556 | _ => None, 557 | }; 558 | 559 | if let Some(data) = path_provide_data { 560 | trace!("FAST PATH - Decision already exist in current database"); 561 | let nix_path = data 562 | .store_path 563 | .join(data.file_entry_name.clone().into()) 564 | .into_owned() 565 | .as_str() 566 | .as_bytes() 567 | .to_vec(); 568 | let ft_attribute = build_fake_fattr(self.allocate_inode(), data.kind); 569 | return self.serve_path(nix_path, target_path, ft_attribute, reply); 570 | } 571 | 572 | 573 | let mut candidates = self.search_in_index(&target_path); 574 | 575 | if !candidates.is_empty() { 576 | let (store_path, ft_entry) = 577 | extract_optimal_path(&mut candidates, |(store_path, _)| { 578 | trace!( 579 | "extracting pop for {}: {}", 580 | store_path.as_str(), 581 | store_path.origin().attr 582 | ); 583 | // Highest popularity comes first, so inverted popularity works here. 584 | let pop = -(*self 585 | .popcount_buffer 586 | .native_build_inputs 587 | .get(&store_path.as_str().to_string()) 588 | .unwrap_or(&0) as i32); 589 | trace!("pop: {pop}"); 590 | pop 591 | }); 592 | 593 | // Ask the user if he want to provide this dependency? 594 | let mut ft_attribute: fuser::FileAttr = ft_entry.node.clone().into(); 595 | let suggestion = (store_path.clone(), ft_entry.clone()); 596 | self.send_ui_event 597 | .send(UserRequest::InteractiveSearch(candidates.clone(), suggestion)) 598 | .expect("Failed to send UI thread a message"); 599 | 600 | 601 | // FIXME: timeouts? 602 | match self.recv_fs_event.recv() { 603 | Ok(FsEventMessage::PackageSuggestion((pkg, ft_entry))) => { 604 | debug!("prompt reply: {:?}", pkg); 605 | // Allocate a file attribute for this file entry. 606 | ft_attribute.ino = self.allocate_inode(); 607 | self.record_resolution( 608 | parent, 609 | name, 610 | Decision::Provide(ProvideData { 611 | file_entry_name: String::from_utf8_lossy(&ft_entry.path).to_string(), 612 | kind: ft_attribute.kind, 613 | store_path: pkg.clone(), 614 | }), 615 | ); 616 | let nix_path = pkg.join_entry(ft_entry.clone()).into_owned().as_str().as_bytes().to_vec(); 617 | let nix_path_as_str = String::from_utf8_lossy(&nix_path); 618 | realize_path(nix_path_as_str.into()) 619 | .expect("Nix path should be realized, database seems incoherent with Nix store."); 620 | 621 | // Now, we want to extract the whole subgraph 622 | // Instead of trying to figure out that subgraph 623 | // We can grab the Nix path and extend the fast working tree with it 624 | // à la lndir. 625 | self.extend_fast_working_tree(&pkg); 626 | return self.serve_path(nix_path, target_path, ft_attribute, reply); 627 | } 628 | Ok(FsEventMessage::IgnorePendingRequests) | _ => { 629 | debug!("ENOENT received from user"); 630 | self.record_resolution(parent, name, Decision::Ignore); 631 | self.recorded_enoent 632 | .insert((parent, name.to_string_lossy().to_string())); 633 | return reply.error(nix::errno::Errno::ENOENT as i32); 634 | } 635 | }; 636 | } else { 637 | // This file potentially don't exist at all 638 | // But it is also possible we just do not have the package for it yet. 639 | // FIXME: provide proper heuristics for this. 640 | debug!("not found in database, recording this ENOENT."); 641 | self.recorded_enoent 642 | .insert((parent, name.to_string_lossy().to_string())); 643 | return reply.error(nix::errno::Errno::ENOENT as i32); 644 | } 645 | } 646 | 647 | fn readlink(&mut self, _req: &fuser::Request<'_>, ino: u64, reply: fuser::ReplyData) { 648 | if let Some(nix_path) = self.nix_paths.get(&ino) { 649 | // Ensure the path is realized, it could have been gc'd between the lookup and the 650 | // readlink. 651 | if realize_path(String::from_utf8_lossy(&nix_path).into()).is_err() { 652 | warn!( 653 | "Failed to realize {} during readlink, it was supposed to be realizable!", 654 | String::from_utf8_lossy(&nix_path) 655 | ); 656 | reply.error(nix::errno::Errno::ENOENT as i32); 657 | } else { 658 | reply.data(nix_path); 659 | } 660 | } 661 | else if let Some(redirection_path) = self.redirections.get(&ino) { 662 | reply.data(redirection_path); 663 | } else { 664 | warn!("Attempt to read a non-existent Nix path, ino={}", ino); 665 | reply.error(nix::errno::Errno::ENOENT as i32); 666 | } 667 | } 668 | } 669 | -------------------------------------------------------------------------------- /src/interactive.rs: -------------------------------------------------------------------------------- 1 | use std::thread; 2 | use std::{ 3 | sync::mpsc::{channel, Sender}, 4 | thread::JoinHandle, 5 | }; 6 | 7 | use log::{debug, info, warn}; 8 | 9 | use crate::cache::{FileTreeEntry, StorePath}; 10 | use crate::fs::FsEventMessage; 11 | 12 | /// Request types between FUSE thread and UI thread 13 | pub enum UserRequest { 14 | /// Order the thread to stop listen for events 15 | Quit, 16 | /// An interactive search request for the given path to the UI thread 17 | /// with a preferred candidate. 18 | InteractiveSearch(Vec<(StorePath, FileTreeEntry)>, (StorePath, FileTreeEntry)), 19 | } 20 | 21 | pub fn prompt_among_choices( 22 | prompt: &str, 23 | choices: Vec 24 | ) -> Option { 25 | loop { 26 | let mut answer = String::new(); 27 | info!("{}", prompt); 28 | for (index, choice) in choices.iter().enumerate() { 29 | info!("{}. {}", index + 1, choice); 30 | } 31 | // TODO: make this non-blocking and interruptible 32 | std::io::stdin() 33 | .read_line(&mut answer) 34 | .ok() 35 | .expect("Failed to read line"); 36 | 37 | if answer.trim().to_lowercase() == "n" || answer.trim().to_lowercase() == "no" || answer.trim() == "" { 38 | return None; 39 | } 40 | 41 | match answer.trim().parse::() { 42 | Ok(k) if k >= 1 && k <= choices.len() => { 43 | return Some(k - 1); 44 | } 45 | _ => { 46 | warn!("Enter a valid choice between 1 and {} or `no`/`n`/press enter for skipping this choice", choices.len()); 47 | continue; 48 | } 49 | } 50 | } 51 | } 52 | 53 | pub fn spawn_ui( 54 | reply_fs: Sender, 55 | automatic: bool, 56 | ) -> (JoinHandle<()>, Sender) { 57 | let (send, recv) = channel(); 58 | 59 | let join_handle = thread::spawn(move || { 60 | info!("UI thread spawned and listening for events"); 61 | loop { 62 | if let Ok(message) = recv.recv() { 63 | match message { 64 | UserRequest::Quit => { 65 | break; 66 | } 67 | UserRequest::InteractiveSearch(candidates, suggested) => { 68 | if automatic { 69 | reply_fs 70 | .send(FsEventMessage::PackageSuggestion(suggested)) 71 | .expect("Failed to send message to FS thread"); 72 | continue; 73 | } 74 | 75 | let choices: Vec = candidates.iter().map(|(c, _)| c.origin().as_ref().clone().attr).collect(); 76 | let potential_index = prompt_among_choices( 77 | "A dependency not found in your search paths was requested, pick a choice", 78 | choices 79 | ); 80 | 81 | match potential_index { 82 | Some(index) => reply_fs.send(FsEventMessage::PackageSuggestion(candidates[index].clone())), 83 | None => reply_fs.send(FsEventMessage::IgnorePendingRequests), 84 | } 85 | .expect("Failed to send message to FS thread"); 86 | 87 | // list all the candidates with numbers 88 | // provide ENOENT option 89 | 90 | // ENOENT 91 | } 92 | } 93 | } 94 | } 95 | }); 96 | 97 | (join_handle, send) 98 | } 99 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use ::nix::sys::signal::Signal::{SIGINT, SIGKILL, SIGTERM}; 2 | use ::nix::unistd::Pid; 3 | use cache::database::read_raw_buffer; 4 | use clap::Parser; 5 | use fuser::spawn_mount2; 6 | use lazy_static::lazy_static; 7 | use log::{debug, info, warn}; 8 | use std::io; 9 | use std::iter; 10 | use std::os::unix::ffi::OsStringExt; 11 | use std::path::PathBuf; 12 | use std::process::Command; 13 | use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; 14 | use std::sync::mpsc::channel; 15 | use std::sync::Arc; 16 | use include_dir::{include_dir, Dir}; 17 | 18 | use crate::cache::StorePath; 19 | use crate::nix::realize_path; 20 | use crate::resolution::{ 21 | load_resolution_db, merge_resolution_db, read_resolution_db, ResolutionDB, Resolution, Decision, 22 | }; 23 | 24 | // mod instrument; 25 | mod cache; 26 | mod fs; 27 | mod interactive; 28 | mod nix; 29 | mod popcount; 30 | mod resolution; 31 | mod runner; 32 | 33 | pub enum EventMessage { 34 | Stop, 35 | Done, 36 | } 37 | 38 | // 2 directories: 39 | // - FUSE filesystem for negative lookups 40 | // - normal filesystem for building the build environment (buildEnv) 41 | 42 | #[derive(Parser, Debug)] 43 | #[command(author, version, about, long_about = None)] 44 | struct Args { 45 | cmd: String, 46 | /// Say yes to everything except if it is recorded as ENOENT. 47 | #[arg(long = "automatic", default_value_t = false)] 48 | automatic: bool, 49 | /// No core resolution 50 | #[arg(long = "naked", default_value_t = false)] 51 | naked: bool, 52 | #[arg(long = "db", default_value_os = cache::cache_dir())] 53 | database: PathBuf, 54 | #[arg(long = "record-to")] 55 | resolution_record_filepath: Option, 56 | #[arg(long = "resolutions-from")] 57 | custom_resolutions_filepath: Option, 58 | /// In case of failures, retry automatically the invocation 59 | #[arg(long = "r", default_value_t = false)] 60 | retry: bool, 61 | /// Print ignored paths 62 | #[arg(long = "print-ignored-paths", default_value_t = false)] 63 | print_ignored_paths: bool 64 | } 65 | 66 | fn get_git_root() -> Option { 67 | // TODO: `git` is not necessarily in the PATH, is it? 68 | let output = Command::new("git") 69 | .args(vec!["rev-parse", "--show-toplevel"]) 70 | .output() 71 | .ok()?; 72 | 73 | if output.status.success() { 74 | Some( 75 | std::ffi::OsString::from_vec(output.stdout) 76 | .as_os_str() 77 | .into(), 78 | ) 79 | } else { 80 | None 81 | } 82 | } 83 | 84 | 85 | static CORE_RESOLUTIONS: Dir = include_dir!("$BUILDXYZ_CORE_RESOLUTIONS"); 86 | lazy_static! { 87 | /// Here are the default search paths by order: 88 | /// $XDG_DATA_DIR/buildxyz 89 | /// "Git root"/.buildxyz if it exist. 90 | /// Current working directory 91 | static ref DEFAULT_RESOLUTION_PATHS: Vec = { 92 | let mut paths = Vec::new(); 93 | let xdg_base_dir = xdg::BaseDirectories::with_prefix("buildxyz").unwrap(); 94 | paths.push( 95 | xdg_base_dir.get_data_home() 96 | ); 97 | if let Some(git_root) = get_git_root() { 98 | paths.push( 99 | git_root.join(".buildxyz") 100 | ) 101 | } 102 | paths.push( 103 | std::env::current_dir().expect("Failed to get current working directory") 104 | ); 105 | paths 106 | }; 107 | } 108 | 109 | fn main() -> Result<(), io::Error> { 110 | let args = Args::parse(); 111 | 112 | stderrlog::new() 113 | //.module(module_path!()) 114 | .verbosity(4) 115 | .init() 116 | .unwrap(); 117 | 118 | // Signal to stop the current program 119 | // If sent twice, uses SIGKILL 120 | let (send_event, recv_event) = channel::(); 121 | let (send_fs_event, recv_fs_event) = channel(); 122 | let (ui_join_handle, send_ui_event) = 123 | interactive::spawn_ui(send_fs_event.clone(), args.automatic); 124 | let mut stop_count = 0; 125 | 126 | let ctrlc_event = send_event.clone(); 127 | ctrlc::set_handler(move || { 128 | println!("stop count: {}", stop_count); 129 | info!("Ctrl-C received..."); 130 | ctrlc_event 131 | .send(EventMessage::Stop) 132 | .expect("Failed to send Ctrl-C event to the main thread"); 133 | }) 134 | .expect("Failed to set Ctrl-C handler"); 135 | // FIXME: register SIGTERM too. 136 | 137 | info!("Mounting the FUSE filesystem in the background..."); 138 | 139 | let fuse_tmpdir = tempfile::tempdir().expect("Failed to create a temporary directory for the FUSE mountpoint"); 140 | let fast_tmpdir = tempfile::tempdir().expect("Failed to create a temporary directory for the fast working tree"); 141 | 142 | // Load all resolution databases in memory. 143 | // Reduce them by merging them in the provided priority order. 144 | // Load *core* resolutions first 145 | let core_resolution_db = if !args.naked { CORE_RESOLUTIONS.find("**/*.toml").unwrap() 146 | .into_iter() 147 | .map(|entry| CORE_RESOLUTIONS.get_file(entry.path()).expect("Failed to find a core resolution file inside the binary, corrupted binary?")) 148 | .filter_map(|file| read_resolution_db(file.contents_utf8().unwrap())) 149 | .fold(ResolutionDB::new(), |left, right| merge_resolution_db(left, right)) 150 | } else { ResolutionDB::new() }; 151 | 152 | let mut resolution_db = std::env::var("BUILDXYZ_RESOLUTION_PATH") 153 | .unwrap_or(String::new()) 154 | .split(":") 155 | .into_iter() 156 | .map(PathBuf::from) 157 | // Default resolution paths are lowest priority. 158 | .chain(DEFAULT_RESOLUTION_PATHS.iter().cloned()) 159 | .map(|searchpath| load_resolution_db(searchpath)) 160 | .flatten() // Filter out all Nones. 161 | .fold(core_resolution_db, |left, right| { 162 | merge_resolution_db(left, right) 163 | }); 164 | 165 | if let Some(custom_resolutions_filepath) = args.custom_resolutions_filepath { 166 | if let Some(custom_resolutions) = read_resolution_db( 167 | &std::fs::read_to_string(custom_resolutions_filepath).expect("Failed to read from custom resolution file") 168 | ) 169 | { 170 | resolution_db = merge_resolution_db(resolution_db, custom_resolutions); 171 | } 172 | } 173 | 174 | if args.print_ignored_paths { 175 | println!("List of ignored paths:"); 176 | for resolution in resolution_db.values() { 177 | let resolution::Resolution::ConstantResolution(data) = resolution; 178 | match data.decision { 179 | resolution::Decision::Ignore => { 180 | println!("\t{}", data.requested_path); 181 | }, 182 | _ => {} 183 | } 184 | } 185 | 186 | return Ok(()); 187 | } 188 | 189 | 190 | let store_paths = resolution_db 191 | .values() 192 | .filter_map(|resolution| { 193 | debug!("store path: {:?}", resolution); 194 | match resolution { 195 | Resolution::ConstantResolution(data) => { 196 | if let Decision::Provide(provide_data) = &data.decision { 197 | return Some(provide_data.store_path.clone()); 198 | } 199 | } 200 | } 201 | 202 | None 203 | }) 204 | .collect::>(); 205 | 206 | for spath in store_paths { 207 | debug!("Ensuring that resolution {} is available in the Nix store", spath.as_str()); 208 | if realize_path(spath.as_str().to_string()).is_err() { 209 | warn!("Failed to realize it, BuildXYZ may fail"); 210 | } 211 | } 212 | 213 | let session = spawn_mount2( 214 | fs::BuildXYZ { 215 | recv_fs_event, 216 | send_ui_event: send_ui_event.clone(), 217 | resolution_record_filepath: args.resolution_record_filepath, 218 | resolution_db, 219 | fast_working_tree: fast_tmpdir.path().to_owned(), 220 | ..Default::default() 221 | }, 222 | fuse_tmpdir 223 | .path() 224 | .to_str() 225 | .expect("Failed to convert the path to a string"), 226 | &[] 227 | 228 | ) 229 | .expect("Error spawning the FUSE filesystem in the background"); 230 | 231 | info!("Running `{}`", args.cmd); 232 | 233 | let retry = Arc::new(AtomicBool::new(args.retry)); 234 | // FIXME uninitialized values are bad. 235 | let current_child_pid = Arc::new(AtomicU32::new(0)); 236 | if let [cmd, cmd_args @ ..] = &args.cmd.split_ascii_whitespace().collect::>()[..] { 237 | let run_join_handle = runner::spawn_instrumented_program( 238 | cmd.to_string(), 239 | // FIXME: ugh ugly 240 | cmd_args 241 | .to_vec() 242 | .into_iter() 243 | .map(|s| s.to_string()) 244 | .collect(), 245 | std::env::vars().collect(), 246 | current_child_pid.clone(), 247 | retry.clone(), 248 | send_event.clone(), 249 | fuse_tmpdir.path(), 250 | fast_tmpdir.path() 251 | ); 252 | 253 | // Main event loop 254 | // We wait for either stop signal or done signal 255 | loop { 256 | match recv_event.recv().expect("Failed to receive message") { 257 | EventMessage::Stop => { 258 | stop_count += 1; 259 | retry.store(false, Ordering::SeqCst); 260 | send_ui_event 261 | .send(interactive::UserRequest::Quit) 262 | .expect("Failed to send message to UI thread"); 263 | let raw_pid = current_child_pid.load(Ordering::SeqCst) as i32; 264 | let pid = Pid::from_raw(raw_pid); 265 | if raw_pid != 0 { 266 | debug!("ENOENT all pending fs requests..."); 267 | send_fs_event 268 | .send(fs::FsEventMessage::IgnorePendingRequests) 269 | .expect("Failed to send message to filesystem threads"); 270 | debug!("Will kill {:?}", pid); 271 | ::nix::sys::signal::kill( 272 | pid, 273 | match stop_count { 274 | 2 => SIGTERM, 275 | k if k >= 3 => SIGKILL, 276 | _ => SIGINT, 277 | }, 278 | ) 279 | .expect("Failed to interrupt the current underlying process"); 280 | } else { 281 | send_event 282 | .send(EventMessage::Done) 283 | .expect("Failed to send event"); 284 | } 285 | } 286 | EventMessage::Done => { 287 | // Ensure we quit the UI thread. 288 | let _ = send_ui_event.send(interactive::UserRequest::Quit); 289 | info!("Waiting for the runner & UI threads to exit..."); 290 | let status_code = run_join_handle 291 | .join() 292 | .expect("Failed to wait for the runner thread"); 293 | ui_join_handle 294 | .join() 295 | .expect("Failed to wait for the UI thread"); 296 | info!("Unmounting the filesystem..."); 297 | session.join(); 298 | 299 | if let Some(code) = status_code { 300 | if code != 0 && args.automatic { 301 | // Exit with the inner process status code 302 | // for proper bookkeeping of errors. 303 | std::process::exit(code); 304 | } 305 | } 306 | 307 | break; 308 | } 309 | } 310 | } 311 | } else { 312 | todo!("Dependent type theory in Rust"); 313 | } 314 | 315 | Ok(()) 316 | } 317 | -------------------------------------------------------------------------------- /src/nix.rs: -------------------------------------------------------------------------------- 1 | use log::trace; 2 | use serde::Deserialize; 3 | use std::process::{Command, Stdio}; 4 | 5 | use error_chain::{bail, error_chain}; 6 | 7 | pub enum StoreKind { 8 | Local, 9 | Remote(String), 10 | } 11 | 12 | error_chain! { 13 | errors { InvalidPath } 14 | } 15 | 16 | /// Ask the store to realize the provided path. 17 | pub fn realize_path(path: String) -> Result<()> { 18 | let nixpkgs_path = env!("BUILDXYZ_NIXPKGS"); 19 | // TODO: send back this information to the meta-panel of the TUI 20 | let output = Command::new("nix-store") 21 | .arg("--realize") 22 | .arg(path) 23 | .env("NIX_PATH", format!("nixpkgs={}", nixpkgs_path)) 24 | .stdin(Stdio::null()) 25 | .output() 26 | .expect("Failed to realize store based on nix-store --realize"); 27 | 28 | if output.status.success() { 29 | Ok(()) 30 | } else { 31 | // TODO: more precise errors. 32 | bail!(ErrorKind::InvalidPath) 33 | } 34 | } 35 | 36 | #[derive(Deserialize)] 37 | struct PathInfo { 38 | #[serde(rename = "closureSize")] 39 | closure_size: Option, 40 | } 41 | 42 | /// Returns `nix path-info -S --store if there's any remote store. 43 | /// If the path is invalid, None is returned. 44 | /// This returns the closure size. 45 | pub fn get_path_size(path: &str, store: StoreKind) -> Option { 46 | let mut cmd0 = Command::new("nix"); 47 | let mut cmd = cmd0.arg("path-info").arg("--json").arg("-S").arg(path); 48 | 49 | cmd = match store { 50 | StoreKind::Local => cmd, 51 | StoreKind::Remote(remote_store) => cmd.arg("--store").arg(remote_store), 52 | }; 53 | 54 | let output = cmd.output().expect("Failed to extract path information"); 55 | 56 | trace!( 57 | "nix path-info output: {}", 58 | String::from_utf8_lossy(&output.stdout) 59 | ); 60 | 61 | if output.status.success() { 62 | let pinfos: Vec = 63 | serde_json::from_slice(&output.stdout).expect("Valid JSON from nix path-info --json"); 64 | pinfos.first().expect("At least one path-info").closure_size 65 | } else { 66 | None 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/popcount.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::collections::HashMap; 3 | 4 | #[derive(Serialize, Deserialize)] 5 | #[serde(rename_all = "camelCase")] 6 | pub struct Popcount { 7 | pub build_inputs: HashMap, 8 | pub propagated_build_inputs: HashMap, 9 | pub native_build_inputs: HashMap, 10 | pub propagated_native_build_inputs: HashMap, 11 | } 12 | -------------------------------------------------------------------------------- /src/resolution.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::{collections::BTreeMap, fs, path::PathBuf}; 3 | use thiserror::Error; 4 | 5 | use crate::cache::StorePath; 6 | 7 | #[derive(Error, Debug)] 8 | pub enum ParseResolutionError { 9 | #[error("missing field `{0}`")] 10 | MissingField(String), 11 | #[error("expected type `{0}` for field `{1}`")] 12 | UnexpectedType(String, String), 13 | } 14 | 15 | type ParseResult = Result; 16 | 17 | /// Resolution is data that enable the tool to automate a situation where 18 | /// a manual decision has to be taken. 19 | 20 | #[derive(Clone, Eq, Hash, PartialEq, Serialize, Deserialize, Debug)] 21 | pub struct ProvideData { 22 | pub kind: fuser::FileType, 23 | pub file_entry_name: String, 24 | pub store_path: StorePath, 25 | } 26 | 27 | fn parse_filetype_kind(v: &str) -> ParseResult { 28 | Ok(match v { 29 | "socket" => fuser::FileType::Socket, 30 | "symlink" => fuser::FileType::Symlink, 31 | "named-pipe" => fuser::FileType::NamedPipe, 32 | "directory" => fuser::FileType::Directory, 33 | "char-device" => fuser::FileType::CharDevice, 34 | "block-device" => fuser::FileType::BlockDevice, 35 | "regular-file" => fuser::FileType::RegularFile, 36 | _ => { 37 | return Err(ParseResolutionError::UnexpectedType( 38 | "fuser::FileType".into(), 39 | "kind".into(), 40 | )) 41 | } 42 | }) 43 | } 44 | 45 | impl ProvideData { 46 | pub fn to_human_toml_table(&self) -> toml::Table { 47 | let mut table = toml::Table::new(); 48 | 49 | table.insert( 50 | "kind".into(), 51 | match self.kind { 52 | fuser::FileType::Socket => "socket", 53 | fuser::FileType::Symlink => "symlink", 54 | fuser::FileType::NamedPipe => "named-pipe", 55 | fuser::FileType::Directory => "directory", 56 | fuser::FileType::CharDevice => "char-device", 57 | fuser::FileType::BlockDevice => "block-device", 58 | fuser::FileType::RegularFile => "regular-file", 59 | } 60 | .into(), 61 | ); 62 | table.insert( 63 | "file_entry_name".into(), 64 | self.file_entry_name.clone().into(), 65 | ); 66 | table.insert( 67 | "store_path".into(), 68 | toml::Table::try_from(&self.store_path).unwrap().into(), 69 | ); 70 | 71 | table 72 | } 73 | 74 | pub fn from_toml(mut data: toml::Table) -> ParseResult { 75 | Ok(ProvideData { 76 | kind: match data.get("kind") { 77 | Some(toml::Value::String(v)) => parse_filetype_kind(v)?, 78 | None => return Err(ParseResolutionError::MissingField("kind".into())), 79 | _ => { 80 | return Err(ParseResolutionError::UnexpectedType( 81 | "string".into(), 82 | "kind".into(), 83 | )) 84 | } 85 | }, 86 | // use the deserializer here. 87 | file_entry_name: data 88 | .remove("file_entry_name") 89 | .map(|v| match v { 90 | toml::Value::String(v) => Ok(v), 91 | _ => Err(ParseResolutionError::UnexpectedType( 92 | "string".into(), 93 | "file_entry_name".into(), 94 | )), 95 | }) 96 | .ok_or_else(|| ParseResolutionError::MissingField("file_entry_name".into()))??, 97 | store_path: data.remove("store_path").expect("missing `store_path` field").try_into() 98 | .unwrap(), 99 | }) 100 | } 101 | } 102 | 103 | #[derive(Serialize, Deserialize, Eq, Hash, PartialEq, Clone, Debug)] 104 | #[serde(tag = "decision")] 105 | pub enum Decision { 106 | /// Provide this store path 107 | Provide(ProvideData), 108 | /// Returns ENOENT 109 | Ignore, 110 | } 111 | 112 | impl Decision { 113 | pub fn to_human_toml_table(&self) -> toml::Table { 114 | let mut table = toml::Table::new(); 115 | 116 | match self { 117 | Self::Provide(data) => { 118 | table.insert("decision".into(), "provide".into()); 119 | table.extend(data.to_human_toml_table()); 120 | } 121 | Self::Ignore => { 122 | table.insert("decision".into(), "ignore".into()); 123 | } 124 | } 125 | 126 | table 127 | } 128 | 129 | pub fn from_toml(decision: toml::Table) -> ParseResult { 130 | Ok(match decision.get("decision") { 131 | Some(toml::Value::String(decision_choice)) => match decision_choice.as_str() { 132 | "ignore" => Self::Ignore, 133 | "provide" => Self::Provide(ProvideData::from_toml(decision)?), 134 | _ => { 135 | return Err(ParseResolutionError::UnexpectedType( 136 | "`ignore` or `provide`".into(), 137 | "decision".into(), 138 | )) 139 | } 140 | }, 141 | None => return Err(ParseResolutionError::MissingField("decision".into())), 142 | _ => { 143 | return Err(ParseResolutionError::UnexpectedType( 144 | "`ignore` or `provide`".into(), 145 | "decision".into(), 146 | )) 147 | } 148 | }) 149 | } 150 | } 151 | 152 | #[derive(Serialize, Deserialize, Eq, Hash, PartialEq, Clone, Debug)] 153 | #[serde(tag = "resolution")] 154 | #[non_exhaustive] 155 | pub enum Resolution { 156 | /// Constant resolution is always issued no matter the context. 157 | ConstantResolution(ResolutionData), 158 | } 159 | 160 | impl Resolution { 161 | pub fn requested_path(&self) -> &String { 162 | match self { 163 | Self::ConstantResolution(res_data) => &res_data.requested_path, 164 | } 165 | } 166 | 167 | pub fn to_human_toml_table(&self) -> toml::Table { 168 | let mut gtable = toml::Table::new(); 169 | 170 | let Self::ConstantResolution(data) = self; 171 | 172 | { 173 | let mut table = toml::Table::new(); 174 | table.insert("resolution".into(), "constant".into()); 175 | table.extend(data.decision.to_human_toml_table()); 176 | gtable.insert(data.requested_path.clone(), table.into()); 177 | } 178 | 179 | gtable 180 | } 181 | 182 | pub fn from_toml_item(resolution: (String, toml::Value)) -> ParseResult<(String, Self)> { 183 | Ok(( 184 | resolution.0.clone(), 185 | Self::ConstantResolution(ResolutionData { 186 | requested_path: resolution.0.clone(), 187 | decision: Decision::from_toml(match resolution.1 { 188 | toml::Value::Table(table) => table, 189 | _ => { 190 | return Err(ParseResolutionError::UnexpectedType( 191 | "a table".into(), 192 | resolution.0, 193 | )) 194 | } 195 | })?, 196 | }), 197 | )) 198 | } 199 | 200 | pub fn from_toml(resolutions: toml::Value) -> ParseResult { 201 | match resolutions { 202 | toml::Value::Table(resolutions_map) => Ok(resolutions_map 203 | .into_iter() 204 | .map(Self::from_toml_item) 205 | .collect::>()?), 206 | _ => Err(ParseResolutionError::UnexpectedType( 207 | "an array of table".into(), 208 | "the whole document".into(), 209 | )), 210 | } 211 | } 212 | } 213 | 214 | #[derive(Serialize, Deserialize, Eq, Hash, PartialEq, Clone, Debug)] 215 | pub struct ResolutionData { 216 | pub requested_path: String, 217 | pub decision: Decision, 218 | } 219 | 220 | // TODO: BTreeMap provide O(log n) search, do we need better? 221 | pub type ResolutionDB = BTreeMap; 222 | 223 | pub fn db_to_human_toml(db: &ResolutionDB) -> toml::Table { 224 | let mut table = toml::Table::new(); 225 | 226 | for item in db.values() { 227 | table.extend(item.to_human_toml_table()); 228 | } 229 | 230 | table 231 | } 232 | 233 | fn locate_resolution_db(search_path: PathBuf) -> Option { 234 | None 235 | } 236 | 237 | pub fn read_resolution_db(data: &str) -> Option { 238 | Resolution::from_toml( 239 | toml::from_str(data) 240 | .expect("Failed to parse the TOML"), 241 | ) 242 | .ok() 243 | } 244 | 245 | /// Search in the provided path for a resolution database. 246 | pub fn load_resolution_db(search_path: PathBuf) -> Option { 247 | locate_resolution_db(search_path).and_then(|filename| read_resolution_db(&std::fs::read_to_string(filename).expect("Failed to read resolution DB from file"))) 248 | } 249 | 250 | /// Unify two set of resolutions, right taking priority over left. 251 | pub fn merge_resolution_db(left: ResolutionDB, right: ResolutionDB) -> ResolutionDB { 252 | left.into_iter().chain(right).collect() 253 | } 254 | -------------------------------------------------------------------------------- /src/runner.rs: -------------------------------------------------------------------------------- 1 | use log::{debug, error, info}; 2 | use std::path::{Path, PathBuf}; 3 | use std::process::Command; 4 | use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; 5 | use std::sync::Arc; 6 | use std::thread; 7 | use std::{collections::HashMap, sync::mpsc::Sender}; 8 | 9 | use crate::EventMessage; 10 | 11 | fn append_search_path(env: &mut HashMap, key: &str, value: PathBuf, insert: bool) { 12 | let entry = env.entry(key.to_string()).and_modify(|env_path| { 13 | debug!("old env: {}={}", key, env_path); 14 | *env_path = format!( 15 | "{env_path}:{value}", 16 | env_path = env_path, 17 | value = value.display() 18 | ); 19 | }); 20 | 21 | if insert { 22 | entry.or_insert_with(|| { 23 | debug!("`{}` was not present before, injecting", key); 24 | format!("{}", value.display()) 25 | }); 26 | } 27 | } 28 | 29 | fn append_search_paths(env: &mut HashMap, 30 | root_path: &Path) { 31 | let bin_path = root_path.join("bin"); 32 | let pkgconfig_path = root_path.join("lib").join("pkgconfig"); 33 | let library_path = root_path.join("lib"); 34 | let include_path = root_path.join("include"); 35 | let cmake_path = root_path.join("cmake"); 36 | let aclocal_path = root_path.join("aclocal"); 37 | let perl_path = root_path.join("perl"); 38 | 39 | append_search_path(env, "PATH", bin_path, true); 40 | 41 | append_search_path(env, "PERL5LIB", perl_path, false); 42 | 43 | append_search_path(env, "PKG_CONFIG_PATH", pkgconfig_path, true); 44 | append_search_path(env, "CMAKE_INCLUDE_PATH", cmake_path, true); 45 | append_search_path(env, "ACLOCAL_PATH", aclocal_path, false); 46 | 47 | // Runtime libraries: 48 | // This is not a workable approach because DT_RUNPATH is after LD_LIBRARY_PATH 49 | // in priority. Anyway, on NixOS, most binaries comes with all the proper 50 | // libraries, on other OS, you must have them in your FHS. 51 | // Therefore, all that remains is handling foreign binaries. 52 | // This is taken care by composing buildxyz with nix-ld for example. 53 | // append_search_path(env, "LD_LIBRARY_PATH", library_path.clone(), false); 54 | 55 | // Build-time libraries 56 | append_search_path(env, "LIBRARY_PATH", library_path.clone(), true); 57 | 58 | env.entry("NIX_CFLAGS_COMPILE".to_string()) 59 | .and_modify(|env_path| { 60 | debug!("old NIX_CFLAGS_COMPILE={}", env_path); 61 | *env_path = format!( 62 | "{env_path} -idirafter {include_path}", 63 | env_path = env_path, 64 | include_path = include_path.display() 65 | ); 66 | debug!("new NIX_CFLAGS_COMPILE={}", env_path); 67 | }); 68 | } 69 | 70 | pub fn spawn_instrumented_program( 71 | cmd: String, 72 | args: Vec, 73 | mut env: HashMap, 74 | current_child_pid: Arc, 75 | should_retry: Arc, 76 | send_to_main: Sender, 77 | mountpoint: &Path, 78 | fast_working_root: &Path 79 | ) -> thread::JoinHandle> { 80 | 81 | // Fast working tree 82 | append_search_paths(&mut env, fast_working_root); 83 | // FUSE 84 | append_search_paths(&mut env, mountpoint); 85 | 86 | thread::spawn(move || { 87 | loop { 88 | debug!("Spawning a child `{}`...", cmd); 89 | let mut child = Command::new(&cmd) 90 | .args(&args) 91 | .env_clear() 92 | .envs(&env) 93 | .spawn() 94 | .expect("Command failed to start"); 95 | 96 | // Send our PID so we can get killed if needed. 97 | current_child_pid.store(child.id(), Ordering::SeqCst); 98 | debug!("Child spawned with PID {}, waiting...", child.id()); 99 | let status = child.wait().expect("Failed to wait for child"); 100 | let success = status.success(); 101 | if !success && should_retry.load(Ordering::SeqCst) { 102 | info!("Command failed but it will be restarted soon."); 103 | } else if !success { 104 | error!("Command failed"); 105 | send_to_main.send(EventMessage::Done) 106 | .expect("Failed to send message to main thread"); 107 | return status.code(); 108 | } else { 109 | info!("Command ended successfully"); 110 | send_to_main 111 | .send(EventMessage::Done) 112 | .expect("Failed to send message to main thread"); 113 | return status.code(); 114 | } 115 | } 116 | }) 117 | } 118 | -------------------------------------------------------------------------------- /tests/flake-module.nix: -------------------------------------------------------------------------------- 1 | { self, ... }: { 2 | perSystem = { lib, pkgs, ... }: { 3 | checks = lib.optionalAttrs pkgs.stdenv.isLinux { 4 | nixos-test = import ./nixos-test.nix { 5 | inherit self pkgs; 6 | }; 7 | }; 8 | }; 9 | } 10 | -------------------------------------------------------------------------------- /tests/lib.nix: -------------------------------------------------------------------------------- 1 | # The first argument to this function is the test module itself Jörg Thalheim, Previous month ◉ add lightning-knd test 2 | test: 3 | # These arguments are provided by `flake.nix` on import, see checkArgs 4 | { pkgs, self }: 5 | let 6 | inherit (pkgs) lib; 7 | nixos-lib = import (pkgs.path + "/nixos/lib") { }; 8 | in 9 | (nixos-lib.runTest { 10 | hostPkgs = pkgs; 11 | # optional to speed up to evaluation by skipping evaluating documentation 12 | defaults.documentation.enable = lib.mkDefault false; 13 | # This makes `self` available in the nixos configuration of our virtual machines. 14 | # This is useful for referencing modules or packages from your own flake as well as importing 15 | # from other flakes. 16 | node.specialArgs = { inherit self; }; 17 | _module.args = { inherit self; }; 18 | imports = [ test ]; 19 | }).config.result 20 | -------------------------------------------------------------------------------- /tests/nixos-test.nix: -------------------------------------------------------------------------------- 1 | (import ./lib.nix) ({ pkgs, ... }: { 2 | name = "from-nixos"; 3 | nodes = { 4 | # self here is set by using specialArgs in `lib.nix` 5 | node1 = { self, ... }: { 6 | environment.systemPackages = [ 7 | self.packages.${pkgs.targetPlatform.system}.buildxyz 8 | ]; 9 | 10 | # Ensure hello closure is here. 11 | system.extraDependencies = [ pkgs.hello ]; 12 | }; 13 | }; 14 | 15 | # This test is still wip 16 | testScript = 17 | '' 18 | start_all() 19 | 20 | node1.succeed("mkdir -p /tmp/buildxyz") 21 | # FIXME: This will not work because we do not have any database yet. 22 | node1.execute("buildxyz hello") 23 | ''; 24 | }) 25 | -------------------------------------------------------------------------------- /treefmt/flake-module.nix: -------------------------------------------------------------------------------- 1 | { inputs, ... }: { 2 | imports = [ 3 | inputs.treefmt-nix.flakeModule 4 | ]; 5 | 6 | perSystem = 7 | { pkgs 8 | , lib 9 | , ... 10 | }: { 11 | treefmt = { 12 | # Used to find the project root 13 | projectRootFile = "flake.lock"; 14 | 15 | programs.rustfmt.enable = true; 16 | 17 | settings.formatter = { 18 | nix = { 19 | command = pkgs.runtimeShell; 20 | options = [ 21 | "-eucx" 22 | '' 23 | ${lib.getExe pkgs.deadnix} --edit "$@" 24 | ${lib.getExe pkgs.nixpkgs-fmt} "$@" 25 | '' 26 | "--" 27 | ]; 28 | includes = [ "*.nix" ]; 29 | }; 30 | 31 | shell = { 32 | command = pkgs.runtimeShell; 33 | options = [ 34 | "-eucx" 35 | '' 36 | ${pkgs.lib.getExe pkgs.shellcheck} --external-sources --source-path=SCRIPTDIR "$@" 37 | ${pkgs.lib.getExe pkgs.shfmt} -i 2 -s -w "$@" 38 | '' 39 | "--" 40 | ]; 41 | includes = [ "*.sh" ]; 42 | }; 43 | }; 44 | }; 45 | }; 46 | } 47 | --------------------------------------------------------------------------------