├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md └── src ├── compression.rs ├── errors.rs ├── lists.rs ├── main.rs └── structs.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | **/*.rs.bk 3 | *.gz 4 | *.br 5 | tags 6 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.17.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler" 16 | version = "1.0.2" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" 19 | 20 | [[package]] 21 | name = "adler32" 22 | version = "1.2.0" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" 25 | 26 | [[package]] 27 | name = "aho-corasick" 28 | version = "0.7.19" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" 31 | dependencies = [ 32 | "memchr", 33 | ] 34 | 35 | [[package]] 36 | name = "atty" 37 | version = "0.2.14" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 40 | dependencies = [ 41 | "hermit-abi", 42 | "libc", 43 | "winapi", 44 | ] 45 | 46 | [[package]] 47 | name = "autocfg" 48 | version = "1.1.0" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 51 | 52 | [[package]] 53 | name = "backtrace" 54 | version = "0.3.66" 55 | source = "registry+https://github.com/rust-lang/crates.io-index" 56 | checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" 57 | dependencies = [ 58 | "addr2line", 59 | "cc", 60 | "cfg-if", 61 | "libc", 62 | "miniz_oxide", 63 | "object", 64 | "rustc-demangle", 65 | ] 66 | 67 | [[package]] 68 | name = "bitflags" 69 | version = "1.3.2" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 72 | 73 | [[package]] 74 | name = "brotli-sys" 75 | version = "0.3.2" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" 78 | dependencies = [ 79 | "cc", 80 | "libc", 81 | ] 82 | 83 | [[package]] 84 | name = "brotli2" 85 | version = "0.3.2" 86 | source = "registry+https://github.com/rust-lang/crates.io-index" 87 | checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" 88 | dependencies = [ 89 | "brotli-sys", 90 | "libc", 91 | ] 92 | 93 | [[package]] 94 | name = "bstr" 95 | version = "0.2.17" 96 | source = "registry+https://github.com/rust-lang/crates.io-index" 97 | checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" 98 | dependencies = [ 99 | "lazy_static 1.4.0", 100 | "memchr", 101 | "regex-automata", 102 | "serde", 103 | ] 104 | 105 | [[package]] 106 | name = "byteorder" 107 | version = "1.4.3" 108 | source = "registry+https://github.com/rust-lang/crates.io-index" 109 | checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" 110 | 111 | [[package]] 112 | name = "cc" 113 | version = "1.0.73" 114 | source = "registry+https://github.com/rust-lang/crates.io-index" 115 | checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" 116 | 117 | [[package]] 118 | name = "cfg-if" 119 | version = "1.0.0" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 122 | 123 | [[package]] 124 | name = "chan" 125 | version = "0.1.23" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "d14956a3dae065ffaa0d92ece848ab4ced88d32361e7fdfbfd653a5c454a1ed8" 128 | dependencies = [ 129 | "rand 0.3.23", 130 | ] 131 | 132 | [[package]] 133 | name = "clap" 134 | version = "3.2.22" 135 | source = "registry+https://github.com/rust-lang/crates.io-index" 136 | checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" 137 | dependencies = [ 138 | "atty", 139 | "bitflags", 140 | "clap_lex", 141 | "indexmap", 142 | "strsim", 143 | "termcolor", 144 | "textwrap", 145 | ] 146 | 147 | [[package]] 148 | name = "clap_lex" 149 | version = "0.2.4" 150 | source = "registry+https://github.com/rust-lang/crates.io-index" 151 | checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" 152 | dependencies = [ 153 | "os_str_bytes", 154 | ] 155 | 156 | [[package]] 157 | name = "crc" 158 | version = "3.0.0" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "53757d12b596c16c78b83458d732a5d1a17ab3f53f2f7412f6fb57cc8a140ab3" 161 | dependencies = [ 162 | "crc-catalog", 163 | ] 164 | 165 | [[package]] 166 | name = "crc-catalog" 167 | version = "2.1.0" 168 | source = "registry+https://github.com/rust-lang/crates.io-index" 169 | checksum = "2d0165d2900ae6778e36e80bbc4da3b5eefccee9ba939761f9c2882a5d9af3ff" 170 | 171 | [[package]] 172 | name = "crc32fast" 173 | version = "1.3.2" 174 | source = "registry+https://github.com/rust-lang/crates.io-index" 175 | checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" 176 | dependencies = [ 177 | "cfg-if", 178 | ] 179 | 180 | [[package]] 181 | name = "csv" 182 | version = "1.1.6" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" 185 | dependencies = [ 186 | "bstr", 187 | "csv-core", 188 | "itoa", 189 | "ryu", 190 | "serde", 191 | ] 192 | 193 | [[package]] 194 | name = "csv-core" 195 | version = "0.1.10" 196 | source = "registry+https://github.com/rust-lang/crates.io-index" 197 | checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" 198 | dependencies = [ 199 | "memchr", 200 | ] 201 | 202 | [[package]] 203 | name = "dirs-next" 204 | version = "2.0.0" 205 | source = "registry+https://github.com/rust-lang/crates.io-index" 206 | checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" 207 | dependencies = [ 208 | "cfg-if", 209 | "dirs-sys-next", 210 | ] 211 | 212 | [[package]] 213 | name = "dirs-sys-next" 214 | version = "0.1.2" 215 | source = "registry+https://github.com/rust-lang/crates.io-index" 216 | checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" 217 | dependencies = [ 218 | "libc", 219 | "redox_users", 220 | "winapi", 221 | ] 222 | 223 | [[package]] 224 | name = "encode_unicode" 225 | version = "1.0.0" 226 | source = "registry+https://github.com/rust-lang/crates.io-index" 227 | checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" 228 | 229 | [[package]] 230 | name = "error-chain" 231 | version = "0.12.4" 232 | source = "registry+https://github.com/rust-lang/crates.io-index" 233 | checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" 234 | dependencies = [ 235 | "backtrace", 236 | "version_check", 237 | ] 238 | 239 | [[package]] 240 | name = "filetime" 241 | version = "0.2.17" 242 | source = "registry+https://github.com/rust-lang/crates.io-index" 243 | checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" 244 | dependencies = [ 245 | "cfg-if", 246 | "libc", 247 | "redox_syscall", 248 | "windows-sys", 249 | ] 250 | 251 | [[package]] 252 | name = "flate2" 253 | version = "1.0.24" 254 | source = "registry+https://github.com/rust-lang/crates.io-index" 255 | checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" 256 | dependencies = [ 257 | "crc32fast", 258 | "miniz_oxide", 259 | ] 260 | 261 | [[package]] 262 | name = "fnv" 263 | version = "1.0.7" 264 | source = "registry+https://github.com/rust-lang/crates.io-index" 265 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 266 | 267 | [[package]] 268 | name = "fuchsia-cprng" 269 | version = "0.1.1" 270 | source = "registry+https://github.com/rust-lang/crates.io-index" 271 | checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" 272 | 273 | [[package]] 274 | name = "getrandom" 275 | version = "0.2.7" 276 | source = "registry+https://github.com/rust-lang/crates.io-index" 277 | checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" 278 | dependencies = [ 279 | "cfg-if", 280 | "libc", 281 | "wasi 0.11.0+wasi-snapshot-preview1", 282 | ] 283 | 284 | [[package]] 285 | name = "gimli" 286 | version = "0.26.2" 287 | source = "registry+https://github.com/rust-lang/crates.io-index" 288 | checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" 289 | 290 | [[package]] 291 | name = "globset" 292 | version = "0.4.9" 293 | source = "registry+https://github.com/rust-lang/crates.io-index" 294 | checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" 295 | dependencies = [ 296 | "aho-corasick", 297 | "bstr", 298 | "fnv", 299 | "log", 300 | "regex", 301 | ] 302 | 303 | [[package]] 304 | name = "hashbrown" 305 | version = "0.12.3" 306 | source = "registry+https://github.com/rust-lang/crates.io-index" 307 | checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 308 | 309 | [[package]] 310 | name = "hermit-abi" 311 | version = "0.1.19" 312 | source = "registry+https://github.com/rust-lang/crates.io-index" 313 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 314 | dependencies = [ 315 | "libc", 316 | ] 317 | 318 | [[package]] 319 | name = "indexmap" 320 | version = "1.9.1" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" 323 | dependencies = [ 324 | "autocfg", 325 | "hashbrown", 326 | ] 327 | 328 | [[package]] 329 | name = "iter-read" 330 | version = "0.3.1" 331 | source = "registry+https://github.com/rust-lang/crates.io-index" 332 | checksum = "c397ca3ea05ad509c4ec451fea28b4771236a376ca1c69fd5143aae0cf8f93c4" 333 | 334 | [[package]] 335 | name = "itoa" 336 | version = "0.4.8" 337 | source = "registry+https://github.com/rust-lang/crates.io-index" 338 | checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" 339 | 340 | [[package]] 341 | name = "lazy_static" 342 | version = "0.2.11" 343 | source = "registry+https://github.com/rust-lang/crates.io-index" 344 | checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" 345 | 346 | [[package]] 347 | name = "lazy_static" 348 | version = "1.4.0" 349 | source = "registry+https://github.com/rust-lang/crates.io-index" 350 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 351 | 352 | [[package]] 353 | name = "libc" 354 | version = "0.2.133" 355 | source = "registry+https://github.com/rust-lang/crates.io-index" 356 | checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" 357 | 358 | [[package]] 359 | name = "log" 360 | version = "0.4.17" 361 | source = "registry+https://github.com/rust-lang/crates.io-index" 362 | checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" 363 | dependencies = [ 364 | "cfg-if", 365 | ] 366 | 367 | [[package]] 368 | name = "memchr" 369 | version = "2.5.0" 370 | source = "registry+https://github.com/rust-lang/crates.io-index" 371 | checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" 372 | 373 | [[package]] 374 | name = "miniz_oxide" 375 | version = "0.5.4" 376 | source = "registry+https://github.com/rust-lang/crates.io-index" 377 | checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" 378 | dependencies = [ 379 | "adler", 380 | ] 381 | 382 | [[package]] 383 | name = "object" 384 | version = "0.29.0" 385 | source = "registry+https://github.com/rust-lang/crates.io-index" 386 | checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" 387 | dependencies = [ 388 | "memchr", 389 | ] 390 | 391 | [[package]] 392 | name = "os_str_bytes" 393 | version = "6.3.0" 394 | source = "registry+https://github.com/rust-lang/crates.io-index" 395 | checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" 396 | 397 | [[package]] 398 | name = "prettytable-rs" 399 | version = "0.9.0" 400 | source = "registry+https://github.com/rust-lang/crates.io-index" 401 | checksum = "5f375cb74c23b51d23937ffdeb48b1fbf5b6409d4b9979c1418c1de58bc8f801" 402 | dependencies = [ 403 | "atty", 404 | "csv", 405 | "encode_unicode", 406 | "lazy_static 1.4.0", 407 | "term", 408 | "unicode-width", 409 | ] 410 | 411 | [[package]] 412 | name = "rand" 413 | version = "0.3.23" 414 | source = "registry+https://github.com/rust-lang/crates.io-index" 415 | checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" 416 | dependencies = [ 417 | "libc", 418 | "rand 0.4.6", 419 | ] 420 | 421 | [[package]] 422 | name = "rand" 423 | version = "0.4.6" 424 | source = "registry+https://github.com/rust-lang/crates.io-index" 425 | checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" 426 | dependencies = [ 427 | "fuchsia-cprng", 428 | "libc", 429 | "rand_core 0.3.1", 430 | "rdrand", 431 | "winapi", 432 | ] 433 | 434 | [[package]] 435 | name = "rand_core" 436 | version = "0.3.1" 437 | source = "registry+https://github.com/rust-lang/crates.io-index" 438 | checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" 439 | dependencies = [ 440 | "rand_core 0.4.2", 441 | ] 442 | 443 | [[package]] 444 | name = "rand_core" 445 | version = "0.4.2" 446 | source = "registry+https://github.com/rust-lang/crates.io-index" 447 | checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" 448 | 449 | [[package]] 450 | name = "rdrand" 451 | version = "0.4.0" 452 | source = "registry+https://github.com/rust-lang/crates.io-index" 453 | checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" 454 | dependencies = [ 455 | "rand_core 0.3.1", 456 | ] 457 | 458 | [[package]] 459 | name = "redox_syscall" 460 | version = "0.2.16" 461 | source = "registry+https://github.com/rust-lang/crates.io-index" 462 | checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" 463 | dependencies = [ 464 | "bitflags", 465 | ] 466 | 467 | [[package]] 468 | name = "redox_users" 469 | version = "0.4.0" 470 | source = "registry+https://github.com/rust-lang/crates.io-index" 471 | checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" 472 | dependencies = [ 473 | "getrandom", 474 | "redox_syscall", 475 | ] 476 | 477 | [[package]] 478 | name = "regex" 479 | version = "1.6.0" 480 | source = "registry+https://github.com/rust-lang/crates.io-index" 481 | checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" 482 | dependencies = [ 483 | "aho-corasick", 484 | "memchr", 485 | "regex-syntax", 486 | ] 487 | 488 | [[package]] 489 | name = "regex-automata" 490 | version = "0.1.10" 491 | source = "registry+https://github.com/rust-lang/crates.io-index" 492 | checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" 493 | 494 | [[package]] 495 | name = "regex-syntax" 496 | version = "0.6.27" 497 | source = "registry+https://github.com/rust-lang/crates.io-index" 498 | checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" 499 | 500 | [[package]] 501 | name = "rustc-demangle" 502 | version = "0.1.21" 503 | source = "registry+https://github.com/rust-lang/crates.io-index" 504 | checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" 505 | 506 | [[package]] 507 | name = "rustversion" 508 | version = "1.0.8" 509 | source = "registry+https://github.com/rust-lang/crates.io-index" 510 | checksum = "24c8ad4f0c00e1eb5bc7614d236a7f1300e3dbd76b68cac8e06fb00b015ad8d8" 511 | 512 | [[package]] 513 | name = "ryu" 514 | version = "1.0.11" 515 | source = "registry+https://github.com/rust-lang/crates.io-index" 516 | checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" 517 | 518 | [[package]] 519 | name = "separator" 520 | version = "0.4.1" 521 | source = "registry+https://github.com/rust-lang/crates.io-index" 522 | checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" 523 | 524 | [[package]] 525 | name = "serde" 526 | version = "1.0.144" 527 | source = "registry+https://github.com/rust-lang/crates.io-index" 528 | checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860" 529 | 530 | [[package]] 531 | name = "size" 532 | version = "0.4.0" 533 | source = "registry+https://github.com/rust-lang/crates.io-index" 534 | checksum = "63c9c8350184b6b037d09dc77df55f1324df076a706d0191b79ec270cc4d756b" 535 | 536 | [[package]] 537 | name = "static-compress" 538 | version = "0.3.3" 539 | dependencies = [ 540 | "brotli2", 541 | "chan", 542 | "clap", 543 | "error-chain", 544 | "filetime", 545 | "flate2", 546 | "globset", 547 | "prettytable-rs", 548 | "separator", 549 | "size", 550 | "stderr", 551 | "zopfli", 552 | ] 553 | 554 | [[package]] 555 | name = "stderr" 556 | version = "0.8.0" 557 | source = "registry+https://github.com/rust-lang/crates.io-index" 558 | checksum = "9ee63545cb3066bed00544996fb5b545426632bf7d14ab3e6567602a49819ef8" 559 | dependencies = [ 560 | "lazy_static 0.2.11", 561 | "time", 562 | ] 563 | 564 | [[package]] 565 | name = "strsim" 566 | version = "0.10.0" 567 | source = "registry+https://github.com/rust-lang/crates.io-index" 568 | checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 569 | 570 | [[package]] 571 | name = "term" 572 | version = "0.7.0" 573 | source = "registry+https://github.com/rust-lang/crates.io-index" 574 | checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" 575 | dependencies = [ 576 | "dirs-next", 577 | "rustversion", 578 | "winapi", 579 | ] 580 | 581 | [[package]] 582 | name = "termcolor" 583 | version = "1.1.3" 584 | source = "registry+https://github.com/rust-lang/crates.io-index" 585 | checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" 586 | dependencies = [ 587 | "winapi-util", 588 | ] 589 | 590 | [[package]] 591 | name = "textwrap" 592 | version = "0.15.1" 593 | source = "registry+https://github.com/rust-lang/crates.io-index" 594 | checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" 595 | 596 | [[package]] 597 | name = "time" 598 | version = "0.1.44" 599 | source = "registry+https://github.com/rust-lang/crates.io-index" 600 | checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" 601 | dependencies = [ 602 | "libc", 603 | "wasi 0.10.0+wasi-snapshot-preview1", 604 | "winapi", 605 | ] 606 | 607 | [[package]] 608 | name = "typed-arena" 609 | version = "2.0.1" 610 | source = "registry+https://github.com/rust-lang/crates.io-index" 611 | checksum = "0685c84d5d54d1c26f7d3eb96cd41550adb97baed141a761cf335d3d33bcd0ae" 612 | 613 | [[package]] 614 | name = "unicode-width" 615 | version = "0.1.10" 616 | source = "registry+https://github.com/rust-lang/crates.io-index" 617 | checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" 618 | 619 | [[package]] 620 | name = "version_check" 621 | version = "0.9.4" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" 624 | 625 | [[package]] 626 | name = "wasi" 627 | version = "0.10.0+wasi-snapshot-preview1" 628 | source = "registry+https://github.com/rust-lang/crates.io-index" 629 | checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" 630 | 631 | [[package]] 632 | name = "wasi" 633 | version = "0.11.0+wasi-snapshot-preview1" 634 | source = "registry+https://github.com/rust-lang/crates.io-index" 635 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 636 | 637 | [[package]] 638 | name = "winapi" 639 | version = "0.3.9" 640 | source = "registry+https://github.com/rust-lang/crates.io-index" 641 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 642 | dependencies = [ 643 | "winapi-i686-pc-windows-gnu", 644 | "winapi-x86_64-pc-windows-gnu", 645 | ] 646 | 647 | [[package]] 648 | name = "winapi-i686-pc-windows-gnu" 649 | version = "0.4.0" 650 | source = "registry+https://github.com/rust-lang/crates.io-index" 651 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 652 | 653 | [[package]] 654 | name = "winapi-util" 655 | version = "0.1.5" 656 | source = "registry+https://github.com/rust-lang/crates.io-index" 657 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 658 | dependencies = [ 659 | "winapi", 660 | ] 661 | 662 | [[package]] 663 | name = "winapi-x86_64-pc-windows-gnu" 664 | version = "0.4.0" 665 | source = "registry+https://github.com/rust-lang/crates.io-index" 666 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 667 | 668 | [[package]] 669 | name = "windows-sys" 670 | version = "0.36.1" 671 | source = "registry+https://github.com/rust-lang/crates.io-index" 672 | checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" 673 | dependencies = [ 674 | "windows_aarch64_msvc", 675 | "windows_i686_gnu", 676 | "windows_i686_msvc", 677 | "windows_x86_64_gnu", 678 | "windows_x86_64_msvc", 679 | ] 680 | 681 | [[package]] 682 | name = "windows_aarch64_msvc" 683 | version = "0.36.1" 684 | source = "registry+https://github.com/rust-lang/crates.io-index" 685 | checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" 686 | 687 | [[package]] 688 | name = "windows_i686_gnu" 689 | version = "0.36.1" 690 | source = "registry+https://github.com/rust-lang/crates.io-index" 691 | checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" 692 | 693 | [[package]] 694 | name = "windows_i686_msvc" 695 | version = "0.36.1" 696 | source = "registry+https://github.com/rust-lang/crates.io-index" 697 | checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" 698 | 699 | [[package]] 700 | name = "windows_x86_64_gnu" 701 | version = "0.36.1" 702 | source = "registry+https://github.com/rust-lang/crates.io-index" 703 | checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" 704 | 705 | [[package]] 706 | name = "windows_x86_64_msvc" 707 | version = "0.36.1" 708 | source = "registry+https://github.com/rust-lang/crates.io-index" 709 | checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" 710 | 711 | [[package]] 712 | name = "zopfli" 713 | version = "0.7.1" 714 | source = "registry+https://github.com/rust-lang/crates.io-index" 715 | checksum = "f1e0d16c30236860686a8f03d36b384dc2fc0675a8916367d2f9a1ecd795eab6" 716 | dependencies = [ 717 | "adler32", 718 | "byteorder", 719 | "crc", 720 | "iter-read", 721 | "log", 722 | "typed-arena", 723 | ] 724 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "static-compress" 3 | version = "0.3.3" 4 | authors = ["Mahmoud Al-Qudsi "] 5 | description = "Create a compressed copy of files matching a glob to serve statically compressed files with a web server" 6 | homepage = "https://github.com/neosmart/static-compress" 7 | repository = "https://github.com/neosmart/static-compress" 8 | readme = "README.md" 9 | keywords = ["nginx", "gzip", "compression", "brotli", "apache"] 10 | categories = ["command-line-utilities", "compression", "web-programming"] 11 | license = "MIT" 12 | 13 | [dependencies] 14 | brotli2 = "0.3.2" 15 | chan = "0.1.23" 16 | clap = "3" 17 | error-chain = "0.12" 18 | filetime = "0.2" 19 | flate2 = "1.0" 20 | globset = "0.4" 21 | size = "0.4.0" 22 | prettytable-rs = "0.9" 23 | separator = "0.4" 24 | stderr = "0.8" 25 | zopfli = "0.7.1" 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 NeoSmart Technologies 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## `static-compress` 2 | 3 | `static-compress` is command-line utility that can be used to aid in the generation of a statically pre-compressed copy of a given directory subtree, useful for serving statically precompressed content via a webserver like nginx or apache. 4 | 5 | `static-compress` currently supports creating statically compressed copies of files matching a given glob (expression) in the gzip and brotli formats. gzip-compressed files can be generated either via the standard `miniz` library, or via the slower-but-higher-compression `zopfli` algorithm, [recently developed by Google](https://en.wikipedia.org/wiki/Zopfli). As of version 0.3, `static-compress` also features [webp](https://developers.google.com/speed/webp/) support for image compression. 6 | 7 | ### Installation 8 | 9 | `static-compress` is available via the cargo package manager on all supported platforms and may be installed by simply executing 10 | 11 | ```bash 12 | cargo install static-compress 13 | ``` 14 | 15 | Pre-built, signed binaries for select platforms can also be found at the static-compress homepage at 16 | https://neosmart.net/static-compress/ 17 | 18 | ### Usage 19 | 20 | USAGE: 21 | 22 | static-compress [OPTIONS] ... 23 | Usage of `static-compress` is straightforward. It is invoked with either a list of files to pre-compress or an expression such as `"*.rs"` (to match all files in the current directory with a `.rs` extension) or `"**/*.png"` (to match `.png` files in all subdirectories). 24 | 25 | No options are required, but optional command line switches are available to influence the behavior of `static-compress`: 26 | 27 | -c, --compressor <[brotli|gzip| The compressor to use, defaulting to gzip 28 | webp|zopfli]> 29 | -e, --extension <.EXT> The extension to use for compressed files. Supplied 30 | automatically if not provided. 31 | -j, --threads The number of simultaneous compressions. 32 | -i, case-insensitive Use case-insensitive matching against patterns. 33 | -q, --quality The algorithm-specific quality parameter to be used. 34 | Automatically set if not provided. 35 | --quiet Suppresses all non-error output. 36 | --no-progress Silences file progress information. 37 | --no-summary Suppress the end-of-run summary. 38 | 39 | Multithreading may be achieved by means of the `-j` switch (akin to `make`), and can be used to specify the number of files to be compressed simultaneously across multiple threads. By default, `static-compress` uses all available threads. 40 | 41 | The algorithm-specific `--quality` switch can be used to set the quality parameter for the chosen compressor (if supported): 42 | 43 | | Compressor | `--quality` range | 44 | | ---------- | ----------------- | 45 | | gzip | 0 - 10 | 46 | | brotli | 0 - 11 | 47 | | webp | 0 - 100 | 48 | | zopfli | *not supported* | 49 | 50 | ### Supported Globs/Expressions 51 | 52 | Supported filters/expressions include `*` to match any filename pattern, `**` to match recursively across all subdirectories, and `?` to substitute any single character. A bracket containing multiple characters will match any one character within the brackets (e.g. `[abc]` will match `a` or `b` but not `aa`), and curly braces can be used to match any of the comma-separated contents (e.g. `{abc,def}`). 53 | 54 | `static-compress` supports the following filter expressions, all of which will match the same file `foo.bar`: 55 | 56 | * `./relative/path/to/foo.bar` 57 | * `/absolute/path/to/foo.bar` 58 | * `relative/*/*/foo.ba?` 59 | * `**/foo.bar` 60 | * `**/foo.ba[rz]` 61 | * `**/{foo|something}.bar` 62 | 63 | **Important Note: Make sure to place expressions in double-quotes to prevent your shell from globbing the expressions!** i.e. use `static-compress "*.html"` and not `static-compress *.html`. The latter may cause an argument overflow in the presence of too many files, and will not use `static-compress`' intelligent globbing, relying on your shell to expand the glob instead! 64 | 65 | ### Supported Compression Methods 66 | 67 | Currently, `static-compress` supports the `gzip` and `brotli` general-purpose compression algortithms for compressing web content. Almost all web servers and web browsers in use today have full `gzip` support. `brotli` is a newer web-compression format [developed by Google](https://en.wikipedia.org/wiki/Brotli), that can be used to achieve higher levels of compression than `gzip`, though compression is more taxing on the server. For that reason, it is especially desirable to be able to pre-compress a given directory tree instead of (re-)compressing files each time they are requested. 68 | 69 | `static-compress` also supports zopfli, which is akin to `gzip -11` ([we jest!](https://www.youtube.com/watch?v=KOO5S4vxi0o)). The only problem is that `zopfli` is ridiculously slow and absolutely not intended to be used for dynamic compression. Again, this is another area where pre-compression is the way to go, and `static-compress` makes it easy to prepare a directory tree to serve zopfli-compressed versions of its contents. Unlike brotli, zopfli is gzip-compatible meaning any browser that supports gzip decompression also supports zopfli - but zopfli is both slower at compressing and typically does not achieve the same compression rates that brotli currently does. (Given the requirement of playing nicely with browsers from the 90s, it's good at what it does.) 70 | 71 | As of version 0.3, `static-compress` also features webp support for image compression. In our testing on a corpus of the approximately 15,000 images uploaded to the NeoSmart website in PNG and JPG formats, webp compression with a quality parameter of 90 (the default `--quality` parameter for webp in `static-compress`) resulted in a 65% reduction in file size with no appreciable increase in artifacts. 72 | 73 | ### Mode of Operation 74 | 75 | `static-compress` is an *intelligent* compressor meant for use in day-to-day web deployment and system administration tasks. The entire point of `static-compress` verses the usage of an extremely fragile and overly-complicated batch script (`find` with `mtime`, `gzip|brotli`, `parallel`, `touch`, and more) is to make life easier and the results more portable/deterministic. `static-compress` can be safely run against any directory tree, and by default it 76 | 77 | * Compresses only files that haven't been previously statically compressed (it sets the modification date of the statically-compressed copy of a file to match the original, and only recompresses if this does not match), 78 | * Does not compress already compressed files (i.e. won't recompress your pre-compressed `.gz` files as `.gz.br`), 79 | * Can be configured to use as many or as few threads as you like for simultaneous compression, 80 | * Can be used to compress an entire directory tree (`static-compress "**"`) or just files matching a certain extension (`static-compress "**/*.html"`) or only matching a certain prefix or subpath (`static-compress "**/tocompress/*"`) 81 | * Sets the modification date of the compressed file equal to the modification date of the original file, so that when the original file is modified the webserver can know not to serve the old/stale compressed file (and so a subsequent `static-compress` run can know to re-compress the file and replace the stale copy). 82 | 83 | ### Web Server Configuration 84 | 85 | Given a subdirectory `optimized`, the contents of which have been pre-compressed in both `gzip` and `brotli` formats via `static-compress optimized/**` and `static-compress optimized/** -c brotli`, the instructions for configuring your web server to use the statically pre-compressed version of the original files is as follows: 86 | 87 | #### nginx: 88 | 89 | To serve gzip-compressed files, nginx must be compiled with the `ngx_http_gzip_static_module` module (included in the default distribution) by specifying `--with-http_gzip_static_module` at build time. Thereafter, the following configuration may be used: 90 | 91 | ```nginx 92 | location optimized { 93 | gzip_static on; 94 | } 95 | ``` 96 | 97 | To serve brotli-compressed files, nginx must be compiled with the `ngx_brotli` module ([available separately](https://github.com/google/ngx_brotli)) by specifying `--add-module ../ngx_brotli` at build time. Thereafter, the following configuration may be used: 98 | 99 | ```nginx 100 | location optimized { 101 | brotli_static on; 102 | } 103 | ``` 104 | 105 | If both the `ngx_brotli` and `ngx_http_gzip_static_module` modules have been installed, the two directives may be safely used in the same `location` block: 106 | 107 | ```nginx 108 | location optimized { 109 | brotli_static on; 110 | gzip_static on; 111 | } 112 | ``` 113 | 114 | Note that the file type options for both modules (``brotli_types`` and `gzip_types`) do not apply to the static option; all files, even those not specified for dynamic compression via these two `_types` options, may be served in these formats if a `.br` or `.gz` file with the same name resides in the same directory. 115 | 116 | ### Acknowledgements, authorship, license, and copyright 117 | 118 | `static-compress` is made freely available to the public under the terms of the MIT license. `static-compress` is open source and would not have been possible without the `flate2` and `zopfli` crate authors, as well as the original creators of the `brotli`, `gzip`, and `zopfli` algorithms. 119 | 120 | `static-compress` is written by Mahmoud Al-Qudsi <[mqudsi@neosmart.net](mailto:mqudsi@neosmart.net)> under the stewardship of NeoSmart Technologies. The name `static-compress` and all other rights not conferred by the MIT license are reserved and copyright of NeoSmart Technologies, 2017-2022. 121 | -------------------------------------------------------------------------------- /src/compression.rs: -------------------------------------------------------------------------------- 1 | extern crate brotli2; 2 | extern crate flate2; 3 | extern crate zopfli; 4 | 5 | use structs::*; 6 | use errors::*; 7 | use std::fs::File; 8 | use std::io::{Read, Write}; 9 | 10 | use std::path::Path; 11 | 12 | impl CompressionFormat for CompressionAlgorithm { 13 | fn extension(&self) -> &'static str { 14 | match self { 15 | &CompressionAlgorithm::Brotli => "br", 16 | &CompressionAlgorithm::GZip => "gz", 17 | &CompressionAlgorithm::WebP => "webp", 18 | &CompressionAlgorithm::Zopfli => "gz", 19 | } 20 | } 21 | } 22 | 23 | impl FileCompressor for CompressionAlgorithm { 24 | fn compress(&self, src: &Path, dst: &Path, quality: Option) -> Result<()> { 25 | match self { 26 | &CompressionAlgorithm::GZip => gzip_compress(src, dst, quality), 27 | &CompressionAlgorithm::Brotli => brotli_compress(src, dst, quality), 28 | &CompressionAlgorithm::WebP => webp_compress(src, dst, quality), 29 | &CompressionAlgorithm::Zopfli => zopfli_compress(src, dst, quality), 30 | // _ => bail!("Compression algorithm not implemented!"), 31 | } 32 | } 33 | } 34 | 35 | fn gzip_compress(src_path: &Path, dst_path: &Path, quality: Option) -> Result<()> { 36 | let mut src = File::open(src_path)?; 37 | let dst = File::create(dst_path)?; 38 | 39 | let level = match quality { 40 | None => flate2::Compression::default(), 41 | Some(0) => flate2::Compression::none(), 42 | Some(1) => flate2::Compression::fast(), 43 | Some(2..=6) => flate2::Compression::default(), 44 | Some(3..=9) => flate2::Compression::best(), 45 | _ => bail!("Invalid --quality parameter specified!"), 46 | }; 47 | 48 | let mut encoder = flate2::write::GzEncoder::new(dst, level); 49 | let mut buf = [0u8; 1024]; 50 | loop { 51 | let bytes_read = src.read(&mut buf).chain_err(|| "Error reading from source file!")?; 52 | match bytes_read { 53 | 0 => break, // End-of-file 54 | l => encoder.write_all(&buf[0..l]).chain_err(|| "Fatal gzip encoder error!")?, 55 | }; 56 | } 57 | 58 | Ok(()) 59 | } 60 | 61 | fn brotli_compress(src_path: &Path, dst_path: &Path, quality: Option) -> Result<()> { 62 | let mut src = File::open(src_path)?; 63 | let dst = File::create(dst_path)?; 64 | 65 | let level = match quality { 66 | None => 6, 67 | Some(q @ 0..=11) => q, 68 | _ => bail!("Invalid --quality parameter specified!"), 69 | }; 70 | 71 | let mut encoder = brotli2::write::BrotliEncoder::new(dst, level as u32); 72 | let mut buf = [0u8; 1024]; 73 | loop { 74 | let bytes_read = src.read(&mut buf).chain_err(|| "Error reading from source file!")?; 75 | match bytes_read { 76 | 0 => break, // End-of-file 77 | l => encoder.write_all(&buf[0..l]).chain_err(|| "Fatal gzip encoder error!")?, 78 | }; 79 | } 80 | 81 | Ok(()) 82 | } 83 | 84 | fn zopfli_compress(src_path: &Path, dst_path: &Path, quality: Option) -> Result<()> { 85 | if quality.is_some() { 86 | bail!("--quality is not implemented for zopfli compression"); 87 | } 88 | 89 | let src = File::open(src_path)?; 90 | let dst = File::create(dst_path)?; 91 | 92 | zopfli::compress(&zopfli::Options::default(), &zopfli::Format::Gzip, src, dst)?; 93 | 94 | Ok(()) 95 | } 96 | 97 | fn webp_compress(src_path: &Path, dst_path: &Path, quality: Option) -> Result<()> { 98 | use std::process::Command; 99 | 100 | let output = Command::new("cwebp") 101 | .arg("-q") 102 | .arg(quality.unwrap_or(90).to_string()) 103 | .arg(src_path.as_os_str()) 104 | .arg("-o") 105 | .arg(dst_path.as_os_str()) 106 | .output() 107 | .chain_err(|| "Error executing cwebp!")?; 108 | 109 | if !output.status.success() { 110 | bail!("Error compressing via webp: {:?}", output); 111 | } 112 | 113 | Ok(()) 114 | } 115 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | error_chain! { 2 | errors { 3 | InvalidParameterValue(pname: &'static str) { 4 | description("An invalid value was supplied for a command line argument.") 5 | display("Invalid value supplied for parameter {}", pname) 6 | } 7 | InvalidUsage 8 | InvalidIncludeFilter 9 | InvalidCharactersInPath 10 | } 11 | foreign_links { 12 | Io(::std::io::Error); 13 | SystemTime(::std::time::SystemTimeError); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/lists.rs: -------------------------------------------------------------------------------- 1 | pub const COMP_EXTS: &'static [&'static str] = &[ 2 | "7z", 3 | "br", 4 | "bz2", 5 | "gz", 6 | "lzh", 7 | "lzma", 8 | "lzx", 9 | "rar", 10 | "sfx", 11 | "xz", 12 | "zip", 13 | "zpaq", 14 | "zz", 15 | ]; 16 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] extern crate error_chain; 2 | #[macro_use] extern crate prettytable; 3 | #[macro_use] extern crate stderr; 4 | extern crate chan; 5 | extern crate clap; 6 | extern crate filetime; 7 | extern crate globset; 8 | extern crate separator; 9 | extern crate size; 10 | 11 | #[macro_use] mod errors; 12 | mod compression; 13 | mod lists; 14 | mod structs; 15 | 16 | use clap::{App, Arg}; 17 | use errors::*; 18 | use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; 19 | use lists::*; 20 | use std::path::{Path, PathBuf}; 21 | use std::sync::Arc; 22 | use std::sync::mpsc; 23 | use structs::*; 24 | 25 | const DEBUG_FILTERS: bool = cfg!(debug_assertions); 26 | #[inline(always)] 27 | fn debug(message: &str) { 28 | if DEBUG_FILTERS { 29 | errstln!("{}", message); 30 | } 31 | } 32 | 33 | quick_main!(run); 34 | 35 | fn run() -> Result<()> { 36 | let matches = App::new("static-compress") 37 | .version("0.3.3") 38 | .about("Create statically-compresed copies of matching files") 39 | .author("Mahmoud Al-Qudsi, NeoSmart Technologies") 40 | .arg(Arg::new("compressor") 41 | .short('c') 42 | .long("compressor") 43 | .value_name("[brotli|gzip|zopfli|webp]") 44 | .help("The compressor to use (default: gzip)") 45 | .takes_value(true)) 46 | .arg(Arg::new("threads") 47 | .short('j') 48 | .long("threads") 49 | .value_name("COUNT") 50 | .help("The number of simultaneous compressions (default: number of cores)") 51 | .takes_value(true)) 52 | .arg(Arg::new("filters") 53 | .value_name("FILTER") 54 | .multiple_occurrences(true) 55 | .required(true)) 56 | .arg(Arg::new("ext") 57 | .short('e') 58 | .value_name("EXT") 59 | .long("extension") 60 | .help("The extension to use for compressed files (default: gz, br, or webp)")) 61 | .arg(Arg::new("quality") 62 | .short('q') 63 | .long("quality") 64 | .takes_value(true) 65 | .help("A quality parameter to be passed to the encoder. Algorithm-specific.")) 66 | .arg(Arg::new("quiet") 67 | .long("quiet") 68 | .takes_value(false) 69 | .help("Does not display progress or end-of-run summary table.")) 70 | .arg(Arg::new("no-progress") 71 | .long("no-progress") 72 | .takes_value(false) 73 | .help("Do not list files as they are compressed.")) 74 | .arg(Arg::new("no-summary") 75 | .long("no-summary") 76 | .takes_value(false) 77 | .help("Hide end-of-run statistics summary.")) 78 | .arg(Arg::new("nocase") 79 | .short('i') 80 | .long("case-insensitive") 81 | .takes_value(false) 82 | .help("Use case-insensitive pattern matching.")) 83 | /*.arg(Arg::new("excludes") 84 | .short('x') 85 | .value_name("FILTER") 86 | .long("exclude") 87 | .multiple(true) 88 | .help("Exclude files matching this glob expression"))*/ 89 | .get_matches(); 90 | 91 | fn get_parameter<'a, T>(matches: &clap::ArgMatches, name: &'static str, default_value: T) -> Result 92 | where T: std::str::FromStr 93 | { 94 | match matches.value_of(name) { 95 | Some(v) => { 96 | Ok(v.parse().map_err(|_| ErrorKind::InvalidParameterValue(name))?) 97 | } 98 | None => Ok(default_value), 99 | } 100 | } 101 | 102 | let case_sensitive = !matches.is_present("nocase"); 103 | let compressor = get_parameter(&matches, "compressor", CompressionAlgorithm::GZip)?; 104 | let show_summary = !matches.contains_id("no-summary") && !matches.contains_id("quiet"); 105 | let show_progress = !matches.contains_id("no-progress") && !matches.contains_id("quiet"); 106 | 107 | let parameters = Arc::new(Parameters { 108 | extension: matches.value_of("ext") 109 | .unwrap_or(compressor.extension()) 110 | .trim_matches(|c: char| c.is_whitespace() || c.is_control() || c == '.') 111 | .to_owned(), 112 | compressor, 113 | quality: match matches.value_of("quality") { 114 | Some(q) => Some(q.parse::().map_err(|_| ErrorKind::InvalidParameterValue("quality"))?), 115 | None => None 116 | }, 117 | show_summary, 118 | show_progress, 119 | threads: get_parameter(&matches, "threads", std::thread::available_parallelism()?.into())?, 120 | }); 121 | 122 | let (send_queue, stats_rx, wait_group) = start_workers(¶meters); 123 | 124 | let mut include_filters: Vec = match matches.values_of("filters") { 125 | Some(values) => Ok(values.map(|s| s.to_owned()).collect()), 126 | None => Err(ErrorKind::InvalidUsage), 127 | }?; 128 | 129 | let mut builder = GlobSetBuilder::new(); 130 | fix_filters(&mut include_filters); 131 | for filter in include_filters.iter() { 132 | let glob = GlobBuilder::new(filter) 133 | .case_insensitive(!case_sensitive) 134 | .literal_separator(true) 135 | .build().map_err(|_| ErrorKind::InvalidIncludeFilter)?; 136 | builder.add(glob); 137 | } 138 | let globset = builder.build().map_err(|_| ErrorKind::InvalidIncludeFilter)?; 139 | 140 | // Convert filters to paths and deal out conversion jobs 141 | dispatch_jobs(send_queue, include_filters, globset/*, exclude_filters*/)?; 142 | 143 | // Wait for all jobs to finish 144 | wait_group.wait(); 145 | 146 | // Merge statistics from all threads 147 | if show_summary { 148 | let mut stats = Statistics::new(); 149 | while let Ok(thread_stats) = stats_rx.recv() { 150 | stats.merge(&thread_stats); 151 | } 152 | 153 | println!("{}", stats); 154 | } 155 | 156 | Ok(()) 157 | } 158 | 159 | type ThreadParam = std::path::PathBuf; 160 | 161 | fn start_workers<'a>(params: &Arc) -> (chan::Sender, mpsc::Receiver, chan::WaitGroup) { 162 | let (tx, rx) = chan::sync::(params.threads); 163 | let (stats_tx, stats_rx) = std::sync::mpsc::channel::(); 164 | let wg = chan::WaitGroup::new(); 165 | 166 | for _ in 0..params.threads { 167 | let local_params = params.clone(); 168 | let local_rx = rx.clone(); 169 | let local_stats_tx = stats_tx.clone(); 170 | let local_wg = wg.clone(); 171 | wg.add(1); 172 | std::thread::spawn(move || { 173 | worker_thread(local_params, local_stats_tx, local_rx); 174 | local_wg.done(); 175 | }); 176 | } 177 | 178 | (tx, stats_rx, wg) 179 | } 180 | 181 | fn yield_file(path: PathBuf, globset: &GlobSet, callback: &F) -> Result<()> 182 | where F: Fn(PathBuf) -> Result<()> 183 | { 184 | if is_hidden(&path)? { 185 | // We are ignoring .files and .directories 186 | // We may add a command-line switch to control this behavior in the future 187 | return Ok(()); 188 | } 189 | 190 | if path.is_dir() { 191 | for child in path.read_dir()? { 192 | let child_path = child?.path(); 193 | yield_file(child_path, globset, callback)?; 194 | } 195 | } 196 | else { 197 | // I'm presuming the binary search in is_blacklisted is faster 198 | // than globset.is_match, but we should benchmark it at some point 199 | if !is_blacklisted(&path)? && globset.is_match(&path) { 200 | callback(path)?; 201 | } 202 | } 203 | 204 | Ok(()) 205 | } 206 | 207 | fn dispatch_jobs(send_queue: chan::Sender, filters: Vec, globset: GlobSet/*, exclude_filters: Vec*/) -> Result<()> { 208 | let paths = extract_paths(&filters)?; 209 | for path in paths { 210 | yield_file(path, &globset, &|path: PathBuf| { 211 | send_queue.send(path); 212 | Ok(()) 213 | })? 214 | } 215 | 216 | Ok(()) 217 | } 218 | 219 | fn worker_thread(params: Arc, stats_tx: mpsc::Sender, rx: chan::Receiver) { 220 | let mut local_stats = Statistics::new(); 221 | 222 | loop { 223 | let src = match rx.recv() { 224 | Some(task) => task, 225 | None => break, // No more tasks 226 | }; 227 | 228 | // In a nested function so we can handle errors centrally 229 | fn compress_single(src: &ThreadParam, params: &Parameters, mut local_stats: &mut Statistics) -> Result<()> { 230 | let dst_path = format!("{}.{}", 231 | src.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?, 232 | params.extension); 233 | let dst = Path::new(&dst_path); 234 | 235 | // Again, in a scope for error handling 236 | |local_stats: &mut Statistics| -> Result<()> { 237 | let src_metadata = std::fs::metadata(src)?; 238 | 239 | // Don't compress files that are already compressed that haven't changed 240 | if let Ok(dst_metadata) = std::fs::metadata(dst) { 241 | // The destination already exists 242 | let src_seconds = src_metadata.modified()?.duration_since(std::time::UNIX_EPOCH)?.as_secs(); 243 | let dst_seconds = dst_metadata.modified()?.duration_since(std::time::UNIX_EPOCH)?.as_secs(); 244 | match src_seconds == dst_seconds { 245 | true => { 246 | local_stats.update(src_metadata.len(), dst_metadata.len(), false); 247 | // No need to recompress 248 | return Ok(()); 249 | }, 250 | false => { 251 | // Return an error if we can't remove the file 252 | std::fs::remove_file(dst)?; 253 | } 254 | }; 255 | } 256 | 257 | if params.show_progress { 258 | println!("{}", src.display()); 259 | } 260 | params.compressor.compress(src.as_path(), dst, params.quality)?; 261 | let dst_metadata = std::fs::metadata(dst)?; 262 | local_stats.update(src_metadata.len(), dst_metadata.len(), true); 263 | let src_modified = filetime::FileTime::from_last_modification_time(&src_metadata); 264 | filetime::set_file_times(dst, filetime::FileTime::zero(), src_modified).unwrap_or_default(); 265 | 266 | Ok(()) 267 | }(&mut local_stats) 268 | .map_err(|e| { 269 | // Try deleting the invalid destination file, but don't care if we can't 270 | std::fs::remove_file(dst).unwrap_or_default(); 271 | e // Bubble up the same error 272 | }) 273 | } 274 | 275 | if let Err(e) = compress_single(&src, ¶ms, &mut local_stats) { 276 | errstln!("Error compressing {}: {}", src.to_string_lossy(), e); 277 | } 278 | } 279 | 280 | if !stats_tx.send(local_stats).is_ok() { 281 | errstln!("Error compiling statistics!"); 282 | } 283 | } 284 | 285 | fn str_search(sorted: &[&str], search_term: &str, case_sensitive: bool) -> std::result::Result { 286 | use std::borrow::Cow; 287 | 288 | let term = match case_sensitive { 289 | true => Cow::from(search_term), 290 | false => if search_term.chars().all(char::is_lowercase) { Cow::from(search_term) } else { Cow::from(search_term.to_lowercase()) }, 291 | }; 292 | 293 | sorted.binary_search_by(|probe| (*probe).cmp(term.as_ref())) 294 | } 295 | 296 | fn is_hidden(path: &Path) -> Result { 297 | let hidden = match path.file_name() { 298 | Some(x) => x.to_str().ok_or(ErrorKind::InvalidCharactersInPath)? 299 | .starts_with("."), 300 | None => false 301 | }; 302 | Ok(hidden) 303 | } 304 | 305 | fn is_blacklisted(path: &Path) -> Result { 306 | let r = match path.extension() { 307 | Some(x) => { 308 | let ext = x.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?; 309 | str_search(COMP_EXTS, &ext, false).is_ok() 310 | }, 311 | None => false, 312 | }; 313 | 314 | return Ok(r); 315 | } 316 | 317 | // Prepends ./ to relative paths 318 | fn fix_filters(filters: &mut Vec) { 319 | for i in 0..filters.len() { 320 | let new_path; 321 | { 322 | let ref path = filters[i]; 323 | match path.chars().next().expect("Received blank filter!") { 324 | '.' | '/' => continue, 325 | _ => new_path = format!("./{}", path) // Use un-prefixed path 326 | } 327 | } 328 | filters[i] = new_path; 329 | } 330 | } 331 | 332 | // Given a list of filters, extracts the directories that should be searched. 333 | // TODO: Also provide info about to what depth they should be recursed. 334 | use std::collections::HashSet; 335 | fn extract_paths(filters: &Vec) -> Result> { 336 | use std::iter::FromIterator; 337 | 338 | let mut dirs = std::collections::HashSet::::new(); 339 | 340 | { 341 | let insert_path = &mut |filter: &String, dir: PathBuf| { 342 | debug(&format!("filter {} mapped to search {}", filter, dir.display())); 343 | dirs.insert(dir); 344 | }; 345 | 346 | for filter in filters { 347 | // Take everything until the first expression 348 | let mut last_char = None::; 349 | let dir; 350 | { 351 | let partial = filter.chars().take_while(|c| match c { 352 | &'?' | &'*' | &'{' | &'[' => false, 353 | c => { last_char = Some(c.clone()); true } 354 | }); 355 | dir = String::from_iter(partial); 356 | } 357 | 358 | let dir = match dir.chars().next() { 359 | Some(c) => match c { 360 | '.' | '/' => PathBuf::from(dir), 361 | _ => { 362 | let mut pb = PathBuf::from("./"); 363 | pb.push(dir); 364 | pb 365 | } 366 | }, 367 | None => { 368 | insert_path(filter, PathBuf::from("./")); 369 | continue; 370 | } 371 | }; 372 | 373 | if dir.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?.ends_with(filter) { 374 | // The "dir" is actually a full path to a single file, return it as-is. 375 | insert_path(filter, dir); 376 | continue; 377 | } 378 | 379 | if last_char == Some('/') { 380 | // Dir is a already a directory, return it as-is. 381 | insert_path(filter, dir); 382 | continue; 383 | } 384 | 385 | // We need to extract the directory from the path we have 386 | let dir = match PathBuf::from(dir).parent() { 387 | Some(parent) => parent.to_path_buf(), 388 | None => PathBuf::from("./"), 389 | }; 390 | 391 | insert_path(filter, dir); 392 | } 393 | } 394 | 395 | debug(&format!("final search paths: {:?}", dirs)); 396 | 397 | Ok(dirs) 398 | } 399 | -------------------------------------------------------------------------------- /src/structs.rs: -------------------------------------------------------------------------------- 1 | use ::*; 2 | use errors::*; 3 | use separator::Separatable; 4 | use size::Size; 5 | use std::path::Path; 6 | 7 | pub struct Parameters { 8 | pub compressor: CompressionAlgorithm, 9 | pub extension: String, 10 | pub quality: Option, 11 | pub threads: usize, 12 | pub show_progress: bool, 13 | pub show_summary: bool, 14 | } 15 | 16 | pub enum CompressionAlgorithm { 17 | Brotli, 18 | GZip, 19 | WebP, 20 | Zopfli, 21 | } 22 | 23 | impl std::str::FromStr for CompressionAlgorithm { 24 | type Err = errors::Error; 25 | fn from_str(s: &str) -> Result { 26 | let r = match s { 27 | "gz" | "gzip" => CompressionAlgorithm::GZip, 28 | "br" | "brotli" => CompressionAlgorithm::Brotli, 29 | "webp" => CompressionAlgorithm::WebP, 30 | "zopfli" => CompressionAlgorithm::Zopfli, 31 | _ => bail!("Unsupported compression algorithm option set!"), 32 | }; 33 | 34 | return Ok(r); 35 | } 36 | } 37 | 38 | pub trait FileCompressor { 39 | fn compress(&self, source: &Path, destination: &Path, quality: Option) -> Result<()>; 40 | } 41 | 42 | pub trait CompressionFormat { 43 | fn extension(&self) -> &'static str; 44 | } 45 | 46 | pub struct Statistics { 47 | total_compressed: u64, 48 | total_compressed_now: u64, 49 | total_file_count: u32, 50 | total_file_count_now: u32, 51 | total_uncompressed: u64, 52 | total_uncompressed_now: u64, 53 | } 54 | 55 | impl Statistics { 56 | pub fn new() -> Statistics { 57 | Statistics { 58 | total_compressed: 0, 59 | total_compressed_now: 0, 60 | total_file_count: 0, 61 | total_file_count_now: 0, 62 | total_uncompressed: 0, 63 | total_uncompressed_now: 0, 64 | } 65 | } 66 | 67 | pub fn update(&mut self, uncompressed_size: u64, compressed_size: u64, newly_compressed: bool) { 68 | if newly_compressed { 69 | self.total_compressed_now += compressed_size; 70 | self.total_file_count_now += 1; 71 | self.total_uncompressed_now += uncompressed_size; 72 | } 73 | 74 | self.total_compressed += compressed_size; 75 | self.total_file_count += 1; 76 | self.total_uncompressed += uncompressed_size; 77 | } 78 | 79 | pub fn merge(&mut self, other: &Statistics) { 80 | self.total_compressed += other.total_compressed; 81 | self.total_compressed_now += other.total_compressed_now; 82 | self.total_file_count += other.total_file_count; 83 | self.total_file_count_now += other.total_file_count_now; 84 | self.total_uncompressed += other.total_uncompressed; 85 | self.total_uncompressed_now += other.total_uncompressed_now; 86 | } 87 | 88 | pub fn savings_ratio(&self) -> f32 { 89 | return self.total_compressed as f32 / self.total_uncompressed as f32; 90 | } 91 | 92 | pub fn savings_ratio_now(&self) -> f32 { 93 | return self.total_compressed_now as f32 / self.total_uncompressed_now as f32; 94 | } 95 | } 96 | 97 | impl std::fmt::Display for Statistics { 98 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 99 | writeln!(f, "")?; 100 | let table = table!(["", "This Run", "Total"], 101 | ["Count", self.total_file_count_now.separated_string(), self.total_file_count.separated_string()], 102 | ["Compressed Size", Size::from_bytes(self.total_compressed_now), Size::from_bytes(self.total_compressed)], 103 | ["Uncompressed Size", Size::from_bytes(self.total_uncompressed_now), Size::from_bytes(self.total_uncompressed)], 104 | ["Total Savings", format!("{:.2}%", 100f32 - 100f32 * self.savings_ratio_now()), format!("{:.2}%", 100f32 - 100f32 * self.savings_ratio())]); 105 | 106 | writeln!(f, "{}", table)?; 107 | Ok(()) 108 | } 109 | } 110 | --------------------------------------------------------------------------------