├── .gitignore ├── LICENSE ├── README.md ├── composer.json ├── composer.lock ├── mirror.config.php.dist ├── mirror.php └── phpstan.neon /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | mirror.config.php 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Nils Adermann, Jordi Boggiano 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is furnished 8 | to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Composer Repository Mirror 2 | 3 | ## Setup 4 | 5 | ### Scripted Setup 6 | 7 | - See https://github.com/peter279k/mirror-kickstarter if you are looking for automated scripts dealing with the setup of mirrors. 8 | 9 | ### Manual Setup 10 | 11 | - `git clone https://github.com/composer/mirror mirror` 12 | - `cd mirror` 13 | - `composer install` 14 | - `cp mirror.config.php.dist mirror.config.php` 15 | - Edit `mirror.config.php` to fit your needs, and mind the TODO entries which MUST be filled-in. 16 | - Run it using supervisord or similar, it is made to shutdown regularly to avoid leaks or getting stuck for any reason. There are 3 scripts you should run: 17 | - `./mirror.php --v1` should be run permanently to sync Composer 1 metadata (if you do not need Composer 1 metadata, you don't have to run this, but then you must set `has_v1_mirror` to `false` in the config) 18 | - `./mirror.php --v2` should be run permanently to sync Composer 2 metadata 19 | - `./mirror.php --gc` should be run once an hour or so with a cron job to clean up old v1 files (if you do not need Composer 1 metadata, you don't need to run this) 20 | 21 | ## Debugging and force-resync of v2 metadata 22 | 23 | In case the v2 metadata gets very outdated because you did not update the mirror for a while, this will be detected 24 | and a resync will happen automatically. 25 | 26 | However, if you want to run a resync manually to see what is going on you can use: 27 | 28 | `./mirror.php --resync -v` 29 | 30 | This will make sure the v2 metadata is in sync again (wait for the script to complete which may take a while) and 31 | then running `./mirror.php --v2` regularly again should get you back on regular updates. 32 | 33 | ## Requirements 34 | 35 | - PHP 7.3+ 36 | - A web server configured: 37 | - to send Last-Modified headers, and respond correctly to `If-Modified-Since` requests with `304 Not Modified` 38 | - to allow HTTP/2 if possible as HTTP/1 performance will be much reduced 39 | - to respond correctly with 404s for missing files 40 | 41 | ## Update 42 | 43 | In your mirror dir: 44 | 45 | - `git pull origin master` 46 | - `composer install` 47 | 48 | Then restart the workers 49 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "require": { 3 | "php": "^8.1", 4 | "ext-curl": "*", 5 | "ext-zlib": "*", 6 | "ext-json": "*", 7 | "symfony/filesystem": "^6 || ^7", 8 | "symfony/finder": "^6 || ^7", 9 | "symfony/lock": "^6 || ^7", 10 | "symfony/http-client": "^6 || ^7" 11 | }, 12 | "require-dev": { 13 | "phpstan/phpstan": "^1.10" 14 | }, 15 | "scripts": { 16 | "phpstan": "phpstan" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /composer.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_readme": [ 3 | "This file locks the dependencies of your project to a known state", 4 | "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", 5 | "This file is @generated automatically" 6 | ], 7 | "content-hash": "a36257acda0dc8a2719d296701e07f68", 8 | "packages": [ 9 | { 10 | "name": "psr/container", 11 | "version": "2.0.2", 12 | "source": { 13 | "type": "git", 14 | "url": "https://github.com/php-fig/container.git", 15 | "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963" 16 | }, 17 | "dist": { 18 | "type": "zip", 19 | "url": "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963", 20 | "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963", 21 | "shasum": "" 22 | }, 23 | "require": { 24 | "php": ">=7.4.0" 25 | }, 26 | "type": "library", 27 | "extra": { 28 | "branch-alias": { 29 | "dev-master": "2.0.x-dev" 30 | } 31 | }, 32 | "autoload": { 33 | "psr-4": { 34 | "Psr\\Container\\": "src/" 35 | } 36 | }, 37 | "notification-url": "https://packagist.org/downloads/", 38 | "license": [ 39 | "MIT" 40 | ], 41 | "authors": [ 42 | { 43 | "name": "PHP-FIG", 44 | "homepage": "https://www.php-fig.org/" 45 | } 46 | ], 47 | "description": "Common Container Interface (PHP FIG PSR-11)", 48 | "homepage": "https://github.com/php-fig/container", 49 | "keywords": [ 50 | "PSR-11", 51 | "container", 52 | "container-interface", 53 | "container-interop", 54 | "psr" 55 | ], 56 | "support": { 57 | "issues": "https://github.com/php-fig/container/issues", 58 | "source": "https://github.com/php-fig/container/tree/2.0.2" 59 | }, 60 | "time": "2021-11-05T16:47:00+00:00" 61 | }, 62 | { 63 | "name": "psr/log", 64 | "version": "3.0.2", 65 | "source": { 66 | "type": "git", 67 | "url": "https://github.com/php-fig/log.git", 68 | "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3" 69 | }, 70 | "dist": { 71 | "type": "zip", 72 | "url": "https://api.github.com/repos/php-fig/log/zipball/f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", 73 | "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", 74 | "shasum": "" 75 | }, 76 | "require": { 77 | "php": ">=8.0.0" 78 | }, 79 | "type": "library", 80 | "extra": { 81 | "branch-alias": { 82 | "dev-master": "3.x-dev" 83 | } 84 | }, 85 | "autoload": { 86 | "psr-4": { 87 | "Psr\\Log\\": "src" 88 | } 89 | }, 90 | "notification-url": "https://packagist.org/downloads/", 91 | "license": [ 92 | "MIT" 93 | ], 94 | "authors": [ 95 | { 96 | "name": "PHP-FIG", 97 | "homepage": "https://www.php-fig.org/" 98 | } 99 | ], 100 | "description": "Common interface for logging libraries", 101 | "homepage": "https://github.com/php-fig/log", 102 | "keywords": [ 103 | "log", 104 | "psr", 105 | "psr-3" 106 | ], 107 | "support": { 108 | "source": "https://github.com/php-fig/log/tree/3.0.2" 109 | }, 110 | "time": "2024-09-11T13:17:53+00:00" 111 | }, 112 | { 113 | "name": "symfony/deprecation-contracts", 114 | "version": "v3.5.0", 115 | "source": { 116 | "type": "git", 117 | "url": "https://github.com/symfony/deprecation-contracts.git", 118 | "reference": "0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1" 119 | }, 120 | "dist": { 121 | "type": "zip", 122 | "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1", 123 | "reference": "0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1", 124 | "shasum": "" 125 | }, 126 | "require": { 127 | "php": ">=8.1" 128 | }, 129 | "type": "library", 130 | "extra": { 131 | "branch-alias": { 132 | "dev-main": "3.5-dev" 133 | }, 134 | "thanks": { 135 | "name": "symfony/contracts", 136 | "url": "https://github.com/symfony/contracts" 137 | } 138 | }, 139 | "autoload": { 140 | "files": [ 141 | "function.php" 142 | ] 143 | }, 144 | "notification-url": "https://packagist.org/downloads/", 145 | "license": [ 146 | "MIT" 147 | ], 148 | "authors": [ 149 | { 150 | "name": "Nicolas Grekas", 151 | "email": "p@tchwork.com" 152 | }, 153 | { 154 | "name": "Symfony Community", 155 | "homepage": "https://symfony.com/contributors" 156 | } 157 | ], 158 | "description": "A generic function and convention to trigger deprecation notices", 159 | "homepage": "https://symfony.com", 160 | "support": { 161 | "source": "https://github.com/symfony/deprecation-contracts/tree/v3.5.0" 162 | }, 163 | "funding": [ 164 | { 165 | "url": "https://symfony.com/sponsor", 166 | "type": "custom" 167 | }, 168 | { 169 | "url": "https://github.com/fabpot", 170 | "type": "github" 171 | }, 172 | { 173 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 174 | "type": "tidelift" 175 | } 176 | ], 177 | "time": "2024-04-18T09:32:20+00:00" 178 | }, 179 | { 180 | "name": "symfony/filesystem", 181 | "version": "v7.1.6", 182 | "source": { 183 | "type": "git", 184 | "url": "https://github.com/symfony/filesystem.git", 185 | "reference": "c835867b3c62bb05c7fe3d637c871c7ae52024d4" 186 | }, 187 | "dist": { 188 | "type": "zip", 189 | "url": "https://api.github.com/repos/symfony/filesystem/zipball/c835867b3c62bb05c7fe3d637c871c7ae52024d4", 190 | "reference": "c835867b3c62bb05c7fe3d637c871c7ae52024d4", 191 | "shasum": "" 192 | }, 193 | "require": { 194 | "php": ">=8.2", 195 | "symfony/polyfill-ctype": "~1.8", 196 | "symfony/polyfill-mbstring": "~1.8" 197 | }, 198 | "require-dev": { 199 | "symfony/process": "^6.4|^7.0" 200 | }, 201 | "type": "library", 202 | "autoload": { 203 | "psr-4": { 204 | "Symfony\\Component\\Filesystem\\": "" 205 | }, 206 | "exclude-from-classmap": [ 207 | "/Tests/" 208 | ] 209 | }, 210 | "notification-url": "https://packagist.org/downloads/", 211 | "license": [ 212 | "MIT" 213 | ], 214 | "authors": [ 215 | { 216 | "name": "Fabien Potencier", 217 | "email": "fabien@symfony.com" 218 | }, 219 | { 220 | "name": "Symfony Community", 221 | "homepage": "https://symfony.com/contributors" 222 | } 223 | ], 224 | "description": "Provides basic utilities for the filesystem", 225 | "homepage": "https://symfony.com", 226 | "support": { 227 | "source": "https://github.com/symfony/filesystem/tree/v7.1.6" 228 | }, 229 | "funding": [ 230 | { 231 | "url": "https://symfony.com/sponsor", 232 | "type": "custom" 233 | }, 234 | { 235 | "url": "https://github.com/fabpot", 236 | "type": "github" 237 | }, 238 | { 239 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 240 | "type": "tidelift" 241 | } 242 | ], 243 | "time": "2024-10-25T15:11:02+00:00" 244 | }, 245 | { 246 | "name": "symfony/finder", 247 | "version": "v7.1.6", 248 | "source": { 249 | "type": "git", 250 | "url": "https://github.com/symfony/finder.git", 251 | "reference": "2cb89664897be33f78c65d3d2845954c8d7a43b8" 252 | }, 253 | "dist": { 254 | "type": "zip", 255 | "url": "https://api.github.com/repos/symfony/finder/zipball/2cb89664897be33f78c65d3d2845954c8d7a43b8", 256 | "reference": "2cb89664897be33f78c65d3d2845954c8d7a43b8", 257 | "shasum": "" 258 | }, 259 | "require": { 260 | "php": ">=8.2" 261 | }, 262 | "require-dev": { 263 | "symfony/filesystem": "^6.4|^7.0" 264 | }, 265 | "type": "library", 266 | "autoload": { 267 | "psr-4": { 268 | "Symfony\\Component\\Finder\\": "" 269 | }, 270 | "exclude-from-classmap": [ 271 | "/Tests/" 272 | ] 273 | }, 274 | "notification-url": "https://packagist.org/downloads/", 275 | "license": [ 276 | "MIT" 277 | ], 278 | "authors": [ 279 | { 280 | "name": "Fabien Potencier", 281 | "email": "fabien@symfony.com" 282 | }, 283 | { 284 | "name": "Symfony Community", 285 | "homepage": "https://symfony.com/contributors" 286 | } 287 | ], 288 | "description": "Finds files and directories via an intuitive fluent interface", 289 | "homepage": "https://symfony.com", 290 | "support": { 291 | "source": "https://github.com/symfony/finder/tree/v7.1.6" 292 | }, 293 | "funding": [ 294 | { 295 | "url": "https://symfony.com/sponsor", 296 | "type": "custom" 297 | }, 298 | { 299 | "url": "https://github.com/fabpot", 300 | "type": "github" 301 | }, 302 | { 303 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 304 | "type": "tidelift" 305 | } 306 | ], 307 | "time": "2024-10-01T08:31:23+00:00" 308 | }, 309 | { 310 | "name": "symfony/http-client", 311 | "version": "v7.1.8", 312 | "source": { 313 | "type": "git", 314 | "url": "https://github.com/symfony/http-client.git", 315 | "reference": "c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a" 316 | }, 317 | "dist": { 318 | "type": "zip", 319 | "url": "https://api.github.com/repos/symfony/http-client/zipball/c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a", 320 | "reference": "c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a", 321 | "shasum": "" 322 | }, 323 | "require": { 324 | "php": ">=8.2", 325 | "psr/log": "^1|^2|^3", 326 | "symfony/deprecation-contracts": "^2.5|^3", 327 | "symfony/http-client-contracts": "^3.4.1", 328 | "symfony/service-contracts": "^2.5|^3" 329 | }, 330 | "conflict": { 331 | "php-http/discovery": "<1.15", 332 | "symfony/http-foundation": "<6.4" 333 | }, 334 | "provide": { 335 | "php-http/async-client-implementation": "*", 336 | "php-http/client-implementation": "*", 337 | "psr/http-client-implementation": "1.0", 338 | "symfony/http-client-implementation": "3.0" 339 | }, 340 | "require-dev": { 341 | "amphp/amp": "^2.5", 342 | "amphp/http-client": "^4.2.1", 343 | "amphp/http-tunnel": "^1.0", 344 | "amphp/socket": "^1.1", 345 | "guzzlehttp/promises": "^1.4|^2.0", 346 | "nyholm/psr7": "^1.0", 347 | "php-http/httplug": "^1.0|^2.0", 348 | "psr/http-client": "^1.0", 349 | "symfony/dependency-injection": "^6.4|^7.0", 350 | "symfony/http-kernel": "^6.4|^7.0", 351 | "symfony/messenger": "^6.4|^7.0", 352 | "symfony/process": "^6.4|^7.0", 353 | "symfony/rate-limiter": "^6.4|^7.0", 354 | "symfony/stopwatch": "^6.4|^7.0" 355 | }, 356 | "type": "library", 357 | "autoload": { 358 | "psr-4": { 359 | "Symfony\\Component\\HttpClient\\": "" 360 | }, 361 | "exclude-from-classmap": [ 362 | "/Tests/" 363 | ] 364 | }, 365 | "notification-url": "https://packagist.org/downloads/", 366 | "license": [ 367 | "MIT" 368 | ], 369 | "authors": [ 370 | { 371 | "name": "Nicolas Grekas", 372 | "email": "p@tchwork.com" 373 | }, 374 | { 375 | "name": "Symfony Community", 376 | "homepage": "https://symfony.com/contributors" 377 | } 378 | ], 379 | "description": "Provides powerful methods to fetch HTTP resources synchronously or asynchronously", 380 | "homepage": "https://symfony.com", 381 | "keywords": [ 382 | "http" 383 | ], 384 | "support": { 385 | "source": "https://github.com/symfony/http-client/tree/v7.1.8" 386 | }, 387 | "funding": [ 388 | { 389 | "url": "https://symfony.com/sponsor", 390 | "type": "custom" 391 | }, 392 | { 393 | "url": "https://github.com/fabpot", 394 | "type": "github" 395 | }, 396 | { 397 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 398 | "type": "tidelift" 399 | } 400 | ], 401 | "time": "2024-11-13T13:40:27+00:00" 402 | }, 403 | { 404 | "name": "symfony/http-client-contracts", 405 | "version": "v3.5.0", 406 | "source": { 407 | "type": "git", 408 | "url": "https://github.com/symfony/http-client-contracts.git", 409 | "reference": "20414d96f391677bf80078aa55baece78b82647d" 410 | }, 411 | "dist": { 412 | "type": "zip", 413 | "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/20414d96f391677bf80078aa55baece78b82647d", 414 | "reference": "20414d96f391677bf80078aa55baece78b82647d", 415 | "shasum": "" 416 | }, 417 | "require": { 418 | "php": ">=8.1" 419 | }, 420 | "type": "library", 421 | "extra": { 422 | "branch-alias": { 423 | "dev-main": "3.5-dev" 424 | }, 425 | "thanks": { 426 | "name": "symfony/contracts", 427 | "url": "https://github.com/symfony/contracts" 428 | } 429 | }, 430 | "autoload": { 431 | "psr-4": { 432 | "Symfony\\Contracts\\HttpClient\\": "" 433 | }, 434 | "exclude-from-classmap": [ 435 | "/Test/" 436 | ] 437 | }, 438 | "notification-url": "https://packagist.org/downloads/", 439 | "license": [ 440 | "MIT" 441 | ], 442 | "authors": [ 443 | { 444 | "name": "Nicolas Grekas", 445 | "email": "p@tchwork.com" 446 | }, 447 | { 448 | "name": "Symfony Community", 449 | "homepage": "https://symfony.com/contributors" 450 | } 451 | ], 452 | "description": "Generic abstractions related to HTTP clients", 453 | "homepage": "https://symfony.com", 454 | "keywords": [ 455 | "abstractions", 456 | "contracts", 457 | "decoupling", 458 | "interfaces", 459 | "interoperability", 460 | "standards" 461 | ], 462 | "support": { 463 | "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.0" 464 | }, 465 | "funding": [ 466 | { 467 | "url": "https://symfony.com/sponsor", 468 | "type": "custom" 469 | }, 470 | { 471 | "url": "https://github.com/fabpot", 472 | "type": "github" 473 | }, 474 | { 475 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 476 | "type": "tidelift" 477 | } 478 | ], 479 | "time": "2024-04-18T09:32:20+00:00" 480 | }, 481 | { 482 | "name": "symfony/lock", 483 | "version": "v7.1.6", 484 | "source": { 485 | "type": "git", 486 | "url": "https://github.com/symfony/lock.git", 487 | "reference": "1b898398007d80b4f32128df4b4f0c07c0368cf4" 488 | }, 489 | "dist": { 490 | "type": "zip", 491 | "url": "https://api.github.com/repos/symfony/lock/zipball/1b898398007d80b4f32128df4b4f0c07c0368cf4", 492 | "reference": "1b898398007d80b4f32128df4b4f0c07c0368cf4", 493 | "shasum": "" 494 | }, 495 | "require": { 496 | "php": ">=8.2", 497 | "psr/log": "^1|^2|^3" 498 | }, 499 | "conflict": { 500 | "doctrine/dbal": "<3.6", 501 | "symfony/cache": "<6.4" 502 | }, 503 | "require-dev": { 504 | "doctrine/dbal": "^3.6|^4", 505 | "predis/predis": "^1.1|^2.0" 506 | }, 507 | "type": "library", 508 | "autoload": { 509 | "psr-4": { 510 | "Symfony\\Component\\Lock\\": "" 511 | }, 512 | "exclude-from-classmap": [ 513 | "/Tests/" 514 | ] 515 | }, 516 | "notification-url": "https://packagist.org/downloads/", 517 | "license": [ 518 | "MIT" 519 | ], 520 | "authors": [ 521 | { 522 | "name": "Jérémy Derussé", 523 | "email": "jeremy@derusse.com" 524 | }, 525 | { 526 | "name": "Symfony Community", 527 | "homepage": "https://symfony.com/contributors" 528 | } 529 | ], 530 | "description": "Creates and manages locks, a mechanism to provide exclusive access to a shared resource", 531 | "homepage": "https://symfony.com", 532 | "keywords": [ 533 | "cas", 534 | "flock", 535 | "locking", 536 | "mutex", 537 | "redlock", 538 | "semaphore" 539 | ], 540 | "support": { 541 | "source": "https://github.com/symfony/lock/tree/v7.1.6" 542 | }, 543 | "funding": [ 544 | { 545 | "url": "https://symfony.com/sponsor", 546 | "type": "custom" 547 | }, 548 | { 549 | "url": "https://github.com/fabpot", 550 | "type": "github" 551 | }, 552 | { 553 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 554 | "type": "tidelift" 555 | } 556 | ], 557 | "time": "2024-10-25T15:34:21+00:00" 558 | }, 559 | { 560 | "name": "symfony/polyfill-ctype", 561 | "version": "v1.31.0", 562 | "source": { 563 | "type": "git", 564 | "url": "https://github.com/symfony/polyfill-ctype.git", 565 | "reference": "a3cc8b044a6ea513310cbd48ef7333b384945638" 566 | }, 567 | "dist": { 568 | "type": "zip", 569 | "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/a3cc8b044a6ea513310cbd48ef7333b384945638", 570 | "reference": "a3cc8b044a6ea513310cbd48ef7333b384945638", 571 | "shasum": "" 572 | }, 573 | "require": { 574 | "php": ">=7.2" 575 | }, 576 | "provide": { 577 | "ext-ctype": "*" 578 | }, 579 | "suggest": { 580 | "ext-ctype": "For best performance" 581 | }, 582 | "type": "library", 583 | "extra": { 584 | "thanks": { 585 | "name": "symfony/polyfill", 586 | "url": "https://github.com/symfony/polyfill" 587 | } 588 | }, 589 | "autoload": { 590 | "files": [ 591 | "bootstrap.php" 592 | ], 593 | "psr-4": { 594 | "Symfony\\Polyfill\\Ctype\\": "" 595 | } 596 | }, 597 | "notification-url": "https://packagist.org/downloads/", 598 | "license": [ 599 | "MIT" 600 | ], 601 | "authors": [ 602 | { 603 | "name": "Gert de Pagter", 604 | "email": "BackEndTea@gmail.com" 605 | }, 606 | { 607 | "name": "Symfony Community", 608 | "homepage": "https://symfony.com/contributors" 609 | } 610 | ], 611 | "description": "Symfony polyfill for ctype functions", 612 | "homepage": "https://symfony.com", 613 | "keywords": [ 614 | "compatibility", 615 | "ctype", 616 | "polyfill", 617 | "portable" 618 | ], 619 | "support": { 620 | "source": "https://github.com/symfony/polyfill-ctype/tree/v1.31.0" 621 | }, 622 | "funding": [ 623 | { 624 | "url": "https://symfony.com/sponsor", 625 | "type": "custom" 626 | }, 627 | { 628 | "url": "https://github.com/fabpot", 629 | "type": "github" 630 | }, 631 | { 632 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 633 | "type": "tidelift" 634 | } 635 | ], 636 | "time": "2024-09-09T11:45:10+00:00" 637 | }, 638 | { 639 | "name": "symfony/polyfill-mbstring", 640 | "version": "v1.31.0", 641 | "source": { 642 | "type": "git", 643 | "url": "https://github.com/symfony/polyfill-mbstring.git", 644 | "reference": "85181ba99b2345b0ef10ce42ecac37612d9fd341" 645 | }, 646 | "dist": { 647 | "type": "zip", 648 | "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/85181ba99b2345b0ef10ce42ecac37612d9fd341", 649 | "reference": "85181ba99b2345b0ef10ce42ecac37612d9fd341", 650 | "shasum": "" 651 | }, 652 | "require": { 653 | "php": ">=7.2" 654 | }, 655 | "provide": { 656 | "ext-mbstring": "*" 657 | }, 658 | "suggest": { 659 | "ext-mbstring": "For best performance" 660 | }, 661 | "type": "library", 662 | "extra": { 663 | "thanks": { 664 | "name": "symfony/polyfill", 665 | "url": "https://github.com/symfony/polyfill" 666 | } 667 | }, 668 | "autoload": { 669 | "files": [ 670 | "bootstrap.php" 671 | ], 672 | "psr-4": { 673 | "Symfony\\Polyfill\\Mbstring\\": "" 674 | } 675 | }, 676 | "notification-url": "https://packagist.org/downloads/", 677 | "license": [ 678 | "MIT" 679 | ], 680 | "authors": [ 681 | { 682 | "name": "Nicolas Grekas", 683 | "email": "p@tchwork.com" 684 | }, 685 | { 686 | "name": "Symfony Community", 687 | "homepage": "https://symfony.com/contributors" 688 | } 689 | ], 690 | "description": "Symfony polyfill for the Mbstring extension", 691 | "homepage": "https://symfony.com", 692 | "keywords": [ 693 | "compatibility", 694 | "mbstring", 695 | "polyfill", 696 | "portable", 697 | "shim" 698 | ], 699 | "support": { 700 | "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.31.0" 701 | }, 702 | "funding": [ 703 | { 704 | "url": "https://symfony.com/sponsor", 705 | "type": "custom" 706 | }, 707 | { 708 | "url": "https://github.com/fabpot", 709 | "type": "github" 710 | }, 711 | { 712 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 713 | "type": "tidelift" 714 | } 715 | ], 716 | "time": "2024-09-09T11:45:10+00:00" 717 | }, 718 | { 719 | "name": "symfony/service-contracts", 720 | "version": "v3.5.0", 721 | "source": { 722 | "type": "git", 723 | "url": "https://github.com/symfony/service-contracts.git", 724 | "reference": "bd1d9e59a81d8fa4acdcea3f617c581f7475a80f" 725 | }, 726 | "dist": { 727 | "type": "zip", 728 | "url": "https://api.github.com/repos/symfony/service-contracts/zipball/bd1d9e59a81d8fa4acdcea3f617c581f7475a80f", 729 | "reference": "bd1d9e59a81d8fa4acdcea3f617c581f7475a80f", 730 | "shasum": "" 731 | }, 732 | "require": { 733 | "php": ">=8.1", 734 | "psr/container": "^1.1|^2.0", 735 | "symfony/deprecation-contracts": "^2.5|^3" 736 | }, 737 | "conflict": { 738 | "ext-psr": "<1.1|>=2" 739 | }, 740 | "type": "library", 741 | "extra": { 742 | "branch-alias": { 743 | "dev-main": "3.5-dev" 744 | }, 745 | "thanks": { 746 | "name": "symfony/contracts", 747 | "url": "https://github.com/symfony/contracts" 748 | } 749 | }, 750 | "autoload": { 751 | "psr-4": { 752 | "Symfony\\Contracts\\Service\\": "" 753 | }, 754 | "exclude-from-classmap": [ 755 | "/Test/" 756 | ] 757 | }, 758 | "notification-url": "https://packagist.org/downloads/", 759 | "license": [ 760 | "MIT" 761 | ], 762 | "authors": [ 763 | { 764 | "name": "Nicolas Grekas", 765 | "email": "p@tchwork.com" 766 | }, 767 | { 768 | "name": "Symfony Community", 769 | "homepage": "https://symfony.com/contributors" 770 | } 771 | ], 772 | "description": "Generic abstractions related to writing services", 773 | "homepage": "https://symfony.com", 774 | "keywords": [ 775 | "abstractions", 776 | "contracts", 777 | "decoupling", 778 | "interfaces", 779 | "interoperability", 780 | "standards" 781 | ], 782 | "support": { 783 | "source": "https://github.com/symfony/service-contracts/tree/v3.5.0" 784 | }, 785 | "funding": [ 786 | { 787 | "url": "https://symfony.com/sponsor", 788 | "type": "custom" 789 | }, 790 | { 791 | "url": "https://github.com/fabpot", 792 | "type": "github" 793 | }, 794 | { 795 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 796 | "type": "tidelift" 797 | } 798 | ], 799 | "time": "2024-04-18T09:32:20+00:00" 800 | } 801 | ], 802 | "packages-dev": [ 803 | { 804 | "name": "phpstan/phpstan", 805 | "version": "1.12.11", 806 | "source": { 807 | "type": "git", 808 | "url": "https://github.com/phpstan/phpstan.git", 809 | "reference": "0d1fc20a962a91be578bcfe7cf939e6e1a2ff733" 810 | }, 811 | "dist": { 812 | "type": "zip", 813 | "url": "https://api.github.com/repos/phpstan/phpstan/zipball/0d1fc20a962a91be578bcfe7cf939e6e1a2ff733", 814 | "reference": "0d1fc20a962a91be578bcfe7cf939e6e1a2ff733", 815 | "shasum": "" 816 | }, 817 | "require": { 818 | "php": "^7.2|^8.0" 819 | }, 820 | "conflict": { 821 | "phpstan/phpstan-shim": "*" 822 | }, 823 | "bin": [ 824 | "phpstan", 825 | "phpstan.phar" 826 | ], 827 | "type": "library", 828 | "autoload": { 829 | "files": [ 830 | "bootstrap.php" 831 | ] 832 | }, 833 | "notification-url": "https://packagist.org/downloads/", 834 | "license": [ 835 | "MIT" 836 | ], 837 | "description": "PHPStan - PHP Static Analysis Tool", 838 | "keywords": [ 839 | "dev", 840 | "static analysis" 841 | ], 842 | "support": { 843 | "docs": "https://phpstan.org/user-guide/getting-started", 844 | "forum": "https://github.com/phpstan/phpstan/discussions", 845 | "issues": "https://github.com/phpstan/phpstan/issues", 846 | "security": "https://github.com/phpstan/phpstan/security/policy", 847 | "source": "https://github.com/phpstan/phpstan-src" 848 | }, 849 | "funding": [ 850 | { 851 | "url": "https://github.com/ondrejmirtes", 852 | "type": "github" 853 | }, 854 | { 855 | "url": "https://github.com/phpstan", 856 | "type": "github" 857 | } 858 | ], 859 | "time": "2024-11-17T14:08:01+00:00" 860 | } 861 | ], 862 | "aliases": [], 863 | "minimum-stability": "stable", 864 | "stability-flags": {}, 865 | "prefer-stable": false, 866 | "prefer-lowest": false, 867 | "platform": { 868 | "php": "^8.1", 869 | "ext-curl": "*", 870 | "ext-zlib": "*", 871 | "ext-json": "*" 872 | }, 873 | "platform-dev": {}, 874 | "plugin-api-version": "2.6.0" 875 | } 876 | -------------------------------------------------------------------------------- /mirror.config.php.dist: -------------------------------------------------------------------------------- 1 | /* TODO */, 6 | // user agent describing your mirror node, if possible include domain name of mirror, and a contact email address 7 | 'user_agent' => /* TODO Mirror for foo.com (mycontact@example.org) */, 8 | // source repository URL 9 | 'repo_url' => 'https://repo.packagist.org', 10 | // source repository hostname (optional, will guess from repo_url) 11 | //'repo_hostname' => 'repo.packagist.org', 12 | // source API URL 13 | 'api_url' => 'https://packagist.org', 14 | // how many times the script will run the mirroring step before exiting 15 | 'iterations' => 120, 16 | // how many seconds to wait between mirror runs 17 | 'iteration_interval' => 5, 18 | // set this to false if you do not run the --v1 mirror job, to ensure that the v2 will then take care of syncing packages.json 19 | 'has_v1_mirror' => true, 20 | ]; 21 | -------------------------------------------------------------------------------- /mirror.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | $config 51 | */ 52 | public function __construct(array $config) 53 | { 54 | $this->target = $config['target_dir']; 55 | $this->url = $config['repo_url'] ?? 'repo.packagist.org'; 56 | $this->apiUrl = $config['api_url'] ?? 'packagist.org'; 57 | $this->hostname = $config['repo_hostname'] ?? parse_url($this->url, PHP_URL_HOST); 58 | $this->userAgent = $config['user_agent']; 59 | $this->syncRootOnV2 = !($config['has_v1_mirror'] ?? true); 60 | $this->gzipOnly = $config['gzip_only'] ?? false; 61 | 62 | if (isset($config['statsd']) && is_array($config['statsd'])) { 63 | $this->statsdConnect($config['statsd'][0], $config['statsd'][1]); 64 | } 65 | 66 | $this->verbose = in_array('-v', $_SERVER['argv']); 67 | if ($this->verbose) { 68 | ini_set('display_errors', 1); 69 | } 70 | } 71 | 72 | public function syncRootOnV2(): void 73 | { 74 | if (!$this->syncRootOnV2) { 75 | return; 76 | } 77 | 78 | $this->initClient(); 79 | 80 | $rootResp = $this->download('/packages.json'); 81 | if ($rootResp->getHeaders()['content-encoding'][0] !== 'gzip') { 82 | throw new \Exception('Expected gzip encoded responses, something is off'); 83 | } 84 | $rootData = $rootResp->getContent(); 85 | $hash = hash('sha256', $rootData); 86 | 87 | if ($hash === $this->getHash('/packages.json')) { 88 | return; 89 | } 90 | 91 | $gzipped = $this->gzencode($rootData, 8); 92 | $this->write('/packages.json', $rootData, $gzipped, $this->strtotime($rootResp->getHeaders()['last-modified'][0])); 93 | $this->output('X'); 94 | 95 | $this->statsdIncrement('mirror.sync_root'); 96 | } 97 | 98 | public function getV2Timestamp(): int 99 | { 100 | $this->initClient(); 101 | 102 | $resp = $this->client->request('GET', $this->apiUrl.'/metadata/changes.json', ['headers' => ['Host' => parse_url($this->apiUrl, PHP_URL_HOST)]]); 103 | $content = json_decode($resp->getContent(false), true); 104 | if ($resp->getStatusCode() === 400 && null !== $content) { 105 | return $content['timestamp']; 106 | } 107 | throw new \Exception('Failed to fetch timestamp from API, got invalid response '.$resp->getStatusCode().': '.$resp->getContent()); 108 | } 109 | 110 | public function syncV2(): bool 111 | { 112 | $this->initClient(); 113 | 114 | $this->statsdIncrement('mirror.run'); 115 | $this->downloaded = 0; 116 | 117 | if (!file_exists($this->getTimestampStorePath())) { 118 | if (!is_writable((string) getcwd())) { 119 | throw new \UnexpectedValueException('Cannot save last timestamp to last_metadata_timestamp in '.getcwd().'. Make sure the directory is writable.'); 120 | } 121 | 122 | return $this->resync($this->getV2Timestamp()); 123 | } elseif (!is_writable($this->getTimestampStorePath())) { 124 | throw new \UnexpectedValueException('Cannot save last timestamp to last_metadata_timestamp in '.getcwd().'. Make sure the file is writable.'); 125 | } 126 | $lastTime = trim((string) file_get_contents($this->getTimestampStorePath())); 127 | if ($lastTime === '') { 128 | throw new \RuntimeException('Failed loading the last timestamp from '.$this->getTimestampStorePath()); 129 | } 130 | 131 | $changesResp = $this->client->request('GET', $this->apiUrl.'/metadata/changes.json?since='.$lastTime, ['headers' => ['Host' => parse_url($this->apiUrl, PHP_URL_HOST)]]); 132 | if ($changesResp->getHeaders()['content-encoding'][0] !== 'gzip') { 133 | throw new \Exception('Expected gzip encoded responses, something is off'); 134 | } 135 | $changes = json_decode($changesResp->getContent(), true); 136 | 137 | if ([] === $changes['actions']) { 138 | $this->output('No work' . PHP_EOL); 139 | $this->writeLastTimestamp($changes['timestamp']); 140 | return true; 141 | } 142 | 143 | if ($changes['actions'][0]['type'] === 'resync') { 144 | return $this->resync($changes['timestamp']); 145 | } 146 | 147 | $requests = []; 148 | foreach ($changes['actions'] as $action) { 149 | if ($action['type'] === 'update') { 150 | // package here can be foo/bar or foo/bar~dev, not strictly a package name 151 | $pkg = $action['package']; 152 | $provPathV2 = '/p2/'.$pkg.'.json'; 153 | $headers = file_exists($this->target.$provPathV2.'.gz') ? ['If-Modified-Since' => gmdate('D, d M Y H:i:s T', (int) filemtime($this->target.$provPathV2.'.gz'))] : []; 154 | $userData = ['path' => $provPathV2, 'minimumFilemtime' => $action['time'], 'retries' => 0]; 155 | $requests[] = ['GET', $this->url.$provPathV2, ['user_data' => $userData, 'headers' => $headers]]; 156 | } elseif ($action['type'] === 'delete') { 157 | $this->delete($action['package']); 158 | } 159 | } 160 | 161 | $result = $this->downloadV2Files($requests); 162 | if (!$result) { 163 | return false; 164 | } 165 | 166 | $this->output(PHP_EOL); 167 | $this->output('Downloaded '.$this->downloaded.' files'.PHP_EOL); 168 | $this->writeLastTimestamp($changes['timestamp']); 169 | 170 | return true; 171 | } 172 | 173 | public function resync(int $timestamp): bool 174 | { 175 | $this->output('Resync requested'.PHP_EOL); 176 | 177 | $listingResp = $this->client->request('GET', $this->apiUrl.'/packages/list.json?'.md5(uniqid()), ['headers' => ['Host' => parse_url($this->apiUrl, PHP_URL_HOST)]]); 178 | if ($listingResp->getHeaders()['content-encoding'][0] !== 'gzip') { 179 | throw new \Exception('Expected gzip encoded responses, something is off'); 180 | } 181 | $list = json_decode($listingResp->getContent(), true); 182 | 183 | // clean up existing files in case we still have outdated packages 184 | if (is_dir($this->target.'/p2')) { 185 | $finder = Finder::create()->directories()->ignoreVCS(true)->in($this->target.'/p2'); 186 | $names = array_flip($list['packageNames']); 187 | 188 | foreach ($finder as $vendorDir) { 189 | $globMatches = glob(((string) $vendorDir).'/*.json.gz'); 190 | if (!is_array($globMatches)) { 191 | throw new \RuntimeException('Failed globbing '.$vendorDir.'/*.json.gz'); 192 | } 193 | foreach ($globMatches as $file) { 194 | if (!preg_match('{/([^/]+/[^/]+?)(~dev)?\.json.gz$}', strtr($file, '\\', '/'), $match)) { 195 | throw new \LogicException('Could not match package name from '.$file); 196 | } 197 | 198 | if (!isset($names[$match[1]])) { 199 | unlink((string) $file); 200 | // also remove the version without .gz suffix if it exists 201 | if (file_exists(substr((string) $file, 0, -3))) { 202 | unlink(substr((string) $file, 0, -3)); 203 | } 204 | continue; 205 | } 206 | 207 | // check for corrupted file if it is older than the bugfix date, and remove if they are corrupted 208 | if (filemtime((string) $file) < 1606297354) { 209 | $contents = $this->decodeFile((string) $file); 210 | if (!isset($contents['packages'][$match[1]])) { 211 | unlink((string) $file); 212 | // also remove the version without .gz suffix if it exists 213 | if (file_exists(substr((string) $file, 0, -3))) { 214 | unlink(substr((string) $file, 0, -3)); 215 | } 216 | continue; 217 | } 218 | } 219 | } 220 | } 221 | } 222 | 223 | // download all package data 224 | $requests = []; 225 | $appendRequest = function ($path) use (&$requests) { 226 | $headers = []; 227 | $filemtime = file_exists($this->target.$path.'.gz') ? filemtime($this->target.$path.'.gz') : 0; 228 | if ($filemtime) { 229 | $headers = ['If-Modified-Since' => gmdate('D, d M Y H:i:s T', $filemtime)]; 230 | } 231 | $userData = ['path' => $path, 'minimumFilemtime' => 0, 'retries' => 0, 'resyncIfNewerThanSource' => true]; 232 | $requests[] = ['GET', $this->url.$path, ['user_data' => $userData, 'headers' => $headers]]; 233 | }; 234 | 235 | foreach ($list['packageNames'] as $pkg) { 236 | $appendRequest('/p2/'.$pkg.'.json'); 237 | $appendRequest('/p2/'.$pkg.'~dev.json'); 238 | } 239 | 240 | $result = $this->downloadV2Files($requests); 241 | if (!$result) { 242 | return false; 243 | } 244 | 245 | $this->output(PHP_EOL); 246 | $this->output('Downloaded '.$this->downloaded.' files'.PHP_EOL); 247 | 248 | $this->writeLastTimestamp($timestamp); 249 | 250 | $this->statsdIncrement('mirror.resync'); 251 | 252 | return true; 253 | } 254 | 255 | private function getTimestampStorePath(): string 256 | { 257 | $timestampStore = __DIR__.'/last_metadata_timestamp'; 258 | 259 | // migrate legacy path to new dir if available 260 | $timestampStoreLegacy = './last_metadata_timestamp'; 261 | if (file_exists($timestampStoreLegacy) && !file_exists($timestampStore)) { 262 | rename($timestampStoreLegacy, $timestampStore); 263 | } 264 | 265 | return $timestampStore; 266 | } 267 | 268 | private function writeLastTimestamp(int $timestamp): void 269 | { 270 | if (false === file_put_contents($this->getTimestampStorePath(), $timestamp)) { 271 | throw new \UnexpectedValueException('Could not save last timestamp to last_metadata_timestamp in '.getcwd().'. Make sure the file/directory is writable'); 272 | } 273 | } 274 | 275 | /** 276 | * @param array}> $requests 277 | */ 278 | private function downloadV2Files(array $requests): bool 279 | { 280 | $hasRetries = false; 281 | 282 | $responseNeedsRetry = function ($response, array $userData) use (&$hasRetries, &$requests): bool { 283 | $is404 = $response->getStatusCode() === 404; 284 | if (!$is404) { 285 | $mtime = $this->strtotime($response->getHeaders(false)['last-modified'][0]); 286 | } 287 | 288 | // got an outdated file, possibly fetched from a mirror which was not yet up to date, so retry after 2sec 289 | if ($is404 || $mtime < $userData['minimumFilemtime']) { 290 | if ($userData['retries'] > 2) { 291 | // 404s after 3 retries should be deemed to have really been deleted, so we stop retrying 292 | if ($is404) { 293 | return false; 294 | } 295 | throw new \Exception('Too many retries, could not update '.$userData['path'].' as the origin server returns an older file ('.$mtime.', expected '.$userData['minimumFilemtime'].')'); 296 | } 297 | $hasRetries = true; 298 | $this->output('R'); 299 | $this->statsdIncrement('mirror.retry_provider_v2'); 300 | $userData['retries']++; 301 | $headers = file_exists($this->target.$userData['path'].'.gz') ? ['If-Modified-Since' => gmdate('D, d M Y H:i:s T', (int) filemtime($this->target.$userData['path'].'.gz'))] : []; 302 | $requests[] = ['GET', $this->url.$userData['path'], ['user_data' => $userData, 'headers' => $headers]]; 303 | 304 | return true; 305 | } 306 | 307 | return false; 308 | }; 309 | 310 | $retryFailedReq = function (\Throwable $e, array $userData) use (&$hasRetries, &$requests): bool { 311 | if ($userData['retries'] > 2) { 312 | return false; 313 | } 314 | 315 | $hasRetries = true; 316 | $this->output('E'); 317 | $this->statsdIncrement('mirror.retry_provider_v2_error'); 318 | $userData['retries']++; 319 | $headers = file_exists($this->target.$userData['path'].'.gz') ? ['If-Modified-Since' => gmdate('D, d M Y H:i:s T', (int) filemtime($this->target.$userData['path'].'.gz'))] : []; 320 | array_unshift($requests, ['GET', $this->url.$userData['path'], ['user_data' => $userData, 'headers' => $headers]]); 321 | 322 | return true; 323 | }; 324 | 325 | while ($requests) { 326 | if ($hasRetries) { 327 | sleep(2); 328 | $hasRetries = false; 329 | } 330 | 331 | $responses = []; 332 | foreach (array_splice($requests, 0, 200) as $req) { 333 | $responses[] = $this->client->request(...$req); 334 | } 335 | 336 | foreach ($this->client->stream($responses) as $response => $chunk) { 337 | try { 338 | if ($chunk->isFirst()) { 339 | if ($response->getStatusCode() === 304) { 340 | $response->cancel(); 341 | $this->downloaded++; 342 | 343 | // retry if the response is an outdated 304 as the mirror we are syncing from 344 | // looks outdated still 345 | $userData = $response->getInfo('user_data'); 346 | if ($responseNeedsRetry($response, $userData)) { 347 | continue; 348 | } 349 | 350 | $this->output('-'); 351 | $this->statsdIncrement('mirror.not_modified'); 352 | continue; 353 | } 354 | 355 | if ($response->getStatusCode() === 404) { 356 | $response->cancel(); 357 | $this->downloaded++; 358 | 359 | // 404s need to be retried just in case the mirror we are syncing from is not yet up to date 360 | // as othwerise this can lead to missing new packages' files as they'll be 404 instead of outdated 304s 361 | $userData = $response->getInfo('user_data'); 362 | if ($responseNeedsRetry($response, $userData)) { 363 | continue; 364 | } 365 | 366 | // ignore 404s for all v2 files as the package might have been deleted already 367 | $this->output('?'); 368 | $this->statsdIncrement('mirror.not_found'); 369 | continue; 370 | } 371 | } 372 | 373 | if ($chunk->isLast()) { 374 | $this->downloaded++; 375 | $userData = $response->getInfo('user_data'); 376 | 377 | $metadata = $response->getContent(); 378 | $decoded = json_decode($metadata, true); 379 | if (null === $decoded) { 380 | throw new \Exception('Invalid JSON received for file '.$userData['path']); 381 | } 382 | 383 | // check the response is for the correct package as safety check against corruption 384 | if (!preg_match('{^/p2/(.+?)(~dev)?.json$}', $userData['path'], $match)) { 385 | throw new \Exception('Invalid path could not be parsed: '.$userData['path']); 386 | } 387 | $packageName = $match[1]; 388 | if (!isset($decoded['packages'][$packageName])) { 389 | throw new \Exception('Invalid response for file '.$userData['path'].', '.$packageName.' could not be found in file content: '.substr($metadata, 0, 300)); 390 | } 391 | foreach ($decoded['packages'][$packageName] as $version) { 392 | $isDevVersion = (bool) preg_match('{^dev-|-dev$}', $version['version_normalized']); 393 | if (($match[2] ?? '') === '~dev' && !$isDevVersion) { 394 | throw new \Exception('Invalid response for file '.$userData['path'].', expected dev versions and got non-dev ones: '.substr($metadata, 0, 300)); 395 | } 396 | if (($match[2] ?? '') === '' && $isDevVersion) { 397 | throw new \Exception('Invalid response for file '.$userData['path'].', expected non-dev versions and got dev ones: '.substr($metadata, 0, 300)); 398 | } 399 | break; 400 | } 401 | 402 | if ($responseNeedsRetry($response, $userData)) { 403 | continue; 404 | } 405 | 406 | $mtime = $this->strtotime($response->getHeaders()['last-modified'][0]); 407 | $gzipped = $this->gzencode($metadata, 7); 408 | $this->write($userData['path'], $metadata, $gzipped, $mtime); 409 | $this->output('M'); 410 | $this->statsdIncrement('mirror.sync_provider_v2'); 411 | } 412 | } catch (\Throwable $e) { 413 | // if it can be retried, we skip it for now 414 | if ($retryFailedReq($e, $response->getInfo('user_data'))) { 415 | $response->cancel(); 416 | $this->downloaded++; 417 | continue; 418 | } 419 | 420 | // abort all responses to avoid triggering any other exception then throw 421 | array_map(function ($r) { $r->cancel(); }, $responses); 422 | 423 | $this->statsdIncrement('mirror.provider_failure'); 424 | $this->downloaded++; 425 | throw $e; 426 | } 427 | } 428 | } 429 | 430 | return true; 431 | } 432 | 433 | public function sync(): bool 434 | { 435 | $this->initClient(); 436 | 437 | $this->statsdIncrement('mirror.run'); 438 | $this->downloaded = 0; 439 | 440 | $rootResp = $this->download('/packages.json'); 441 | if ($rootResp->getHeaders()['content-encoding'][0] !== 'gzip') { 442 | throw new \Exception('Expected gzip encoded responses, something is off'); 443 | } 444 | $rootData = $rootResp->getContent(); 445 | $hash = hash('sha256', $rootData); 446 | 447 | if ($hash === $this->getHash('/packages.json')) { 448 | $this->output('No work' . PHP_EOL); 449 | return true; 450 | } 451 | 452 | $rootJson = json_decode($rootData, true); 453 | if (null === $rootJson) { 454 | throw new \Exception('Invalid JSON received for file /packages.json: '.$rootData); 455 | } 456 | 457 | $requests = []; 458 | $listingsToWrite = []; 459 | 460 | foreach ($rootJson['provider-includes'] as $listing => $opts) { 461 | $listing = str_replace('%hash%', $opts['sha256'], $listing); 462 | if (file_exists($this->target.'/'.$listing.'.gz')) { 463 | continue; 464 | } 465 | 466 | $listingResp = $this->download('/'.$listing); 467 | 468 | $listingData = $listingResp->getContent(); 469 | if (hash('sha256', $listingData) !== $opts['sha256']) { 470 | throw new \Exception('Invalid hash received for file /'.$listing); 471 | } 472 | 473 | $listingJson = json_decode($listingData, true); 474 | if (null === $listingJson) { 475 | throw new \Exception('Invalid JSON received for file /'.$listing.': '.$listingData); 476 | } 477 | 478 | foreach ($listingJson['providers'] as $pkg => $opts) { 479 | $provPath = '/p/'.$pkg.'$'.$opts['sha256'].'.json'; 480 | $provAltPath = '/p/'.$pkg.'.json'; 481 | 482 | if (file_exists($this->target.$provPath.'.gz')) { 483 | continue; 484 | } 485 | 486 | $userData = [$provPath, $provAltPath, $opts['sha256']]; 487 | $requests[] = ['GET', $this->url.$provPath, ['user_data' => $userData]]; 488 | } 489 | 490 | $listingsToWrite['/'.$listing] = [$listingData, $this->strtotime($listingResp->getHeaders()['last-modified'][0])]; 491 | } 492 | 493 | while ($requests) { 494 | $responses = []; 495 | foreach (array_splice($requests, 0, 200) as $req) { 496 | $responses[] = $this->client->request(...$req); 497 | } 498 | 499 | foreach ($this->client->stream($responses) as $response => $chunk) { 500 | try { 501 | if ($chunk->isFirst()) { 502 | if ($response->getStatusCode() === 304) { 503 | $this->output('-'); 504 | $this->statsdIncrement('mirror.not_modified'); 505 | $response->cancel(); 506 | $this->downloaded++; 507 | continue; 508 | } 509 | } 510 | 511 | if ($chunk->isLast()) { 512 | $this->downloaded++; 513 | $userData = $response->getInfo('user_data'); 514 | 515 | // provider v1 516 | $providerData = $response->getContent(); 517 | if (null === json_decode($providerData, true)) { 518 | throw new \Exception('Invalid JSON received for file '.$userData[0]); 519 | } 520 | if (hash('sha256', $providerData) !== $userData[2]) { 521 | throw new \Exception('Invalid hash received for file '.$userData[0]); 522 | } 523 | 524 | $mtime = $this->strtotime($response->getHeaders()['last-modified'][0]); 525 | $gzipped = $this->gzencode($providerData, 7); 526 | $this->write($userData[0], $providerData, $gzipped, $mtime); 527 | $this->write($userData[1], $providerData, $gzipped, $mtime); 528 | $this->output('P'); 529 | $this->statsdIncrement('mirror.sync_provider'); 530 | } 531 | } catch (\Throwable $e) { 532 | // abort all responses to avoid triggering any other exception then throw 533 | array_map(function ($r) { $r->cancel(); }, $responses); 534 | 535 | $this->statsdIncrement('mirror.provider_failure'); 536 | $this->downloaded++; 537 | 538 | throw $e; 539 | } 540 | } 541 | } 542 | 543 | foreach ($listingsToWrite as $listing => $listingData) { 544 | $gzipped = $this->gzencode($listingData[0], 8); 545 | $this->write($listing, $listingData[0], $gzipped, $listingData[1]); 546 | $this->output('L'); 547 | $this->statsdIncrement('mirror.sync_listing'); 548 | } 549 | 550 | $gzipped = $this->gzencode($rootData, 8); 551 | $this->write('/packages.json', $rootData, $gzipped, $this->strtotime($rootResp->getHeaders()['last-modified'][0])); 552 | $this->output('X'); 553 | $this->statsdIncrement('mirror.sync_root'); 554 | 555 | $this->output(PHP_EOL); 556 | $this->output('Downloaded '.$this->downloaded.' files'.PHP_EOL); 557 | 558 | return true; 559 | } 560 | 561 | public function gc(): void 562 | { 563 | // GC is only for v1 metadata, so abort if v1 is not enabled 564 | if ($this->syncRootOnV2) { 565 | return; 566 | } 567 | 568 | // build up array of safe files/packages 569 | $safeFiles = []; 570 | $safePackages = []; 571 | 572 | $rootFile = $this->target.'/packages.json.gz'; 573 | if (!file_exists($rootFile)) { 574 | return; 575 | } 576 | $rootJson = $this->decodeFile($rootFile); 577 | 578 | foreach ($rootJson['provider-includes'] as $listing => $opts) { 579 | $listing = str_replace('%hash%', $opts['sha256'], $listing).'.gz'; 580 | $safeFiles['/'.$listing] = true; 581 | 582 | $listingJson = $this->decodeFile($this->target.'/'.$listing); 583 | foreach ($listingJson['providers'] as $pkg => $opts) { 584 | $provPath = '/p/'.$pkg.'$'.$opts['sha256'].'.json.gz'; 585 | $safeFiles[$provPath] = true; 586 | $safePackages[(string) $pkg] = true; 587 | } 588 | } 589 | 590 | $this->cleanOldFiles($safeFiles, $safePackages); 591 | } 592 | 593 | /** 594 | * @param array $safeFiles 595 | * @param array $safePackages 596 | */ 597 | private function cleanOldFiles(array $safeFiles, array $safePackages): void 598 | { 599 | $finder = Finder::create()->directories()->ignoreVCS(true)->in($this->target.'/p'); 600 | foreach ($finder as $vendorDir) { 601 | // clean up hashed provider files which are >10min old and not safe 602 | $vendorFiles = Finder::create()->files()->ignoreVCS(true) 603 | ->name('/\$[a-f0-9]+\.json\.gz$/') 604 | ->date('until 10minutes ago') 605 | ->in((string) $vendorDir); 606 | 607 | foreach ($vendorFiles as $file) { 608 | $key = strtr(str_replace($this->target, '', (string) $file), '\\', '/'); 609 | if (!isset($safeFiles[$key])) { 610 | unlink((string) $file); 611 | // also remove the version without .gz suffix if it exists 612 | if (file_exists(substr((string) $file, 0, -3))) { 613 | unlink(substr((string) $file, 0, -3)); 614 | } 615 | } 616 | } 617 | 618 | // clean up foo/bar.json static files which are >10min old (to make sure newly synced ones do not get wiped before providers are written) if that package is not found anymore in the listings 619 | $vendorFiles = Finder::create()->files()->ignoreVCS(true) 620 | ->name('/^[A-Za-z0-9_.-]+\.json\.gz$/') 621 | ->date('until 10minutes ago') 622 | ->in((string) $vendorDir); 623 | 624 | foreach ($vendorFiles as $file) { 625 | $key = basename($vendorDir).'/'.basename((string) $file, '.json.gz'); 626 | if (!isset($safePackages[$key])) { 627 | unlink((string) $file); 628 | // also remove the version without .gz suffix if it exists 629 | if (file_exists(substr((string) $file, 0, -3))) { 630 | unlink(substr((string) $file, 0, -3)); 631 | } 632 | } 633 | } 634 | } 635 | 636 | // clean up old provider listings 637 | $finder = Finder::create()->depth(0)->files()->name('provider-*.json.gz')->ignoreVCS(true)->in($this->target.'/p')->date('until 10minutes ago'); 638 | foreach ($finder as $provider) { 639 | $key = strtr(str_replace($this->target, '', (string) $provider), '\\', '/'); 640 | if (!isset($safeFiles[$key])) { 641 | unlink((string) $provider); 642 | // also remove the version without .gz suffix if it exists 643 | if (file_exists(substr((string) $provider, 0, -3))) { 644 | unlink(substr((string) $provider, 0, -3)); 645 | } 646 | } 647 | } 648 | } 649 | 650 | private function output(string $str): void 651 | { 652 | if ($this->verbose) { 653 | echo $str; 654 | } 655 | } 656 | 657 | private function getHash(string $file): string|null 658 | { 659 | if (file_exists($this->target.$file)) { 660 | $hash = hash_file('sha256', $this->target.$file); 661 | if (false === $hash) { 662 | throw new \RuntimeException('Failed hashing file '.$this->target.$file); 663 | } 664 | 665 | return $hash; 666 | } 667 | 668 | return null; 669 | } 670 | 671 | private function download(string $file): ResponseInterface 672 | { 673 | $this->downloaded++; 674 | 675 | try { 676 | $resp = $this->client->request('GET', $this->url.$file); 677 | // trigger throws if needed 678 | $resp->getContent(); 679 | } catch (TransportException $e) { 680 | // retry once 681 | usleep(10000); 682 | $resp = $this->client->request('GET', $this->url.$file); 683 | // trigger throws if needed 684 | $resp->getContent(); 685 | } 686 | 687 | if ($resp->getStatusCode() >= 300) { 688 | throw new \RuntimeException('Failed to fetch '.$file.' => '.$resp->getStatusCode() .' '. $resp->getContent()); 689 | } 690 | 691 | return $resp; 692 | } 693 | 694 | private function write(string $file, string $content, string $gzipped, int $mtime): void 695 | { 696 | $path = $this->target.$file; 697 | 698 | if (!is_dir(dirname($path))) { 699 | mkdir(dirname($path), 0777, true); 700 | } 701 | 702 | if (!$this->gzipOnly || $file === '/packages.json') { 703 | file_put_contents($path.'.tmp', $content); 704 | touch($path.'.tmp', $mtime); 705 | rename($path.'.tmp', $path); 706 | } 707 | file_put_contents($path.'.gz.tmp', $gzipped); 708 | touch($path.'.gz.tmp', $mtime); 709 | rename($path.'.gz.tmp', $path.'.gz'); 710 | } 711 | 712 | private function delete(string $packageName): void 713 | { 714 | $this->output('D'); 715 | $files = [ 716 | $this->target.'/p2/'.$packageName.'.json', 717 | $this->target.'/p2/'.$packageName.'.json.gz', 718 | $this->target.'/p2/'.$packageName.'~dev.json', 719 | $this->target.'/p2/'.$packageName.'~dev.json.gz', 720 | ]; 721 | foreach ($files as $file) { 722 | if (file_exists($file)) { 723 | unlink($file); 724 | } 725 | } 726 | } 727 | 728 | public function statsdIncrement(string $metric): void 729 | { 730 | static $loggedStatsdError = null; 731 | 732 | if ($this->statsdSocket) { 733 | $message = $metric.':1|c'; 734 | try { 735 | fwrite($this->statsdSocket, $message); 736 | } catch (\ErrorException $e) { 737 | if (!$loggedStatsdError) { 738 | trigger_error('Statsd not responding: '.$e->getMessage(), E_USER_WARNING); 739 | $loggedStatsdError = true; 740 | } 741 | } 742 | } 743 | } 744 | 745 | private function statsdConnect(string $ip, int $port): void 746 | { 747 | try { 748 | $socket = fsockopen('udp://' . $ip, $port, $errno, $errstr, 1); 749 | } catch (\ErrorException $e) { 750 | trigger_error( 751 | 'StatsD server connection failed: ' . $e->getMessage(), 752 | E_USER_WARNING 753 | ); 754 | return; 755 | } 756 | if ($socket === false) { 757 | trigger_error( 758 | 'StatsD server connection failed (' . $errno . ') ' . $errstr, 759 | E_USER_WARNING 760 | ); 761 | return; 762 | } 763 | 764 | stream_set_timeout($socket, 1); 765 | 766 | $this->statsdSocket = $socket; 767 | } 768 | 769 | private function initClient(): void 770 | { 771 | if (isset($this->client)) { 772 | return; 773 | } 774 | 775 | $this->client = HttpClient::create([ 776 | 'headers' => [ 777 | 'User-Agent' => $this->userAgent, 778 | 'Host' => $this->hostname, 779 | ], 780 | 'timeout' => 120, 781 | 'max_duration' => 120, 782 | 'http_version' => '2.0', 783 | ]); 784 | } 785 | 786 | private function decodeFile(string $path): mixed 787 | { 788 | $contents = file_get_contents($path); 789 | if (false === $contents) { 790 | throw new \RuntimeException('Failed reading '.$path); 791 | } 792 | $contents = gzdecode($contents); 793 | if (false === $contents) { 794 | throw new \RuntimeException('Failed gzdecoding '.$path); 795 | } 796 | 797 | return json_decode($contents, true); 798 | } 799 | 800 | private function gzencode(string $data, int $level): string 801 | { 802 | $gzipped = gzencode($data, $level); 803 | if (false === $gzipped) { 804 | throw new \RuntimeException('Failed gzencoding '.$data); 805 | } 806 | 807 | return $gzipped; 808 | } 809 | 810 | private function strtotime(string $str): int 811 | { 812 | $timestamp = strtotime($str); 813 | if (false === $timestamp) { 814 | throw new \RuntimeException('Failed parsing time '.$str); 815 | } 816 | 817 | return $timestamp; 818 | } 819 | } 820 | 821 | $lockName = 'mirror'; 822 | $config = require __DIR__ . '/mirror.config.php'; 823 | $isGC = false; 824 | $isV2 = false; 825 | $isV1 = false; 826 | $isResync = false; 827 | 828 | if (in_array('--gc', $_SERVER['argv'])) { 829 | $lockName .= '-gc'; 830 | $isGC = true; 831 | } elseif (in_array('--v2', $_SERVER['argv'])) { 832 | $lockName .= '-v2'; 833 | $isV2 = true; 834 | } elseif (in_array('--v1', $_SERVER['argv'])) { 835 | $isV1 = true; 836 | // default mode 837 | } elseif (in_array('--resync', $_SERVER['argv'])) { 838 | // resync uses same lock name as --v2 to make sure they can not run in parallel 839 | $lockName .= '-v2'; 840 | $isResync = true; 841 | } else { 842 | throw new \RuntimeException('Missing one of --gc, --v1 or --v2 modes'); 843 | } 844 | 845 | $lockFactory = new LockFactory(new FlockStore(sys_get_temp_dir())); 846 | $lock = $lockFactory->createLock($lockName, 3600); 847 | 848 | // if resync is running, we wait for the lock to be 849 | // acquired in case a v2 process is still running 850 | // otherwise abort immediately 851 | if (!$lock->acquire($isResync)) { 852 | // sleep so supervisor assumes a correct start and we avoid restarting too quickly, then exit 853 | sleep(3); 854 | exit(0); 855 | } 856 | 857 | try { 858 | $mirror = new Mirror($config); 859 | if ($isGC) { 860 | $mirror->gc(); 861 | $lock->release(); 862 | exit(0); 863 | } 864 | if ($isResync) { 865 | $mirror->resync($mirror->getV2Timestamp()); 866 | $lock->release(); 867 | exit(0); 868 | } 869 | 870 | $iterations = $config['iterations']; 871 | $hasSyncedRoot = false; 872 | 873 | while ($iterations--) { 874 | if ($isV2) { 875 | // sync root only once in a while as on a v2 only repo it rarely changes 876 | if (($iterations % 20) === 0 || $hasSyncedRoot === false) { 877 | $mirror->syncRootOnV2(); 878 | $hasSyncedRoot = true; 879 | } 880 | if (!$mirror->syncV2()) { 881 | $lock->release(); 882 | exit(1); 883 | } 884 | } elseif ($isV1) { 885 | if (!$mirror->sync()) { 886 | $lock->release(); 887 | exit(1); 888 | } 889 | } 890 | sleep($config['iteration_interval']); 891 | $lock->refresh(); 892 | } 893 | } catch (\Throwable $e) { 894 | // sleep so supervisor assumes a correct start and we avoid restarting too quickly, then rethrow 895 | if (isset($mirror)) { 896 | $mirror->statsdIncrement('mirror.hard_failure'); 897 | } 898 | sleep(3); 899 | echo 'Mirror '.($isV2 ? 'v2' : '').' job failed at '.date('Y-m-d H:i:s').PHP_EOL; 900 | echo '['.get_class($e).'] '.$e->getMessage().PHP_EOL; 901 | throw $e; 902 | } finally { 903 | $lock->release(); 904 | } 905 | 906 | exit(0); 907 | -------------------------------------------------------------------------------- /phpstan.neon: -------------------------------------------------------------------------------- 1 | includes: 2 | - ./vendor/phpstan/phpstan/conf/bleedingEdge.neon 3 | 4 | parameters: 5 | level: 8 6 | 7 | reportUnmatchedIgnoredErrors: false 8 | treatPhpDocTypesAsCertain: false 9 | 10 | paths: 11 | - ./mirror.php 12 | --------------------------------------------------------------------------------