├── .circleci └── config.yml ├── .editorconfig ├── .env ├── .env.postgres.local ├── .gitignore ├── Dockerfile ├── FUNDING.yml ├── LICENSE ├── README.md ├── bin └── console ├── composer.json ├── composer.lock ├── composer.phar ├── config ├── apache │ └── symfony.conf ├── bootstrap.php ├── bundles.php ├── packages │ ├── cache.yaml │ ├── dev │ │ └── monolog.yaml │ ├── doctrine.yaml │ ├── doctrine_migrations.yaml │ ├── framework.yaml │ ├── monolog.yaml │ ├── prod │ │ ├── doctrine.yaml │ │ └── routing.yaml │ ├── routing.yaml │ ├── stg │ │ ├── doctrine.yaml │ │ └── routing.yaml │ ├── test │ │ ├── framework.yaml │ │ ├── monolog.yaml │ │ └── twig.yaml │ └── twig.yaml ├── php │ └── php.ini ├── routes.yaml ├── routes │ ├── annotations.yaml │ └── dev │ │ └── framework.yaml └── services.yaml ├── data └── .gitkeep ├── deploy ├── migrate-db.sh ├── task_entrypoint.sh └── web_entrypoint.sh ├── docker-compose.yml ├── local_data └── .gitkeep ├── packages └── Readme.md ├── run-cron.sh ├── src ├── Command │ ├── BuildCommand.php │ ├── RefreshCommand.php │ └── UpdateCommand.php ├── Controller │ ├── .gitignore │ ├── MainController.php │ └── OpenSearchController.php ├── Entity │ ├── Package.php │ ├── PackageData.php │ ├── PackageRepository.php │ ├── Plugin.php │ ├── Request.php │ ├── RequestRepository.php │ └── Theme.php ├── EventListener │ └── ExceptionListener.php ├── Kernel.php ├── Migrations │ ├── Version20200924213211.php │ ├── Version20201029165342.php │ ├── Version20201104150851.php │ └── Version20231025122432.php ├── Persistence │ └── RetrySafeEntityManager.php ├── Service │ ├── Builder.php │ └── Update.php ├── Storage │ ├── Database.php │ ├── Filesystem.php │ └── PackageStore.php └── Twig │ └── PackageExtension.php ├── symfony.lock ├── var └── cache │ └── twig │ └── .gitkeep └── web ├── .htaccess ├── assets ├── css │ ├── foundation.css │ └── style.css ├── img │ ├── fork.png │ └── home.svg └── js │ └── main.js ├── favicon.png ├── index.php └── templates ├── footer.twig ├── index.twig ├── layout.twig ├── opensearch.twig ├── search.twig └── searchbar.twig /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | # Hold off on 8.2.x+ until the fix for https://github.com/CircleCI-Public/aws-ecr-orb/issues/256 5 | # is in a tagged version. 6 | aws-ecr: circleci/aws-ecr@8.1.3 7 | aws-ecs: circleci/aws-ecs@3.2.0 8 | 9 | executors: 10 | outlandish: 11 | machine: 12 | image: ubuntu-2004:edge 13 | 14 | workflows: 15 | deploy-staging: 16 | jobs: 17 | - aws-ecr/build-and-push-image: 18 | executor: outlandish 19 | context: 20 | - ecs-deploys 21 | filters: 22 | branches: 23 | only: 24 | - develop 25 | repo: 'staging-wpackagist' 26 | tag: 'staging,staging-${CIRCLE_SHA1}' 27 | extra-build-args: '--build-arg env=stg' 28 | - aws-ecs/deploy-service-update: 29 | context: 30 | - ecs-deploys 31 | requires: 32 | - aws-ecr/build-and-push-image 33 | family: 'ol-ecs-staging-wpackagist' 34 | cluster: 'ol-ecs-staging-shared' 35 | service-name: 'staging-wpackagist' 36 | 37 | deploy-production: 38 | jobs: 39 | - aws-ecr/build-and-push-image: 40 | executor: outlandish 41 | context: 42 | - ecs-deploys 43 | filters: 44 | branches: 45 | only: 46 | - main 47 | repo: 'production-wpackagist' 48 | tag: 'production,production-${CIRCLE_SHA1}' 49 | extra-build-args: '--build-arg env=prod' 50 | - aws-ecs/deploy-service-update: 51 | context: 52 | - ecs-deploys 53 | requires: 54 | - aws-ecr/build-and-push-image 55 | family: 'ol-ecs-production-wpackagist' 56 | cluster: 'ol-ecs-production-shared' 57 | service-name: 'production-wpackagist' 58 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig helps developers define and maintain consistent 2 | # coding styles between different editors and IDEs 3 | # editorconfig.org 4 | 5 | root = true 6 | 7 | [*] 8 | 9 | # Change these settings to your own preference 10 | indent_style = space 11 | indent_size = 4 12 | 13 | # We recommend you to keep these unchanged 14 | end_of_line = lf 15 | charset = utf-8 16 | trim_trailing_whitespace = true 17 | insert_final_newline = true -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | # In all environments, the following files are loaded if they exist, 2 | # the latter taking precedence over the former: 3 | # 4 | # * .env contains default values for the environment variables needed by the app 5 | # * .env.local uncommitted file with local overrides 6 | # * .env.$APP_ENV committed environment-specific defaults 7 | # * .env.$APP_ENV.local uncommitted environment-specific overrides 8 | # 9 | # Real environment variables win over .env files. 10 | # 11 | # DO NOT DEFINE PRODUCTION SECRETS IN THIS FILE NOR IN ANY OTHER COMMITTED FILES. 12 | # 13 | # Run "composer dump-env prod" to compile .env files for production use (requires symfony/flex >=1.2). 14 | # https://symfony.com/doc/current/best_practices.html#use-environment-variables-for-infrastructure-configuration 15 | 16 | ###> symfony/framework-bundle ### 17 | APP_ENV=dev 18 | APP_SECRET=replace-with-your-secret 19 | #TRUSTED_PROXIES=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 20 | #TRUSTED_HOSTS='^(localhost|example\.com)$' 21 | ###< symfony/framework-bundle ### 22 | 23 | ###> doctrine/doctrine-bundle ### 24 | # Format described at https://www.doctrine-project.org/projects/doctrine-dbal/en/latest/reference/configuration.html#connecting-using-a-url 25 | # For an SQLite database, use: "sqlite:///%kernel.project_dir%/var/data.db" 26 | # For a PostgreSQL database, use: "postgresql://db_user:db_password@127.0.0.1:5432/db_name?serverVersion=11&charset=utf8" 27 | # IMPORTANT: You MUST configure your server version, either here or in config/packages/doctrine.yaml 28 | DATABASE_URL=postgresql://wpackagist_local:wpLocalPassword@db:/wpackagist_local 29 | ###< doctrine/doctrine-bundle ### 30 | 31 | # Only used if you turn off using the database, e.g. for isolated local testing. 32 | PACKAGE_PATH=/var/www/html/packages 33 | 34 | REDIS_HOST=redis 35 | 36 | PUBLIC_DIR=web 37 | -------------------------------------------------------------------------------- /.env.postgres.local: -------------------------------------------------------------------------------- 1 | POSTGRES_HOST=db 2 | POSTGRES_PORT=5432 3 | POSTGRES_DB=wpackagist_local 4 | POSTGRES_USER=wpackagist_local 5 | POSTGRES_PASSWORD=wpLocalPassword 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /data/packages.sqlite 3 | /vendor 4 | /packages/* 5 | !/packages/Readme.md 6 | 7 | ###> symfony/framework-bundle ### 8 | /.env.local 9 | /.env.local.php 10 | /.env.*.local 11 | /config/secrets/prod/prod.decrypt.private.php 12 | /public/bundles/ 13 | /vendor/ 14 | ###< symfony/framework-bundle ### 15 | 16 | # Postgres' config is an exception to the above and should be checked 17 | # in for easy local Docker testing. 18 | !/.env.postgres.local 19 | 20 | # Symfony sidesteps permissions dances with cache:warmup now, but apparently Twig 21 | # does not. Pre-create these directories to hopefully let web tasks run as an 22 | # Apache-specific user and not implode at runtime when they want to write cache files. 23 | /var/* 24 | !/var/cache/twig/.gitkeep 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM php:8.1-apache 2 | 3 | ARG env 4 | RUN test -n "$env" 5 | 6 | # Install the AWS CLI - needed to load in secrets safely from S3. See https://aws.amazon.com/blogs/security/how-to-manage-secrets-for-amazon-ec2-container-service-based-applications-by-using-amazon-s3-and-docker/ 7 | RUN apt-get update -qq && apt-get install -y awscli && \ 8 | rm -rf /var/lib/apt/lists/* /var/cache/apk/* 9 | 10 | # Install svn client, a requirement for the current native exec approach; git for 11 | # Composer pulls; libpq-dev for Postgres; libicu-dev for intl; libonig-dev for mbstring. 12 | RUN apt-get update -qq && \ 13 | apt-get install -y git libicu-dev libonig-dev libpq-dev libzip-dev subversion zip && \ 14 | rm -rf /var/lib/apt/lists/* /var/cache/apk/* 15 | 16 | # intl recommended by something in the Doctrine/Symfony stack for improved performance. 17 | RUN docker-php-ext-configure pgsql -with-pgsql=/usr/local/pgsql \ 18 | && docker-php-ext-install intl mbstring pdo_pgsql zip 19 | 20 | RUN docker-php-ext-enable opcache 21 | 22 | RUN pecl install redis && rm -rf /tmp/pear && docker-php-ext-enable redis 23 | 24 | # Get latest Composer 25 | RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer 26 | 27 | # Set up virtual host. 28 | COPY config/apache/symfony.conf /etc/apache2/sites-available/ 29 | RUN a2enmod rewrite \ 30 | && a2enmod remoteip \ 31 | && a2dissite 000-default \ 32 | && a2ensite symfony \ 33 | && echo ServerName localhost >> /etc/apache2/apache2.conf 34 | 35 | COPY . /var/www/html 36 | 37 | # Configure PHP to e.g. not hit 128M memory limit. 38 | COPY ./config/php/php.ini /usr/local/etc/php/ 39 | 40 | # Ensure Apache can run as www-data and still write to these when the Docker build creates them as root. 41 | RUN mkdir /tmp/twig 42 | RUN chmod -R 777 /tmp/twig 43 | 44 | RUN APP_ENV=${env} composer install --no-interaction --quiet --optimize-autoloader --no-dev 45 | -------------------------------------------------------------------------------- /FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: outlandishideas 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Outlandish LLP 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | WordPress Packagist 2 | === 3 | 4 | This is the repository for [wpackagist.org](https://wpackagist.org) which allows WordPress plugins and themes to be 5 | managed along with other dependencies using [Composer](https://getcomposer.org). 6 | 7 | More info and usage instructions at [wpackagist.org](https://wpackagist.org) or follow us on 8 | Twitter [@wpackagist](https://twitter.com/wpackagist). 9 | 10 | For support and discussion, please use the issue tracker above. 11 | 12 | ## Usage 13 | 14 | Example composer.json: 15 | 16 | ```json 17 | { 18 | "name": "acme/brilliant-wordpress-site", 19 | "description": "My brilliant WordPress site", 20 | "repositories":[ 21 | { 22 | "type":"composer", 23 | "url":"https://wpackagist.org", 24 | "only": ["wpackagist-plugin/*", "wpackagist-theme/*"] 25 | } 26 | ], 27 | "require": { 28 | "aws/aws-sdk-php":"*", 29 | "wpackagist-plugin/akismet":"dev-trunk", 30 | "wpackagist-plugin/wordpress-seo":">=7.0.2", 31 | "wpackagist-theme/hueman":"*" 32 | }, 33 | "autoload": { 34 | "psr-0": { 35 | "Acme": "src/" 36 | } 37 | } 38 | } 39 | ``` 40 | 41 | ## WordPress core 42 | 43 | This does not provide WordPress itself. 44 | 45 | See https://github.com/fancyguy/webroot-installer or https://github.com/roots/wordpress. 46 | 47 | ## How it works 48 | 49 | WPackagist implements the `wordpress-plugin` and `wordpress-theme` Composer Installers 50 | (https://github.com/composer/installers). 51 | 52 | It essentially provides a lookup table from package (theme or plugin) name to WordPress.org 53 | SVN repository. Versions correspond to different tags in their repository, with the special 54 | `dev-trunk` version being mapped to `trunk`. 55 | 56 | Note that to maintain Composer v1 compatibility (as well as v2) 57 | for `dev-` versions, for now we need to use the `VersionParser` from 58 | `composer/composer` v1.x and not a newer release branch. Correct resolution 59 | of these depends on the legacy behaviour where `dev-trunk` et al. correspond to 60 | 61 | "version_normalized":"9999999-dev" 62 | 63 | The lookup table is provided as a hierarchy of static JSON files. The entry point to these 64 | files can be found at https://wpackagist.org/packages.json, which consists of a series of 65 | sub-tables (each as its own JSON file). These sub-tables are grouped by last commit 66 | date (trying to keep them roughly the same size), and contain references to individual packages. 67 | Each package has its own JSON file detailing its versions; these can be found in 68 | https://wpackagist.org/p/wpackagist-{theme|plugin}/{package-name-and-hash}.json. 69 | 70 | ### Version format limitations 71 | 72 | Currently, Wpackagist can only process packages with up to 4 parts in their version numbers, in line with 73 | the internal handling of Composer v1.x. 74 | 75 | ## Running Wpackagist 76 | 77 | ### Installing 78 | 79 | 1. Make sure you have Composer dependencies installed, including extensions. 80 | 2. Make `.env.local`, overriding anything you want to from `.env`. 81 | 3. (Only if you're going to skip using a database for `PackageStore`): ensure sure your `PACKAGE_PATH` directory is writable. 82 | 4. Run `composer install` to install dependencies. 83 | 5. Populate the database and package files (see steps below). 84 | 5. Point your Web server to [`web`](web/). A [`.htaccess`](web/.htaccess) is provided for Apache. 85 | 86 | ### Updating the database 87 | 88 | The first database population may easily take hours. Be patient. 89 | 90 | 0. `bin/console doctrine:migrations:migrate`: Ensure the database schema is up to date with the code. 91 | 1. `bin/console refresh`: Query the WordPress.org SVN in order to find new and updated packages. 92 | 2. `bin/console update`: Update the version information for packages identified in `2`. Uses the WordPress.org API. 93 | 3. `bin/console build`: Rebuild all `PackageStore` data. 94 | 95 | ## Running locally with Docker 96 | 97 | This may be simpler than setting up native dependencies, but is 98 | experimental. 99 | 100 | To prepare environment variables: 101 | 102 | cp .env .env.local 103 | 104 | and edit as necessary. 105 | 106 | To set up and update the database: 107 | 108 | docker-compose run --rm cron composer install 109 | docker-compose run --rm cron deploy/migrate-db.sh 110 | docker-compose run --rm cron 111 | 112 | To start a web server on `localhost:30100`: 113 | 114 | docker-compose up web adminer 115 | 116 | #### Services 117 | 118 | * Web: [localhost:30100](http://localhost:30100) 119 | * Adminer: [localhost:30101](http://localhost:30101) (See credentials in `.env.postgres.local`) 120 | 121 | ## Live deployments 122 | 123 | CircleCI is used to deploy the live app on ECS. 124 | 125 | Automatic deploys run: 126 | 127 | * from `develop` to [Staging](https://staging-wpackagist.out.re); 128 | * from `main` to [Production](https://wpackagist.org/) 129 | 130 | See [.circleci/config.yml](./.circleci/config.yml) for full configuration. 131 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | getParameterOption(['--env', '-e'], null, true)) { 23 | putenv('APP_ENV='.$_SERVER['APP_ENV'] = $_ENV['APP_ENV'] = $env); 24 | } 25 | 26 | if ($input->hasParameterOption('--no-debug', true)) { 27 | putenv('APP_DEBUG='.$_SERVER['APP_DEBUG'] = $_ENV['APP_DEBUG'] = '0'); 28 | } 29 | 30 | require dirname(__DIR__).'/config/bootstrap.php'; 31 | 32 | if ($_SERVER['APP_DEBUG']) { 33 | umask(0000); 34 | 35 | if (class_exists(Debug::class)) { 36 | Debug::enable(); 37 | } 38 | } 39 | 40 | $kernel = new Kernel($_SERVER['APP_ENV'], (bool) $_SERVER['APP_DEBUG']); 41 | $application = new Application($kernel); 42 | 43 | $application->run($input); 44 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "outlandish/wpackagist", 3 | "description": "Install and manage WordPress plugins with Composer", 4 | "config": { 5 | "platform": { 6 | "php": "8.0.28" 7 | }, 8 | "sort-packages": true, 9 | "allow-plugins": { 10 | "symfony/flex": true 11 | } 12 | }, 13 | "minimum-stability": "stable", 14 | "require": { 15 | "php": "^8.0", 16 | "ext-intl": "*", 17 | "ext-json": "*", 18 | "ext-mbstring": "*", 19 | "ext-pdo": "*", 20 | "ext-pdo_pgsql": "*", 21 | "ext-pdo_sqlite": "*", 22 | "ext-redis": "*", 23 | "ext-simplexml": "*", 24 | "babdev/pagerfanta-bundle": "^3.0", 25 | "composer/composer": "^1.10.19", 26 | "composer/package-versions-deprecated": "^1.11", 27 | "doctrine/dbal": "^2.10.2", 28 | "doctrine/doctrine-bundle": "^2.6", 29 | "doctrine/doctrine-migrations-bundle": "^3.2", 30 | "doctrine/orm": "^2.12", 31 | "guzzlehttp/guzzle-services": "^1.1.3", 32 | "pagerfanta/doctrine-orm-adapter": "^3.7", 33 | "pagerfanta/twig": "^3.0", 34 | "rarst/wporg-client": "~0.5", 35 | "symfony/config": "^5.0", 36 | "symfony/console": "^5.0", 37 | "symfony/dotenv": "^5.0", 38 | "symfony/filesystem": "^5.0", 39 | "symfony/flex": "^1.6", 40 | "symfony/form": "^5.0", 41 | "symfony/monolog-bundle": "^3.6", 42 | "symfony/security-csrf": "^5.0", 43 | "symfony/twig-bundle": "^5.4", 44 | "symfony/yaml": "^5.0", 45 | "twig/extra-bundle": "^3.4", 46 | "twig/twig": "^3.14" 47 | }, 48 | "require-dev": { 49 | "roave/security-advisories": "dev-latest" 50 | }, 51 | "conflict": { 52 | "silex/silex": "*", 53 | "symfony/symfony": "*" 54 | }, 55 | "autoload": { 56 | "psr-4": { 57 | "Outlandish\\Wpackagist\\": "src/" 58 | } 59 | }, 60 | "scripts": { 61 | "auto-scripts": { 62 | "cache:clear": "symfony-cmd", 63 | "assets:install %PUBLIC_DIR%": "symfony-cmd" 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /composer.phar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outlandishideas/wpackagist/466f50d4dcdcaeaaec12b2429028a7f159ee92bc/composer.phar -------------------------------------------------------------------------------- /config/apache/symfony.conf: -------------------------------------------------------------------------------- 1 | 2 | DocumentRoot /var/www/html/web 3 | # Just use 'localhost' instead of exposing user supplied value 4 | UseCanonicalName on 5 | 6 | 7 | AllowOverride None 8 | Order Allow,Deny 9 | Allow from All 10 | 11 | RewriteEngine On 12 | RewriteCond %{REQUEST_FILENAME} !-f 13 | RewriteRule ^ index.php [QSA,L] 14 | 15 | 16 | -------------------------------------------------------------------------------- /config/bootstrap.php: -------------------------------------------------------------------------------- 1 | =1.2) 13 | if (is_array($env = @include dirname(__DIR__).'/.env.local.php') && (!isset($env['APP_ENV']) || ($_SERVER['APP_ENV'] ?? $_ENV['APP_ENV'] ?? $env['APP_ENV']) === $env['APP_ENV'])) { 14 | (new Dotenv(false))->populate($env); 15 | } else { 16 | // load all the .env files 17 | (new Dotenv(false))->loadEnv(dirname(__DIR__).'/.env'); 18 | } 19 | 20 | $_SERVER += $_ENV; 21 | $_SERVER['APP_ENV'] = $_ENV['APP_ENV'] = ($_SERVER['APP_ENV'] ?? $_ENV['APP_ENV'] ?? null) ?: 'dev'; 22 | $_SERVER['APP_DEBUG'] = $_SERVER['APP_DEBUG'] ?? $_ENV['APP_DEBUG'] ?? 'prod' !== $_SERVER['APP_ENV']; 23 | $_SERVER['APP_DEBUG'] = $_ENV['APP_DEBUG'] = (int) $_SERVER['APP_DEBUG'] || filter_var($_SERVER['APP_DEBUG'], FILTER_VALIDATE_BOOLEAN) ? '1' : '0'; 24 | -------------------------------------------------------------------------------- /config/bundles.php: -------------------------------------------------------------------------------- 1 | ['all' => true], 5 | Doctrine\Bundle\DoctrineBundle\DoctrineBundle::class => ['all' => true], 6 | Doctrine\Bundle\MigrationsBundle\DoctrineMigrationsBundle::class => ['all' => true], 7 | Symfony\Bundle\FrameworkBundle\FrameworkBundle::class => ['all' => true], 8 | Symfony\Bundle\MonologBundle\MonologBundle::class => ['all' => true], 9 | Symfony\Bundle\TwigBundle\TwigBundle::class => ['all' => true], 10 | Twig\Extra\TwigExtraBundle\TwigExtraBundle::class => ['all' => true], 11 | ]; 12 | -------------------------------------------------------------------------------- /config/packages/cache.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | cache: 3 | # Unique name of your app: used to compute stable namespaces for cache keys. 4 | prefix_seed: "wpackagist/%env(APP_ENV)%" 5 | 6 | # The "app" cache stores to the filesystem by default. 7 | # The data in this cache should persist between deploys. 8 | # Other options include: 9 | 10 | # Redis 11 | app: cache.adapter.redis 12 | system: cache.adapter.redis 13 | default_redis_provider: "redis://%env(REDIS_HOST)%" 14 | 15 | # APCu (not recommended with heavy random-write workloads as memory fragmentation can cause perf issues) 16 | #app: cache.adapter.apcu 17 | 18 | # Namespaced pools use the above "app" backend by default 19 | pools: 20 | doctrine.result_cache_pool: 21 | adapter: cache.app 22 | doctrine.system_cache_pool: 23 | adapter: cache.system 24 | -------------------------------------------------------------------------------- /config/packages/dev/monolog.yaml: -------------------------------------------------------------------------------- 1 | monolog: 2 | channels: [deprecation] 3 | handlers: 4 | main: 5 | type: fingers_crossed 6 | handler: stream 7 | level: info 8 | buffer_size: 50 # How many messages should be saved? Prevent memory leaks 9 | channels: ["!deprecation"] 10 | excluded_http_codes: [404] 11 | stream: 12 | type: stream 13 | path: "php://stdout" 14 | file: 15 | type: stream 16 | path: "%kernel.logs_dir%/%kernel.environment%.log" 17 | level: debug 18 | channels: ["!event"] # Include deprecations only in local, file-based logs. 19 | # uncomment to get logging in your browser 20 | # you may have to allow bigger header sizes in your Web server configuration 21 | #firephp: 22 | # type: firephp 23 | # level: info 24 | #chromephp: 25 | # type: chromephp 26 | # level: info 27 | console: 28 | type: console 29 | process_psr_3_messages: false 30 | channels: ["!console", "!deprecation", "!doctrine", "!event"] 31 | -------------------------------------------------------------------------------- /config/packages/doctrine.yaml: -------------------------------------------------------------------------------- 1 | doctrine: 2 | dbal: 3 | # Ensure any URL-special characters in username or password are encoded 4 | # in the env var so that `resolve:` works. 5 | url: '%env(resolve:DATABASE_URL)%' 6 | 7 | # IMPORTANT: You MUST configure your server version, 8 | # either here or in the DATABASE_URL env var (see .env file) 9 | server_version: '12.8' 10 | orm: 11 | auto_generate_proxy_classes: true 12 | naming_strategy: doctrine.orm.naming_strategy.underscore_number_aware 13 | auto_mapping: true 14 | mappings: 15 | Outlandish\Wpackagist\Entity: 16 | type: annotation 17 | dir: '%kernel.project_dir%/src/Entity' 18 | is_bundle: false 19 | prefix: Outlandish\Wpackagist\Entity 20 | alias: App 21 | -------------------------------------------------------------------------------- /config/packages/doctrine_migrations.yaml: -------------------------------------------------------------------------------- 1 | doctrine_migrations: 2 | migrations_paths: 3 | # namespace is arbitrary but should be different from App\Migrations 4 | # as migrations classes should NOT be autoloaded 5 | 'DoctrineMigrations': '%kernel.project_dir%/src/Migrations' 6 | -------------------------------------------------------------------------------- /config/packages/framework.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | secret: '%env(APP_SECRET)%' 3 | #csrf_protection: true 4 | #http_method_override: true 5 | 6 | # Enables session support. Note that the session will ONLY be started if you read or write from it. 7 | # Remove or comment this section to explicitly disable session support. 8 | session: 9 | handler_id: null 10 | cookie_secure: auto 11 | cookie_samesite: lax 12 | 13 | #esi: true 14 | #fragments: true 15 | php_errors: 16 | log: true 17 | -------------------------------------------------------------------------------- /config/packages/monolog.yaml: -------------------------------------------------------------------------------- 1 | monolog: 2 | channels: [deprecation] 3 | handlers: 4 | main: 5 | type: fingers_crossed 6 | handler: stream 7 | level: info 8 | buffer_size: 50 # How many messages should be saved? Prevent memory leaks 9 | channels: ["!deprecation"] 10 | excluded_http_codes: [404] 11 | stream: 12 | type: stream 13 | path: "php://stdout" 14 | console: 15 | type: console 16 | level: info 17 | process_psr_3_messages: false 18 | channels: ["!deprecation", "!doctrine", "!event"] 19 | verbosity_levels: 20 | VERBOSITY_NORMAL: INFO 21 | -------------------------------------------------------------------------------- /config/packages/prod/doctrine.yaml: -------------------------------------------------------------------------------- 1 | doctrine: 2 | orm: 3 | auto_generate_proxy_classes: false 4 | metadata_cache_driver: 5 | type: pool 6 | pool: doctrine.system_cache_pool 7 | query_cache_driver: 8 | type: pool 9 | pool: doctrine.system_cache_pool 10 | result_cache_driver: 11 | type: pool 12 | pool: doctrine.result_cache_pool 13 | -------------------------------------------------------------------------------- /config/packages/prod/routing.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | router: 3 | strict_requirements: null 4 | -------------------------------------------------------------------------------- /config/packages/routing.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | router: 3 | utf8: true 4 | -------------------------------------------------------------------------------- /config/packages/stg/doctrine.yaml: -------------------------------------------------------------------------------- 1 | doctrine: 2 | orm: 3 | auto_generate_proxy_classes: false 4 | metadata_cache_driver: 5 | type: pool 6 | pool: doctrine.system_cache_pool 7 | query_cache_driver: 8 | type: pool 9 | pool: doctrine.system_cache_pool 10 | result_cache_driver: 11 | type: pool 12 | pool: doctrine.result_cache_pool 13 | -------------------------------------------------------------------------------- /config/packages/stg/routing.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | router: 3 | strict_requirements: null 4 | -------------------------------------------------------------------------------- /config/packages/test/framework.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | test: true 3 | session: 4 | storage_id: session.storage.mock_file 5 | -------------------------------------------------------------------------------- /config/packages/test/monolog.yaml: -------------------------------------------------------------------------------- 1 | monolog: 2 | handlers: 3 | main: 4 | type: fingers_crossed 5 | action_level: error 6 | handler: nested 7 | excluded_http_codes: [404, 405] 8 | channels: ["!event"] 9 | nested: 10 | type: stream 11 | path: "%kernel.logs_dir%/%kernel.environment%.log" 12 | level: debug 13 | -------------------------------------------------------------------------------- /config/packages/test/twig.yaml: -------------------------------------------------------------------------------- 1 | twig: 2 | strict_variables: true 3 | -------------------------------------------------------------------------------- /config/packages/twig.yaml: -------------------------------------------------------------------------------- 1 | twig: 2 | cache: '/tmp/twig' 3 | default_path: '%kernel.project_dir%/web/templates' 4 | -------------------------------------------------------------------------------- /config/php/php.ini: -------------------------------------------------------------------------------- 1 | memory_limit = -1 2 | date.timezone = "UTC" 3 | 4 | ; See https://secure.php.net/manual/en/opcache.configuration.php#ini.opcache.max-accelerated-files 5 | ; and https://www.scalingphpbook.com/blog/2014/02/14/best-zend-opcache-settings.html 6 | opcache.max_accelerated_files = 7963 7 | 8 | ; We have a good amount of memory available, so increase this from the default 128MB 9 | opcache.memory_consumption = 384 10 | 11 | ; Increase from default 8MB 12 | opcache.interned_strings_buffer = 32 13 | 14 | ; As recommended by https://secure.php.net/manual/en/opcache.installation.php and 15 | ; https://www.scalingphpbook.com/blog/2014/02/14/best-zend-opcache-settings.html 16 | opcache.fast_shutdown = 1 17 | 18 | ; Task definition artifacts are immutable on ECS and cache check time is 0 on local -> may as well make tasks faster 19 | opcache.enable_cli = 1 20 | 21 | ; Trialling this 25/5/23, as Apache might(?) not have been compression JSON returned from PHP as we wanted 22 | ; without it. We should monitor response times & load and decide whether to keep this. 23 | zlib.output_compression = 1 24 | -------------------------------------------------------------------------------- /config/routes.yaml: -------------------------------------------------------------------------------- 1 | home: 2 | path: / 3 | controller: Outlandish\Wpackagist\Controller\MainController::home 4 | methods: GET 5 | 6 | search: 7 | path: /search 8 | controller: Outlandish\Wpackagist\Controller\MainController::search 9 | methods: GET 10 | 11 | update: 12 | path: /update 13 | controller: Outlandish\Wpackagist\Controller\MainController::update 14 | methods: POST 15 | 16 | opensearch: 17 | path: /opensearch.xml 18 | controller: Outlandish\Wpackagist\Controller\OpenSearchController::go 19 | methods: GET 20 | -------------------------------------------------------------------------------- /config/routes/annotations.yaml: -------------------------------------------------------------------------------- 1 | controllers: 2 | resource: ../../src/Controller/ 3 | type: annotation 4 | 5 | kernel: 6 | resource: ../../src/Kernel.php 7 | type: annotation 8 | -------------------------------------------------------------------------------- /config/routes/dev/framework.yaml: -------------------------------------------------------------------------------- 1 | _errors: 2 | resource: '@FrameworkBundle/Resources/config/routing/errors.xml' 3 | prefix: /_error 4 | -------------------------------------------------------------------------------- /config/services.yaml: -------------------------------------------------------------------------------- 1 | # This file is the entry point to configure your own services. 2 | # Files in the packages/ subdirectory configure your dependencies. 3 | 4 | # Put parameters here that don't need to change on each machine where the app is deployed 5 | # https://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration 6 | parameters: 7 | wpackagist.packages.path: '%env(resolve:PACKAGE_PATH)%' # Not used any more except for a subset of local dev. 8 | 9 | services: 10 | # default configuration for services in *this* file 11 | _defaults: 12 | autowire: true # Automatically injects dependencies in your services. 13 | autoconfigure: true # Automatically registers your services as commands, event subscribers, etc. 14 | public: false # Allows optimizing the container by removing unused services; this also means 15 | # fetching services directly from the container via $container->get() won't work. 16 | # The best practice is to be explicit about your dependencies anyway. 17 | 18 | request_metadata: 19 | class: Doctrine\ORM\Mapping\ClassMetadata 20 | arguments: 21 | $entityName: Outlandish\Wpackagist\Entity\Request 22 | 23 | Doctrine\ORM\Configuration: 24 | alias: 'doctrine.orm.default_configuration' 25 | 26 | Doctrine\ORM\EntityManagerInterface: 27 | alias: Outlandish\Wpackagist\Persistence\RetrySafeEntityManager 28 | 29 | doctrine.orm.default_entity_manager: 30 | alias: Outlandish\Wpackagist\Persistence\RetrySafeEntityManager 31 | public: true 32 | 33 | doctrine.orm.default_entity_manager.abstract: 34 | alias: Outlandish\Wpackagist\Persistence\RetrySafeEntityManager 35 | public: true 36 | 37 | Outlandish\Wpackagist\Entity\RequestRepository: 38 | arguments: 39 | $class: '@request_metadata' 40 | 41 | # makes classes in src/ available to be used as services 42 | # this creates a service per class whose id is the fully-qualified class name 43 | Outlandish\Wpackagist\: 44 | resource: '../src/*' 45 | exclude: '../src/{DependencyInjection,Entity,Migrations,Tests,Kernel.php}' 46 | 47 | Outlandish\Wpackagist\Storage\PackageStore: 48 | alias: Outlandish\Wpackagist\Storage\Database 49 | 50 | Outlandish\Wpackagist\Storage\Filesystem: 51 | arguments: ['%wpackagist.packages.path%'] 52 | 53 | # controllers are imported separately to make sure services can be injected 54 | # as action arguments even if you don't extend any base controller class 55 | Outlandish\Wpackagist\Controller\: 56 | resource: '../src/Controller' 57 | tags: ['controller.service_arguments'] 58 | 59 | # add more service definitions when explicit configuration is needed 60 | # please note that last definitions always *replace* previous ones 61 | 62 | Outlandish\Wpackagist\EventListener\ExceptionListener: 63 | tags: 64 | - { name: kernel.event_listener, event: kernel.exception } 65 | -------------------------------------------------------------------------------- /data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outlandishideas/wpackagist/466f50d4dcdcaeaaec12b2429028a7f159ee92bc/data/.gitkeep -------------------------------------------------------------------------------- /deploy/migrate-db.sh: -------------------------------------------------------------------------------- 1 | echo "Running DB migrations..." 2 | bin/console doctrine:migrations:migrate --no-interaction --allow-no-migration --env=$APP_ENV 3 | -------------------------------------------------------------------------------- /deploy/task_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script is taken from https://aws.amazon.com/blogs/security/how-to-manage-secrets-for-amazon-ec2-container-service-based-applications-by-using-amazon-s3-and-docker/ 4 | # and is used to set up app secrets in ECS without exposing them as widely as using ECS env vars directly would. 5 | 6 | # Check that the environment variable has been set correctly 7 | if [ -z "$SECRETS_URI" ]; then 8 | echo >&2 'error: missing SECRETS_URI environment variable' 9 | exit 1 10 | fi 11 | 12 | # Load the S3 secrets file contents into the environment variables 13 | export $(aws s3 cp ${SECRETS_URI} - | grep -v '^#' | xargs) 14 | 15 | echo "Dumping env..." 16 | composer dump-env "${APP_ENV}" 17 | 18 | echo "Running DB migrations..." 19 | bin/console doctrine:migrations:migrate --no-interaction --allow-no-migration --env=$APP_ENV 20 | 21 | echo "Starting task..." 22 | # Call the normal CLI entry-point script, passing on script name and any other arguments 23 | docker-php-entrypoint "$@" 24 | -------------------------------------------------------------------------------- /deploy/web_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script is taken from https://aws.amazon.com/blogs/security/how-to-manage-secrets-for-amazon-ec2-container-service-based-applications-by-using-amazon-s3-and-docker/ 4 | # and is used to set up app secrets in ECS without exposing them as widely as using ECS env vars directly would. 5 | 6 | # Check that the environment variable has been set correctly 7 | if [ -z "$SECRETS_URI" ]; then 8 | echo >&2 'error: missing SECRETS_URI environment variable' 9 | exit 1 10 | fi 11 | 12 | # Load the S3 secrets file contents into the environment variables 13 | export $(aws s3 cp ${SECRETS_URI} - | grep -v '^#' | xargs) 14 | 15 | echo "Dumping env..." 16 | composer dump-env "${APP_ENV}" 17 | 18 | # Includes Doctrine proxies – https://stackoverflow.com/a/36685804/2803757 19 | echo "Clearing & warming cache..." 20 | bin/console cache:clear --no-debug --env=$APP_ENV 21 | 22 | echo "Running DB migrations..." 23 | bin/console doctrine:migrations:migrate --no-interaction --allow-no-migration --env=$APP_ENV 24 | 25 | chmod -R 777 /tmp/twig 26 | 27 | echo "Starting Apache..." 28 | # Call the normal web server entry-point script 29 | apache2-foreground "$@" 30 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Experimental docker-compose for local use. Access `web` at localhost:30100. 2 | 3 | version: "3.7" 4 | 5 | volumes: 6 | local_data: {} 7 | 8 | services: 9 | db: 10 | image: postgres:12.8-alpine 11 | volumes: 12 | - ./local_data:/var/lib/postgresql/ 13 | ports: 14 | - "5432:5432" 15 | env_file: 16 | - .env.postgres.local 17 | redis: 18 | image: redis:6.2 19 | cron: 20 | build: 21 | context: . 22 | args: 23 | env: dev 24 | entrypoint: docker-php-entrypoint 25 | command: /var/www/html/run-cron.sh 26 | volumes: 27 | - .:/var/www/html 28 | env_file: 29 | - .env.local 30 | depends_on: 31 | - db 32 | - redis 33 | web: 34 | build: 35 | context: . 36 | args: 37 | env: dev 38 | ports: 39 | - "30100:80" 40 | volumes: 41 | - .:/var/www/html 42 | env_file: 43 | - .env.local 44 | depends_on: 45 | - db 46 | - redis 47 | adminer: 48 | image: adminer 49 | ports: 50 | - "30101:8080" 51 | -------------------------------------------------------------------------------- /local_data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outlandishideas/wpackagist/466f50d4dcdcaeaaec12b2429028a7f159ee92bc/local_data/.gitkeep -------------------------------------------------------------------------------- /packages/Readme.md: -------------------------------------------------------------------------------- 1 | Local reconfigured in non-Database PackageStore mode only. ECS will use the DB. 2 | -------------------------------------------------------------------------------- /run-cron.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Starting refresh..." 3 | APP_ENV=${APP_ENV} php bin/console refresh 4 | echo "Starting update..." 5 | APP_ENV=${APP_ENV} php bin/console update 6 | echo "Starting build..." 7 | APP_ENV=${APP_ENV} php bin/console build 8 | 9 | echo "Done!" 10 | -------------------------------------------------------------------------------- /src/Command/BuildCommand.php: -------------------------------------------------------------------------------- 1 | builder = $builder; 36 | $this->entityManager = $entityManager; 37 | $this->storage = $storage; 38 | 39 | parent::__construct($name); 40 | } 41 | 42 | protected function configure() 43 | { 44 | $this 45 | ->setName('build') 46 | ->setDescription('Build packages.json from DB'); 47 | } 48 | 49 | protected function execute(InputInterface $input, OutputInterface $output): int 50 | { 51 | $output->writeln("Building packages"); 52 | 53 | $start = new \DateTime(); 54 | 55 | $this->doBuild( 56 | allowMoreTries: 1, 57 | output: $output, 58 | ); 59 | 60 | $this->showProviders($output); 61 | 62 | $interval = $start->diff(new \DateTime()); 63 | $output->writeln("Wrote package data in " . $interval->format('%Hh %Im %Ss')); 64 | 65 | return 0; 66 | } 67 | 68 | protected function doBuild(int $allowMoreTries, OutputInterface $output) 69 | { 70 | $output->writeln('Starting main build...'); 71 | 72 | /** @var PackageRepository $packageRepo */ 73 | $packageRepo = $this->entityManager->getRepository(Package::class); 74 | 75 | // ensure all packages have the right provider group assigned 76 | $packageRepo->updateProviderGroups(); 77 | 78 | $packages = $packageRepo->findActive(); 79 | // once we have the packages, we don't need them to be tracked any more 80 | $this->entityManager->clear(); 81 | 82 | $this->storage->prepare(); 83 | 84 | $providerGroups = []; 85 | 86 | $progressBar = new ProgressBar($output, count($packages)); 87 | 88 | foreach ($packages as $package) { 89 | $this->builder->updatePackage($package); 90 | $progressBar->advance(); 91 | $group = $package->getProviderGroup(); 92 | if (!array_key_exists($group, $providerGroups)) { 93 | $providerGroups[$group] = []; 94 | } 95 | $providerGroups[$group][] = $package->getPackageName(); 96 | } 97 | 98 | $progressBar->finish(); 99 | 100 | $output->writeln(''); 101 | 102 | $output->writeln('Finalising package data...'); 103 | 104 | /** 105 | * In rare edge cases this can hit a `UniqueConstraintViolationException` and would previously crash the 106 | * process. We now allow retries of the whole command if this happens. {@see Database} still logs a 107 | * warning so we can easily evaluate frequency of these events, without adding log noise if it's 108 | * recoverable. For now we allow only 1 retry per build command run. 109 | */ 110 | try { 111 | $this->storage->persist(); 112 | } catch (UniqueConstraintViolationException $exception) { 113 | if ($allowMoreTries > 0) { 114 | $output->writeln('Caught a `UniqueConstraintViolationException` while finalising package data. Retrying...'); 115 | $this->doBuild( 116 | allowMoreTries: $allowMoreTries - 1, 117 | output: $output, 118 | ); 119 | } else { 120 | throw $exception; 121 | } 122 | } 123 | 124 | // now all of the packages are up-to-date, rebuild all of the provider groups and the root 125 | foreach ($providerGroups as $group => $groupPackageNames) { 126 | $this->builder->updateProviderGroup($group, $groupPackageNames); 127 | } 128 | $this->storage->persist(); 129 | $this->builder->updateRoot(); 130 | 131 | // final persist, to write everything that needs writing 132 | $this->storage->persist(true); 133 | } 134 | 135 | /** 136 | * @param OutputInterface $output 137 | */ 138 | protected function showProviders(OutputInterface $output) 139 | { 140 | $groups = $this->storage->loadAllProviders(); 141 | ksort($groups); 142 | 143 | $table = new Table($output); 144 | $table->setHeaders(['provider', 'packages', 'size']); 145 | 146 | $totalSize = 0; 147 | $totalProviders = 0; 148 | foreach ($groups as $group => $content) { 149 | $json = json_decode($content); 150 | 151 | // Get size in bytes, without resorting to e.g. filesystem operations. 152 | // https://stackoverflow.com/a/9718273/2803757 153 | $count = count((array)$json->providers); 154 | $filesize = mb_strlen($content, '8bit'); 155 | $totalSize += $filesize; 156 | $totalProviders += $count; 157 | $table->addRow([ 158 | $group, 159 | $count, 160 | Helper::formatMemory($filesize), 161 | ]); 162 | } 163 | 164 | $table->addRow(new TableSeparator()); 165 | $table->addRow([ 166 | 'Total', 167 | $totalProviders, 168 | Helper::formatMemory($totalSize), 169 | ]); 170 | 171 | $table->render(); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/Command/RefreshCommand.php: -------------------------------------------------------------------------------- 1 | connection = $entityManager->getConnection(); 25 | $this->entityManager = $entityManager; 26 | 27 | parent::__construct($name); 28 | } 29 | 30 | protected function configure() 31 | { 32 | $this 33 | ->setName('refresh') 34 | ->setDescription('Refresh list of plugins and themes from WP SVN') 35 | ->addOption( 36 | 'svn', 37 | null, 38 | InputOption::VALUE_REQUIRED, 39 | 'Path to svn executable', 40 | 'svn' 41 | ); 42 | } 43 | 44 | protected function execute(InputInterface $input, OutputInterface $output): int 45 | { 46 | $svn = $input->getOption('svn'); 47 | 48 | $types = [ 49 | 'plugin' => Plugin::class, 50 | 'theme' => Theme::class, 51 | ]; 52 | 53 | $updateStmt = $this->connection->prepare('UPDATE packages SET last_committed = :date, provider_group = :group WHERE class_name = :class_name AND name = :name'); 54 | $insertStmt = $this->connection->prepare('INSERT INTO packages (class_name, name, last_committed, provider_group, is_active) VALUES (:class_name, :name, :date, :group, true)'); 55 | 56 | foreach ($types as $type => $class_name) { 57 | /** @var Plugin|Theme $class_name */ 58 | $url = $class_name::getSvnBaseUrl(); 59 | $output->writeln("Fetching full $type list from $url"); 60 | 61 | $xmlLines = []; 62 | exec("$svn ls --xml $url 2>&1", $xmlLines, $returnCode); 63 | if ($returnCode > 0) { 64 | $output->writeln("Error code $returnCode from svn command"); 65 | 66 | return $returnCode; // error code 67 | } 68 | $xml = simplexml_load_string(implode("\n", $xmlLines)); 69 | 70 | $output->writeln("Updating database"); 71 | 72 | $this->connection->beginTransaction(); 73 | $newCount = 0; 74 | foreach ($xml->list->entry as $entry) { 75 | $date = date('Y-m-d H:i:s', strtotime((string) $entry->commit->date)); 76 | $params = [ 77 | ':class_name' => $class_name, 78 | ':name' => (string) $entry->name, 79 | ':date' => $date, 80 | ':group' => Package::makeComposerProviderGroup($date) 81 | ]; 82 | 83 | $updateStmt->execute($params); 84 | if ($updateStmt->rowCount() == 0) { 85 | $insertStmt->execute($params); 86 | $newCount++; 87 | } 88 | } 89 | $this->connection->commit(); 90 | 91 | $updateCount = $this->entityManager->getRepository(Package::class) 92 | ->getNewlyRefreshedCount($class_name); 93 | 94 | $output->writeln("Found $newCount new and $updateCount updated {$type}s"); 95 | } 96 | 97 | return 0; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/Command/UpdateCommand.php: -------------------------------------------------------------------------------- 1 | updateService = $updateService; 20 | 21 | parent::__construct($name); 22 | } 23 | 24 | protected function configure() 25 | { 26 | $this 27 | ->setName('update') 28 | ->setDescription('Update version info for individual plugins') 29 | ->addOption( 30 | 'name', 31 | null, 32 | InputOption::VALUE_OPTIONAL, 33 | 'Name of package to update', 34 | null 35 | ); 36 | } 37 | 38 | /** 39 | * Parse the $version => $tag from the developers tab of wordpress.org 40 | * Advantages: 41 | * * Checks for invalid and inactive plugins (and ignore it until next SVN commit) 42 | * * Use the parsing mechanism of wordpress.org, which is more robust 43 | * 44 | * Disadvantages: 45 | * * Much slower 46 | * * Subject to changes without notice 47 | * 48 | * Wordpress.org APIs do not list versions history 49 | * @link http://codex.wordpress.org/WordPress.org_API 50 | * 51 | *
  • Development Version (svn)
    • 52 | *
  • VERSION (svn)
    • 53 | */ 54 | protected function execute(InputInterface $input, OutputInterface $output): int 55 | { 56 | $name = $input->getOption('name'); 57 | $logger = new ConsoleLogger($output); 58 | if ($name) { 59 | $this->updateService->updateOne($logger, $name); 60 | } else { 61 | $this->updateService->updateAll($logger); 62 | } 63 | 64 | return 0; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/Controller/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outlandishideas/wpackagist/466f50d4dcdcaeaaec12b2429028a7f159ee92bc/src/Controller/.gitignore -------------------------------------------------------------------------------- /src/Controller/MainController.php: -------------------------------------------------------------------------------- 1 | formFactory = $formFactory; 49 | $this->requestRepository = $requestRepository; 50 | $this->storage = $storage; 51 | } 52 | 53 | /** 54 | * @Route("packages.json", name="json_index") 55 | */ 56 | public function packageIndexJson(): Response 57 | { 58 | $response = new Response($this->storage->loadRoot()); 59 | $response->headers->set('Content-Type', 'application/json'); 60 | 61 | return $response; 62 | } 63 | 64 | /** 65 | * @Route("p/{provider}${hash}.json", name="json_provider", requirements={"hash"="[0-9a-f]{64}"}) 66 | * @param string $provider 67 | * @param string $hash 68 | * @return Response 69 | */ 70 | public function providerJson(string $provider, string $hash): Response 71 | { 72 | $data = $this->storage->loadProvider($provider, $hash); 73 | 74 | if (empty($data)) { 75 | throw new NotFoundHttpException(); 76 | } 77 | 78 | $response = new Response($data); 79 | $response->headers->set('Content-Type', 'application/json'); 80 | 81 | return $response; 82 | } 83 | 84 | /** 85 | * @Route("p/{dir}/{package}${hash}.json", name="json_package", requirements={"hash"="[0-9a-f]{64}"}) 86 | * @param string $dir Directory: wpackagist-plugin or wpackagist-theme. 87 | * @param string $package 88 | * @param string $hash 89 | * @return Response 90 | */ 91 | public function packageJson(string $package, string $hash, string $dir): Response 92 | { 93 | $dir = str_replace('.', '', $dir); 94 | 95 | if (!in_array($dir, ['wpackagist-plugin', 'wpackagist-theme'], true)) { 96 | throw new BadRequestException('Unexpected package path'); 97 | } 98 | 99 | $data = $this->storage->loadPackage("{$dir}/{$package}", $hash); 100 | 101 | if (empty($data)) { 102 | throw new NotFoundHttpException(); 103 | } 104 | 105 | $response = new Response($data); 106 | $response->headers->set('Content-Type', 'application/json'); 107 | 108 | return $response; 109 | } 110 | 111 | public function home(Request $request): Response 112 | { 113 | return $this->render('index.twig', [ 114 | 'title' => 'WPackagist: Manage your WordPress® plugins and themes with Composer', 115 | 'searchForm' => $this->getForm()->handleRequest($request)->createView(), 116 | ]); 117 | } 118 | 119 | public function search(Request $request, EntityManagerInterface $entityManager): Response 120 | { 121 | $queryBuilder = new QueryBuilder($entityManager); 122 | 123 | $form = $this->getForm(); 124 | $form->handleRequest($request); 125 | 126 | $data = $form->getData(); 127 | $type = $data['type'] ?? null; 128 | $query = empty($data['q']) ? null : trim($data['q']); 129 | // Existing encoding is mb-guessed since PHP 8. 130 | $query = mb_convert_encoding($query, 'UTF-8'); 131 | 132 | $data = [ 133 | 'title' => "WPackagist: Search packages", 134 | 'searchForm' => $form->createView(), 135 | 'currentPageResults' => '', 136 | 'error' => '', 137 | ]; 138 | 139 | // TODO move search query logic to PackageRepository. 140 | $queryBuilder 141 | ->select('p'); 142 | 143 | switch ($type) { 144 | case 'theme': 145 | $queryBuilder->from(Theme::class, 'p'); 146 | break; 147 | case 'plugin': 148 | $queryBuilder->from(Plugin::class, 'p'); 149 | break; 150 | default: 151 | $queryBuilder->from(Package::class, 'p'); 152 | } 153 | 154 | if (!empty($query)) { 155 | $queryBuilder 156 | ->andWhere($queryBuilder->expr()->orX( 157 | $queryBuilder->expr()->like('p.name', ':name'), 158 | $queryBuilder->expr()->like('p.displayName', ':name') 159 | )) 160 | ->addSelect('CASE WHEN p.name = :nameWithoutWildcards THEN 0 ELSE 1 END AS HIDDEN sortCondition') 161 | ->addOrderBy('sortCondition, p.name', 'ASC') 162 | ->setParameter('name', "%{$query}%") 163 | ->setParameter('nameWithoutWildcards', $query); 164 | } else { 165 | $queryBuilder 166 | ->addOrderBy('p.lastCommitted', 'DESC'); 167 | } 168 | 169 | $adapter = new QueryAdapter($queryBuilder); 170 | $pagerfanta = new Pagerfanta($adapter); 171 | $pagerfanta->setMaxPerPage(30); 172 | $page = $request->query->get('page'); 173 | if (!is_numeric($page)) { 174 | $page = 1; 175 | } 176 | $pagerfanta->setCurrentPage(intval($page)); 177 | 178 | $data['pager'] = $pagerfanta; 179 | $data['currentPageResults'] = $pagerfanta->getCurrentPageResults(); 180 | 181 | return $this->render('search.twig', $data); 182 | } 183 | 184 | public function update( 185 | Request $request, 186 | EntityManagerInterface $entityManager, 187 | LoggerInterface $logger, 188 | Service\Update $updateService, 189 | Storage\PackageStore $storage, 190 | Service\Builder $builder 191 | ): Response 192 | { 193 | $storage->prepare(true); 194 | 195 | // first run the update command 196 | $name = $request->get('name'); 197 | 198 | if (is_array($name)) { 199 | $name = reset($name); 200 | } 201 | 202 | if (!trim($name)) { 203 | return new Response('Invalid Request',400); 204 | } 205 | 206 | /** @var PackageRepository $packageRepo */ 207 | $packageRepo = $entityManager->getRepository(Package::class); 208 | 209 | $package = $packageRepo->findOneBy(['name' => $name]); 210 | if (!$package) { 211 | return new Response('Not Found',404); 212 | } 213 | 214 | $requestCount = $this->requestRepository->getRequestCountByIp($request->getClientIp(), 0); 215 | if ($requestCount > 5) { 216 | return new Response('Too many requests. Try again in an hour.', 403); 217 | } 218 | 219 | $safeName = $package->getName(); 220 | 221 | try { 222 | $this->doSingleUpdate($logger, $builder, $updateService, $storage, $packageRepo, $safeName); 223 | } catch (UniqueConstraintViolationException $exception) { 224 | // Permit exactly 1 retry for now, after a random 0.5 – 3 second wait. 225 | usleep(random_int(500000, 3000000)); 226 | $entityManager->refresh($package); 227 | $this->doSingleUpdate($logger, $builder, $updateService, $storage, $packageRepo, $safeName); 228 | } 229 | 230 | return new RedirectResponse('/search?q=' . $safeName); 231 | } 232 | 233 | private function getForm(): FormInterface 234 | { 235 | if (!isset($this->form)) { 236 | $this->form = $this->formFactory 237 | // A named builder with blank name enables not having a param prefix like `formName[fieldName]`. 238 | ->createNamedBuilder('', FormType::class, null, ['csrf_protection' => false]) 239 | ->setAction('search') 240 | ->setMethod('GET') 241 | ->add('q', SearchType::class) 242 | ->add('type', ChoiceType::class, [ 243 | 'choices' => [ 244 | 'All packages' => 'any', 245 | 'Plugins' => 'plugin', 246 | 'Themes' => 'theme', 247 | ], 248 | ]) 249 | ->add('search', SubmitType::class) 250 | ->getForm(); 251 | } 252 | 253 | return $this->form; 254 | } 255 | 256 | private function doSingleUpdate( 257 | LoggerInterface $logger, 258 | Service\Builder $builder, 259 | Service\Update $updateService, 260 | Storage\PackageStore $storage, 261 | PackageRepository $packageRepo, 262 | string $safeName 263 | ): void 264 | { 265 | $package = $updateService->updateOne($logger, $safeName); 266 | if ($package && !empty($package->getVersions()) && $package->isActive()) { 267 | // update just the package 268 | $builder->updatePackage($package); 269 | $storage->persist(); 270 | 271 | // then update the corresponding group and the root provider, using all packages in the same group 272 | $group = $package->getProviderGroup(); 273 | $groupPackageNames = $packageRepo->findActivePackageNamesByGroup($group); 274 | $builder->updateProviderGroup($group, $groupPackageNames); 275 | $storage->persist(); 276 | $builder->updateRoot(); 277 | } 278 | 279 | // updates are complete, so persist everything 280 | $storage->persist(true); 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /src/Controller/OpenSearchController.php: -------------------------------------------------------------------------------- 1 | render('opensearch.twig', ['host' => $request->getHttpHost()]); 14 | $response->headers->add(['Content-Type' => 'application/opensearchdescription+xml']); 15 | 16 | return $response; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Entity/Package.php: -------------------------------------------------------------------------------- 1 | getName() . '/'; 125 | } 126 | 127 | /** 128 | * @return string package shortname 129 | */ 130 | public function getName(): string 131 | { 132 | return $this->name; 133 | } 134 | 135 | public function getSvnUrl(): string 136 | { 137 | return static::getSvnBaseUrl() . "{$this->getName()}/"; 138 | } 139 | 140 | /** 141 | * @return string "wpackagist-TYPE/PACKAGE" 142 | */ 143 | public function getPackageName(): string 144 | { 145 | return $this->getVendorName() . '/' . $this->getName(); 146 | } 147 | 148 | /** 149 | * @return DateTime 150 | */ 151 | public function getLastCommitted(): DateTime 152 | { 153 | return $this->lastCommitted; 154 | } 155 | 156 | /** 157 | * @return DateTime|null 158 | */ 159 | public function getLastFetched(): ?DateTime 160 | { 161 | return $this->lastFetched; 162 | } 163 | 164 | /** 165 | * @return array 3-dimensional associative array of Composer format data for every 166 | * package, ready to be JSON-encoded, where keys are: 167 | * * first level = package name; 168 | * * second level = version; 169 | * * third level = Composer metadata item, e.g. 'version_normalized'. 170 | */ 171 | public function getVersionData(): array 172 | { 173 | if (empty($this->versions)) { // May be null when read from persisted data, or empty array 174 | return []; 175 | } 176 | 177 | $versions = []; 178 | foreach ($this->versions as $version => $tag) { 179 | try { 180 | $versions[$version] = $this->getPackageVersion($version); 181 | } catch (\UnexpectedValueException $e) { 182 | //skip packages with weird version numbers 183 | } 184 | } 185 | 186 | return $versions; 187 | } 188 | 189 | /** 190 | * @param $version 191 | * @return array Associative array of Composer format data, ready to be JSON-encoded. 192 | * @throws \UnexpectedValueException 193 | */ 194 | public function getPackageVersion($version) 195 | { 196 | $versionParser = new VersionParser(); 197 | $normalizedVersion = $versionParser->normalize($version); 198 | 199 | $tag = $this->versions[$version]; 200 | 201 | $package = [ 202 | 'name' => $this->getPackageName(), 203 | 'version' => $version, 204 | 'version_normalized' => $normalizedVersion, 205 | 'uid' => hash('sha256', $this->id . '|' . $normalizedVersion), 206 | ]; 207 | 208 | if ($version === 'dev-trunk') { 209 | $package['time'] = $this->getLastCommitted()->format('Y-m-d H:i:s'); 210 | } 211 | 212 | if ($url = $this->getDownloadUrl($version)) { 213 | $package['dist'] = [ 214 | 'type' => 'zip', 215 | 'url' => $url, 216 | ]; 217 | } 218 | 219 | if (($url = $this->getSvnUrl()) && $tag) { 220 | $package['source'] = [ 221 | 'type' => 'svn', 222 | 'url' => $this->getSvnUrl(), 223 | 'reference' => $tag, 224 | ]; 225 | } 226 | 227 | if ($url = $this->getHomepageUrl()) { 228 | $package['homepage'] = $url; 229 | } 230 | 231 | if ($type = $this->getComposerType()) { 232 | $package['require']['composer/installers'] = '^1.0 || ^2.0'; 233 | $package['type'] = $type; 234 | } 235 | 236 | return $package; 237 | } 238 | 239 | public function getVersions(): ?array 240 | { 241 | return $this->versions; 242 | } 243 | 244 | public function isActive(): bool 245 | { 246 | return $this->isActive; 247 | } 248 | 249 | public function setIsActive(bool $active) 250 | { 251 | $this->isActive = $active; 252 | } 253 | 254 | public function setVersions(array $versions) 255 | { 256 | $this->versions = $versions; 257 | } 258 | 259 | public function setDisplayName(string $displayName) 260 | { 261 | if ($displayName) { 262 | $displayName = mb_substr($displayName, 0, 255); 263 | } 264 | $this->displayName = $displayName; 265 | } 266 | 267 | public function setLastFetched(\DateTime $lastFetched) 268 | { 269 | $this->lastFetched = $lastFetched; 270 | } 271 | 272 | public function setLastCommitted(\DateTime $lastCommitted) 273 | { 274 | $this->lastCommitted = $lastCommitted; 275 | } 276 | 277 | public function getId(): int 278 | { 279 | return $this->id; 280 | } 281 | 282 | /** 283 | * @return string Short type: 'plugin' or 'theme'. 284 | */ 285 | public function getType(): string 286 | { 287 | return str_replace('wordpress-', '', $this->getComposerType()); 288 | } 289 | 290 | public function getProviderGroup() 291 | { 292 | return $this->providerGroup; 293 | } 294 | 295 | /** 296 | * Return a string to split packages in more-or-less even groups 297 | * of their last modification. Minimizes groups modifications. 298 | * 299 | * @param DateTime|string $date 300 | * @return string 301 | */ 302 | public static function makeComposerProviderGroup($date) 303 | { 304 | if (empty($date)) { 305 | return 'old'; 306 | } 307 | 308 | if (is_string($date)) { 309 | $date = new DateTime($date); 310 | } 311 | 312 | if ($date >= new DateTime('monday last week')) { 313 | return 'this-week'; 314 | } 315 | 316 | if ($date >= new DateTime(date('Y') . '-01-01')) { 317 | // split current by chunks of 3 months, current month included 318 | // past chunks will never be update this year 319 | $month = $date->format('n'); 320 | $month = ceil($month / 3) * 3; 321 | $month = str_pad($month, 2, '0', STR_PAD_LEFT); 322 | 323 | return $date->format('Y-') . $month; 324 | } 325 | 326 | if ($date >= new DateTime(self::PROVIDER_GROUP_OLD_CUTOFF)) { 327 | // split by years, limit at 2011 so we never update 'old' again 328 | return $date->format('Y'); 329 | } 330 | 331 | // 2010 and older is about 2M, which is manageable 332 | // Still some packages ? Probably junk/errors 333 | return 'old'; 334 | } 335 | } 336 | -------------------------------------------------------------------------------- /src/Entity/PackageData.php: -------------------------------------------------------------------------------- 1 | value; 52 | } 53 | 54 | /** 55 | * @return string 56 | */ 57 | public function getName(): string 58 | { 59 | return $this->name; 60 | } 61 | 62 | /** 63 | * @return string 64 | */ 65 | public function getType(): string 66 | { 67 | return $this->type; 68 | } 69 | 70 | /** 71 | * @return string 72 | */ 73 | public function getHash(): string 74 | { 75 | return $this->hash; 76 | } 77 | 78 | /** 79 | * @return bool 80 | */ 81 | public function getIsLatest(): bool 82 | { 83 | return $this->isLatest; 84 | } 85 | 86 | public function setType(string $type): void 87 | { 88 | $this->type = $type; 89 | } 90 | 91 | public function setName(string $name): void 92 | { 93 | $this->name = $name; 94 | } 95 | 96 | public function setHash(string $hash): void 97 | { 98 | $this->hash = $hash; 99 | } 100 | 101 | public function setValue(string $value): void 102 | { 103 | $this->value = $value; 104 | } 105 | 106 | public function setIsLatest(bool $latest): void 107 | { 108 | $this->isLatest = $latest; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/Entity/PackageRepository.php: -------------------------------------------------------------------------------- 1 | getEntityManager(); 18 | 19 | // build the groups, trying to keep them of roughly equal numbers of packages 20 | $year = date('Y'); 21 | $groups = [ 22 | ['group' => 'this-week', 'start' => new \DateTimeImmutable('monday last week')], 23 | ['group' => $year . '-12', 'start' => new \DateTimeImmutable($year . '-10-01')], 24 | ['group' => $year . '-09', 'start' => new \DateTimeImmutable($year . '-07-01')], 25 | ['group' => $year . '-06', 'start' => new \DateTimeImmutable($year . '-04-01')], 26 | ['group' => $year . '-03', 'start' => new \DateTimeImmutable($year . '-01-01')], 27 | ]; 28 | for ($y=$year-1; $y>=$treatAsOldBeforeDate->format('Y'); $y--) { 29 | $groups[] = ['group' => $y, 'start' => new \DateTimeImmutable($y . '-01-01')]; 30 | } 31 | 32 | foreach ($groups as $i => $group) { 33 | $groups[$i]['start'] = $group['start']->format('Y-m-d 00:00:00'); 34 | if ($i === 0) { 35 | $groups[$i]['end'] = (new \DateTimeImmutable())->format('Y-m-d 23:59:59'); 36 | } else { 37 | $groups[$i]['end'] = $groups[$i-1]['start']; 38 | } 39 | } 40 | 41 | $query = (new QueryBuilder($em))->update(Package::class, 'p') 42 | ->set('p.providerGroup', ':group') 43 | ->where('p.lastCommitted BETWEEN :start AND :end') 44 | ->getQuery(); 45 | foreach ($groups as $group) { 46 | $query->execute($group); 47 | } 48 | 49 | 50 | $oldPackagesQuery = (new QueryBuilder($em))->update(Package::class, 'p') 51 | ->set('p.providerGroup', ':group') 52 | ->where('p.lastCommitted < :cutoffDate') 53 | ->getQuery(); 54 | $oldPackagesQuery->execute([ 55 | 'group' => 'old', 56 | 'cutoffDate' => $treatAsOldBeforeDate->format('Y-m-d'), 57 | ]); 58 | } 59 | 60 | /** 61 | * Get packages that have never been fetched or have been updated since last 62 | * being fetched or that are inactive but have been updated in the past 90 days 63 | * and haven't been fetched in the past 7 days. 64 | * 65 | * @return Package[] 66 | */ 67 | public function findDueUpdate(): array 68 | { 69 | $entityManager = $this->getEntityManager(); 70 | $dql = << :twoHours 75 | OR (p.isActive = false AND p.lastCommitted > :threeMonthsAgo AND p.lastFetched < :oneWeekAgo) 76 | EOT; 77 | $dateFormat = $this->getEntityManager()->getConnection()->getDatabasePlatform()->getDateTimeFormatString(); 78 | $query = $entityManager->createQuery($dql) 79 | // This seems to be how Doctrine wants its 'interval' type bound – not a DateInterval 80 | ->setParameter('twoHours', '2 hour') 81 | ->setParameter('threeMonthsAgo', (new \DateTime())->sub(new \DateInterval('P3M'))->format($dateFormat)) 82 | ->setParameter('oneWeekAgo', (new \DateTime())->sub(new \DateInterval('P1W'))->format($dateFormat)); 83 | 84 | return $query->getResult(); 85 | } 86 | 87 | /** 88 | * @param string|null $packageName 89 | * @return Package[] 90 | */ 91 | public function findActive(?string $packageName = null): array 92 | { 93 | $entityManager = $this->getEntityManager(); 94 | 95 | $qb = new QueryBuilder($entityManager); 96 | $qb = $qb->select('p') 97 | ->from(Package::class, 'p') 98 | ->where('p.versions IS NOT NULL') 99 | ->andWhere('p.isActive = true'); 100 | 101 | if ($packageName) { 102 | $qb = $qb->andWhere('p.name = :name') 103 | ->setParameter('name', $packageName); 104 | } 105 | 106 | $qb = $qb->orderBy('p.name', 'ASC'); 107 | 108 | return $qb->getQuery()->getResult(); 109 | } 110 | 111 | public function findActivePackageNamesByGroup($group): array 112 | { 113 | $entityManager = $this->getEntityManager(); 114 | 115 | $qb = new QueryBuilder($entityManager); 116 | $qb = $qb->select('partial p.{id, name}') 117 | ->from(Package::class, 'p') 118 | ->where('p.versions IS NOT NULL') 119 | ->andWhere('p.providerGroup = :group') 120 | ->andWhere('p.isActive = true'); 121 | $qb->setParameter('group', $group); 122 | 123 | $packages = $qb->getQuery()->getResult(); 124 | return array_map(function (Package $package) { 125 | return $package->getPackageName(); 126 | }, $packages); 127 | } 128 | 129 | public function getNewlyRefreshedCount(string $className): int 130 | { 131 | $qb = new QueryBuilder($this->getEntityManager()); 132 | $qb = $qb->select('count(p.id)') 133 | ->from(Package::class, 'p') 134 | ->where('p INSTANCE OF :className') 135 | ->andWhere('p.lastFetched < p.lastCommitted') 136 | ->setParameter('className', $className); 137 | 138 | return $qb->getQuery()->getSingleScalarResult(); 139 | } 140 | 141 | /** 142 | * @param string $providerGroupMain Provider group without prefix, e.g. '2020', '2021-03', 'this-week'. 143 | * @return int 144 | */ 145 | public function getCountByGroup(string $providerGroupMain): int 146 | { 147 | $qb = new QueryBuilder($this->getEntityManager()); 148 | $qb = $qb->select('count(p.id)') 149 | ->from(Package::class, 'p') 150 | ->where('p.providerGroup = :providerGroup') 151 | ->setParameter('providerGroup', $providerGroupMain); 152 | 153 | return $qb->getQuery()->getSingleScalarResult(); 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/Entity/Plugin.php: -------------------------------------------------------------------------------- 1 | versions[$version] === 'trunk'; 20 | 21 | //Assemble file name and append ?timestamp= variable to the trunk version to avoid Composer cache when plugin/theme author only updates the trunk 22 | $filename = ($isTrunk ? $this->getName() : $this->getName().'.'.$version) . '.zip' . ($isTrunk ? '?timestamp=' . urlencode($this->lastCommitted->format('U')) : ''); 23 | 24 | return "https://downloads.wordpress.org/plugin/{$filename}"; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Entity/Request.php: -------------------------------------------------------------------------------- 1 | requestCount++; 43 | $this->lastRequest = new \DateTime(); 44 | } 45 | 46 | public function getRequestCount(): int 47 | { 48 | return $this->requestCount; 49 | } 50 | 51 | public function setIpAddress(string $ipAddress): void 52 | { 53 | $this->ipAddress = $ipAddress; 54 | } 55 | 56 | public function setRequestCount(int $requestCount): void 57 | { 58 | $this->requestCount = $requestCount; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Entity/RequestRepository.php: -------------------------------------------------------------------------------- 1 | logger = $logger; 26 | 27 | parent::__construct($em, $class); 28 | } 29 | 30 | /** 31 | * Get a count of sensitive requests for an IP, incrementing the counter as a side effect. 32 | * 33 | * @param string $ip 34 | * @param int $previousTries Work around record contention edge case while avoiding risk of an infinite loop. 35 | * @return int The number of requests within the past 24 hours 36 | * @throws \Doctrine\DBAL\DBALException 37 | */ 38 | public function getRequestCountByIp(string $ip, int $previousTries): int 39 | { 40 | $em = $this->getEntityManager(); 41 | $qb = new QueryBuilder($em); 42 | 43 | $oneHourAgo = (new \DateTime())->sub(new \DateInterval('PT1H')); 44 | 45 | $qb->select('r') 46 | ->from(Request::class, 'r') 47 | ->where('r.ipAddress = :ip') 48 | ->andWhere('r.lastRequest >= :cutoff') 49 | ->setParameter('ip', $ip) 50 | ->setParameter('cutoff', $oneHourAgo); 51 | 52 | // `wrapInTransaction()` auto-flushes on commit / return, but we still saw a rare edge case where an insert 53 | // led to a unique IP constraint violation. For now we are logging a warning when this happens and trying 54 | // a second time. 55 | // See https://doctrine2.readthedocs.io/en/latest/reference/transactions-and-concurrency.html#approach-2-explicitly 56 | $requestItem = $em->wrapInTransaction(function () use ($em, $qb, $ip, $previousTries, $oneHourAgo) { 57 | $requestHistory = $qb->getQuery()->getResult(); 58 | 59 | if (empty($requestHistory)) { 60 | $this->deleteOldRequestCounts($ip, $oneHourAgo); 61 | 62 | $requestItem = new Request(); 63 | $requestItem->setIpAddress($ip); 64 | } else { 65 | $requestItem = $requestHistory[0]; 66 | } 67 | 68 | $requestItem->addRequest(); 69 | $em->persist($requestItem); 70 | 71 | try { 72 | $em->flush(); 73 | } catch (UniqueConstraintViolationException $exception) { 74 | $logLevel = $previousTries === 0 ? LogLevel::WARNING : LogLevel::ERROR; 75 | $this->logger->log( 76 | $logLevel, 77 | sprintf( 78 | 'UniqueConstraintViolationException led to access insert retry for IP %s', 79 | $ip 80 | ) 81 | ); 82 | 83 | if ($previousTries > 0) { 84 | // "Fail safe" by simulating a very high request count if something is going persistently wrong. 85 | $dummyRequest = new Request(); 86 | $dummyRequest->setRequestCount(100000); 87 | return $dummyRequest; 88 | } 89 | 90 | // If we hit a locking edge case just once, try a 2nd time. 91 | return $this->getRequestCountByIp($ip, $previousTries + 1); 92 | } 93 | 94 | return $requestItem; 95 | }); 96 | 97 | return $requestItem->getRequestCount(); 98 | } 99 | 100 | /** 101 | * Remove expired entries for the provided IP address. 102 | * 103 | * @param string $ip 104 | * @param \DateTime $cutoff 105 | */ 106 | private function deleteOldRequestCounts(string $ip, \DateTime $cutoff): void 107 | { 108 | $em = $this->getEntityManager(); 109 | $qb = new QueryBuilder($em); 110 | $qb->delete(Request::class, 'r') 111 | ->where('r.ipAddress = :ip') 112 | ->andWhere('r.lastRequest < :cutoff') 113 | ->setParameter('ip', $ip) 114 | ->setParameter('cutoff', $cutoff) 115 | ->getQuery() 116 | ->execute(); 117 | 118 | // Ensure the old record is deleted at DB level before the insert coming up, so we don't 119 | // fall foul of the IP uniqueness constraint. 120 | $em->flush(); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/Entity/Theme.php: -------------------------------------------------------------------------------- 1 | getName().".$version.zip"; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/EventListener/ExceptionListener.php: -------------------------------------------------------------------------------- 1 | logger = $logger; 22 | } 23 | 24 | public function onKernelException(ExceptionEvent $event) 25 | { 26 | // Let Symfony's default error tracing happen in dev. 27 | if ($_SERVER['APP_ENV'] === 'dev') { 28 | return; 29 | } 30 | 31 | $exception = $event->getThrowable(); 32 | $response = new Response(); 33 | 34 | if ($exception instanceof HttpExceptionInterface) { 35 | $response->setStatusCode($exception->getStatusCode()); 36 | $response->headers->replace($exception->getHeaders()); 37 | } else { 38 | $response->setStatusCode(Response::HTTP_INTERNAL_SERVER_ERROR); 39 | } 40 | 41 | // Reduce log severity for scan junk & actual not found requests, inc. unexpected methods like 42 | // `POST /` and made up HTTP methods, e.g. `Invalid method override "__CONSTRUCT"`. 43 | $notFound = ( 44 | $exception instanceof BadRequestHttpException || 45 | $exception instanceof MethodNotAllowedHttpException || 46 | $exception instanceof NotFoundHttpException 47 | ); 48 | 49 | $this->logger->log( 50 | $notFound ? LogLevel::INFO : LogLevel::CRITICAL, 51 | sprintf('%s – %s', get_class($exception), $exception->getMessage()) 52 | ); 53 | 54 | $response->setContent($notFound ? 'The requested page could not be found.' : 'Something went wrong.'); 55 | 56 | $event->setResponse($response); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/Kernel.php: -------------------------------------------------------------------------------- 1 | getProjectDir().'/config/bundles.php'; 19 | foreach ($contents as $class => $envs) { 20 | if ($envs[$this->environment] ?? $envs['all'] ?? false) { 21 | yield new $class(); 22 | } 23 | } 24 | } 25 | 26 | public function getProjectDir(): string 27 | { 28 | return \dirname(__DIR__); 29 | } 30 | 31 | protected function configureContainer(ContainerConfigurator $container): void 32 | { 33 | $container->import('../config/{packages}/*.yaml'); 34 | $container->import('../config/{packages}/'.$this->environment.'/*.yaml'); 35 | 36 | if (is_file(\dirname(__DIR__).'/config/services.yaml')) { 37 | $container->import('../config/services.yaml'); 38 | $container->import('../config/{services}_'.$this->environment.'.yaml'); 39 | } elseif (is_file($path = \dirname(__DIR__).'/config/services.php')) { 40 | (require $path)($container->withPath($path), $this); 41 | } 42 | } 43 | 44 | protected function configureRoutes(RoutingConfigurator $routes): void 45 | { 46 | $routes->import('../config/{routes}/'.$this->environment.'/*.yaml'); 47 | $routes->import('../config/{routes}/*.yaml'); 48 | 49 | if (is_file(\dirname(__DIR__).'/config/routes.yaml')) { 50 | $routes->import('../config/routes.yaml'); 51 | } elseif (is_file($path = \dirname(__DIR__).'/config/routes.php')) { 52 | (require $path)($routes->withPath($path), $this); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/Migrations/Version20200924213211.php: -------------------------------------------------------------------------------- 1 | addSql('CREATE TABLE packages (id SERIAL PRIMARY KEY, class_name VARCHAR(50) NOT NULL, name VARCHAR(255) NOT NULL, last_committed TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, last_fetched TIMESTAMP(0) WITHOUT TIME ZONE, versions JSON, is_active BOOLEAN NOT NULL, display_name VARCHAR(255))'); 23 | $this->addSql('CREATE INDEX last_committed_idx ON packages (last_committed)'); 24 | $this->addSql('CREATE INDEX last_fetched_idx ON packages (last_fetched)'); 25 | $this->addSql('CREATE UNIQUE INDEX type_and_name_unique ON packages (class_name, name)'); 26 | $this->addSql('CREATE TABLE requests (id SERIAL PRIMARY KEY, ip_address VARCHAR(15) NOT NULL, last_request TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, request_count INT NOT NULL)'); 27 | $this->addSql('CREATE UNIQUE INDEX UNIQ_7B85D65122FFD58C ON requests (ip_address)'); 28 | $this->addSql('CREATE TABLE state (key VARCHAR(50) NOT NULL, value VARCHAR(50) NOT NULL, PRIMARY KEY(key))'); 29 | 30 | // Sly actually-a-fixture to maintain compatibility with the previous Sqlite auto setup. 31 | $this->addSql("INSERT INTO state (key, value) VALUES ('build_required', '')"); 32 | } 33 | 34 | public function down(Schema $schema) : void 35 | { 36 | $this->addSql('DROP TABLE packages'); 37 | $this->addSql('DROP TABLE requests'); 38 | $this->addSql('DROP TABLE state'); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Migrations/Version20201029165342.php: -------------------------------------------------------------------------------- 1 | addSql('CREATE TABLE package_data (type VARCHAR(10) NOT NULL, name VARCHAR(200) NOT NULL, hash VARCHAR(64) NOT NULL, value TEXT DEFAULT NULL, is_latest BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY(type, name, hash))'); 24 | $this->addSql('CREATE INDEX package_data_is_latest_idx ON package_data (is_latest)'); 25 | $this->addSql('DROP TABLE state'); 26 | $this->addSql('ALTER TABLE packages ALTER class_name TYPE VARCHAR(255)'); 27 | 28 | $this->addSql("UPDATE packages SET class_name = replace(class_name, '\\Package\\', '\\Entity\\')"); 29 | } 30 | 31 | public function down(Schema $schema) : void 32 | { 33 | // this down() migration is auto-generated, please modify it to your needs 34 | $this->addSql('CREATE TABLE state (key VARCHAR(50) NOT NULL, value VARCHAR(50) NOT NULL, PRIMARY KEY(key))'); 35 | $this->addSql('DROP TABLE package_data'); 36 | $this->addSql('ALTER TABLE packages ALTER class_name TYPE VARCHAR(50)'); 37 | 38 | $this->addSql("UPDATE packages SET class_name = replace(class_name, '\\Entity\\', '\\Package\\')"); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Migrations/Version20201104150851.php: -------------------------------------------------------------------------------- 1 | addSql('ALTER TABLE packages ADD provider_group VARCHAR(255) NOT NULL default \'old\''); 24 | $this->addSql('CREATE INDEX provider_group_idx ON packages (provider_group)'); 25 | $this->addSql('CREATE INDEX package_is_active_idx ON packages (is_active)'); 26 | $year = date('Y'); 27 | $groups = [ 28 | 'this-week' => new \DateTime('monday last week'), 29 | $year . '-12' => new \DateTime($year . '-10-01'), 30 | $year . '-09' => new \DateTime($year . '-07-01'), 31 | $year . '-06' => new \DateTime($year . '-04-01'), 32 | $year . '-03' => new \DateTime($year . '-01-01'), 33 | ]; 34 | for ($y=$year-1; $y>=2011; $y--) { 35 | $groups[$y] = new \DateTime($y . '-01-01'); 36 | } 37 | foreach ($groups as $key=>$date) { 38 | $this->addSql( 39 | 'UPDATE packages SET provider_group = :group WHERE provider_group = \'old\' AND last_committed >= :date', 40 | ['group' => $key, 'date' => $date->format('Y-m-d')] 41 | ); 42 | } 43 | } 44 | 45 | public function down(Schema $schema) : void 46 | { 47 | // this down() migration is auto-generated, please modify it to your needs 48 | $this->addSql('ALTER TABLE packages DROP provider_group'); 49 | $this->addSql('DROP INDEX package_is_active_idx'); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Migrations/Version20231025122432.php: -------------------------------------------------------------------------------- 1 | addSql('CREATE INDEX package_class_and_last_committed_idx ON packages (class_name, last_committed)'); 23 | $this->addSql('ALTER INDEX last_committed_idx RENAME TO package_last_committed_idx'); 24 | $this->addSql('ALTER INDEX last_fetched_idx RENAME TO package_last_fetched_idx'); 25 | $this->addSql('ALTER INDEX provider_group_idx RENAME TO package_provider_group_idx'); 26 | $this->addSql('ALTER INDEX type_and_name_unique RENAME TO package_type_and_name_unique'); 27 | } 28 | 29 | public function down(Schema $schema): void 30 | { 31 | $this->addSql('DROP INDEX package_class_and_last_committed_idx'); 32 | $this->addSql('ALTER INDEX package_last_fetched_idx RENAME TO last_fetched_idx'); 33 | $this->addSql('ALTER INDEX package_last_committed_idx RENAME TO last_committed_idx'); 34 | $this->addSql('ALTER INDEX package_provider_group_idx RENAME TO provider_group_idx'); 35 | $this->addSql('ALTER INDEX package_type_and_name_unique RENAME TO type_and_name_unique'); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/Persistence/RetrySafeEntityManager.php: -------------------------------------------------------------------------------- 1 | connection = $connection; 48 | $this->logger = $logger; 49 | $this->ormConfig = $ormConfig; 50 | 51 | $this->entityManager = $this->buildEntityManager(); 52 | parent::__construct($this->entityManager); 53 | } 54 | 55 | public function transactional($func): mixed 56 | { 57 | $retries = 0; 58 | do { 59 | $this->beginTransaction(); 60 | 61 | try { 62 | $ret = $func(); 63 | 64 | $this->flush(); 65 | $this->commit(); 66 | 67 | return $ret; 68 | } catch (RetryableException $ex) { 69 | $this->rollback(); 70 | $this->close(); 71 | 72 | $this->logger->warning('RetrySafeEntityManager rolling back from ' . get_class($ex)); 73 | usleep(random_int(0, 200000)); // Wait between 0 and 0.2 seconds before retrying 74 | 75 | $this->resetManager(); 76 | ++$retries; 77 | } catch (\Exception $ex) { 78 | $this->rollback(); 79 | $this->logger->error( 80 | 'RetrySafeEntityManager bailing out having hit ' . get_class($ex) . ': ' . $ex->getMessage() 81 | ); 82 | 83 | throw $ex; 84 | } 85 | } while ($retries < $this->maxLockRetries); 86 | 87 | $this->logger->error('RetrySafeEntityManager bailing out after ' . $this->maxLockRetries . ' tries'); 88 | 89 | throw $ex; 90 | } 91 | 92 | /** 93 | * Attempt a persist the normal way, and if the underlying EM is closed, make a new one 94 | * and try a second time. We were forced to take this approach because the properties 95 | * tracking a closed EM are annotated private. 96 | * 97 | * {@inheritDoc} 98 | */ 99 | public function persist($object): void 100 | { 101 | try { 102 | $this->entityManager->persist($object); 103 | } catch (EntityManagerClosed $closedException) { 104 | $this->logger->warning('EM closed. RetrySafeEntityManager::persist() trying with a new instance'); 105 | $this->resetManager(); 106 | $this->entityManager->persist($object); 107 | } 108 | } 109 | 110 | /** 111 | * Attempt a flush the normal way, and if the underlying EM is closed, make a new one 112 | * and try a second time. We were forced to take this approach because the properties 113 | * tracking a closed EM are annotated private. 114 | * 115 | * {@inheritDoc} 116 | */ 117 | public function flush($entity = null): void 118 | { 119 | try { 120 | $this->entityManager->flush($entity); 121 | } catch (EntityManagerClosed $closedException) { 122 | $this->logger->warning('EM closed. RetrySafeEntityManager::flush() trying with a new instance'); 123 | $this->resetManager(); 124 | $this->entityManager->flush($entity); 125 | } 126 | } 127 | 128 | public function resetManager(): void 129 | { 130 | $this->entityManager = $this->buildEntityManager(); 131 | } 132 | 133 | /** 134 | * We need to override the base `EntityManager` call with the equivalent so that repositories 135 | * contain the retry-safe EM (i.e. `$this` in our current context) and not the default one. 136 | */ 137 | public function getRepository($className) 138 | { 139 | return $this->ormConfig->getRepositoryFactory()->getRepository($this, $className); 140 | } 141 | 142 | private function buildEntityManager(): EntityManagerInterface 143 | { 144 | return EntityManager::create($this->connection, $this->ormConfig); 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/Service/Builder.php: -------------------------------------------------------------------------------- 1 | entityManager = $entityManager; 20 | $this->storage = $storage; 21 | } 22 | 23 | /** 24 | * Doesn't prepare or finalise storage – call these methods once before and 25 | * after calling this on all packages if updating many. 26 | * 27 | * @param Package $package 28 | */ 29 | public function updatePackage(Package $package): void 30 | { 31 | $packageName = $package->getPackageName(); 32 | 33 | $content = json_encode(['packages' => [$packageName => $package->getVersionData()]]); 34 | $packageSha256 = hash('sha256', $content); 35 | $this->storage->savePackage($packageName, $packageSha256, $content); 36 | } 37 | 38 | /** 39 | * @param string $providerGroupName 40 | * @param string[] $packageNames 41 | */ 42 | public function updateProviderGroup(string $providerGroupName, array $packageNames): void 43 | { 44 | $packagesJson = $this->storage->loadAllPackages($packageNames); 45 | $providerJson = []; 46 | foreach ($packagesJson as $packageName => $packageJson) { 47 | $sha256 = hash('sha256', $packageJson); 48 | $providerJson[$packageName] = ['sha256' => $sha256]; 49 | } 50 | ksort($providerJson); 51 | $providerDataJson = json_encode(['providers' => $providerJson]); 52 | $providersSha256 = hash('sha256', $providerDataJson); 53 | $this->storage->saveProvider("providers-$providerGroupName", $providersSha256, $providerDataJson); 54 | } 55 | 56 | public function updateRoot(): void 57 | { 58 | $providers = $this->storage->loadAllProviders(); 59 | $includes = []; 60 | $providerFormat = 'p/%package%$%hash%.json'; 61 | foreach ($providers as $name => $value) { 62 | $sha256 = hash('sha256', $value); 63 | 64 | // Skip/delete old providers, e.g. 3-monthly groups for past years, 65 | // allowing grouping to smoothly switch over at the start of a new year. 66 | // Packages themselves automatically update provider group e.g. from '2020-12' 67 | // to '2020' when the next year starts. `continue`ing here implicitly removes 68 | // items from `root` while `deactivateProvider()` deletes or deactivates the 69 | // individual provider. 70 | if ($this->providerIsOutdated($name)) { 71 | $this->storage->deactivateProvider($name, $sha256); 72 | continue; 73 | } 74 | 75 | $includes[str_replace('%package%', $name, $providerFormat)] = ['sha256' => $sha256]; 76 | } 77 | 78 | ksort($includes); 79 | 80 | $content = json_encode([ 81 | 'packages' => [], 82 | 'providers-url' => '/' . $providerFormat, 83 | 'provider-includes' => $includes, 84 | 'available-package-patterns' => ['wpackagist-plugin/*', 'wpackagist-theme/*'], 85 | ]); 86 | $this->storage->saveRoot($content); 87 | } 88 | 89 | /** 90 | * @param string $name Provider group name, e.g. 'providers-2021-03', 'providers-this-week'. 91 | * @return bool Whether group name should now be retired. 92 | */ 93 | private function providerIsOutdated(string $name): bool 94 | { 95 | $packageCount = $this->entityManager->getRepository(Package::class) 96 | ->getCountByGroup(str_replace('providers-', '', $name)); 97 | 98 | return $packageCount === 0; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/Service/Update.php: -------------------------------------------------------------------------------- 1 | entityManager = $entityManager; 29 | $this->repo = $entityManager->getRepository(Package::class); 30 | } 31 | 32 | public function updateAll(LoggerInterface $logger): void 33 | { 34 | $packages = $this->repo->findDueUpdate(); 35 | $this->update($logger, $packages); 36 | } 37 | 38 | /** 39 | * @param LoggerInterface $logger 40 | * @param string $name 41 | * @param int $allowMoreTries How many more times we may try to complete the update. 42 | * Defaults to 1 and is decremented by 1 on a retry, which 43 | * calls the same method again. The idea is to check the repo 44 | * for a fresh copy to update if it seems like 2 threads have 45 | * tried to work on the same data simultaneously and hit a unique 46 | * lock violation as a result. 47 | * @return Package|null 48 | * @throws UniqueConstraintViolationException if a unique lock was violated *and* we have no 49 | * tries left. 50 | */ 51 | public function updateOne(LoggerInterface $logger, string $name, int $allowMoreTries = 1): ?Package 52 | { 53 | try { 54 | $package = $this->repo->findOneBy(['name' => $name]); 55 | } catch (EntityManagerClosed $exception) { 56 | $logger->warning('EntityManagerClosed on repo find, resetting...'); 57 | if ($this->entityManager instanceof RetrySafeEntityManager) { 58 | $this->entityManager->resetManager(); 59 | $package = $this->repo->findOneBy(['name' => $name]); 60 | } 61 | } 62 | 63 | if ($package) { 64 | try { 65 | $this->update($logger, [$package]); 66 | } catch (UniqueConstraintViolationException $exception) { 67 | if ($allowMoreTries > 0) { 68 | return $this->updateOne($logger, $name, $allowMoreTries - 1); 69 | } 70 | 71 | // Else we are out of tries. 72 | throw $exception; 73 | } 74 | } 75 | 76 | return $package; 77 | } 78 | 79 | /** 80 | * @param LoggerInterface $logger 81 | * @param Package[] $packages 82 | */ 83 | protected function update(LoggerInterface $logger, array $packages): void 84 | { 85 | $count = count($packages); 86 | $versionParser = new VersionParser(); 87 | 88 | $wporgClient = WporgClient::getClient(); 89 | 90 | $logger->info("Updating {$count} packages"); 91 | 92 | foreach ($packages as $index => $package) { 93 | $percent = $index / $count * 100; 94 | 95 | $name = $package->getName(); 96 | 97 | $info = null; 98 | $fields = ['versions']; 99 | try { 100 | if ($package instanceof Plugin) { 101 | $info = $wporgClient->getPlugin($name, $fields); 102 | } else { 103 | $info = $wporgClient->getTheme($name, $fields); 104 | } 105 | 106 | $logger->info(sprintf("%04.1f%% Fetched %s %s", $percent, $package->getType(), $name)); 107 | } catch (CommandClientException $exception) { 108 | $res = $exception->getResponse(); 109 | $this->deactivate($package, $res->getStatusCode() . ': ' . $res->getReasonPhrase(), $logger); 110 | continue; 111 | } catch (GuzzleException $exception) { 112 | $logger->warning("Skipped {$package->getType()} '{$name}' due to error: '{$exception->getMessage()}'"); 113 | } 114 | 115 | if (empty($info)) { 116 | // Plugin is not active 117 | $this->deactivate($package, 'not active', $logger); 118 | 119 | continue; 120 | } 121 | 122 | //get versions as [version => url] 123 | $versions = $info['versions'] ?: []; 124 | 125 | //current version of plugin not present in tags so add it 126 | if (empty($versions[$info['version']])) { 127 | $logger->info('Adding trunk psuedo-version for ' . $name); 128 | 129 | //add to front of array 130 | $versions = array_reverse($versions, true); 131 | $versions[$info['version']] = 'trunk'; 132 | $versions = array_reverse($versions, true); 133 | } 134 | 135 | //all plugins have a dev-trunk version 136 | if ($package instanceof Plugin) { 137 | unset($versions['trunk']); 138 | $versions['dev-trunk'] = 'trunk'; 139 | } 140 | 141 | foreach ($versions as $version => $url) { 142 | try { 143 | //make sure versions are parseable by Composer 144 | $versionParser->normalize($version); 145 | if ($package instanceof Theme) { 146 | //themes have different SVN folder structure 147 | $versions[$version] = $version; 148 | } elseif ($url !== 'trunk') { 149 | //add ref to SVN tag 150 | $versions[$version] = 'tags/' . $version; 151 | } // else do nothing, for 'trunk'. 152 | } catch (\UnexpectedValueException $e) { 153 | // Version is invalid – we've seen this e.g. with 5 numeric parts. 154 | $logger->info(sprintf( 155 | 'Skipping invalid version %s for %s %s', 156 | $version, 157 | $package->getType(), 158 | $name 159 | )); 160 | unset($versions[$version]); 161 | } 162 | } 163 | 164 | if ($versions) { 165 | $package->setLastFetched(new \DateTime()); 166 | $package->setVersions($versions); 167 | $package->setIsActive(true); 168 | $package->setDisplayName($info['name']); 169 | $this->entityManager->persist($package); 170 | } else { 171 | // Package is not active 172 | $this->deactivate($package, 'no versions found', $logger); 173 | } 174 | } 175 | $this->entityManager->flush(); 176 | } 177 | 178 | private function deactivate(Package $package, string $reason, LoggerInterface $logger): void 179 | { 180 | $package->setLastFetched(new \DateTime()); 181 | $package->setIsActive(false); 182 | $this->entityManager->persist($package); 183 | $logger->info(sprintf("Deactivated %s %s because %s", $package->getType(), $package->getName(), $reason)); 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/Storage/Database.php: -------------------------------------------------------------------------------- 1 | entityManager = $entityManager; 29 | $this->logger = $logger; 30 | } 31 | 32 | protected function loadEntity($type, $name, $hash): ?string 33 | { 34 | $data = $this->getRepository()->findOneBy(['type' => $type, 'name' => $name, 'hash' => $hash]); 35 | if ($data) { 36 | return $data->getValue(); 37 | } 38 | 39 | return null; 40 | } 41 | 42 | /** 43 | * @param string $packageName 44 | * @param string $hash 45 | * @return string|null Blank if not found. 46 | */ 47 | public function loadPackage(string $packageName, string $hash): ?string 48 | { 49 | return $this->loadEntity(self::TYPE_PACKAGE, $packageName, $hash); 50 | } 51 | 52 | /** 53 | * @param string $name 54 | * @param string $hash 55 | * @return string|null Blank if not found. 56 | */ 57 | public function loadProvider(string $name, string $hash): ?string 58 | { 59 | return $this->loadEntity(self::TYPE_PROVIDER, $name, $hash); 60 | } 61 | 62 | /** 63 | * @return string|null Blank if not found. 64 | */ 65 | public function loadRoot(): ?string 66 | { 67 | return $this->loadEntity(self::TYPE_ROOT, '', ''); 68 | } 69 | 70 | public function loadLatestEntities($type, $names = null): array 71 | { 72 | // we're not interested in the models, only the keyed values, so don't use the repository 73 | $qb = new QueryBuilder($this->entityManager); 74 | $qb->select('p.name, p.value') 75 | ->from(PackageData::class, 'p') 76 | ->where('p.type = :type') 77 | ->andWhere('p.isLatest = true') 78 | ->setParameter('type', $type); 79 | if ($names) { 80 | $qb->andWhere($qb->expr()->in('p.name', $names)); 81 | } 82 | $data = $qb->getQuery()->getResult(AbstractQuery::HYDRATE_ARRAY); 83 | $values = []; 84 | foreach ($data as $datum) { 85 | $values[$datum['name']] = $datum['value']; 86 | } 87 | 88 | return $values; 89 | } 90 | 91 | public function loadAllPackages($packageNames): array 92 | { 93 | return $this->loadLatestEntities(self::TYPE_PACKAGE, $packageNames); 94 | } 95 | 96 | public function loadAllProviders(): array 97 | { 98 | return $this->loadLatestEntities(self::TYPE_PROVIDER); 99 | } 100 | 101 | protected function saveEntity(string $type, string $name, string $hash, string $json): bool 102 | { 103 | // Update or insert as needed. 104 | /** @var PackageData[] $data */ 105 | $data = $this->getRepository()->findBy(['type' => $type, 'name' => $name]); 106 | $match = null; 107 | 108 | // ensure there is only one 'latest' package data for this entity 109 | foreach ($data as $datum) { 110 | if ($datum->getHash() === $hash) { 111 | $match = $datum; 112 | } elseif ($datum->getIsLatest()) { 113 | $datum->setIsLatest(false); 114 | $this->entityManager->persist($datum); 115 | } else { 116 | $this->entityManager->detach($datum); 117 | } 118 | } 119 | 120 | $changed = false; 121 | if (!$match) { 122 | $match = new PackageData(); 123 | $match->setType($type); 124 | $match->setName($name); 125 | $match->setHash($hash); 126 | $changed = true; 127 | } 128 | if ($json !== $match->getValue()) { 129 | $match->setValue($json); 130 | $changed = true; 131 | } 132 | if (!$match->getIsLatest()) { 133 | $match->setIsLatest(true); 134 | $changed = true; 135 | } 136 | 137 | if ($changed) { 138 | $this->entityManager->persist($match); 139 | } else { 140 | $this->entityManager->detach($match); 141 | } 142 | 143 | return true; 144 | } 145 | 146 | public function savePackage(string $packageName, string $hash, string $json): bool 147 | { 148 | return $this->saveEntity(self::TYPE_PACKAGE, $packageName, $hash, $json); 149 | } 150 | 151 | public function saveProvider(string $name, string $hash, string $json): bool 152 | { 153 | return $this->saveEntity(self::TYPE_PROVIDER, $name, $hash, $json); 154 | } 155 | 156 | /** 157 | * Just sets `isLatest` false on the provider for now. Note that a 'final' `persist()` will 158 | * later hard delete these providers in a full update. 159 | * 160 | * @inheritDoc 161 | * 162 | * @see Database::persist() 163 | */ 164 | public function deactivateProvider(string $name, string $hash): void 165 | { 166 | $qb = new QueryBuilder($this->entityManager); 167 | $qb->update(PackageData::class, 'p') 168 | ->set('p.isLatest', ':newIsLatest') 169 | ->where('p.isLatest = true') 170 | ->andWhere('p.type = :type') 171 | ->andWhere('p.name = :name') 172 | ->getQuery() 173 | ->execute([ 174 | 'newIsLatest' => false, 175 | 'type' => 'provider', 176 | 'name' => $name, 177 | ]); 178 | } 179 | 180 | public function saveRoot(string $json): bool 181 | { 182 | return $this->saveEntity(self::TYPE_ROOT, '', '', $json); 183 | } 184 | 185 | public function prepare($partial = false): void 186 | { 187 | } 188 | 189 | /** 190 | * @throws UniqueConstraintViolationException in possible DB connectivity error edge case. 191 | */ 192 | public function persist($final = false): void 193 | { 194 | try { 195 | $this->entityManager->flush(); 196 | } catch (UniqueConstraintViolationException $exception) { 197 | // This was downgraded to warning level because in cases which *do* represent a high 198 | // severity event, the re-thrown exception will still lead to an error that shows up 199 | // in alarms etc. In cases where we can recover without issue and the exception is caught, 200 | // we're better off not adding noise to our monitoring channels. 201 | $this->logger->warning( 202 | 'UniqueConstraintViolationException in Database::persist(): ' . 203 | $exception->getTraceAsString() 204 | ); 205 | // Leave the exception unhandled so we don't keep processing with incomplete package data. 206 | throw $exception; 207 | } 208 | 209 | if ($final) { 210 | $qb = new QueryBuilder($this->entityManager); 211 | $qb->delete(PackageData::class, 'p') 212 | ->where('p.isLatest = false') 213 | ->getQuery() 214 | ->execute(); 215 | } 216 | } 217 | 218 | protected function getRepository(): ObjectRepository 219 | { 220 | if (!$this->repository) { 221 | $this->repository = $this->entityManager->getRepository(PackageData::class); 222 | } 223 | 224 | return $this->repository; 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /src/Storage/Filesystem.php: -------------------------------------------------------------------------------- 1 | basePath = $basePath; 23 | } 24 | 25 | public function prepare($partial = false): void 26 | { 27 | $this->isPartial = $partial; 28 | 29 | if (!$partial) { 30 | $this->packageDir = 'p.new' . date('YmdHis'); 31 | $fs = new \Symfony\Component\Filesystem\Filesystem(); 32 | $fs->mkdir("{$this->basePath}/{$this->packageDir}/wpackagist-plugin"); 33 | $fs->mkdir("{$this->basePath}/{$this->packageDir}/wpackagist-theme"); 34 | } 35 | } 36 | 37 | protected function readFile($path) 38 | { 39 | $contents = null; 40 | if (file_exists($path)) { 41 | $contents = file_get_contents($path); 42 | } 43 | return $contents; 44 | } 45 | 46 | protected function writeFile($path, $json) 47 | { 48 | return file_put_contents($path, $json) > 0; 49 | } 50 | 51 | protected function getResourcePath($packageName, $hash): string 52 | { 53 | return "{$this->basePath}/{$this->packageDir}/{$packageName}\${$hash}.json"; 54 | } 55 | 56 | public function loadPackage(string $packageName, string $hash): ?string 57 | { 58 | return $this->readFile($this->getResourcePath($packageName, $hash)); 59 | } 60 | 61 | public function savePackage(string $packageName, string $hash, string $json): bool 62 | { 63 | $this->packages[$packageName] = $json; 64 | return $this->writeFile($this->getResourcePath($packageName, $hash), $json); 65 | } 66 | 67 | public function loadAllPackages($packageNames): array 68 | { 69 | $json = []; 70 | $toFind = []; 71 | foreach ($packageNames as $name) { 72 | if (array_key_exists($name, $this->packages)) { 73 | $json[$name] = $this->packages[$name]; 74 | } else { 75 | $toFind[] = $name; 76 | } 77 | } 78 | 79 | if ($toFind) { 80 | $finder = new Finder(); 81 | $finder->files()->in("{$this->basePath}/{$this->packageDir}")->depth('> 0'); 82 | foreach ($finder as $file) { 83 | if (preg_match('/^(.+)\$[0-9a-f]{64}\.json$/', $file->getRelativePathname(), $matches)) { 84 | $packageName = str_replace('\\', '/', $matches[1]); 85 | if (in_array($packageName, $toFind)) { 86 | $json[$packageName] = $file->getContents(); 87 | } 88 | } 89 | } 90 | } 91 | return $json; 92 | } 93 | 94 | public function loadAllProviders(): array 95 | { 96 | $finder = new Finder(); 97 | $finder->files()->in("{$this->basePath}/{$this->packageDir}")->depth(0); 98 | $providers = []; 99 | foreach ($finder as $file) { 100 | if (preg_match('/^(.+)\$[0-9a-f]{64}\.json$/', $file->getRelativePathname(), $matches)) { 101 | $packageName = str_replace('\\', '/', $matches[1]); 102 | $providers[$packageName] = $file->getContents(); 103 | } 104 | } 105 | return $providers; 106 | } 107 | 108 | public function loadProvider(string $group, string $hash): ?string 109 | { 110 | return $this->readFile($this->getResourcePath($group, $hash)); 111 | } 112 | 113 | public function saveProvider(string $name, string $hash, string $json): bool 114 | { 115 | return $this->writeFile($this->getResourcePath($name, $hash), $json); 116 | } 117 | 118 | public function deactivateProvider(string $name, string $hash): void 119 | { 120 | $path = $this->getResourcePath($name, $hash); 121 | if (file_exists($path)) { 122 | unlink($path); 123 | } 124 | } 125 | 126 | public function loadRoot(): ?string 127 | { 128 | return $this->readFile("{$this->basePath}/packages.json"); 129 | } 130 | 131 | public function saveRoot(string $json): bool 132 | { 133 | return $this->writeFile("{$this->basePath}/packages.json", $json); 134 | } 135 | 136 | public function persist($final = false): void 137 | { 138 | if ($final && !$this->isPartial) { 139 | $fs = new \Symfony\Component\Filesystem\Filesystem(); 140 | $fs->rename("{$this->basePath}/p", "{$this->basePath}/p.old"); 141 | $fs->rename("{$this->basePath}/{$this->packageDir}", "{$this->basePath}/p"); 142 | $fs->remove("{$this->basePath}/p.old"); 143 | $this->packageDir = 'p'; 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/Storage/PackageStore.php: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /web/assets/js/main.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function () { 2 | $('.search-result__refresh-form').on('submit', function (event) { 3 | // Disable refresh buttons while refreshing to prevent double submit crashes. 4 | $('.search-result__refresh-button').prop('disabled', true); 5 | }); 6 | 7 | //click on a version sug to display an info box row 8 | $('.js-version').on('click', function (event) { 9 | event.preventDefault(); 10 | 11 | var $element = $(this), 12 | $parentRow = $element.closest('tr'), 13 | name = $parentRow.find('[data-name]').data('name'), 14 | type = $parentRow.find('[data-type]').data('type'), 15 | version = $element.data('version'), 16 | copyString = '"wpackagist-' + type + '/' + name + '":"' + version + '"'; 17 | 18 | //remove any existing info boxes 19 | $('.js-composer-info').remove(); 20 | 21 | //add info box in next row 22 | $parentRow.after(" \ 23 |
    \ 24 |
    \ 25 | \ 26 |
    \ 27 |
    \ 28 | \ 29 |
    \ 30 |
    \ 31 | "); 32 | 33 | //select text for copying 34 | $('.js-composer-info .js-copy').val(copyString).select(); 35 | }); 36 | 37 | //extra versions toggle 38 | $('.js-toggle-more').on('click', function (event) { 39 | event.preventDefault(); 40 | 41 | var $element = $(this), 42 | $siblingsToToggle = $element.siblings('[data-hide]'); 43 | 44 | $siblingsToToggle.toggleClass('hide'); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /web/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/outlandishideas/wpackagist/466f50d4dcdcaeaaec12b2429028a7f159ee92bc/web/favicon.png -------------------------------------------------------------------------------- /web/index.php: -------------------------------------------------------------------------------- 1 | handle($request); 35 | $response->send(); 36 | $kernel->terminate($request, $response); 37 | -------------------------------------------------------------------------------- /web/templates/footer.twig: -------------------------------------------------------------------------------- 1 |
    2 |

    3 | 4 | 13 |

    14 |

    An Outlandish experiment.

    15 |

    16 | The WordPress® trademark is the intellectual property of the WordPress Foundation. Uses of the WordPress® 17 | name in this website are for identification purposes only and do not imply an endorsement by WordPress Foundation. 18 | WPackagist is not endorsed or owned by, or affiliated with, the WordPress Foundation. 19 |

    20 |
    21 | -------------------------------------------------------------------------------- /web/templates/index.twig: -------------------------------------------------------------------------------- 1 | {% extends "layout.twig" %} 2 | 3 | {% block title %} 4 | {{ title }} 5 | {% endblock %} 6 | 7 | {% block content %} 8 |

    This site mirrors the WordPress® plugin and theme directories as a Composer repository. 12 |

    13 | 14 | {% include 'searchbar.twig' %} 15 | 16 |
    17 |
    18 |

    How do I use it?

    19 | 20 |
      21 |
    1. Add the repository to your composer.json
      2. 22 | 23 |
    2. Add the desired plugins and themes to your requirements using 24 | wpackagist-plugin or wpackagist-theme as 25 | the vendor name.
      3. 26 | 27 |
    3. Run $ composer.phar update
      4. 28 | 29 |
    4. Packages are 30 | installed to wp-content/plugins/ or 31 | wp-content/themes/ (unless otherwise specified by 32 | installer-paths) 33 |
      5. 34 |
    35 | 36 |

    Example

    37 | 
    {
 38 |     "name": "acme/brilliant-wordpress-site",
 39 |     "description": "My brilliant WordPress site",
 40 | 
        "repositories":[
 41 |         {
 42 |             "type":"composer",
 43 |             "url":"https://wpackagist.org",
 44 |             "only": [
 45 |                 "wpackagist-plugin/*",
 46 |                 "wpackagist-theme/*"
 47 |             ]
 48 |         }
 49 |     ],
 50 | 
        "require": {
 51 |         "aws/aws-sdk-php":"*",
 52 | 
            "wpackagist-plugin/akismet":"dev-trunk",
 53 |         "wpackagist-plugin/wordpress-seo":">=7.0.2",
 54 |         "wpackagist-theme/hueman":"*"
 55 | 
        },
 56 |     "autoload": {
 57 |         "psr-0": {
 58 |             "Acme": "src/"
 59 |         }
 60 |     },
 61 |     "extra": {
 62 |         "installer-paths": {
 63 | 
                "wp-content/mu-plugins/{$name}/": [
 64 |                 "wpackagist-plugin/akismet"
 65 |             ],
 66 | 
                "wp-content/plugins/{$name}/": [
 67 |                 "type:wordpress-plugin"
 68 |             ]
 69 |         }
 70 |     }
 71 | }
    72 | 73 |

    This example composer.json file adds the Wpackagist 74 | repository and includes the latest version of Akismet (installed as 75 | a must-use plugin), at least version 7.0.2 of Wordpress SEO, and the latest 76 | Hueman theme along with the Amazon Web Services SDK from the main 77 | Packagist repository.

    78 | 79 |

    Find out more about using Composer 81 | including custom 83 | install paths.

    84 | 85 |

    The old vendor prefix wpackagist is now 86 | removed in favour of 87 | wpackagist-plugin.

    88 |
    89 | 90 |
    91 |

    Why use Composer?

    92 | 93 |
    94 | “Composer is a tool for dependency management in PHP. It 95 | allows you to declare the dependent libraries your project needs 96 | and it will install them in your project for you.” 97 | 98 |
    99 | — getcomposer.org 101 |
    102 |
    103 | 104 |
      105 |
    • Avoid committing plugins and themes into source control.
      • 106 | 107 |
    • Avoid having to use git submodules.
      • 108 | 109 |
    • Manage WordPress® and non-WordPress® project libraries with the 110 | same tools.
      • 111 | 112 |
    • Could eventually be used to manage dependencies between 113 | plugins.
      • 114 |
    115 | 116 |

    How does the repository work?

    117 | 118 |
      119 |
    • Scans the WordPress® Subversion repository every hour for plugins and themes. Search and click to make any newer versions available. 122 |
      • 123 | 124 |
    • Fetches the tags for each updated package and maps 125 | those to versions.
      • 126 | 127 |
    • For plugins, adds trunk as a dev version.
      • 128 | 129 |
    • Rebuilds the composer package JSON files.
      • 130 |
    131 | 132 |

    Known issues

    133 | 134 |
      135 |
    • Requires Composer 1.0.0-alpha7 or more recent
      • 136 | 137 |
    • Version strings which Composer cannot parse are ignored. All 138 | plugins have at least the trunk build available.
      • 139 | 140 |
    • Themes do not have a trunk version. It is recommended to use 141 | "*" as the required version.
      • 142 | 143 |
    • Even when packages are present on SVN, they won’t be available 144 | if they are not published on wordpress.org. 145 | Try searching for your plugin before reporting a 146 | bug.
      • 147 | 148 |
    • You can also check for open issues.
      • 149 |
    150 | 151 |

    WordPress® Core

    152 | 153 |

    See 154 | fancyguy/webroot-installer or 155 | roots/wordpress for installing 156 | WordPress® itself using Composer.

    157 | 158 |

    Contribute or get support

    159 | 160 |

    Please visit our GitHub page.

    161 | 162 |
    163 |
    164 | {% endblock %} 165 | -------------------------------------------------------------------------------- /web/templates/layout.twig: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {% block title %}{% endblock %} 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 |
    22 |
    23 | 24 |

    WPackagist

    25 | 26 | {% block content %}{% endblock %} 27 | 28 | {% include 'footer.twig' %} 29 |
    30 |
    31 | Fork WPackagist on GitHub 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /web/templates/opensearch.twig: -------------------------------------------------------------------------------- 1 | 2 | 3 | WPackagist 4 | Search Wordpress® plugins and themes 5 | 6 | 7 | -------------------------------------------------------------------------------- /web/templates/search.twig: -------------------------------------------------------------------------------- 1 | {% extends "layout.twig" %} 2 | 3 | {% block title %} 4 | {{ title }} 5 | {% endblock %} 6 | 7 | {% block content %} 8 | {% include 'searchbar.twig' %} 9 | 10 | {% if error %} 11 |
    {{ error }}
    12 | {% endif %} 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | {% for package in currentPageResults %} 27 | 28 | 31 | 34 | 37 | 40 | 57 | 64 | 70 | 71 | {% else %} 72 | 73 | 74 | 75 | {% endfor %} 76 | 77 | 78 |
    TypeNameLast committedLast fetchedVersionsActiveRefresh
    29 | {{ package.type | capitalize }} 30 | 32 | {{ package.name | e }} 33 | 35 | {{ package.lastCommitted ? package.lastCommitted | date : 'Not Committed' }} 36 | 38 | {{ package.lastFetched ? package.lastFetched | date : 'Not fetched' }} 39 | 41 | {% set versions = package.versions | format_versions %} 42 | {% for version in versions %} 43 | {# Separator allowing to toggle the show more version #} 44 | {% if (loop.index == 2 and loop.length > 4) %} 45 | ... 46 | {% endif %} 47 | {# Hide extra versions, keep showing only the last 3 and the dev-trunk #} 48 | {% if loop.index >= 2 and loop.index <= loop.length - 3 %} 49 | {{ version }} 50 | {% else %} 51 | {{ version }} 52 | {% endif %} 53 | {% else %} 54 | No version available. 55 | {% endfor %} 56 | 58 | {% if package.isActive %} 59 | ✔ 60 | {% else %} 61 | ✘ 62 | {% endif %} 63 | 65 |
    66 | 67 | 68 |
    69 |
    No results.
    79 | 80 | {{ pagerfanta(pager, 'default') }} 81 | 82 |
    83 | If a package has no version and/or is not active, please check it is visible on 84 | wordpress.org before reporting a 85 | bug. 86 |
    87 | {% endblock %} 88 | -------------------------------------------------------------------------------- /web/templates/searchbar.twig: -------------------------------------------------------------------------------- 1 | {{ form_start(searchForm, {'attr': {'class': 'row collapse'}}) }} 2 |
    3 |
    4 |
    5 | {{ form_widget(searchForm.q, {'required': false, 'attr': {'autofocus': 'true'}}) }} 6 |
    7 |
    8 |
    9 |
    10 |
    11 |
    12 | {{ form_widget(searchForm.type) }} 13 |
    14 |
    15 | {{ form_widget(searchForm.search, { 'label': 'Search »','attr': {'class': 'primary-color postfix'}}) }} 16 |
    17 |
    18 |
    19 | {{ form_end(searchForm) }} 20 | --------------------------------------------------------------------------------