├── .phplint.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.es.md ├── README.md ├── bin └── rfclinc ├── composer.json ├── config └── config-sample.php ├── public ├── .htaccess └── index.php ├── sql ├── mysql │ └── initial.sql ├── postgres │ └── initial.sql └── sqlite │ └── initial.sql └── src ├── Application ├── Cli │ ├── Application.php │ ├── Commands │ │ ├── CatalogCommand.php │ │ └── UpdateCommand.php │ ├── OutputLogger.php │ └── ProgressByHit.php ├── Config.php ├── Providers │ ├── ConfigServiceProvider.php │ └── DataGatewayServiceProvider.php ├── SetUpContainerTrait.php └── Web │ ├── Application.php │ └── Controllers │ └── ListedRfcController.php ├── DataGateway ├── CatalogGatewayInterface.php ├── FactoryInterface.php ├── ListedRfcGatewayInterface.php ├── NotFoundException.php ├── NullOptimizer.php ├── OptimizerInterface.php ├── Pdo │ ├── AbstractPdoGateway.php │ ├── CatalogGateway.php │ ├── ListedRfcGateway.php │ ├── PdoFactory.php │ ├── RfcLogGateway.php │ └── TransactionOptimizer.php ├── PersistenceException.php └── RfcLogGatewayInterface.php ├── Domain ├── Catalog.php ├── ListedRfc.php ├── RfcLog.php └── VersionDate.php ├── Downloader ├── DownloaderInterface.php └── PhpDownloader.php ├── Updater ├── Blob.php ├── Importer.php ├── IndexInterpreter.php ├── PackedBlobReader.php └── Updater.php └── Util ├── CommandReader.php ├── FileReader.php ├── ReaderInterface.php ├── ShellWhich.php └── TemporaryFilename.php /.phplint.yml: -------------------------------------------------------------------------------- 1 | # config file for phplint 2 | # see https://github.com/overtrue/phplint 3 | 4 | path: ./ 5 | cache: build/phplint.cache 6 | jobs: 10 7 | extensions: 8 | - php 9 | exclude: 10 | - vendor 11 | - build 12 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at eclipxe13@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are welcome. We accept pull requests on [GitHub](https://github.com/phpcfdi/rfclinc). 4 | 5 | This project adheres to a 6 | [Contributor Code of Conduct](https://github.com/phpcfdi/rfclinc/blob/master/CODE_OF_CONDUCT.md). 7 | By participating in this project and its community, you are expected to uphold this code. 8 | 9 | ## Team members 10 | 11 | * [Carlos C Soto](https://github.com/eclipxe13) - original author and maintainer 12 | * [GitHub constributors](https://github.com/phpcfdi/rfclinc/graphs/contributors) 13 | 14 | ## Communication Channels 15 | 16 | You can find help and discussion in the following places: 17 | 18 | * GitHub Issues: 19 | 20 | ## Reporting Bugs 21 | 22 | Bugs are tracked in our project's [issue tracker](https://github.com/phpcfdi/rfclinc/issues). 23 | 24 | When submitting a bug report, please include enough information for us to reproduce the bug. 25 | A good bug report includes the following sections: 26 | 27 | * Expected outcome 28 | * Actual outcome 29 | * Steps to reproduce, including sample code 30 | * Any other information that will help us debug and reproduce the issue, including stack traces, system/environment information, and screenshots 31 | 32 | **Please do not include passwords or any personally identifiable information in your bug report and sample code.** 33 | 34 | ## Fixing Bugs 35 | 36 | We welcome pull requests to fix bugs! 37 | 38 | If you see a bug report that you'd like to fix, please feel free to do so. 39 | Following the directions and guidelines described in the "Adding New Features" 40 | section below, you may create bugfix branches and send us pull requests. 41 | 42 | ## Adding New Features 43 | 44 | If you have an idea for a new feature, it's a good idea to check out our 45 | [issues](https://github.com/phpcfdi/rfclinc/issues) or active 46 | [pull requests](https://github.com/phpcfdi/rfclinc/pulls) 47 | first to see if the feature is already being worked on. 48 | If not, feel free to submit an issue first, asking whether the feature is beneficial to the project. 49 | This will save you from doing a lot of development work only to have your feature rejected. 50 | We don't enjoy rejecting your hard work, but some features just don't fit with the goals of the project. 51 | 52 | When you do begin working on your feature, here are some guidelines to consider: 53 | 54 | * Your pull request description should clearly detail the changes you have made. 55 | * Follow our code style using `squizlabs/php_codesniffer` and `friendsofphp/php-cs-fixer`. 56 | * Please **write tests** for any new features you add. 57 | * Please **ensure that tests pass** before submitting your pull request. We have Travis CI automatically running tests for pull requests. However, running the tests locally will help save time. 58 | * **Use topic/feature branches.** Please do not ask us to pull from your master branch. 59 | * **Submit one feature per pull request.** If you have multiple features you wish to submit, please break them up into separate pull requests. 60 | * **Send coherent history**. Make sure each individual commit in your pull request is meaningful. If you had to make multiple intermediate commits while developing, please squash them before submitting. 61 | 62 | ## Check the code style 63 | 64 | If you are having issues with coding standars use `php-cs-fixer` and `phpcbf` 65 | 66 | ```shell 67 | vendor/bin/php-cs-fixer fix -v 68 | vendor/bin/phpcbf src/ tests/ 69 | ``` 70 | 71 | ## Running Tests 72 | 73 | The following tests must pass before we will accept a pull request. 74 | If any of these do not pass, it will result in a complete build failure. 75 | Before you can run these, be sure to `composer install` or `composer update`. 76 | 77 | ```shell 78 | vendor/bin/phplint src/ tests/ 79 | vendor/bin/php-cs-fixer fix -v --dry-run 80 | vendor/bin/phpcs -sp src/ tests/ 81 | vendor/bin/phpunit --coverage-text 82 | ``` 83 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Carlos C Soto 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.es.md: -------------------------------------------------------------------------------- 1 | # phpcfdi/rfclinc 2 | 3 | > RFC inscritos no cancelados [README.md](README.md) 4 | 5 | Esta librería permite mantener un listado local de RFC inscritos no cancelados. 6 | 7 | Los RFC que se encuentren en la lista de inscritos no cancelados son los RFC a los que se les puede 8 | emitir un Comprobante Fiscal Digital por Internet (el receptor del CFDI). 9 | 10 | El propósito de crear esta libería nace de la necesidad de mantener una copia actualizada de este listado 11 | así como mantener un listado de los cambios en el mismo. 12 | 13 | Actualmente el SAT provee una forma de consultar un RFC en la lista en 14 | https://portalsat.plataforma.sat.gob.mx/ConsultaRFC/ 15 | El problema de este recurso es que se encuentra detrás de un captcha y no ha publicado su consumo 16 | por servicios web, algún otro tipo de API, ni permite web scraping. 17 | 18 | La librería no contiene en sí misma la información del SAT del Listado de RFC inscritos no cancelados. 19 | Lo que contiene es una forma de automatizar su descarga desde el contenedor de Azure 20 | https://cfdisat.blob.core.windows.net/lco 21 | 22 | La lista se encuentra disponible desde 2016-01-08 23 | 24 | Los pasos genéricos son: 25 | 26 | 1. A partir de una fecha establecida, por ejemplo 2018-02-11 27 | 2. Cargar el índice de blobs desde https://cfdisat.blob.core.windows.net/lco?restype=container&comp=list&prefix=l_RFC_2018_02_11 28 | 3. Por cada blob encontrado 29 | 30 | Por cada blob 31 | 32 | 1. Descargar el blob 33 | 2. Verificar la descarga con su digestión md5 34 | 3. Descomprimir el listado 35 | 4. Desempaquetar (smime) el listado 36 | 5. Procesar cada línea 37 | 38 | Los pasos de descomprimir, desempaquetar y procesar se realizan en una sola pasada usando unix pipes. 39 | 40 | Para generar esta tarea la librería depende de las utilerías `gunzip` para descomprimir, 41 | `openssl` para desempaquetar e `iconv` para transformar de `iso8859-1` a `utf-8` 42 | 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # phpcfdi/rfclinc 2 | 3 | [![Source Code][badge-source]][source] 4 | [![Latest Version][badge-release]][release] 5 | [![Software License][badge-license]][license] 6 | [![Build Status][badge-build]][build] 7 | [![Scrutinizer][badge-quality]][quality] 8 | [![Coverage Status][badge-coverage]][coverage] 9 | [![Total Downloads][badge-downloads]][downloads] 10 | 11 | > RFC inscritos no cancelados 12 | 13 | This is a PHP library to maintain the list of L_RFC (Listado de RFC inscritos no cancelados) 14 | as published by SAT but tracking changes. 15 | 16 | [Para mayor información consulta el archivo README en español](README.es.md) 17 | 18 | 19 | ## Installation 20 | 21 | Use [composer](https://getcomposer.org/) 22 | ```shell 23 | composer require phpcfdi/rfclinc 24 | ``` 25 | 26 | 27 | ## PHP Support 28 | 29 | This library is compatible with PHP versions 7.0 and above. 30 | Please, try to use the full potential of the language. 31 | 32 | 33 | ## Contributing 34 | 35 | Contributions are welcome! Please read [CONTRIBUTING][] for details 36 | and don't forget to take a look in the [TODO][] and [CHANGELOG][] files. 37 | 38 | 39 | ## Copyright and License 40 | 41 | The phpcfdi/rfclinc library is copyright © [Carlos C Soto](http://eclipxe.com.mx) 42 | and licensed for use under the MIT License (MIT). Please see [LICENSE][] for more information. 43 | 44 | 45 | [contributing]: https://github.com/phpcfdi/rfclinc/blob/master/CONTRIBUTING.md 46 | [changelog]: https://github.com/phpcfdi/rfclinc/blob/master/docs/CHANGELOG.md 47 | [todo]: https://github.com/phpcfdi/rfclinc/blob/master/docs/TODO.md 48 | 49 | [source]: https://github.com/phpcfdi/rfclinc 50 | [release]: https://github.com/phpcfdi/rfclinc/releases 51 | [license]: https://github.com/phpcfdi/rfclinc/blob/master/LICENSE 52 | [build]: https://travis-ci.org/phpcfdi/rfclinc?branch=master 53 | [quality]: https://scrutinizer-ci.com/g/phpcfdi/rfclinc/ 54 | [coverage]: https://scrutinizer-ci.com/g/phpcfdi/rfclinc/code-structure/master/code-coverage 55 | [downloads]: https://packagist.org/packages/phpcfdi/rfclinc 56 | 57 | [badge-source]: http://img.shields.io/badge/source-phpcfdi/rfclinc-blue.svg?style=flat-square 58 | [badge-release]: https://img.shields.io/github/release/phpcfdi/rfclinc.svg?style=flat-square 59 | [badge-license]: https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square 60 | [badge-build]: https://img.shields.io/travis/phpcfdi/rfclinc/master.svg?style=flat-square 61 | [badge-quality]: https://img.shields.io/scrutinizer/g/phpcfdi/rfclinc/master.svg?style=flat-square 62 | [badge-coverage]: https://img.shields.io/scrutinizer/coverage/g/phpcfdi/rfclinc/master.svg?style=flat-square 63 | [badge-downloads]: https://img.shields.io/packagist/dt/phpcfdi/rfclinc.svg?style=flat-square 64 | -------------------------------------------------------------------------------- /bin/rfclinc: -------------------------------------------------------------------------------- 1 | #!/bin/env php 2 | //bin/ 14 | __DIR__ . '/../autoload.php', // running from vendor/bin/ 15 | ]; 16 | foreach ($autoloaders as $autoloaderFile) { 17 | if (file_exists($autoloaderFile) && is_readable($autoloaderFile)) { 18 | /** @noinspection PhpIncludeInspection */ 19 | require $autoloaderFile; 20 | break; 21 | } 22 | } 23 | 24 | // run application 25 | Application::createApplication()->run(); 26 | }); 27 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "phpcfdi/rfclinc", 3 | "description": "Listado de RFC Inscritos No Cancelados (php library and application)", 4 | "keywords": ["sat", "mexico", "lrfc"], 5 | "homepage": "https://github.com/phpcfdi/rfclinc", 6 | "license": "MIT", 7 | "authors": [ 8 | { 9 | "name": "Carlos C Soto", 10 | "email": "eclipxe13@gmail.com", 11 | "homepage": "http://eclipxe.com.mx" 12 | } 13 | ], 14 | "require": { 15 | "php": ">=7.0", 16 | "ext-libxml": "*", 17 | "ext-simplexml": "*", 18 | "ext-pdo": "*", 19 | "silex/silex": "~2.2", 20 | "symfony/console": "~3.4", 21 | "psr/log": "^1.0", 22 | "eclipxe/engineworks-progress-status": "^1.0", 23 | "pimple/pimple": "^3.2", 24 | "symfony/http-foundation": "^3.4" 25 | }, 26 | "require-dev": { 27 | "ext-json": "*", 28 | "ext-sqlite3": "*", 29 | "symfony/browser-kit": "^3.4", 30 | "phpunit/phpunit": "^6.2", 31 | "overtrue/phplint": "^1.0", 32 | "squizlabs/php_codesniffer": "^3.0", 33 | "friendsofphp/php-cs-fixer": "^2.4", 34 | "phpstan/phpstan-shim": "^0.9.1", 35 | "maglnet/composer-require-checker": "^0.1.6" 36 | }, 37 | "autoload": { 38 | "psr-4": { 39 | "PhpCfdi\\RfcLinc\\": "src/" 40 | } 41 | }, 42 | "autoload-dev": { 43 | "psr-4": { 44 | "PhpCfdi\\RfcLinc\\Tests\\": "tests/" 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /config/config-sample.php: -------------------------------------------------------------------------------- 1 | 'development', 11 | 'db.dsn' => 'sqlite:' . realpath(__DIR__ . '/../tests/assets/database.sqlite3'), 12 | 'db.username' => '', 13 | 'db.password' => '', 14 | ]; 15 | -------------------------------------------------------------------------------- /public/.htaccess: -------------------------------------------------------------------------------- 1 | 2 | Options -MultiViews 3 | RewriteEngine On 4 | RewriteCond %{REQUEST_FILENAME} !-d 5 | RewriteCond %{REQUEST_FILENAME} !-f 6 | RewriteRule ^ index.php [QSA,L] 7 | 8 | -------------------------------------------------------------------------------- /public/index.php: -------------------------------------------------------------------------------- 1 | run(); 10 | -------------------------------------------------------------------------------- /sql/mysql/initial.sql: -------------------------------------------------------------------------------- 1 | drop table if exists versions; 2 | create table catalogs ( 3 | version int unsigned not null primary key, 4 | records int unsigned not null default 0, 5 | inserted int unsigned not null default 0, 6 | updated int unsigned not null default 0, 7 | deleted int unsigned not null default 0 8 | ) engine=MyISAM; 9 | 10 | drop table if exists rfcs; 11 | create table rfcs ( 12 | rfc varchar(17) not null primary key, 13 | sncf bool not null default false, 14 | sub bool not null default false, 15 | since int unsigned not null default 0, 16 | deleted bool not null default false 17 | ) engine=MyISAM; 18 | 19 | drop table if exists rfclogs; 20 | create table rfclogs ( 21 | version int unsigned not null, 22 | rfc varchar(17) not null, 23 | action tinyint(1) unsigned not null /* tinyint(1) values goes from (0-255) */ 24 | ) engine=MyISAM; 25 | -------------------------------------------------------------------------------- /sql/postgres/initial.sql: -------------------------------------------------------------------------------- 1 | drop table if exists catalogs; 2 | create table catalogs ( 3 | version integer not null primary key, 4 | records integer not null default 0, 5 | inserted integer not null default 0, 6 | updated integer not null default 0, 7 | deleted integer not null default 0 8 | ); 9 | 10 | drop table if exists rfcs; 11 | create table rfcs ( 12 | rfc varchar(17) not null primary key, 13 | sncf bool not null default false, 14 | sub bool not null default false, 15 | since integer not null default 0, 16 | deleted bool not null default false 17 | ); 18 | 19 | drop table if exists rfclogs; 20 | create table rfclogs ( 21 | version integer not null, 22 | rfc varchar(17) not null, 23 | action smallint not null /* smallint values goes from (-32768 to +32767) */ 24 | ); 25 | -------------------------------------------------------------------------------- /sql/sqlite/initial.sql: -------------------------------------------------------------------------------- 1 | drop table if exists catalogs; 2 | create table catalogs ( 3 | version int primary key not null, 4 | records int not null default 0, 5 | inserted int not null default 0, 6 | updated int not null default 0, 7 | deleted int not null default 0 8 | ); 9 | 10 | drop table if exists rfcs; 11 | create table rfcs ( 12 | rfc text primary key not null, 13 | sncf int not null default 0, 14 | sub int not null default 0, 15 | since int not null, 16 | deleted int not null default 0 17 | ); 18 | 19 | drop table if exists rfclogs; 20 | create table rfclogs ( 21 | version int not null, 22 | rfc text not null, 23 | action int not null 24 | ); 25 | -------------------------------------------------------------------------------- /src/Application/Cli/Application.php: -------------------------------------------------------------------------------- 1 | container = $container; 24 | } 25 | 26 | public function container(): Container 27 | { 28 | return $this->container; 29 | } 30 | 31 | public static function createApplication(): self 32 | { 33 | // create 34 | $containter = new Container(); 35 | $app = new static($containter); 36 | 37 | static::setUpContainer($containter); 38 | 39 | // pass container to avoid gateway creation 40 | $app->add(new UpdateCommand($containter)); 41 | $app->add(new CatalogCommand($containter)); 42 | 43 | return $app; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Application/Cli/Commands/CatalogCommand.php: -------------------------------------------------------------------------------- 1 | container = $container; 27 | } 28 | 29 | public function gateways(): FactoryInterface 30 | { 31 | return $this->container['gateways']; 32 | } 33 | 34 | public function config(): Config 35 | { 36 | return $this->container['config']; 37 | } 38 | 39 | protected function configure() 40 | { 41 | $this->setDescription('Obtain catalog information'); 42 | $this->addArgument('catalog', InputArgument::REQUIRED, 'Catalog (as date) to request, can use "latest"'); 43 | } 44 | 45 | public function getLatestCatalog(): Catalog 46 | { 47 | $latest = $this->gateways()->catalog()->latest(); 48 | if ($latest instanceof Catalog) { 49 | return $latest; 50 | } 51 | throw new \RuntimeException('There are no latest catalog in the database'); 52 | } 53 | 54 | protected function getCatalogByDateString(string $dateString): Catalog 55 | { 56 | return $this->gateways()->catalog()->get( 57 | VersionDate::createFromString($dateString) 58 | ); 59 | } 60 | 61 | protected function execute(InputInterface $input, OutputInterface $output) 62 | { 63 | // input arguments 64 | $argumentCatalog = (string) $input->getArgument('catalog'); 65 | 66 | // create logger 67 | $logger = new OutputLogger($output); 68 | 69 | // report working info 70 | $logger->info('Required catalog: ' . $argumentCatalog); 71 | 72 | // show debug messages of database config 73 | $config = $this->config(); 74 | $logger->debug(sprintf('DB: [%s], Username: [%s]', $config->dbDsn(), $config->dbUsername())); 75 | 76 | // get data 77 | if ('latest' === $argumentCatalog) { 78 | $catalog = $this->getLatestCatalog(); 79 | } else { 80 | $catalog = $this->getCatalogByDateString($argumentCatalog); 81 | } 82 | 83 | // print data 84 | $logger->notice(sprintf( 85 | 'Catalog: %s, Active: %s, Inserted: %s, Updated: %s, Deleted: %s', 86 | $catalog->date()->format(), 87 | number_format($catalog->records(), 0), 88 | number_format($catalog->inserted(), 0), 89 | number_format($catalog->updated(), 0), 90 | number_format($catalog->deleted(), 0) 91 | )); 92 | 93 | return 0; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/Application/Cli/Commands/UpdateCommand.php: -------------------------------------------------------------------------------- 1 | container = $container; 31 | } 32 | 33 | public function gateways(): FactoryInterface 34 | { 35 | return $this->container['gateways']; 36 | } 37 | 38 | public function config(): Config 39 | { 40 | return $this->container['config']; 41 | } 42 | 43 | protected function configure() 44 | { 45 | $this->setDescription('Update the database with a new catalog'); 46 | $this->addArgument('date', InputArgument::REQUIRED, 'Update the database to this date'); 47 | $this->addArgument( 48 | 'blobs', 49 | InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 50 | 'Use this blobs instead of downloading' 51 | ); 52 | $this->addOption( 53 | 'report-every', 54 | 're', 55 | InputOption::VALUE_OPTIONAL, 56 | 'Update the database to this date', 57 | '500000' 58 | ); 59 | } 60 | 61 | public function getOptionReportEvery(string $value): int 62 | { 63 | $inputReportEvery = $value; 64 | if (! is_numeric($inputReportEvery)) { 65 | $inputReportEvery = 5000000; 66 | } else { 67 | $inputReportEvery = (int) $inputReportEvery; 68 | } 69 | return $inputReportEvery; 70 | } 71 | 72 | public function getArgumentDate(string $value): VersionDate 73 | { 74 | return VersionDate::createFromString($value); 75 | } 76 | 77 | /** 78 | * @return VersionDate|null 79 | */ 80 | public function getLatestVersionDate() 81 | { 82 | $latest = $this->gateways()->catalog()->latest(); 83 | if ($latest instanceof Catalog) { 84 | return $latest->date(); 85 | } 86 | return null; 87 | } 88 | 89 | protected function execute(InputInterface $input, OutputInterface $output) 90 | { 91 | // input arguments 92 | $reportEvery = $this->getOptionReportEvery((string) $input->getOption('report-every')); 93 | $date = $this->getArgumentDate((string) $input->getArgument('date')); 94 | 95 | // create logger 96 | $logger = new OutputLogger($output); 97 | 98 | // report working info 99 | $logger->notice('Update date: ' . $date->format()); 100 | $logger->debug(sprintf('Report progress every %d lines', $reportEvery)); 101 | 102 | // show debug messages of database config 103 | $config = $this->config(); 104 | $logger->debug(sprintf('DB: [%s], Username: [%s]', $config->dbDsn(), $config->dbUsername())); 105 | 106 | // verify previous version 107 | $latestDate = $this->getLatestVersionDate(); 108 | if (null !== $latestDate) { 109 | $logger->debug(sprintf('Latest catalog is %s', $latestDate->format())); 110 | if ($date->compare($latestDate) <= 0) { 111 | throw new \RuntimeException(sprintf( 112 | 'The update date %s is less or equal to the latest catalog %s', 113 | $date->format(), 114 | $latestDate->format() 115 | )); 116 | } 117 | } else { 118 | $logger->debug('Cannot found any previus catalog'); 119 | } 120 | 121 | // check upper boud 122 | $today = VersionDate::createFromString('today'); 123 | if ($date->compare($today) > 0) { 124 | throw new \RuntimeException(sprintf( 125 | 'The update date %s is greater than today %s', 126 | $date->format(), 127 | $today->format() 128 | )); 129 | } 130 | 131 | $updater = $this->createUpdater($date); 132 | if (! $output->isQuiet()) { 133 | $updater->setLogger($logger); 134 | 135 | $progress = new ProgressByHit(null, [], $reportEvery); 136 | $progress->attach($logger); 137 | $updater->setProgress($progress); 138 | } 139 | 140 | // separate into a different method to allow mock & test 141 | $blobFiles = $input->getArgument('blobs'); 142 | if (is_array($blobFiles) && count($blobFiles)) { 143 | $blobs = []; 144 | foreach ($blobFiles as $index => $blobSourceFile) { 145 | $blobFile = (string) realpath($blobSourceFile); 146 | if ('' === $blobFile) { 147 | throw new \RuntimeException('Cannot find file ' . $blobSourceFile); 148 | } 149 | $blobs[] = new Blob($blobFile, 'file://' . $blobFile, ''); 150 | } 151 | $this->runUpdaterWithBlobs($updater, ...$blobs); 152 | } else { 153 | $this->runUpdater($updater); 154 | } 155 | 156 | return 0; 157 | } 158 | 159 | public function runUpdater(Updater $updater) 160 | { 161 | $updater->run(); 162 | } 163 | 164 | public function runUpdaterWithBlobs(Updater $updater, Blob ...$blobs) 165 | { 166 | $updater->runBlobs(...$blobs); 167 | } 168 | 169 | public function createUpdater(VersionDate $date): Updater 170 | { 171 | return new Updater($date, $this->gateways()); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/Application/Cli/OutputLogger.php: -------------------------------------------------------------------------------- 1 | output = $output; 23 | } 24 | 25 | public function output(): OutputInterface 26 | { 27 | return $this->output; 28 | } 29 | 30 | public function debug($message, array $context = []) 31 | { 32 | if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_NORMAL) { 33 | parent::debug($message, $context); 34 | } 35 | } 36 | 37 | public function log($level, $message, array $context = []) 38 | { 39 | $style = $this->styleFromLogLevel((string) $level); 40 | $message = $this->decorate($message, $style); 41 | $this->output->writeln($message); 42 | } 43 | 44 | public function styleFromLogLevel(string $level): string 45 | { 46 | if (LogLevel::WARNING === $level) { 47 | return 'info'; 48 | } 49 | if (LogLevel::NOTICE === $level) { 50 | return 'comment'; 51 | } 52 | if (LogLevel::INFO === $level || LogLevel::DEBUG === $level) { 53 | return ''; 54 | } 55 | return 'error'; 56 | } 57 | 58 | public function decorate(string $message, string $type = ''): string 59 | { 60 | $decorated = str_replace('<', '\<', $message); 61 | if ('' !== $type) { 62 | $decorated = '<' . $type . '>' . $decorated . ''; 63 | } 64 | return $decorated; 65 | } 66 | 67 | /** 68 | * @param SplSubject|ProgressInterface $subject 69 | */ 70 | public function update(SplSubject $subject) 71 | { 72 | if (! $subject instanceof ProgressInterface) { 73 | return; 74 | } 75 | 76 | $this->info($this->statusToString($subject->getStatus())); 77 | } 78 | 79 | public function statusToString(Status $status) 80 | { 81 | return vsprintf('Processed %s lines in %s [%d / sec]', [ 82 | number_format($status->getValue()), 83 | $status->getIntervalElapsed()->format('%h:%I:%S'), 84 | number_format($status->getSpeed()), 85 | ]); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/Application/Cli/ProgressByHit.php: -------------------------------------------------------------------------------- 1 | setHits($hits); 19 | } 20 | 21 | public function hits(): int 22 | { 23 | return $this->hits; 24 | } 25 | 26 | public function setHits(int $hits) 27 | { 28 | if ($hits < 0) { 29 | throw new \DomainException('Expected integer greater or equal than zero'); 30 | } 31 | $this->hits = $hits; 32 | } 33 | 34 | public function shouldNotifyChange(Status $currentStatus, Status $newStatus): bool 35 | { 36 | $hits = $this->hits(); 37 | if (0 === $hits) { 38 | return false; 39 | } 40 | $current = floor($currentStatus->getValue() / $hits); 41 | $new = floor($newStatus->getValue() / $hits); 42 | return ($current !== $new); 43 | } 44 | 45 | public function getObservers() 46 | { 47 | $observers = parent::getObservers(); 48 | return $observers; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Application/Config.php: -------------------------------------------------------------------------------- 1 | environmentProduction = $environmentProduction; 34 | $this->dbDsn = $dbDsn; 35 | $this->dbUsername = $dbUsername; 36 | $this->dbPassword = $dbPassword; 37 | } 38 | 39 | public static function createFromConfigFile(string $filename): self 40 | { 41 | try { 42 | if (! file_exists($filename)) { 43 | throw new \RuntimeException('It does not exists'); 44 | } 45 | if (is_dir($filename)) { 46 | throw new \RuntimeException('It is a directory'); 47 | } 48 | if (! is_readable($filename)) { 49 | throw new \RuntimeException('It is not readable'); 50 | } 51 | /** @noinspection PhpIncludeInspection */ 52 | $settings = require $filename; 53 | if (! is_array($settings)) { 54 | throw new \RuntimeException('The file did not return an array'); 55 | } 56 | return self::createFromArray($settings); 57 | } catch (\Throwable $exception) { 58 | throw new \RuntimeException("Cannot read config file $filename", 0, $exception); 59 | } 60 | } 61 | 62 | public static function createFromArray(array $values): self 63 | { 64 | return new self( 65 | 'production' === ($values[static::KEY_ENVIRONMENT] ?? ''), 66 | (string) ($values[static::KEY_DB_DSN] ?? ''), 67 | (string) ($values[static::KEY_DB_USERNAME] ?? ''), 68 | (string) ($values[static::KEY_DB_PASSWORD] ?? '') 69 | ); 70 | } 71 | 72 | public function isEnvironmentProduction(): bool 73 | { 74 | return $this->environmentProduction; 75 | } 76 | 77 | public function dbDsn(): string 78 | { 79 | return $this->dbDsn; 80 | } 81 | 82 | public function dbUsername(): string 83 | { 84 | return $this->dbUsername; 85 | } 86 | 87 | public function dbPassword(): string 88 | { 89 | return $this->dbPassword; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/Application/Providers/ConfigServiceProvider.php: -------------------------------------------------------------------------------- 1 | configFiles = $this->defaultConfigFiles(); 20 | } else { 21 | $this->configFiles = [$configFile]; 22 | } 23 | } 24 | 25 | public function register(Container $container) 26 | { 27 | $container['config'] = function (Container $container) { 28 | foreach ($this->configFiles() as $file) { 29 | $config = $this->tryCreateConfigFromFile($file); 30 | if ($config instanceof Config) { 31 | $container['debug'] = ! $config->isEnvironmentProduction(); 32 | return $config; 33 | } 34 | } 35 | throw new \RuntimeException('Cannot locate any valid config file'); 36 | }; 37 | } 38 | 39 | /** @return string[] */ 40 | public function configFiles(): array 41 | { 42 | return $this->configFiles; 43 | } 44 | 45 | /** @return string[] */ 46 | public function defaultConfigFiles(): array 47 | { 48 | return [ 49 | getcwd() . '/.rfcLinc.php', 50 | getcwd() . 'rfcLinc.config.php', 51 | // __DIR__ is src/Application/Providers 52 | dirname(__DIR__, 3) . '/config/config.php', 53 | ]; 54 | } 55 | 56 | /** 57 | * @param string $filename 58 | * @return Config|null 59 | */ 60 | public function tryCreateConfigFromFile(string $filename) 61 | { 62 | try { 63 | return Config::createFromConfigFile($filename); 64 | } catch (\Throwable $exception) { 65 | return null; 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/Application/Providers/DataGatewayServiceProvider.php: -------------------------------------------------------------------------------- 1 | createPdo($config)); 20 | }; 21 | } 22 | 23 | public function createPdo(Config $config): PDO 24 | { 25 | if ('' === $config->dbDsn()) { 26 | throw new \RuntimeException('No database DSN is configured'); 27 | } 28 | 29 | try { 30 | return new PDO($config->dbDsn(), $config->dbUsername(), $config->dbPassword()); 31 | } catch (\Throwable $exception) { 32 | throw new \RuntimeException(sprintf( 33 | "Unable to create PDO using\nDSN: %s,\nUsername: '%s',\nPassword: %s.", 34 | $config->dbDsn(), 35 | $config->dbUsername(), 36 | ('' === $config->dbPassword()) ? 'empty' : 'not empty' 37 | ), 0, $exception); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Application/SetUpContainerTrait.php: -------------------------------------------------------------------------------- 1 | register(new ConfigServiceProvider()); 17 | $container->register(new DataGatewayServiceProvider()); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Application/Web/Application.php: -------------------------------------------------------------------------------- 1 | get('lrfc/{id}', function (SilexApp $app, string $id) { 23 | $controller = new ListedRfcController($app['gateways']); 24 | return $controller->get($id); 25 | }); 26 | 27 | return $app; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/Application/Web/Controllers/ListedRfcController.php: -------------------------------------------------------------------------------- 1 | gateways = $gateways; 20 | } 21 | 22 | public function gateways(): FactoryInterface 23 | { 24 | return $this->gateways; 25 | } 26 | 27 | public function get(string $id): JsonResponse 28 | { 29 | try { 30 | $gateways = $this->gateways(); 31 | $listedRfc = $gateways->listedRfc()->get($id); 32 | $rfclogs = $gateways->rfclog()->byRfc($id); 33 | 34 | return new JsonResponse([ 35 | 'rfc' => $listedRfc->rfc(), 36 | 'since' => $listedRfc->since()->format(), 37 | 'sncf' => $listedRfc->sncf(), 38 | 'sub' => $listedRfc->sub(), 39 | 'active' => ! $listedRfc->deleted(), 40 | 'logs' => array_map(function (RfcLog $rfcLog) { 41 | return [ 42 | 'date' => $rfcLog->date()->format(), 43 | 'action' => $rfcLog->action(), 44 | ]; 45 | }, $rfclogs), 46 | ], JsonResponse::HTTP_OK); 47 | } catch (NotFoundException $exception) { 48 | return new JsonResponse(['error' => $exception->getMessage()], JsonResponse::HTTP_NOT_FOUND); 49 | } catch (\Throwable $exception) { 50 | return new JsonResponse(['error' => $exception->getMessage()], JsonResponse::HTTP_INTERNAL_SERVER_ERROR); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/DataGateway/CatalogGatewayInterface.php: -------------------------------------------------------------------------------- 1 | pdo = $pdo; 22 | $this->preparedStatements = []; 23 | } 24 | 25 | public function preparedStatements(string $query): PDOStatement 26 | { 27 | $statement = $this->preparedStatements[$query] ?? null; 28 | if ($statement instanceof PDOStatement) { 29 | return $statement; 30 | } 31 | 32 | $statement = $this->pdo->prepare($query); 33 | if (false === $statement) { 34 | throw new \LogicException("Cannot prepare the statement: $query"); 35 | } 36 | $this->preparedStatements[$query] = $statement; 37 | 38 | return $statement; 39 | } 40 | 41 | public function executePrepared(string $query, array $arguments = [], string $exceptionMessage = ''): PDOStatement 42 | { 43 | $statement = $this->preparedStatements($query); 44 | if (! $statement->execute($arguments)) { 45 | $exceptionMessage = $exceptionMessage ? : 'Error retrieving data from database'; 46 | throw new PersistenceException($exceptionMessage); 47 | } 48 | 49 | return $statement; 50 | } 51 | 52 | public function queryValue(string $query, array $arguments = [], $defaultValue = null) 53 | { 54 | $stmt = $this->executePrepared($query, $arguments); 55 | $values = $stmt->fetch(PDO::FETCH_NUM); 56 | if (is_array($values) && array_key_exists(0, $values)) { 57 | return $values[0]; 58 | } 59 | 60 | return $defaultValue; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/DataGateway/Pdo/CatalogGateway.php: -------------------------------------------------------------------------------- 1 | queryValue($query, ['version' => $date->timestamp()]); 20 | if (null === $count) { 21 | throw new PersistenceException('Cannot get count from catalogs'); 22 | } 23 | 24 | return (1 === (int) $count); 25 | } 26 | 27 | public function get(VersionDate $date): Catalog 28 | { 29 | $query = 'select version, records, inserted, updated, deleted from catalogs where (version = :version);'; 30 | $stmt = $this->executePrepared($query, [ 31 | 'version' => $date->timestamp(), 32 | ], 'Cannot get one record from catalogs'); 33 | $values = $stmt->fetch(PDO::FETCH_ASSOC); 34 | if (false === $values) { 35 | throw new NotFoundException("The version {$date->format()} does not exists"); 36 | } 37 | 38 | return $this->createVersionFromArray($date, $values); 39 | } 40 | 41 | public function latest() 42 | { 43 | $query = 'select version, records, inserted, updated, deleted from catalogs order by version desc limit 1;'; 44 | $stmt = $this->executePrepared($query, [], 'Cannot get the latest version'); 45 | $values = $stmt->fetch(PDO::FETCH_ASSOC); 46 | if (false === $values) { 47 | return null; 48 | } 49 | 50 | return $this->createVersionFromArray(VersionDate::createFromTimestamp((int) $values['version']), $values); 51 | } 52 | 53 | public function nextAfter(VersionDate $date) 54 | { 55 | $query = 'select version, records, inserted, updated, deleted from catalogs' 56 | . ' where (version > :version) order by version limit 1;'; 57 | $stmt = $this->executePrepared( 58 | $query, 59 | ['version' => $date->timestamp()], 60 | sprintf('Cannot get the version after %s', $date->format()) 61 | ); 62 | $values = $stmt->fetch(PDO::FETCH_ASSOC); 63 | if (false === $values) { 64 | return null; 65 | } 66 | 67 | return $this->createVersionFromArray(VersionDate::createFromTimestamp((int) $values['version']), $values); 68 | } 69 | 70 | public function previousBefore(VersionDate $date) 71 | { 72 | $query = 'select version, records, inserted, updated, deleted from catalogs' 73 | . ' where (version < :version) order by version desc limit 1;'; 74 | $stmt = $this->executePrepared( 75 | $query, 76 | ['version' => $date->timestamp()], 77 | sprintf('Cannot get the version before %s', $date->format()) 78 | ); 79 | $values = $stmt->fetch(PDO::FETCH_ASSOC); 80 | if (false === $values) { 81 | return null; 82 | } 83 | 84 | return $this->createVersionFromArray(VersionDate::createFromTimestamp((int) $values['version']), $values); 85 | } 86 | 87 | public function insert(Catalog $catalog) 88 | { 89 | $query = 'insert into catalogs (version, records, inserted, updated, deleted)' 90 | . ' values (:version, :records, :inserted, :updated, :deleted);'; 91 | $this->executePrepared($query, [ 92 | 'version' => $catalog->date()->timestamp(), 93 | 'records' => $catalog->records(), 94 | 'inserted' => $catalog->inserted(), 95 | 'updated' => $catalog->updated(), 96 | 'deleted' => $catalog->deleted(), 97 | ], 'Cannot insert into catalogs'); 98 | } 99 | 100 | public function update(Catalog $catalog) 101 | { 102 | $query = 'update catalogs set records = :records, inserted = :inserted, updated = :updated, deleted = :deleted' 103 | . ' where (version = :version);'; 104 | $this->executePrepared($query, [ 105 | 'version' => $catalog->date()->timestamp(), 106 | 'records' => $catalog->records(), 107 | 'inserted' => $catalog->inserted(), 108 | 'updated' => $catalog->updated(), 109 | 'deleted' => $catalog->deleted(), 110 | ], 'Cannot update into catalogs'); 111 | } 112 | 113 | public function delete(VersionDate $date) 114 | { 115 | $query = 'delete from catalogs where (version = :version);'; 116 | $this->executePrepared($query, ['version' => $date->timestamp()], 'Cannot delete from catalogs'); 117 | } 118 | 119 | private function createVersionFromArray(VersionDate $date, array $values): Catalog 120 | { 121 | return new Catalog( 122 | $date, 123 | (int) $values['records'], 124 | (int) $values['inserted'], 125 | (int) $values['updated'], 126 | (int) $values['deleted'] 127 | ); 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/DataGateway/Pdo/ListedRfcGateway.php: -------------------------------------------------------------------------------- 1 | queryValue($query, ['rfc' => $rfc]); 20 | if (null === $value) { 21 | throw new PersistenceException('Cannot get count from rfc list'); 22 | } 23 | return (1 === (int) $value); 24 | } 25 | 26 | public function get(string $rfc): ListedRfc 27 | { 28 | $query = 'select rfc, since, sncf, sub, deleted from rfcs where (rfc = :rfc);'; 29 | $stmt = $this->executePrepared($query, ['rfc' => $rfc], 'Cannot get one record from rfc list'); 30 | $values = $stmt->fetch(PDO::FETCH_ASSOC); 31 | if (false === $values) { 32 | throw new NotFoundException("The RFC $rfc does not exists"); 33 | } 34 | return $this->createListedRfcFromArray($rfc, $values); 35 | } 36 | 37 | public function insert(ListedRfc $listedRfc) 38 | { 39 | $query = 'insert into rfcs (rfc, since, sncf, sub, deleted)' 40 | . ' values (:rfc, :since, :sncf, :sub, :deleted);'; 41 | $this->executePrepared($query, [ 42 | 'rfc' => $listedRfc->rfc(), 43 | 'since' => $listedRfc->since()->timestamp(), 44 | 'sncf' => (int) $listedRfc->sncf(), 45 | 'sub' => (int) $listedRfc->sub(), 46 | 'deleted' => (int) $listedRfc->deleted(), 47 | ], 'Cannot insert into rfc list'); 48 | } 49 | 50 | public function update(ListedRfc $listedRfc) 51 | { 52 | $query = 'update rfcs set sncf = :sncf, sub = :sub, deleted = :deleted' 53 | . ' where (rfc = :rfc);'; 54 | $this->executePrepared($query, [ 55 | 'rfc' => $listedRfc->rfc(), 56 | 'sncf' => (int) $listedRfc->sncf(), 57 | 'sub' => (int) $listedRfc->sub(), 58 | 'deleted' => (int) $listedRfc->deleted(), 59 | ], 'Cannot update into rfc list'); 60 | } 61 | 62 | public function delete(string $rfc) 63 | { 64 | $query = 'delete from rfcs where (rfc = :rfc);'; 65 | $this->executePrepared($query, ['rfc' => $rfc], 'Cannot delete from rfc list'); 66 | } 67 | 68 | public function markAllAsDeleted() 69 | { 70 | $query = 'update rfcs set deleted = :deleted;'; 71 | $this->executePrepared($query, ['deleted' => 1], 'Cannot mark all as deleted on rfc list'); 72 | } 73 | 74 | public function eachDeleted() 75 | { 76 | $query = 'select rfc from rfcs where (deleted = :deleted);'; 77 | $stmt = $this->executePrepared($query, ['deleted' => 1], 'Cannot get all deleted from rfc list'); 78 | while (false !== $values = $stmt->fetch(PDO::FETCH_ASSOC)) { 79 | yield $values['rfc']; 80 | } 81 | } 82 | 83 | public function countDeleted(bool $deleted): int 84 | { 85 | $query = 'select count(*) from rfcs where (deleted = :deleted);'; 86 | return (int) $this->queryValue($query, ['deleted' => (int) $deleted]); 87 | } 88 | 89 | private function createListedRfcFromArray(string $rfc, array $values): ListedRfc 90 | { 91 | return new ListedRfc( 92 | $rfc, 93 | VersionDate::createFromTimestamp((int) $values['since']), 94 | (bool) $values['sncf'], 95 | (bool) $values['sub'], 96 | (bool) $values['deleted'] 97 | ); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/DataGateway/Pdo/PdoFactory.php: -------------------------------------------------------------------------------- 1 | pdo = $pdo; 35 | if (null === $optimizer) { 36 | $optimizer = $this->createOptimizerByDriver($pdo); 37 | } 38 | $this->optimizer = $optimizer; 39 | } 40 | 41 | public function createOptimizerByDriver(PDO $pdo): OptimizerInterface 42 | { 43 | $driver = (string) $pdo->getAttribute($pdo::ATTR_DRIVER_NAME); 44 | if (in_array($driver, ['sqlite', 'pgsql', 'mysql'], true)) { 45 | return new TransactionOptimizer($pdo); 46 | } 47 | return new NullOptimizer(); 48 | } 49 | 50 | public function pdo(): PDO 51 | { 52 | return $this->pdo; 53 | } 54 | 55 | public function catalog(): CatalogGatewayInterface 56 | { 57 | if (null === $this->catalog) { 58 | $this->catalog = new CatalogGateway($this->pdo); 59 | } 60 | return $this->catalog; 61 | } 62 | 63 | public function listedRfc(): ListedRfcGatewayInterface 64 | { 65 | if (null === $this->listedRfc) { 66 | $this->listedRfc = new ListedRfcGateway($this->pdo); 67 | } 68 | return $this->listedRfc; 69 | } 70 | 71 | public function rfclog(): RfcLogGatewayInterface 72 | { 73 | if (null === $this->rfcLog) { 74 | $this->rfcLog = new RfcLogGateway($this->pdo); 75 | } 76 | return $this->rfcLog; 77 | } 78 | 79 | public function optimizer(): OptimizerInterface 80 | { 81 | return $this->optimizer; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/DataGateway/Pdo/RfcLogGateway.php: -------------------------------------------------------------------------------- 1 | openStatementByRfc($rfc); 18 | while (false !== $values = $stmt->fetch(PDO::FETCH_ASSOC)) { 19 | yield $this->createRfcLogFromArray($values); 20 | } 21 | } 22 | 23 | public function byRfc(string $rfc): array 24 | { 25 | $list = []; 26 | $stmt = $this->openStatementByRfc($rfc); 27 | while (false !== $values = $stmt->fetch(PDO::FETCH_ASSOC)) { 28 | $list[] = $this->createRfcLogFromArray($values); 29 | } 30 | return $list; 31 | } 32 | 33 | public function insert(RfcLog $rfcLog) 34 | { 35 | $query = 'insert into rfclogs (version, rfc, action)' 36 | . ' values (:version, :rfc, :action);'; 37 | $this->executePrepared($query, [ 38 | 'version' => $rfcLog->date()->timestamp(), 39 | 'rfc' => $rfcLog->rfc(), 40 | 'action' => $rfcLog->action(), 41 | ], 'Cannot insert into rfc logs list'); 42 | } 43 | 44 | private function openStatementByRfc(string $rfc): PDOStatement 45 | { 46 | $query = 'select version, rfc, action from rfclogs' 47 | . ' where (rfc = :rfc) order by version'; 48 | return $this->executePrepared($query, ['rfc' => $rfc], 'Cannot get logs from rfc list'); 49 | } 50 | 51 | private function createRfcLogFromArray(array $values): RfcLog 52 | { 53 | return new RfcLog( 54 | VersionDate::createFromTimestamp((int) $values['version']), 55 | (string) $values['rfc'], 56 | (int) $values['action'] 57 | ); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/DataGateway/Pdo/TransactionOptimizer.php: -------------------------------------------------------------------------------- 1 | pdo = $pdo; 18 | } 19 | 20 | public function prepare() 21 | { 22 | $this->pdo->beginTransaction(); 23 | } 24 | 25 | public function finish() 26 | { 27 | $this->pdo->commit(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/DataGateway/PersistenceException.php: -------------------------------------------------------------------------------- 1 | date = $date; 27 | $this->records = $records; 28 | $this->inserted = $inserted; 29 | $this->updated = $updated; 30 | $this->deleted = $deleted; 31 | } 32 | 33 | public function date(): VersionDate 34 | { 35 | return $this->date; 36 | } 37 | 38 | public function records(): int 39 | { 40 | return $this->records; 41 | } 42 | 43 | public function inserted(): int 44 | { 45 | return $this->inserted; 46 | } 47 | 48 | public function updated(): int 49 | { 50 | return $this->updated; 51 | } 52 | 53 | public function deleted(): int 54 | { 55 | return $this->deleted; 56 | } 57 | 58 | public function setRecords(int $records) 59 | { 60 | $this->records = $records; 61 | } 62 | 63 | public function setInserted(int $inserted) 64 | { 65 | $this->inserted = $inserted; 66 | } 67 | 68 | public function setUpdated(int $updated) 69 | { 70 | $this->updated = $updated; 71 | } 72 | 73 | public function setDeleted(int $deleted) 74 | { 75 | $this->deleted = $deleted; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Domain/ListedRfc.php: -------------------------------------------------------------------------------- 1 | rfc = $rfc; 32 | $this->since = $since; 33 | $this->sncf = $sncf; 34 | $this->sub = $sub; 35 | $this->deleted = $deleted; 36 | } 37 | 38 | public function rfc(): string 39 | { 40 | return $this->rfc; 41 | } 42 | 43 | public function since(): VersionDate 44 | { 45 | return $this->since; 46 | } 47 | 48 | public function sncf(): bool 49 | { 50 | return $this->sncf; 51 | } 52 | 53 | public function sub(): bool 54 | { 55 | return $this->sub; 56 | } 57 | 58 | public function deleted(): bool 59 | { 60 | return $this->deleted; 61 | } 62 | 63 | public function setSncf(bool $sncf) 64 | { 65 | $this->sncf = $sncf; 66 | } 67 | 68 | public function setSub(bool $sub) 69 | { 70 | $this->sub = $sub; 71 | } 72 | 73 | public function setDeleted(bool $deleted) 74 | { 75 | $this->deleted = $deleted; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Domain/RfcLog.php: -------------------------------------------------------------------------------- 1 | date = $date; 33 | $this->rfc = $rfc; 34 | $this->action = $action; 35 | } 36 | 37 | public function date(): VersionDate 38 | { 39 | return $this->date; 40 | } 41 | 42 | public function rfc(): string 43 | { 44 | return $this->rfc; 45 | } 46 | 47 | public function action(): int 48 | { 49 | return $this->action; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Domain/VersionDate.php: -------------------------------------------------------------------------------- 1 | dateIsValid($year, $month, $day)) { 17 | throw new \InvalidArgumentException('The combination of year-month-day is not valid'); 18 | } 19 | $this->ymd = [ 20 | str_pad((string) $year, 4, '0', STR_PAD_LEFT), 21 | str_pad((string) $month, 2, '0', STR_PAD_LEFT), 22 | str_pad((string) $day, 2, '0', STR_PAD_LEFT), 23 | ]; 24 | $this->timestamp = (int) strtotime($this->format() . 'T00:00:00+00:00'); 25 | } 26 | 27 | /** 28 | * Create a VersionDate based only on the date part information given. 29 | * It will not consider any information about time or timezone. 30 | * You can use terms like 'today'. 31 | * 32 | * @param string $date 33 | * @return VersionDate 34 | */ 35 | public static function createFromString(string $date): self 36 | { 37 | $dt = new \DateTime($date); 38 | return new self((int) $dt->format('Y'), (int) $dt->format('m'), (int) $dt->format('d')); 39 | } 40 | 41 | /** 42 | * Create a VersionDate based on the timestamp. 43 | * The timestamp is considered the seconds since 1970-01-01 44 | * Only the date part is taken, this function consider the timestamp as UTC 45 | * 46 | * @param int $timestamp 47 | * @return VersionDate 48 | */ 49 | public static function createFromTimestamp(int $timestamp): self 50 | { 51 | return static::createFromString(gmdate('Y-m-d', $timestamp)); 52 | } 53 | 54 | public static function dateIsValid(int $year, int $month, int $day) 55 | { 56 | return checkdate($month, $day, $year); 57 | } 58 | 59 | public function format(string $separator = '-'): string 60 | { 61 | return implode($separator, $this->ymd); 62 | } 63 | 64 | public function timestamp(): int 65 | { 66 | return $this->timestamp; 67 | } 68 | 69 | public function year(): int 70 | { 71 | return (int) $this->ymd[0]; 72 | } 73 | 74 | public function month(): int 75 | { 76 | return (int) $this->ymd[1]; 77 | } 78 | 79 | public function day(): int 80 | { 81 | return (int) $this->ymd[2]; 82 | } 83 | 84 | public function compare(self $to): int 85 | { 86 | return $this->timestamp() <=> $to->timestamp(); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/Downloader/DownloaderInterface.php: -------------------------------------------------------------------------------- 1 | [ 13 | 'protocol_version' => '1.1', 14 | ], 15 | ])); 16 | } 17 | 18 | public function downloadAs(string $url, string $filename) 19 | { 20 | copy($url, $filename); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Updater/Blob.php: -------------------------------------------------------------------------------- 1 | name = $name; 27 | $this->url = $url; 28 | $this->contentMd5 = $contentMd5; 29 | $this->md5 = $this->convertMd5BlobToMd5Standard($contentMd5); 30 | } 31 | 32 | public static function convertMd5BlobToMd5Standard(string $stringBase64): string 33 | { 34 | // base64 -> decoded string -> split to bytes -> map to hex -> implode all hex 35 | return implode( 36 | array_map( 37 | function (string $input): string { 38 | return bin2hex($input); 39 | }, 40 | str_split( 41 | base64_decode($stringBase64) ? : '' // base64 can return FALSE 42 | ) ? : [] // str_split can return FALSE 43 | ) 44 | ); 45 | } 46 | 47 | public function name(): string 48 | { 49 | return $this->name; 50 | } 51 | 52 | public function url(): string 53 | { 54 | return $this->url; 55 | } 56 | 57 | public function contentMd5(): string 58 | { 59 | return $this->contentMd5; 60 | } 61 | 62 | public function md5(): string 63 | { 64 | return $this->md5; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/Updater/Importer.php: -------------------------------------------------------------------------------- 1 | catalog = $catalog; 29 | $this->gateways = $gateways; 30 | $this->progress = $progress ?: new NullProgress(); 31 | } 32 | 33 | public function catalog(): Catalog 34 | { 35 | return $this->catalog; 36 | } 37 | 38 | public function gateways(): FactoryInterface 39 | { 40 | return $this->gateways; 41 | } 42 | 43 | public function progress(): ProgressInterface 44 | { 45 | return $this->progress; 46 | } 47 | 48 | public function incrementInserted() 49 | { 50 | $this->catalog->setInserted($this->catalog->inserted() + 1); 51 | } 52 | 53 | public function incrementUpdated() 54 | { 55 | $this->catalog->setUpdated($this->catalog->updated() + 1); 56 | } 57 | 58 | public function incrementDeleted() 59 | { 60 | $this->catalog->setDeleted($this->catalog->deleted() + 1); 61 | } 62 | 63 | public function importReader(ReaderInterface $reader): int 64 | { 65 | $processedLines = 0; 66 | while (true) { 67 | $line = $reader->readLine(); 68 | if (false === $line) { // line: end of line 69 | break; 70 | } 71 | if ($this->importLine($line)) { 72 | $processedLines = $processedLines + 1; 73 | $this->progress->increase(); 74 | } 75 | } 76 | return $processedLines; 77 | } 78 | 79 | public function importLine(string $line): bool 80 | { 81 | $input = str_getcsv($line, '|'); 82 | if (3 === count($input)) { 83 | // do a simple check of the last item (can only be 1 or 0) 84 | // todo: benchmark to test using in_array 85 | if ('0' === $input[2] || '1' === $input[2]) { 86 | $this->importRecord($input[0], '1' === $input[1], '1' === $input[2]); 87 | return true; 88 | } 89 | } 90 | return false; 91 | } 92 | 93 | public function importRecord(string $rfc, bool $sncf, bool $sub) 94 | { 95 | $gwRfc = $this->gateways->listedRfc(); 96 | /* 97 | * This has been tested 98 | * Is less expensive to perform exists + get + update than get + update 99 | * in mysql and pgsql, sqlite is almost the same 100 | */ 101 | if ($gwRfc->exists($rfc)) { 102 | // update 103 | $this->performUpdate($gwRfc->get($rfc), $sncf, $sub); 104 | } else { 105 | // insert 106 | $this->performInsert(new ListedRfc($rfc, $this->catalog->date(), $sncf, $sub, false)); 107 | } 108 | } 109 | 110 | public function performInsert(ListedRfc $listedRfc) 111 | { 112 | $this->gateways->listedRfc()->insert($listedRfc); 113 | $this->createLog($listedRfc->rfc(), RfcLog::ACTION_CREATED); 114 | $this->incrementInserted(); 115 | } 116 | 117 | public function performUpdate(ListedRfc $listedRfc, bool $sncf, bool $sub) 118 | { 119 | $changed = false; 120 | if ($sncf !== $listedRfc->sncf()) { 121 | $listedRfc->setSncf($sncf); 122 | $this->createLog($listedRfc->rfc(), $sncf ? RfcLog::ACTION_CHANGE_SNCF_ON : RfcLog::ACTION_CHANGE_SNCF_OFF); 123 | $changed = true; 124 | } 125 | if ($sub !== $listedRfc->sub()) { 126 | $listedRfc->setSub($sub); 127 | $this->createLog($listedRfc->rfc(), $sub ? RfcLog::ACTION_CHANGE_SUB_ON : RfcLog::ACTION_CHANGE_SUB_OFF); 128 | $changed = true; 129 | } 130 | 131 | // change delete status and store 132 | $listedRfc->setDeleted(false); 133 | $this->gateways->listedRfc()->update($listedRfc); 134 | 135 | // only increment counter if a significant value changed 136 | if ($changed) { 137 | $this->incrementUpdated(); 138 | } 139 | } 140 | 141 | public function performDelete(string $rfc) 142 | { 143 | $this->createLog($rfc, RfcLog::ACTION_REMOVED); 144 | $this->incrementDeleted(); 145 | } 146 | 147 | public function createLog(string $rfc, int $type) 148 | { 149 | $this->gateways->rfclog()->insert( 150 | new RfcLog($this->catalog->date(), $rfc, $type) 151 | ); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/Updater/IndexInterpreter.php: -------------------------------------------------------------------------------- 1 | {'Blobs'}) || ! isset($xml->{'Blobs'}->{'Blob'})) { 23 | return []; 24 | } 25 | $blobs = []; 26 | foreach ($xml->{'Blobs'}->{'Blob'} as $xmlBlob) { 27 | $blobs[] = $this->blobFromSimpleXml($xmlBlob); 28 | } 29 | return $blobs; 30 | } 31 | 32 | public function blobFromSimpleXml(SimpleXMLElement $xmlBlob): Blob 33 | { 34 | return new Blob( 35 | (string) $xmlBlob->{'Name'}, 36 | (string) $xmlBlob->{'Url'}, 37 | (string) $xmlBlob->{'Properties'}->{'Content-MD5'} 38 | ); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Updater/PackedBlobReader.php: -------------------------------------------------------------------------------- 1 | reader = new CommandReader(); 22 | $this->commands = $this->commandPaths(); 23 | } 24 | 25 | public function commandPaths(): array 26 | { 27 | $which = new ShellWhich(); 28 | $commands = [ 29 | 'gunzip' => $which('gunzip'), 30 | 'openssl' => $which('openssl'), 31 | 'iconv' => $which('iconv'), 32 | 'sed' => $which('sed'), 33 | ]; 34 | foreach ($commands as $command => $path) { 35 | if ('' === $path) { 36 | throw new \InvalidArgumentException("Cannot find $command, it is required to update"); 37 | } 38 | } 39 | return $commands; 40 | } 41 | 42 | private function createCommandString($filename): string 43 | { 44 | $command = implode(' | ', [ 45 | $this->commands['gunzip'] . ' --stdout ' . escapeshellarg($filename), 46 | $this->commands['openssl'] . ' smime -verify -inform der -noverify 2> /dev/null', 47 | $this->commands['iconv'] . ' --from iso8859-1 --to utf-8', 48 | $this->commands['sed'] . ' ' . escapeshellarg('s/\r$//'), 49 | ]); 50 | return $command; 51 | } 52 | 53 | public function open(string $source) 54 | { 55 | $command = $this->createCommandString($source); 56 | $this->reader = new CommandReader(); 57 | $this->reader->open($command); 58 | } 59 | 60 | public function readLine() 61 | { 62 | return $this->reader->readLine(); 63 | } 64 | 65 | public function close() 66 | { 67 | $this->reader->close(); 68 | } 69 | 70 | public function isOpen(): bool 71 | { 72 | return $this->reader->isOpen(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/Updater/Updater.php: -------------------------------------------------------------------------------- 1 | date = $date; 47 | $this->gateways = $gateways; 48 | $this->downloader = new PhpDownloader(); 49 | $this->indexInterpreter = new IndexInterpreter(); 50 | $this->logger = new NullLogger(); 51 | $this->progress = new NullProgress(); 52 | } 53 | 54 | public function hasImporter(): bool 55 | { 56 | return ($this->importer instanceof Importer); 57 | } 58 | 59 | public function importer(): Importer 60 | { 61 | if ($this->importer instanceof Importer) { 62 | return $this->importer; 63 | } 64 | throw new \LogicException('There is no importer, did you call run() method?'); 65 | } 66 | 67 | public function catalog(): Catalog 68 | { 69 | return $this->importer()->catalog(); 70 | } 71 | 72 | public function progress(): ProgressInterface 73 | { 74 | return $this->progress; 75 | } 76 | 77 | public function date(): VersionDate 78 | { 79 | return $this->date; 80 | } 81 | 82 | public function gateways(): FactoryInterface 83 | { 84 | return $this->gateways; 85 | } 86 | 87 | public function downloader(): DownloaderInterface 88 | { 89 | return $this->downloader; 90 | } 91 | 92 | public function indexInterpreter(): IndexInterpreter 93 | { 94 | return $this->indexInterpreter; 95 | } 96 | 97 | public function logger(): LoggerInterface 98 | { 99 | return $this->logger; 100 | } 101 | 102 | public function setDownloader(DownloaderInterface $downloader) 103 | { 104 | $this->downloader = $downloader; 105 | } 106 | 107 | public function setIndexInterpreter(IndexInterpreter $indexInterpreter) 108 | { 109 | $this->indexInterpreter = $indexInterpreter; 110 | } 111 | 112 | public function setLogger(LoggerInterface $logger) 113 | { 114 | $this->logger = $logger; 115 | } 116 | 117 | public function setProgress(ProgressInterface $progress) 118 | { 119 | $this->progress = $progress; 120 | } 121 | 122 | public function run(): int 123 | { 124 | $indexUrl = $this->indexUrl(); 125 | $this->logger->info("Processing {$indexUrl}..."); 126 | 127 | $this->logger->debug("Downloading {$indexUrl}..."); 128 | $indexContents = $this->downloader->download($indexUrl); 129 | 130 | $this->logger->debug('Obtaining blobs...'); 131 | $blobs = $this->indexInterpreter->obtainBlobs($indexContents); 132 | $blobsCount = count($blobs); 133 | 134 | $this->logger->debug("Processing $blobsCount blobs..."); 135 | $processedLines = $this->runBlobs(...$blobs); 136 | 137 | $this->logger->info(sprintf('Processed %s lines', number_format($processedLines))); 138 | return $processedLines; 139 | } 140 | 141 | public function runBlobs(Blob ...$blobs): int 142 | { 143 | $processedLines = 0; 144 | $this->processBegin(); 145 | foreach ($blobs as $blob) { 146 | $processedLines = $processedLines + $this->processBlob($blob); 147 | } 148 | $this->processEnd(); 149 | 150 | return $processedLines; 151 | } 152 | 153 | public function processBegin() 154 | { 155 | $this->logger->notice('Starting general process...'); 156 | 157 | // obtain or create version 158 | $gwCatalogs = $this->gateways->catalog(); 159 | if ($gwCatalogs->exists($this->date)) { 160 | throw new \RuntimeException('The version is already in the catalog, it was not expected to exists'); 161 | } 162 | // start optimizations 163 | $this->gateways->optimizer()->prepare(); 164 | 165 | // create and store version 166 | $catalog = new Catalog($this->date, 0, 0, 0, 0); 167 | $gwCatalogs->insert($catalog); 168 | 169 | // create importer 170 | $this->importer = new Importer($catalog, $this->gateways, $this->progress()); 171 | 172 | // set all records as deleted 173 | $this->gateways->listedRfc()->markAllAsDeleted(); 174 | 175 | $this->logger->debug('General process started'); 176 | } 177 | 178 | public function processBlob(Blob $blob): int 179 | { 180 | // create temp file 181 | $downloaded = new TemporaryFilename(); 182 | $filename = (string) $downloaded; 183 | $url = $blob->url(); 184 | $expectedMd5 = $blob->md5(); 185 | 186 | $this->logger->info("Downloading $url..."); 187 | 188 | // download the resourse 189 | $this->logger->debug("Downloading $url into $filename..."); 190 | $downloadStart = time(); 191 | $this->downloader->downloadAs($url, $filename); 192 | $downloadElapsed = time() - $downloadStart; 193 | $this->logger->debug("Download $url takes $downloadElapsed seconds"); 194 | 195 | // check the md5 checksum 196 | if ('' !== $expectedMd5) { 197 | $this->logger->debug("Checking $expectedMd5 on $filename..."); 198 | $this->checkFileMd5($filename, $expectedMd5); 199 | } 200 | 201 | // process file 202 | $this->logger->debug("Opening $filename (as packed data)..."); 203 | $reader = $this->createReaderForPackedFile($filename); 204 | $processedLines = $this->processReader($reader); 205 | $this->logger->debug("Closing $filename..."); 206 | $reader->close(); 207 | 208 | $this->logger->notice(sprintf('Blob %s process %s lines', $url, number_format($processedLines))); 209 | 210 | // clear the resource 211 | $this->logger->debug("Removing $filename..."); 212 | $downloaded->unlink(); 213 | 214 | return $processedLines; 215 | } 216 | 217 | public function processReader(ReaderInterface $reader): int 218 | { 219 | return $this->importer()->importReader($reader); 220 | } 221 | 222 | public function processEnd() 223 | { 224 | $importer = $this->importer(); 225 | $catalog = $importer->catalog(); 226 | $gwRfc = $this->gateways->listedRfc(); 227 | 228 | // count how many were deleted and log 229 | $this->logger->debug('Checking deletes...'); 230 | foreach ($gwRfc->eachDeleted() as $rfc) { 231 | $importer->performDelete($rfc); 232 | } 233 | $this->logger->debug(sprintf('Found %s lines deleted', number_format($catalog->deleted()))); 234 | 235 | $active = $gwRfc->countDeleted(false); 236 | $catalog->setRecords($active); 237 | $this->logger->info(sprintf('Found %s RFC active', number_format($active))); 238 | 239 | // store current version 240 | $this->logger->debug('Saving version...'); 241 | $this->gateways->catalog()->update($catalog); 242 | 243 | // end optimizations 244 | $this->gateways->optimizer()->finish(); 245 | $this->logger->notice('General process finish'); 246 | } 247 | 248 | public function checkFileMd5(string $filename, string $expectedMd5) 249 | { 250 | // check md5 251 | $md5file = (string) md5_file($filename); 252 | if ($md5file !== $expectedMd5) { 253 | throw new \RuntimeException(sprintf( 254 | 'The MD5 from file "%s" does not match with "%s"', 255 | $md5file, 256 | $expectedMd5 257 | )); 258 | } 259 | } 260 | 261 | public function createReaderForPackedFile(string $filename): ReaderInterface 262 | { 263 | $reader = new PackedBlobReader(); 264 | $reader->open($filename); 265 | return $reader; 266 | } 267 | 268 | public function indexUrl(): string 269 | { 270 | return $this->buildIndexUrl($this->date); 271 | } 272 | 273 | public static function buildIndexUrl(VersionDate $date): string 274 | { 275 | return static::URL_BLOBS_LIST . '&prefix=l_RFC_' . $date->format('_'); 276 | } 277 | } 278 | -------------------------------------------------------------------------------- /src/Util/CommandReader.php: -------------------------------------------------------------------------------- 1 | close(); 18 | } 19 | 20 | public function open(string $command) 21 | { 22 | $this->close(); 23 | $process = proc_open( 24 | $command, 25 | [ 26 | 0 => ['pipe', 'r'], 27 | 1 => ['pipe', 'w'], 28 | 2 => ['pipe', 'w'], 29 | ], 30 | $pipes 31 | ); 32 | 33 | // if the process is not a resource 34 | if (! is_resource($process)) { 35 | throw new \RuntimeException('Cannot create a reader from the command'); 36 | } 37 | 38 | // close stdin and stderr 39 | foreach ([0, 2] as $pipeIndex) { 40 | if (isset($pipes[$pipeIndex]) && is_resource($pipes[$pipeIndex])) { 41 | fclose($pipes[$pipeIndex]); 42 | } 43 | } 44 | 45 | // setup object 46 | $this->process = $process; 47 | $this->inputPipe = $pipes[1]; 48 | } 49 | 50 | public function readLine() 51 | { 52 | if (! is_resource($this->process)) { 53 | throw new \RuntimeException('File is not open (command)'); 54 | } 55 | if (! is_resource($this->inputPipe)) { 56 | throw new \RuntimeException('File is not open (pipe)'); 57 | } 58 | return stream_get_line($this->inputPipe, 1024, PHP_EOL); 59 | } 60 | 61 | public function close() 62 | { 63 | if (is_resource($this->inputPipe)) { 64 | fclose($this->inputPipe); 65 | $this->inputPipe = null; 66 | } 67 | if (is_resource($this->process)) { 68 | $status = proc_get_status($this->process) ? : []; 69 | if ($status['running'] ?? false) { 70 | proc_terminate($this->process); 71 | } 72 | proc_close($this->process); 73 | $this->process = null; 74 | } 75 | } 76 | 77 | public function isOpen(): bool 78 | { 79 | return is_resource($this->process); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/Util/FileReader.php: -------------------------------------------------------------------------------- 1 | close(); 15 | } 16 | 17 | public function open(string $source) 18 | { 19 | $file = fopen($source, 'r'); 20 | if (! is_resource($file)) { 21 | throw new \RuntimeException('Cannot create a reader from the file'); 22 | } 23 | $this->file = $file; 24 | } 25 | 26 | public function readLine() 27 | { 28 | if (! is_resource($this->file)) { 29 | throw new \RuntimeException('File is not open'); 30 | } 31 | if (feof($this->file)) { 32 | return false; 33 | } 34 | $line = fgets($this->file); 35 | return (false !== $line) ? rtrim($line, PHP_EOL) : false; 36 | } 37 | 38 | public function close() 39 | { 40 | if (is_resource($this->file)) { 41 | fclose($this->file); 42 | } 43 | $this->file = null; 44 | } 45 | 46 | public function isOpen(): bool 47 | { 48 | return is_resource($this->file); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Util/ReaderInterface.php: -------------------------------------------------------------------------------- 1 | deleteOnDestruct = $deleteOnDestruct; 18 | $this->filename = (string) tempnam($dir, $prefix); 19 | if ('' === $this->filename) { 20 | throw new \RuntimeException('Cannot create the temporary filename'); 21 | } 22 | } 23 | 24 | public function __destruct() 25 | { 26 | if ($this->deleteOnDestruct) { 27 | $this->unlink(); 28 | } 29 | } 30 | 31 | public function __toString() 32 | { 33 | return $this->filename(); 34 | } 35 | 36 | public function filename(): string 37 | { 38 | return $this->filename; 39 | } 40 | 41 | public function deleteOnDestruct(): bool 42 | { 43 | return $this->deleteOnDestruct; 44 | } 45 | 46 | public function setDeleteOnDestruct(bool $deleteOnDestruct) 47 | { 48 | $this->deleteOnDestruct = $deleteOnDestruct; 49 | } 50 | 51 | public function unlink() 52 | { 53 | if (file_exists($this->filename) && ! is_dir($this->filename)) { 54 | unlink($this->filename); 55 | } 56 | } 57 | } 58 | --------------------------------------------------------------------------------