├── .gitignore ├── .php_cs.dist ├── .travis.yml ├── Makefile ├── README.md ├── composer.json ├── examples └── twitter-streaming.php ├── phpunit.xml.dist ├── src └── StreamingJsonParser.php └── test └── StreamingJsonParserTest.php /.gitignore: -------------------------------------------------------------------------------- 1 | /.php_cs.cache 2 | /composer.lock 3 | /vendor/ 4 | -------------------------------------------------------------------------------- /.php_cs.dist: -------------------------------------------------------------------------------- 1 | setRiskyAllowed(true) 5 | ->setRules([ 6 | "@PSR1" => true, 7 | "@PSR2" => true, 8 | "braces" => [ 9 | "allow_single_line_closure" => true, 10 | "position_after_functions_and_oop_constructs" => "same", 11 | ], 12 | "array_syntax" => ["syntax" => "short"], 13 | "cast_spaces" => true, 14 | "combine_consecutive_unsets" => true, 15 | "function_to_constant" => true, 16 | "no_multiline_whitespace_before_semicolons" => true, 17 | "no_unused_imports" => true, 18 | "no_useless_else" => true, 19 | "no_useless_return" => true, 20 | "no_whitespace_before_comma_in_array" => true, 21 | "no_whitespace_in_blank_line" => true, 22 | "non_printable_character" => true, 23 | "normalize_index_brace" => true, 24 | "ordered_imports" => true, 25 | "php_unit_construct" => true, 26 | "php_unit_dedicate_assert" => true, 27 | "php_unit_fqcn_annotation" => true, 28 | "phpdoc_summary" => true, 29 | "phpdoc_types" => true, 30 | "psr4" => true, 31 | "return_type_declaration" => ["space_before" => "none"], 32 | "short_scalar_cast" => true, 33 | "single_blank_line_before_namespace" => true, 34 | ]) 35 | ->setFinder( 36 | PhpCsFixer\Finder::create() 37 | ->in(__DIR__ . "/examples") 38 | ->in(__DIR__ . "/src") 39 | ->in(__DIR__ . "/test") 40 | ); 41 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | language: php 4 | 5 | php: 6 | - 7.0 7 | - 7.1 8 | - nightly 9 | 10 | matrix: 11 | allow_failures: 12 | - php: nightly 13 | fast_finish: true 14 | 15 | before_install: 16 | - phpenv config-rm xdebug.ini 17 | 18 | install: 19 | # --ignore-platform-reqs, because https://github.com/FriendsOfPHP/PHP-CS-Fixer/pull/2722 20 | - composer update -n --prefer-dist --ignore-platform-reqs 21 | - composer require satooshi/php-coveralls dev-master --ignore-platform-reqs 22 | 23 | - mkdir -p coverage/cov coverage/bin 24 | - wget https://phar.phpunit.de/phpcov.phar -O coverage/bin/phpcov 25 | - chmod +x coverage/bin/phpcov 26 | 27 | - composer show 28 | 29 | script: 30 | - phpdbg -qrr vendor/bin/phpunit --verbose --coverage-php coverage/cov/main.cov 31 | - PHP_CS_FIXER_IGNORE_ENV=1 php vendor/bin/php-cs-fixer --diff --dry-run -v fix 32 | 33 | after_script: 34 | - phpdbg -qrr coverage/bin/phpcov merge --clover build/logs/clover.xml coverage/cov 35 | - php vendor/bin/coveralls -v 36 | 37 | cache: 38 | directories: 39 | - $HOME/.composer/cache/files 40 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PHP_BIN := php 2 | COMPOSER_BIN := composer 3 | 4 | COVERAGE = coverage 5 | SRCS = lib test 6 | 7 | find_php_files = $(shell find $(1) -type f -name "*.php") 8 | src = $(foreach d,$(SRCS),$(call find_php_files,$(d))) 9 | 10 | .PHONY: test 11 | test: setup phpunit code-style 12 | 13 | .PHONY: clean 14 | clean: clean-coverage clean-vendor 15 | 16 | .PHONY: clean-coverage 17 | clean-coverage: 18 | test ! -e coverage || rm -r coverage 19 | 20 | .PHONY: clean-vendor 21 | clean-vendor: 22 | test ! -e vendor || rm -r vendor 23 | 24 | .PHONY: setup 25 | setup: vendor/autoload.php 26 | 27 | .PHONY: deps-update 28 | deps-update: 29 | $(COMPOSER_BIN) update 30 | 31 | .PHONY: phpunit 32 | phpunit: setup 33 | $(PHP_BIN) vendor/bin/phpunit 34 | 35 | .PHONY: code-style 36 | code-style: setup 37 | PHP_CS_FIXER_IGNORE_ENV=1 $(PHP_BIN) vendor/bin/php-cs-fixer --diff -v fix 38 | 39 | composer.lock: composer.json 40 | $(COMPOSER_BIN) install 41 | touch $@ 42 | 43 | vendor/autoload.php: composer.lock 44 | $(COMPOSER_BIN) install 45 | touch $@ 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # streaming-json 2 | 3 | A streaming JSON parser for Amp. 4 | 5 | ## Installation 6 | 7 | ``` 8 | composer require kelunik/streaming-json 9 | ``` 10 | 11 | ## Usage 12 | 13 | ```php 14 | $parser = new StreamingJsonParser($inputStream); 15 | 16 | while (yield $parser->advance()) { 17 | $parsedItem = $parser->getCurrent(); 18 | } 19 | ``` 20 | 21 | Options can be passed to the constructor just like for `json_decode`. The parser will consume the passed input stream and is itself an `Amp\Iterator` that allows consumption of all parsed items. Any malformed message will fail the parser. If the input stream ends, the parser will try to parse the last item and will complete the iterator successfully or fail it, depending on whether the last item was malformed or not. 22 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kelunik/streaming-json", 3 | "description": "A streaming JSON parser for Amp.", 4 | "type": "library", 5 | "license": "MIT", 6 | "authors": [ 7 | { 8 | "name": "Niklas Keller", 9 | "email": "me@kelunik.com" 10 | } 11 | ], 12 | "autoload": { 13 | "psr-4": { 14 | "Kelunik\\StreamingJson\\": "src" 15 | } 16 | }, 17 | "require": { 18 | "amphp/amp": "^2", 19 | "amphp/socket": "^0.10", 20 | "amphp/byte-stream": "^1", 21 | "daverandom/exceptional-json": "^1.0.1" 22 | }, 23 | "require-dev": { 24 | "amphp/artax": "^3", 25 | "amphp/phpunit-util": "^1", 26 | "friendsofphp/php-cs-fixer": "^2.3" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/twitter-streaming.php: -------------------------------------------------------------------------------- 1 | setOption(Client::OP_TRANSFER_TIMEOUT, 0); 31 | $client->setOption(Client::OP_MAX_BODY_BYTES, 0); 32 | 33 | $params = [ 34 | "oauth_consumer_key" => $consumerKey, 35 | "oauth_nonce" => $nonce, 36 | "oauth_signature_method" => "HMAC-SHA1", 37 | "oauth_timestamp" => $timestamp, 38 | "oauth_token" => $token, 39 | "oauth_version" => "1.0", 40 | ]; 41 | 42 | $authorization = "OAuth "; 43 | 44 | foreach ($params as $key => $param) { 45 | $authorization .= rawurlencode($key) . '="' . rawurlencode($param) . '", '; 46 | } 47 | 48 | $uri = "https://stream.twitter.com/1.1/statuses/filter.json?track=" . \rawurlencode($argv[1] ?? "php"); 49 | 50 | $queryParams = (new Uri($uri))->getAllQueryParameters(); 51 | $encodedParams = []; 52 | 53 | foreach (array_merge($params, $queryParams) as $key => $value) { 54 | $encodedParams[\rawurlencode($key)] = \rawurlencode(\is_array($value) ? \current($value) : $value); 55 | } 56 | 57 | ksort($encodedParams); 58 | 59 | $signingData = "POST&" . \rawurlencode(\strtok($uri, "?")) . "&" . \rawurlencode(\http_build_query($encodedParams)); 60 | $signature = base64_encode(hash_hmac("sha1", $signingData, \rawurlencode($consumerSecret) . "&" . \rawurlencode($tokenSecret), true)); 61 | 62 | /** @var Response $response */ 63 | $response = yield $client->request( 64 | (new Request($uri, "POST")) 65 | ->withHeader("authorization", $authorization . 'oauth_signature="' . \rawurlencode($signature) . '"') 66 | ); 67 | 68 | print "Status: " . $response->getStatus() . PHP_EOL; 69 | 70 | if ($response->getStatus() !== 200) { 71 | exit(1); 72 | } 73 | 74 | $parser = new StreamingJsonParser($response->getBody(), true); 75 | 76 | while (yield $parser->advance()) { 77 | var_dump($parser->getCurrent()); 78 | } 79 | 80 | exit(0); 81 | }); 82 | -------------------------------------------------------------------------------- /phpunit.xml.dist: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | test 18 | 19 | 20 | 21 | 22 | src 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/StreamingJsonParser.php: -------------------------------------------------------------------------------- 1 | assoc = $assoc; 26 | $this->depth = $depth; 27 | $this->options = $options; 28 | 29 | $this->source = $inputStream; 30 | $this->emitter = new Emitter; 31 | $this->iterator = $this->emitter->iterate(); 32 | $this->backpressure = new Success; 33 | 34 | call(function () { 35 | return $this->pipe(); 36 | })->onResolve(function ($error) { 37 | $this->handleStreamEnd($error); 38 | }); 39 | } 40 | 41 | private function pipe(): \Generator { 42 | $buffer = ""; 43 | 44 | while (null !== $chunk = yield $this->source->read()) { 45 | $buffer .= $chunk; 46 | 47 | while (($pos = \strpos($buffer, "\r\n")) !== false) { 48 | $this->handleLine(\substr($buffer, 0, $pos)); 49 | $buffer = \substr($buffer, $pos + 1); 50 | } 51 | 52 | yield $this->backpressure; 53 | } 54 | 55 | if ($buffer !== "") { 56 | $this->handleLine($buffer); 57 | } 58 | } 59 | 60 | private function handleStreamEnd(\Throwable $error = null) { 61 | if ($this->emitter === null) { // Previously failed already 62 | return; 63 | } 64 | 65 | $emitter = $this->emitter; 66 | $this->emitter = null; 67 | 68 | if ($error) { 69 | $emitter->fail($error); 70 | } else { 71 | $emitter->complete(); 72 | } 73 | } 74 | 75 | private function handleLine(string $line) { 76 | try { 77 | $decodedLine = decode($line, $this->assoc, $this->depth, $this->options); 78 | $this->backpressure = $this->emitter->emit($decodedLine); 79 | } catch (DecodeErrorException $e) { 80 | $emitter = $this->emitter; 81 | $this->emitter = null; 82 | $emitter->fail($e); 83 | } 84 | } 85 | 86 | /** @inheritdoc */ 87 | public function advance(): Promise { 88 | return $this->iterator->advance(); 89 | } 90 | 91 | /** @inheritdoc */ 92 | public function getCurrent() { 93 | return $this->iterator->getCurrent(); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /test/StreamingJsonParserTest.php: -------------------------------------------------------------------------------- 1 | "bar"]), 18 | \json_encode(["foo" => "baz"]), 19 | \json_encode(["foo" => "random-longer-value"]), 20 | ]); 21 | 22 | $stream = new IteratorStream(fromIterable(\str_split($payload, 8), 10)); 23 | $parser = new StreamingJsonParser($stream); 24 | 25 | $i = 0; 26 | 27 | while (yield $parser->advance()) { 28 | $current = $parser->getCurrent(); 29 | 30 | $this->assertInternalType("object", $current); 31 | $this->assertObjectHasAttribute("foo", $current); 32 | 33 | $i++; 34 | } 35 | 36 | $this->assertSame(3, $i); 37 | }); 38 | } 39 | 40 | public function testBackpressure() { 41 | // Reads exactly the first item and stops then when no item is consumed 42 | $this->expectOutputString(\str_repeat("r", 15)); 43 | 44 | Loop::run(function () { 45 | $payload = implode("\r\n", [ 46 | \json_encode(["foo" => "bar"]), 47 | \json_encode(["foo" => "baz"]), 48 | \json_encode(["foo" => "random-longer-value"]), 49 | ]); 50 | 51 | $stream = new class(new IteratorStream(fromIterable(\str_split($payload, 1), 1))) implements InputStream { 52 | private $inputStream; 53 | 54 | public function __construct(InputStream $inputStream) { 55 | $this->inputStream = $inputStream; 56 | } 57 | 58 | public function read(): Promise { 59 | echo "r"; 60 | return $this->inputStream->read(); 61 | } 62 | }; 63 | 64 | $parser = new StreamingJsonParser($stream); 65 | 66 | Loop::delay(1000, function () use ($payload, $parser) { 67 | Loop::stop(); 68 | }); 69 | }); 70 | } 71 | } 72 | --------------------------------------------------------------------------------