45 | Articles
46 | Articles
Loading ....
47 |
48 | EOT);
49 | $response->headers->set('Content-Type', 'text/html');
50 |
51 | $response->send();
52 |
53 | break;
54 | case '/articles.json':
55 | $entityManager = EntityManagerFactory::getEntityManagerFactory();
56 | $action = new ArticleListAction();
57 | $response = $action($entityManager);
58 | $response->send();
59 |
60 | $memoryUsage = memory_get_usage(true);
61 | $memoryPeakUsage = memory_get_peak_usage(true);
62 |
63 | file_put_contents(
64 | __DIR__ . '/../var/memory-usage.txt',
65 | bytes($memoryUsage) . PHP_EOL . bytes($memoryPeakUsage)
66 | );
67 | break;
68 | case '/symfony-articles.json':
69 | $entityManager = EntityManagerFactory::getEntityManagerFactory();
70 | $action = new ArticleListSymfonyAction();
71 | $response = $action($entityManager);
72 | $response->send();
73 |
74 | $memoryUsage = memory_get_usage(true);
75 | $memoryPeakUsage = memory_get_peak_usage(true);
76 |
77 | file_put_contents(
78 | __DIR__ . '/../var/memory-usage-symfony.txt',
79 | bytes($memoryUsage) . PHP_EOL . bytes($memoryPeakUsage)
80 | );
81 |
82 | break;
83 | case '/old-articles.json':
84 | $entityManager = EntityManagerFactory::getEntityManagerFactory();
85 | $action = new ArticleListOldAction();
86 | $response = $action($entityManager);
87 | $response->send();
88 |
89 | $memoryUsage = memory_get_usage(true);
90 | $memoryPeakUsage = memory_get_peak_usage(true);
91 |
92 | file_put_contents(
93 | __DIR__ . '/../var/memory-usage-old.txt',
94 | bytes($memoryUsage) . PHP_EOL . bytes($memoryPeakUsage)
95 | );
96 |
97 | break;
98 | case '/old-iterable-articles.json':
99 | $entityManager = EntityManagerFactory::getEntityManagerFactory();
100 | $action = new ArticleListOldIterableAction();
101 | $response = $action($entityManager);
102 | $response->send();
103 |
104 | $memoryUsage = memory_get_usage(true);
105 | $memoryPeakUsage = memory_get_peak_usage(true);
106 |
107 | file_put_contents(
108 | __DIR__ . '/../var/memory-usage-old-iterable.txt',
109 | bytes($memoryUsage) . PHP_EOL . bytes($memoryPeakUsage)
110 | );
111 |
112 | break;
113 | default:
114 | $response = new Response();
115 | $response->setStatusCode(404);
116 | $response->setContent('Error 404 - Page not found.');
117 |
118 | $response->send();
119 |
120 | break;
121 | }
122 |
--------------------------------------------------------------------------------
/public/script.js:
--------------------------------------------------------------------------------
1 | async function* loadLineByLine(url) {
2 | const utf8Decoder = new TextDecoder('utf-8');
3 | const response = await fetch(url);
4 | const reader = response.body.getReader();
5 | let { value: chunk, done: readerDone } = await reader.read();
6 | chunk = chunk ? utf8Decoder.decode(chunk) : '';
7 |
8 | const re = /\n|\r|\r\n/gm;
9 | let startIndex = 0;
10 | let result;
11 |
12 | for (;;) {
13 | let result = re.exec(chunk);
14 | if (!result) {
15 | if (readerDone) {
16 | break;
17 | }
18 | let remainder = chunk.substr(startIndex);
19 | ({ value: chunk, done: readerDone } = await reader.read());
20 | chunk = remainder + (chunk ? utf8Decoder.decode(chunk) : '');
21 | startIndex = re.lastIndex = 0;
22 | continue;
23 | }
24 |
25 | yield chunk.substring(startIndex, result.index);
26 | startIndex = re.lastIndex;
27 | }
28 |
29 | if (startIndex < chunk.length) {
30 | // last line didn't end in a newline char
31 | yield chunk.substr(startIndex);
32 | }
33 | }
34 |
35 | async function run(itemCallback, contentCallback) {
36 | let content = '';
37 |
38 | for await (let line of loadLineByLine('articles.json')) {
39 | try {
40 | const object = JSON.parse(line.replace(/\,$/, ''));
41 |
42 | itemCallback(object);
43 | } catch (e) {
44 | content += line;
45 |
46 | continue;
47 | }
48 | }
49 |
50 | contentCallback(content);
51 | }
52 |
53 | const list = document.getElementById('list');
54 | const loader = document.getElementById('loading');
55 | let counter = 0;
56 |
57 | run((object) => {
58 | const tr = document.createElement('tr');
59 | tr.innerHTML =
60 | '' + object.id + ' | '
61 | + '' + object.title + ' | '
62 | + '' + object.description + ' | ';
63 |
64 | list.append(tr);
65 |
66 | ++counter;
67 | loader.innerText = 'Loaded ' + counter;
68 | }, (content) => {
69 | loader.innerText = 'Loaded: ' + content;
70 | });
71 |
--------------------------------------------------------------------------------
/src/Controller/ArticleListAction.php:
--------------------------------------------------------------------------------
1 | findArticles($entityManager);
15 |
16 | return new StreamedResponse(function() use ($articles) {
17 | // defining our json structure but replaces the articles with a placeholder
18 | $jsonStructure = json_encode([
19 | 'embedded' => [
20 | 'articles' => ['__REPLACES_ARTICLES__'],
21 | ],
22 | 'total' => 100_000,
23 | ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
24 |
25 | // split by placeholder
26 | [$before, $after] = explode('"__REPLACES_ARTICLES__"', $jsonStructure, 2);
27 |
28 | // send first before part of the json
29 | echo $before . PHP_EOL;
30 |
31 | // stream article one by one as own json
32 | foreach ($articles as $count => $article) {
33 | if ($count !== 0) {
34 | echo ',' . PHP_EOL; // if not first element we need a separator
35 | }
36 |
37 | if ($count % 500 === 0 && $count !== 100_000) { // flush response after every 500
38 | flush();
39 | }
40 |
41 | echo json_encode($article, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
42 | }
43 |
44 | // send the after part of the json as last
45 | echo PHP_EOL;
46 | echo $after;
47 |
48 | }, 200, ['Content-Type' => 'application/json']);
49 | }
50 |
51 | private function findArticles(EntityManagerInterface $entityManager): iterable
52 | {
53 | $queryBuilder = $entityManager->createQueryBuilder();
54 | $queryBuilder->from(Article::class, 'article');
55 | $queryBuilder->select('article.id')
56 | ->addSelect('article.title')
57 | ->addSelect('article.description');
58 |
59 | return $queryBuilder->getQuery()->toIterable();
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/Controller/ArticleListOldAction.php:
--------------------------------------------------------------------------------
1 | findArticles($entityManager);
16 |
17 | return JsonResponse::fromJsonString(json_encode([
18 | 'embedded' => [
19 | 'articles' => $articles,
20 | 'total' => 100_000,
21 | ],
22 | ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
23 | }
24 |
25 | private function findArticles(EntityManagerInterface $entityManager): iterable
26 | {
27 | $queryBuilder = $entityManager->createQueryBuilder();
28 | $queryBuilder->from(Article::class, 'article');
29 | $queryBuilder->select('article.id')
30 | ->addSelect('article.title')
31 | ->addSelect('article.description');
32 |
33 | return $queryBuilder->getQuery()->getResult();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/Controller/ArticleListOldIterableAction.php:
--------------------------------------------------------------------------------
1 | findArticles($entityManager));
16 |
17 | return JsonResponse::fromJsonString(json_encode([
18 | 'embedded' => [
19 | 'articles' => $articles,
20 | 'total' => 100_000,
21 | ],
22 | ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
23 | }
24 |
25 | private function findArticles(EntityManagerInterface $entityManager): iterable
26 | {
27 | $queryBuilder = $entityManager->createQueryBuilder();
28 | $queryBuilder->from(Article::class, 'article');
29 | $queryBuilder->select('article.id')
30 | ->addSelect('article.title')
31 | ->addSelect('article.description');
32 |
33 | return $queryBuilder->getQuery()->toIterable();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/Controller/ArticleListSymfonyAction.php:
--------------------------------------------------------------------------------
1 | [
15 | 'articles' => $this->findArticles($entityManager),
16 | ],
17 | 'total' => 100_000,
18 | ]);
19 | }
20 |
21 | private function findArticles(EntityManagerInterface $entityManager): \Generator
22 | {
23 | $queryBuilder = $entityManager->createQueryBuilder();
24 | $queryBuilder->from(Article::class, 'article');
25 | $queryBuilder->select('article.id')
26 | ->addSelect('article.title')
27 | ->addSelect('article.description');
28 |
29 | $count = 0;
30 | foreach ($queryBuilder->getQuery()->toIterable() as $key => $value) {
31 | yield $key => $value;
32 |
33 | ++$count;
34 | if ($count % 500 === 0 && $count !== 100_000) { // flush response after every 500
35 | flush();
36 | }
37 | }
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/Controller/StreamedJsonResponse.php:
--------------------------------------------------------------------------------
1 |
19 | *
20 | * Example usage:
21 | *
22 | * function loadArticles(): \Generator
23 | * // some streamed loading
24 | * yield ['title' => 'Article 1'];
25 | * yield ['title' => 'Article 2'];
26 | * yield ['title' => 'Article 3'];
27 | * // recommended to use flush() after every specific number of items
28 | * }),
29 | *
30 | * $response = new StreamedJsonResponse(
31 | * // json structure with generators in which will be streamed
32 | * [
33 | * '_embedded' => [
34 | * 'articles' => loadArticles(), // any generator which you want to stream as list of data
35 | * ],
36 | * ],
37 | * );
38 | */
39 | class StreamedJsonResponse extends StreamedResponse
40 | {
41 | private const PLACEHOLDER = '__symfony_json__';
42 |
43 | /**
44 | * @param mixed[] $data JSON Data containing PHP generators which will be streamed as list of data
45 | * @param int $status The HTTP status code (200 "OK" by default)
46 | * @param array $headers An array of HTTP headers
47 | * @param int $encodingOptions Flags for the json_encode() function
48 | */
49 | public function __construct(
50 | private readonly array $data,
51 | int $status = 200,
52 | array $headers = [],
53 | private int $encodingOptions = JsonResponse::DEFAULT_ENCODING_OPTIONS,
54 | ) {
55 | parent::__construct($this->stream(...), $status, $headers);
56 |
57 | if (!$this->headers->get('Content-Type')) {
58 | $this->headers->set('Content-Type', 'application/json');
59 | }
60 | }
61 |
62 | private function stream(): void
63 | {
64 | $generators = [];
65 | $structure = $this->data;
66 |
67 | array_walk_recursive($structure, function (&$item, $key) use (&$generators) {
68 | if (self::PLACEHOLDER === $key) {
69 | // if the placeholder is already in the structure it should be replaced with a new one that explode
70 | // works like expected for the structure
71 | $generators[] = $item;
72 | }
73 |
74 | // generators should be used but for better DX all kind of Traversable are supported
75 | if ($item instanceof \Traversable || $item instanceof \JsonSerializable) {
76 | $generators[] = $item;
77 | $item = self::PLACEHOLDER;
78 | } elseif (self::PLACEHOLDER === $item) {
79 | // if the placeholder is already in the structure it should be replaced with a new one that explode
80 | // works like expected for the structure
81 | $generators[] = $item;
82 | }
83 | });
84 |
85 | $jsonEncodingOptions = \JSON_THROW_ON_ERROR | $this->encodingOptions;
86 | $keyEncodingOptions = $jsonEncodingOptions & ~\JSON_NUMERIC_CHECK;
87 |
88 | $jsonParts = explode('"'.self::PLACEHOLDER.'"', json_encode($structure, $jsonEncodingOptions));
89 |
90 | foreach ($generators as $index => $generator) {
91 | // send first and between parts of the structure
92 | echo $jsonParts[$index];
93 |
94 | if (self::PLACEHOLDER === $generator || $generator instanceof \JsonSerializable) {
95 | // the placeholders already in the structure are rendered here
96 | echo json_encode($generator, $jsonEncodingOptions);
97 |
98 | continue;
99 | }
100 |
101 | $isFirstItem = true;
102 | $startTag = '[';
103 | foreach ($generator as $key => $item) {
104 | if ($isFirstItem) {
105 | $isFirstItem = false;
106 | // depending on the first elements key the generator is detected as a list or map
107 | // we can not check for a whole list or map because that would hurt the performance
108 | // of the streamed response which is the main goal of this response class
109 | if (0 !== $key) {
110 | $startTag = '{';
111 | }
112 |
113 | echo $startTag;
114 | } else {
115 | // if not first element of the generic, a separator is required between the elements
116 | echo ',';
117 | }
118 |
119 | if ('{' === $startTag) {
120 | echo json_encode((string) $key, $keyEncodingOptions).':';
121 | }
122 |
123 | echo json_encode($item, $jsonEncodingOptions);
124 | }
125 |
126 | echo '[' === $startTag ? ']' : '}';
127 | }
128 |
129 | // send last part of the structure
130 | echo $jsonParts[array_key_last($jsonParts)];
131 | }
132 | }
133 |
--------------------------------------------------------------------------------
/src/Doctrine/EntityManagerFactory.php:
--------------------------------------------------------------------------------
1 | 'pdo_sqlite',
34 | 'path' => static::$DATABASE_FILE,
35 | ];
36 |
37 | $cache = new FilesystemAdapter('doctrine', 0, __DIR__ . '/../../var/cache');
38 |
39 | $config = ORMSetup::createConfiguration(
40 | false,
41 | __DIR__ . '/../../var/cache',
42 | $cache
43 | );
44 |
45 | $namespaces = [
46 | __DIR__ . '/../../config/doctrine' => 'App\Entity',
47 | ];
48 |
49 | $driver = new SimplifiedXmlDriver($namespaces);
50 |
51 | $config->setMetadataDriverImpl($driver);
52 |
53 | $eventManager = new EventManager();
54 | $eventManager->addEventListener(Events::loadClassMetadata, new ResolveTargetEntityListener());
55 | static::$entityManager = EntityManager::create($connection, $config, $eventManager);
56 |
57 | if (!file_exists(static::$DATABASE_FILE)) {
58 | $schemaTool = new SchemaTool(static::$entityManager);
59 | $classes = static::$entityManager->getMetadataFactory()->getAllMetadata();
60 | $schemaTool->createSchema($classes);
61 |
62 | // load fixtures
63 | for ($i = 0; $i <= 100_000; ++$i) {
64 | $article = new Article('Title ' . $i, 'Description ' . $i . PHP_EOL . 'More description text ....');
65 | static::$entityManager->persist($article);
66 |
67 | if ($i % 100 === 0) {
68 | static::$entityManager->flush();
69 | static::$entityManager->clear();
70 | }
71 | }
72 |
73 | static::$entityManager->flush();
74 | static::$entityManager->clear();
75 | }
76 |
77 | return static::$entityManager;
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/src/Entity/Article.php:
--------------------------------------------------------------------------------
1 | title = $title;
16 | $this->description = $description;
17 | }
18 |
19 | public function getId(): string
20 | {
21 | return $this->title;
22 | }
23 |
24 | public function getTitle(): string
25 | {
26 | return $this->title;
27 | }
28 |
29 | public function getDescription(): string
30 | {
31 | return $this->description;
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/var/memory-usage-old-iterable.txt:
--------------------------------------------------------------------------------
1 | 62.11 MB
2 | 71.79 MB
--------------------------------------------------------------------------------
/var/memory-usage-old.txt:
--------------------------------------------------------------------------------
1 | 64.21 MB
2 | 73.89 MB
--------------------------------------------------------------------------------
/var/memory-usage-symfony.txt:
--------------------------------------------------------------------------------
1 | 2.10 MB
2 | 2.10 MB
--------------------------------------------------------------------------------
/var/memory-usage.txt:
--------------------------------------------------------------------------------
1 | 2.10 MB
2 | 2.10 MB
--------------------------------------------------------------------------------
/var/test.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexander-schranz/efficient-json-streaming-with-symfony-doctrine/650abab20ce807872aa5a3c92617f80b18eebd68/var/test.sqlite
--------------------------------------------------------------------------------