├── .env.sample
├── .gitignore
├── LICENSE
├── README.md
├── bin
    └── console
├── cache
    └── .gitignore
├── composer.json
├── composer.lock
└── src
    ├── Console
        └── Commands
        │   └── SyncCommand.php
    ├── Database
        ├── BigQuery.php
        └── Mysql.php
    ├── Doctrine
        ├── BigQueryDateTimeType.php
        └── BigQueryDateType.php
    └── Services
        └── SyncService.php


/.env.sample:
--------------------------------------------------------------------------------
 1 | BQ_PROJECT_ID=memed-project
 2 | BQ_KEY_FILE=key.json
 3 | BQ_DATASET=medicine
 4 | 
 5 | DB_DATABASE_NAME=medicine
 6 | DB_USERNAME=mysql_user
 7 | DB_PASSWORD=secret
 8 | DB_HOST=mysql-server
 9 | DB_PORT=3306
10 | 
11 | IGNORE_COLUMNS=password,hidden_column,another_column


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | vendor/
2 | key.json
3 | keyfile.json
4 | .env


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | Copyright © 2016 Memed SA <contato@memed.com.br>
 3 | 
 4 | Permission is hereby granted, free of charge, to any person obtaining a copy
 5 | of this software and associated documentation files (the “Software”), to deal
 6 | in the Software without restriction, including without limitation the rights
 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 8 | copies of the Software, and to permit persons to whom the Software is
 9 | furnished to do so, subject to the following conditions:
10 | 
11 | The above copyright notice and this permission notice shall be included in
12 | all copies or substantial portions of the Software.
13 | 
14 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 | THE SOFTWARE.


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | <p align="center"><img src="https://cloud.githubusercontent.com/assets/2197005/19776979/f4abd1be-9c54-11e6-9842-212f26e765a5.png" alt="MySQL to Google BigQuery Logo" /></p>
  2 | 
  3 | <h1 align="center">MySQL to Google BigQuery Sync Tool</h1>
  4 | 
  5 | ## Table of Contents
  6 | 
  7 | + [How it works](#how-it-works)
  8 | + [Requirements](#requirements)
  9 | + [Usage](#usage)
 10 | + [Credits](#credits)
 11 | + [License](#license)
 12 | 
 13 | ## How it works
 14 | 
 15 | Steps when no order column has been supplied:
 16 | 
 17 | + Count MySQL table rows
 18 | + Count BigQuery table rows
 19 | + MySQL rows > BigQuery rows?
 20 | + Get the rows diff, split in batches of XXXXX rows/batch
 21 | 
 22 | Steps when order column has been supplied:
 23 | 
 24 | + Get max value for order column from MySQL table
 25 | + Get max value for order column from BigQuery table
 26 | + Max value MySQL > Max value BigQuery?
 27 | + Delete all rows with order column value = max value BigQuery 
 28 | to make sure no duplicate records are being created in BigQuery
 29 | + Get max value for order column from BigQuery table
 30 | + Get the rows diff based on new max value BigQuery, 
 31 | split in batches of XXXXX rows/batch
 32 | 
 33 | Final three steps:
 34 | 
 35 | + Dump MySQL rows to a JSON
 36 | + Send JSON to BigQuery
 37 | + Repeat until all batches are sent
 38 | 
 39 | Tip: Create a cron job for keep syncing the tables using an interval like 15 minutes (respect the Load Jobs [quota policy](https://cloud.google.com/bigquery/quota-policy))
 40 | 
 41 | ## Requirements
 42 | 
 43 | The following PHP versions are supported:
 44 | 
 45 | + PHP 7
 46 | + HHVM
 47 | + PDO Extension with MySQL driver
 48 | 
 49 | ## Usage
 50 | 
 51 | Download the library using [composer](https://packagist.org/packages/memeddev/mysql-to-google-bigquery):
 52 | 
 53 | ```bash
 54 | $ composer require memeddev/mysql-to-google-bigquery
 55 | ```
 56 | 
 57 | Now, define some environment variables or create a `.env` file on the root of the project, replacing the values:
 58 | 
 59 | ```text
 60 | BQ_PROJECT_ID=bigquery-project-id
 61 | BQ_KEY_FILE=google-service-account-json-key-file.json
 62 | BQ_DATASET=bigquery-dataset-name
 63 | 
 64 | DB_DATABASE_NAME=mysql-database-name
 65 | DB_USERNAME=mysql_username
 66 | DB_PASSWORD=mysql_password
 67 | DB_HOST=mysql-host
 68 | DB_PORT=3306
 69 | 
 70 | IGNORE_COLUMNS=password,hidden_column,another_column
 71 | ```
 72 | 
 73 | PS: To create the `Google Service Account JSON Key File`, access [https://console.cloud.google.com/apis/credentials/serviceaccountkey](https://console.cloud.google.com/apis/credentials/serviceaccountkey)
 74 | 
 75 | Run:
 76 | 
 77 | ```bash
 78 | vendor/bin/console sync table-name
 79 | ```
 80 | 
 81 | If you want to auto create the table on BigQuery:
 82 | 
 83 | ```bash
 84 | vendor/bin/console sync table-name --create-table
 85 | ```
 86 | 
 87 | If you want to delete (and create) the table on BigQuery for a full dump:
 88 | 
 89 | ```bash
 90 | vendor/bin/console sync table-name --delete-table
 91 | ```
 92 | 
 93 | ## Credits
 94 | 
 95 | :heart: Memed SA ([memed.com.br](https://memed.com.br))
 96 | 
 97 | ## License
 98 | 
 99 | MIT license, see [LICENSE](LICENSE)
100 | 


--------------------------------------------------------------------------------
/bin/console:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env php
 2 | <?php
 3 | foreach (['../../../autoload.php', '../../autoload.php', '../vendor/autoload.php', 'vendor/autoload.php'] as $autoload) {
 4 |     $autoload = __DIR__ . '/' . $autoload;
 5 |     if (file_exists($autoload)) {
 6 |         require $autoload;
 7 |         break;
 8 |     }
 9 | }
10 | 
11 | use Symfony\Component\Console\Application;
12 | use MysqlToGoogleBigQuery\Console\Commands\SyncCommand;
13 | 
14 | // Load the .env definitions
15 | if (file_exists(getcwd(). '/.env')) {
16 |     $dotenv = new Dotenv\Dotenv(getcwd());
17 |     $dotenv->load();
18 | }
19 | 
20 | $application = new Application();
21 | 
22 | // Lista de comandos
23 | $application->add(new SyncCommand());
24 | 
25 | $application->run();


--------------------------------------------------------------------------------
/cache/.gitignore:
--------------------------------------------------------------------------------
1 | *
2 | !.gitignore
3 | 


--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "memeddev/mysql-to-google-bigquery",
 3 |     "description": "MySQL to Google BigQuery Sync Tool",
 4 |     "keywords": ["mysql","google","bigquery","sync","data","load","send","tool"],
 5 |     "homepage": "http://github.com/memeddev/mysql-to-google-bigquery",
 6 |     "license": "MIT",
 7 |     "authors": [
 8 |         {
 9 |             "name": "Memed SA",
10 |             "email": "contato@memed.com.br"
11 |         }
12 |     ],
13 |     "require": {
14 |         "symfony/console": "^3.1",
15 |         "vlucas/phpdotenv": "^2.4",
16 |         "php-di/php-di": "^5.4",
17 |         "google/cloud": "^0.11.1",
18 |         "doctrine/dbal": "^2.5",
19 |         "facile-it/doctrine-mysql-come-back": "^1.6"
20 |     },
21 |     "autoload": {
22 |         "psr-4": {
23 |             "MysqlToGoogleBigQuery\\": "src/"
24 |         }
25 |     },
26 |     "config": {
27 |         "bin-dir": "bin"
28 |     },
29 |     "bin": ["bin/console"]
30 | }
31 | 


--------------------------------------------------------------------------------
/composer.lock:
--------------------------------------------------------------------------------
   1 | {
   2 |     "_readme": [
   3 |         "This file locks the dependencies of your project to a known state",
   4 |         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
   5 |         "This file is @generated automatically"
   6 |     ],
   7 |     "hash": "cbe217867f5f220c9de91ebd9ad9bb13",
   8 |     "content-hash": "6b4a14419099eab83d60987606b6bd63",
   9 |     "packages": [
  10 |         {
  11 |             "name": "container-interop/container-interop",
  12 |             "version": "1.1.0",
  13 |             "source": {
  14 |                 "type": "git",
  15 |                 "url": "https://github.com/container-interop/container-interop.git",
  16 |                 "reference": "fc08354828f8fd3245f77a66b9e23a6bca48297e"
  17 |             },
  18 |             "dist": {
  19 |                 "type": "zip",
  20 |                 "url": "https://api.github.com/repos/container-interop/container-interop/zipball/fc08354828f8fd3245f77a66b9e23a6bca48297e",
  21 |                 "reference": "fc08354828f8fd3245f77a66b9e23a6bca48297e",
  22 |                 "shasum": ""
  23 |             },
  24 |             "type": "library",
  25 |             "autoload": {
  26 |                 "psr-4": {
  27 |                     "Interop\\Container\\": "src/Interop/Container/"
  28 |                 }
  29 |             },
  30 |             "notification-url": "https://packagist.org/downloads/",
  31 |             "license": [
  32 |                 "MIT"
  33 |             ],
  34 |             "description": "Promoting the interoperability of container objects (DIC, SL, etc.)",
  35 |             "time": "2014-12-30 15:22:37"
  36 |         },
  37 |         {
  38 |             "name": "doctrine/annotations",
  39 |             "version": "v1.3.0",
  40 |             "source": {
  41 |                 "type": "git",
  42 |                 "url": "https://github.com/doctrine/annotations.git",
  43 |                 "reference": "30e07cf03edc3cd3ef579d0dd4dd8c58250799a5"
  44 |             },
  45 |             "dist": {
  46 |                 "type": "zip",
  47 |                 "url": "https://api.github.com/repos/doctrine/annotations/zipball/30e07cf03edc3cd3ef579d0dd4dd8c58250799a5",
  48 |                 "reference": "30e07cf03edc3cd3ef579d0dd4dd8c58250799a5",
  49 |                 "shasum": ""
  50 |             },
  51 |             "require": {
  52 |                 "doctrine/lexer": "1.*",
  53 |                 "php": "^5.6 || ^7.0"
  54 |             },
  55 |             "require-dev": {
  56 |                 "doctrine/cache": "1.*",
  57 |                 "phpunit/phpunit": "^5.6.1"
  58 |             },
  59 |             "type": "library",
  60 |             "extra": {
  61 |                 "branch-alias": {
  62 |                     "dev-master": "1.4.x-dev"
  63 |                 }
  64 |             },
  65 |             "autoload": {
  66 |                 "psr-4": {
  67 |                     "Doctrine\\Common\\Annotations\\": "lib/Doctrine/Common/Annotations"
  68 |                 }
  69 |             },
  70 |             "notification-url": "https://packagist.org/downloads/",
  71 |             "license": [
  72 |                 "MIT"
  73 |             ],
  74 |             "authors": [
  75 |                 {
  76 |                     "name": "Roman Borschel",
  77 |                     "email": "roman@code-factory.org"
  78 |                 },
  79 |                 {
  80 |                     "name": "Benjamin Eberlei",
  81 |                     "email": "kontakt@beberlei.de"
  82 |                 },
  83 |                 {
  84 |                     "name": "Guilherme Blanco",
  85 |                     "email": "guilhermeblanco@gmail.com"
  86 |                 },
  87 |                 {
  88 |                     "name": "Jonathan Wage",
  89 |                     "email": "jonwage@gmail.com"
  90 |                 },
  91 |                 {
  92 |                     "name": "Johannes Schmitt",
  93 |                     "email": "schmittjoh@gmail.com"
  94 |                 }
  95 |             ],
  96 |             "description": "Docblock Annotations Parser",
  97 |             "homepage": "http://www.doctrine-project.org",
  98 |             "keywords": [
  99 |                 "annotations",
 100 |                 "docblock",
 101 |                 "parser"
 102 |             ],
 103 |             "time": "2016-10-24 11:45:47"
 104 |         },
 105 |         {
 106 |             "name": "doctrine/cache",
 107 |             "version": "v1.6.0",
 108 |             "source": {
 109 |                 "type": "git",
 110 |                 "url": "https://github.com/doctrine/cache.git",
 111 |                 "reference": "f8af318d14bdb0eff0336795b428b547bd39ccb6"
 112 |             },
 113 |             "dist": {
 114 |                 "type": "zip",
 115 |                 "url": "https://api.github.com/repos/doctrine/cache/zipball/f8af318d14bdb0eff0336795b428b547bd39ccb6",
 116 |                 "reference": "f8af318d14bdb0eff0336795b428b547bd39ccb6",
 117 |                 "shasum": ""
 118 |             },
 119 |             "require": {
 120 |                 "php": "~5.5|~7.0"
 121 |             },
 122 |             "conflict": {
 123 |                 "doctrine/common": ">2.2,<2.4"
 124 |             },
 125 |             "require-dev": {
 126 |                 "phpunit/phpunit": "~4.8|~5.0",
 127 |                 "predis/predis": "~1.0",
 128 |                 "satooshi/php-coveralls": "~0.6"
 129 |             },
 130 |             "type": "library",
 131 |             "extra": {
 132 |                 "branch-alias": {
 133 |                     "dev-master": "1.6.x-dev"
 134 |                 }
 135 |             },
 136 |             "autoload": {
 137 |                 "psr-4": {
 138 |                     "Doctrine\\Common\\Cache\\": "lib/Doctrine/Common/Cache"
 139 |                 }
 140 |             },
 141 |             "notification-url": "https://packagist.org/downloads/",
 142 |             "license": [
 143 |                 "MIT"
 144 |             ],
 145 |             "authors": [
 146 |                 {
 147 |                     "name": "Roman Borschel",
 148 |                     "email": "roman@code-factory.org"
 149 |                 },
 150 |                 {
 151 |                     "name": "Benjamin Eberlei",
 152 |                     "email": "kontakt@beberlei.de"
 153 |                 },
 154 |                 {
 155 |                     "name": "Guilherme Blanco",
 156 |                     "email": "guilhermeblanco@gmail.com"
 157 |                 },
 158 |                 {
 159 |                     "name": "Jonathan Wage",
 160 |                     "email": "jonwage@gmail.com"
 161 |                 },
 162 |                 {
 163 |                     "name": "Johannes Schmitt",
 164 |                     "email": "schmittjoh@gmail.com"
 165 |                 }
 166 |             ],
 167 |             "description": "Caching library offering an object-oriented API for many cache backends",
 168 |             "homepage": "http://www.doctrine-project.org",
 169 |             "keywords": [
 170 |                 "cache",
 171 |                 "caching"
 172 |             ],
 173 |             "time": "2015-12-31 16:37:02"
 174 |         },
 175 |         {
 176 |             "name": "doctrine/collections",
 177 |             "version": "v1.3.0",
 178 |             "source": {
 179 |                 "type": "git",
 180 |                 "url": "https://github.com/doctrine/collections.git",
 181 |                 "reference": "6c1e4eef75f310ea1b3e30945e9f06e652128b8a"
 182 |             },
 183 |             "dist": {
 184 |                 "type": "zip",
 185 |                 "url": "https://api.github.com/repos/doctrine/collections/zipball/6c1e4eef75f310ea1b3e30945e9f06e652128b8a",
 186 |                 "reference": "6c1e4eef75f310ea1b3e30945e9f06e652128b8a",
 187 |                 "shasum": ""
 188 |             },
 189 |             "require": {
 190 |                 "php": ">=5.3.2"
 191 |             },
 192 |             "require-dev": {
 193 |                 "phpunit/phpunit": "~4.0"
 194 |             },
 195 |             "type": "library",
 196 |             "extra": {
 197 |                 "branch-alias": {
 198 |                     "dev-master": "1.2.x-dev"
 199 |                 }
 200 |             },
 201 |             "autoload": {
 202 |                 "psr-0": {
 203 |                     "Doctrine\\Common\\Collections\\": "lib/"
 204 |                 }
 205 |             },
 206 |             "notification-url": "https://packagist.org/downloads/",
 207 |             "license": [
 208 |                 "MIT"
 209 |             ],
 210 |             "authors": [
 211 |                 {
 212 |                     "name": "Roman Borschel",
 213 |                     "email": "roman@code-factory.org"
 214 |                 },
 215 |                 {
 216 |                     "name": "Benjamin Eberlei",
 217 |                     "email": "kontakt@beberlei.de"
 218 |                 },
 219 |                 {
 220 |                     "name": "Guilherme Blanco",
 221 |                     "email": "guilhermeblanco@gmail.com"
 222 |                 },
 223 |                 {
 224 |                     "name": "Jonathan Wage",
 225 |                     "email": "jonwage@gmail.com"
 226 |                 },
 227 |                 {
 228 |                     "name": "Johannes Schmitt",
 229 |                     "email": "schmittjoh@gmail.com"
 230 |                 }
 231 |             ],
 232 |             "description": "Collections Abstraction library",
 233 |             "homepage": "http://www.doctrine-project.org",
 234 |             "keywords": [
 235 |                 "array",
 236 |                 "collections",
 237 |                 "iterator"
 238 |             ],
 239 |             "time": "2015-04-14 22:21:58"
 240 |         },
 241 |         {
 242 |             "name": "doctrine/common",
 243 |             "version": "v2.6.1",
 244 |             "source": {
 245 |                 "type": "git",
 246 |                 "url": "https://github.com/doctrine/common.git",
 247 |                 "reference": "a579557bc689580c19fee4e27487a67fe60defc0"
 248 |             },
 249 |             "dist": {
 250 |                 "type": "zip",
 251 |                 "url": "https://api.github.com/repos/doctrine/common/zipball/a579557bc689580c19fee4e27487a67fe60defc0",
 252 |                 "reference": "a579557bc689580c19fee4e27487a67fe60defc0",
 253 |                 "shasum": ""
 254 |             },
 255 |             "require": {
 256 |                 "doctrine/annotations": "1.*",
 257 |                 "doctrine/cache": "1.*",
 258 |                 "doctrine/collections": "1.*",
 259 |                 "doctrine/inflector": "1.*",
 260 |                 "doctrine/lexer": "1.*",
 261 |                 "php": "~5.5|~7.0"
 262 |             },
 263 |             "require-dev": {
 264 |                 "phpunit/phpunit": "~4.8|~5.0"
 265 |             },
 266 |             "type": "library",
 267 |             "extra": {
 268 |                 "branch-alias": {
 269 |                     "dev-master": "2.7.x-dev"
 270 |                 }
 271 |             },
 272 |             "autoload": {
 273 |                 "psr-4": {
 274 |                     "Doctrine\\Common\\": "lib/Doctrine/Common"
 275 |                 }
 276 |             },
 277 |             "notification-url": "https://packagist.org/downloads/",
 278 |             "license": [
 279 |                 "MIT"
 280 |             ],
 281 |             "authors": [
 282 |                 {
 283 |                     "name": "Roman Borschel",
 284 |                     "email": "roman@code-factory.org"
 285 |                 },
 286 |                 {
 287 |                     "name": "Benjamin Eberlei",
 288 |                     "email": "kontakt@beberlei.de"
 289 |                 },
 290 |                 {
 291 |                     "name": "Guilherme Blanco",
 292 |                     "email": "guilhermeblanco@gmail.com"
 293 |                 },
 294 |                 {
 295 |                     "name": "Jonathan Wage",
 296 |                     "email": "jonwage@gmail.com"
 297 |                 },
 298 |                 {
 299 |                     "name": "Johannes Schmitt",
 300 |                     "email": "schmittjoh@gmail.com"
 301 |                 }
 302 |             ],
 303 |             "description": "Common Library for Doctrine projects",
 304 |             "homepage": "http://www.doctrine-project.org",
 305 |             "keywords": [
 306 |                 "annotations",
 307 |                 "collections",
 308 |                 "eventmanager",
 309 |                 "persistence",
 310 |                 "spl"
 311 |             ],
 312 |             "time": "2015-12-25 13:18:31"
 313 |         },
 314 |         {
 315 |             "name": "doctrine/dbal",
 316 |             "version": "v2.5.5",
 317 |             "source": {
 318 |                 "type": "git",
 319 |                 "url": "https://github.com/doctrine/dbal.git",
 320 |                 "reference": "9f8c05cd5225a320d56d4bfdb4772f10d045a0c9"
 321 |             },
 322 |             "dist": {
 323 |                 "type": "zip",
 324 |                 "url": "https://api.github.com/repos/doctrine/dbal/zipball/9f8c05cd5225a320d56d4bfdb4772f10d045a0c9",
 325 |                 "reference": "9f8c05cd5225a320d56d4bfdb4772f10d045a0c9",
 326 |                 "shasum": ""
 327 |             },
 328 |             "require": {
 329 |                 "doctrine/common": ">=2.4,<2.7-dev",
 330 |                 "php": ">=5.3.2"
 331 |             },
 332 |             "require-dev": {
 333 |                 "phpunit/phpunit": "4.*",
 334 |                 "symfony/console": "2.*||^3.0"
 335 |             },
 336 |             "suggest": {
 337 |                 "symfony/console": "For helpful console commands such as SQL execution and import of files."
 338 |             },
 339 |             "bin": [
 340 |                 "bin/doctrine-dbal"
 341 |             ],
 342 |             "type": "library",
 343 |             "extra": {
 344 |                 "branch-alias": {
 345 |                     "dev-master": "2.5.x-dev"
 346 |                 }
 347 |             },
 348 |             "autoload": {
 349 |                 "psr-0": {
 350 |                     "Doctrine\\DBAL\\": "lib/"
 351 |                 }
 352 |             },
 353 |             "notification-url": "https://packagist.org/downloads/",
 354 |             "license": [
 355 |                 "MIT"
 356 |             ],
 357 |             "authors": [
 358 |                 {
 359 |                     "name": "Roman Borschel",
 360 |                     "email": "roman@code-factory.org"
 361 |                 },
 362 |                 {
 363 |                     "name": "Benjamin Eberlei",
 364 |                     "email": "kontakt@beberlei.de"
 365 |                 },
 366 |                 {
 367 |                     "name": "Guilherme Blanco",
 368 |                     "email": "guilhermeblanco@gmail.com"
 369 |                 },
 370 |                 {
 371 |                     "name": "Jonathan Wage",
 372 |                     "email": "jonwage@gmail.com"
 373 |                 }
 374 |             ],
 375 |             "description": "Database Abstraction Layer",
 376 |             "homepage": "http://www.doctrine-project.org",
 377 |             "keywords": [
 378 |                 "database",
 379 |                 "dbal",
 380 |                 "persistence",
 381 |                 "queryobject"
 382 |             ],
 383 |             "time": "2016-09-09 19:13:33"
 384 |         },
 385 |         {
 386 |             "name": "doctrine/inflector",
 387 |             "version": "v1.1.0",
 388 |             "source": {
 389 |                 "type": "git",
 390 |                 "url": "https://github.com/doctrine/inflector.git",
 391 |                 "reference": "90b2128806bfde671b6952ab8bea493942c1fdae"
 392 |             },
 393 |             "dist": {
 394 |                 "type": "zip",
 395 |                 "url": "https://api.github.com/repos/doctrine/inflector/zipball/90b2128806bfde671b6952ab8bea493942c1fdae",
 396 |                 "reference": "90b2128806bfde671b6952ab8bea493942c1fdae",
 397 |                 "shasum": ""
 398 |             },
 399 |             "require": {
 400 |                 "php": ">=5.3.2"
 401 |             },
 402 |             "require-dev": {
 403 |                 "phpunit/phpunit": "4.*"
 404 |             },
 405 |             "type": "library",
 406 |             "extra": {
 407 |                 "branch-alias": {
 408 |                     "dev-master": "1.1.x-dev"
 409 |                 }
 410 |             },
 411 |             "autoload": {
 412 |                 "psr-0": {
 413 |                     "Doctrine\\Common\\Inflector\\": "lib/"
 414 |                 }
 415 |             },
 416 |             "notification-url": "https://packagist.org/downloads/",
 417 |             "license": [
 418 |                 "MIT"
 419 |             ],
 420 |             "authors": [
 421 |                 {
 422 |                     "name": "Roman Borschel",
 423 |                     "email": "roman@code-factory.org"
 424 |                 },
 425 |                 {
 426 |                     "name": "Benjamin Eberlei",
 427 |                     "email": "kontakt@beberlei.de"
 428 |                 },
 429 |                 {
 430 |                     "name": "Guilherme Blanco",
 431 |                     "email": "guilhermeblanco@gmail.com"
 432 |                 },
 433 |                 {
 434 |                     "name": "Jonathan Wage",
 435 |                     "email": "jonwage@gmail.com"
 436 |                 },
 437 |                 {
 438 |                     "name": "Johannes Schmitt",
 439 |                     "email": "schmittjoh@gmail.com"
 440 |                 }
 441 |             ],
 442 |             "description": "Common String Manipulations with regard to casing and singular/plural rules.",
 443 |             "homepage": "http://www.doctrine-project.org",
 444 |             "keywords": [
 445 |                 "inflection",
 446 |                 "pluralize",
 447 |                 "singularize",
 448 |                 "string"
 449 |             ],
 450 |             "time": "2015-11-06 14:35:42"
 451 |         },
 452 |         {
 453 |             "name": "doctrine/lexer",
 454 |             "version": "v1.0.1",
 455 |             "source": {
 456 |                 "type": "git",
 457 |                 "url": "https://github.com/doctrine/lexer.git",
 458 |                 "reference": "83893c552fd2045dd78aef794c31e694c37c0b8c"
 459 |             },
 460 |             "dist": {
 461 |                 "type": "zip",
 462 |                 "url": "https://api.github.com/repos/doctrine/lexer/zipball/83893c552fd2045dd78aef794c31e694c37c0b8c",
 463 |                 "reference": "83893c552fd2045dd78aef794c31e694c37c0b8c",
 464 |                 "shasum": ""
 465 |             },
 466 |             "require": {
 467 |                 "php": ">=5.3.2"
 468 |             },
 469 |             "type": "library",
 470 |             "extra": {
 471 |                 "branch-alias": {
 472 |                     "dev-master": "1.0.x-dev"
 473 |                 }
 474 |             },
 475 |             "autoload": {
 476 |                 "psr-0": {
 477 |                     "Doctrine\\Common\\Lexer\\": "lib/"
 478 |                 }
 479 |             },
 480 |             "notification-url": "https://packagist.org/downloads/",
 481 |             "license": [
 482 |                 "MIT"
 483 |             ],
 484 |             "authors": [
 485 |                 {
 486 |                     "name": "Roman Borschel",
 487 |                     "email": "roman@code-factory.org"
 488 |                 },
 489 |                 {
 490 |                     "name": "Guilherme Blanco",
 491 |                     "email": "guilhermeblanco@gmail.com"
 492 |                 },
 493 |                 {
 494 |                     "name": "Johannes Schmitt",
 495 |                     "email": "schmittjoh@gmail.com"
 496 |                 }
 497 |             ],
 498 |             "description": "Base library for a lexer that can be used in Top-Down, Recursive Descent Parsers.",
 499 |             "homepage": "http://www.doctrine-project.org",
 500 |             "keywords": [
 501 |                 "lexer",
 502 |                 "parser"
 503 |             ],
 504 |             "time": "2014-09-09 13:34:57"
 505 |         },
 506 |         {
 507 |             "name": "facile-it/doctrine-mysql-come-back",
 508 |             "version": "v1.6.3",
 509 |             "source": {
 510 |                 "type": "git",
 511 |                 "url": "https://github.com/facile-it/doctrine-mysql-come-back.git",
 512 |                 "reference": "8ffda796865ff91fe1f7de3fc8888c35d854447c"
 513 |             },
 514 |             "dist": {
 515 |                 "type": "zip",
 516 |                 "url": "https://api.github.com/repos/facile-it/doctrine-mysql-come-back/zipball/8ffda796865ff91fe1f7de3fc8888c35d854447c",
 517 |                 "reference": "8ffda796865ff91fe1f7de3fc8888c35d854447c",
 518 |                 "shasum": ""
 519 |             },
 520 |             "require": {
 521 |                 "doctrine/dbal": ">=2.3,<2.6-dev",
 522 |                 "php": "^5.4 || ^7.0"
 523 |             },
 524 |             "require-dev": {
 525 |                 "friendsofphp/php-cs-fixer": "^1.11",
 526 |                 "phpunit/phpunit": "^4.8 || ^5.2"
 527 |             },
 528 |             "type": "library",
 529 |             "autoload": {
 530 |                 "psr-4": {
 531 |                     "Facile\\DoctrineMySQLComeBack\\Doctrine\\DBAL\\": "src/"
 532 |                 }
 533 |             },
 534 |             "notification-url": "https://packagist.org/downloads/",
 535 |             "license": [
 536 |                 "Apache License Version 2.0"
 537 |             ],
 538 |             "authors": [
 539 |                 {
 540 |                     "name": "Luca Bo",
 541 |                     "email": "luca.boeri@facile.it"
 542 |                 }
 543 |             ],
 544 |             "description": "Auto reconnect on Doctrine MySql has gone away exceptions on doctrine/dbal",
 545 |             "keywords": [
 546 |                 "Connection",
 547 |                 "doctrine",
 548 |                 "exception",
 549 |                 "has gone away",
 550 |                 "mysql",
 551 |                 "reconnect",
 552 |                 "refresh"
 553 |             ],
 554 |             "time": "2016-09-30 10:55:31"
 555 |         },
 556 |         {
 557 |             "name": "firebase/php-jwt",
 558 |             "version": "v3.0.0",
 559 |             "source": {
 560 |                 "type": "git",
 561 |                 "url": "https://github.com/firebase/php-jwt.git",
 562 |                 "reference": "fa8a06e96526eb7c0eeaa47e4f39be59d21f16e1"
 563 |             },
 564 |             "dist": {
 565 |                 "type": "zip",
 566 |                 "url": "https://api.github.com/repos/firebase/php-jwt/zipball/fa8a06e96526eb7c0eeaa47e4f39be59d21f16e1",
 567 |                 "reference": "fa8a06e96526eb7c0eeaa47e4f39be59d21f16e1",
 568 |                 "shasum": ""
 569 |             },
 570 |             "require": {
 571 |                 "php": ">=5.3.0"
 572 |             },
 573 |             "type": "library",
 574 |             "autoload": {
 575 |                 "psr-4": {
 576 |                     "Firebase\\JWT\\": "src"
 577 |                 }
 578 |             },
 579 |             "notification-url": "https://packagist.org/downloads/",
 580 |             "license": [
 581 |                 "BSD-3-Clause"
 582 |             ],
 583 |             "authors": [
 584 |                 {
 585 |                     "name": "Neuman Vong",
 586 |                     "email": "neuman+pear@twilio.com",
 587 |                     "role": "Developer"
 588 |                 },
 589 |                 {
 590 |                     "name": "Anant Narayanan",
 591 |                     "email": "anant@php.net",
 592 |                     "role": "Developer"
 593 |                 }
 594 |             ],
 595 |             "description": "A simple library to encode and decode JSON Web Tokens (JWT) in PHP. Should conform to the current spec.",
 596 |             "homepage": "https://github.com/firebase/php-jwt",
 597 |             "time": "2015-07-22 18:31:08"
 598 |         },
 599 |         {
 600 |             "name": "google/auth",
 601 |             "version": "v0.10",
 602 |             "source": {
 603 |                 "type": "git",
 604 |                 "url": "https://github.com/google/google-auth-library-php.git",
 605 |                 "reference": "760e3fbe4064c0525c22e27e5374eada3c103da8"
 606 |             },
 607 |             "dist": {
 608 |                 "type": "zip",
 609 |                 "url": "https://api.github.com/repos/google/google-auth-library-php/zipball/760e3fbe4064c0525c22e27e5374eada3c103da8",
 610 |                 "reference": "760e3fbe4064c0525c22e27e5374eada3c103da8",
 611 |                 "shasum": ""
 612 |             },
 613 |             "require": {
 614 |                 "firebase/php-jwt": "~2.0|~3.0",
 615 |                 "guzzlehttp/guzzle": "~5.3|~6.0",
 616 |                 "guzzlehttp/psr7": "~1.2",
 617 |                 "php": ">=5.4",
 618 |                 "psr/cache": "^1.0",
 619 |                 "psr/http-message": "^1.0"
 620 |             },
 621 |             "require-dev": {
 622 |                 "friendsofphp/php-cs-fixer": "^1.11",
 623 |                 "phpunit/phpunit": "3.7.*"
 624 |             },
 625 |             "type": "library",
 626 |             "autoload": {
 627 |                 "classmap": [
 628 |                     "src/"
 629 |                 ],
 630 |                 "psr-4": {
 631 |                     "Google\\Auth\\": "src"
 632 |                 }
 633 |             },
 634 |             "notification-url": "https://packagist.org/downloads/",
 635 |             "license": [
 636 |                 "Apache-2.0"
 637 |             ],
 638 |             "description": "Google Auth Library for PHP",
 639 |             "homepage": "http://github.com/google/google-auth-library-php",
 640 |             "keywords": [
 641 |                 "Authentication",
 642 |                 "google",
 643 |                 "oauth2"
 644 |             ],
 645 |             "time": "2016-08-02 22:00:48"
 646 |         },
 647 |         {
 648 |             "name": "google/cloud",
 649 |             "version": "v0.11.1",
 650 |             "source": {
 651 |                 "type": "git",
 652 |                 "url": "https://github.com/GoogleCloudPlatform/google-cloud-php.git",
 653 |                 "reference": "7207d5b9bfb55cc732b082588f488150594141c2"
 654 |             },
 655 |             "dist": {
 656 |                 "type": "zip",
 657 |                 "url": "https://api.github.com/repos/GoogleCloudPlatform/google-cloud-php/zipball/7207d5b9bfb55cc732b082588f488150594141c2",
 658 |                 "reference": "7207d5b9bfb55cc732b082588f488150594141c2",
 659 |                 "shasum": ""
 660 |             },
 661 |             "require": {
 662 |                 "google/auth": "0.10",
 663 |                 "guzzlehttp/guzzle": "~5.2|~6.0",
 664 |                 "guzzlehttp/psr7": "^1.2",
 665 |                 "monolog/monolog": "~1",
 666 |                 "php": ">=5.5",
 667 |                 "psr/http-message": "1.0.*",
 668 |                 "rize/uri-template": "~0.3"
 669 |             },
 670 |             "require-dev": {
 671 |                 "erusev/parsedown": "^1.6",
 672 |                 "james-heinrich/getid3": "^1.9",
 673 |                 "league/json-guard": "^0.3",
 674 |                 "phpdocumentor/reflection": "^3.0",
 675 |                 "phpunit/phpunit": "4.8.*",
 676 |                 "squizlabs/php_codesniffer": "2.*",
 677 |                 "symfony/console": "^3.0",
 678 |                 "vierbergenlars/php-semver": "^3.0"
 679 |             },
 680 |             "suggest": {
 681 |                 "google/gax": "Required to support gRPC",
 682 |                 "google/proto-client-php": "Required to support gRPC",
 683 |                 "james-heinrich/getid3": "Allows the Google Cloud Speech client to determine sample rate and encoding of audio inputs"
 684 |             },
 685 |             "type": "library",
 686 |             "autoload": {
 687 |                 "psr-4": {
 688 |                     "Google\\Cloud\\": "src"
 689 |                 }
 690 |             },
 691 |             "notification-url": "https://packagist.org/downloads/",
 692 |             "license": [
 693 |                 "Apache-2.0"
 694 |             ],
 695 |             "authors": [
 696 |                 {
 697 |                     "name": "John Pedrie",
 698 |                     "email": "john@pedrie.com"
 699 |                 },
 700 |                 {
 701 |                     "name": "Dave Supplee",
 702 |                     "email": "dwsupplee@gmail.com"
 703 |                 }
 704 |             ],
 705 |             "description": "Google Cloud Client Library",
 706 |             "homepage": "http://github.com/GoogleCloudPlatform/google-cloud-php",
 707 |             "keywords": [
 708 |                 "big query",
 709 |                 "bigquery",
 710 |                 "cloud",
 711 |                 "datastore",
 712 |                 "gcs",
 713 |                 "google",
 714 |                 "google api",
 715 |                 "google api client",
 716 |                 "google apis",
 717 |                 "google apis client",
 718 |                 "google cloud",
 719 |                 "google cloud platform",
 720 |                 "natural language",
 721 |                 "pub sub",
 722 |                 "pubsub",
 723 |                 "speech",
 724 |                 "stackdriver logging",
 725 |                 "storage",
 726 |                 "translate",
 727 |                 "vision"
 728 |             ],
 729 |             "time": "2016-10-13 20:41:02"
 730 |         },
 731 |         {
 732 |             "name": "guzzlehttp/guzzle",
 733 |             "version": "6.2.2",
 734 |             "source": {
 735 |                 "type": "git",
 736 |                 "url": "https://github.com/guzzle/guzzle.git",
 737 |                 "reference": "ebf29dee597f02f09f4d5bbecc68230ea9b08f60"
 738 |             },
 739 |             "dist": {
 740 |                 "type": "zip",
 741 |                 "url": "https://api.github.com/repos/guzzle/guzzle/zipball/ebf29dee597f02f09f4d5bbecc68230ea9b08f60",
 742 |                 "reference": "ebf29dee597f02f09f4d5bbecc68230ea9b08f60",
 743 |                 "shasum": ""
 744 |             },
 745 |             "require": {
 746 |                 "guzzlehttp/promises": "^1.0",
 747 |                 "guzzlehttp/psr7": "^1.3.1",
 748 |                 "php": ">=5.5"
 749 |             },
 750 |             "require-dev": {
 751 |                 "ext-curl": "*",
 752 |                 "phpunit/phpunit": "^4.0",
 753 |                 "psr/log": "^1.0"
 754 |             },
 755 |             "type": "library",
 756 |             "extra": {
 757 |                 "branch-alias": {
 758 |                     "dev-master": "6.2-dev"
 759 |                 }
 760 |             },
 761 |             "autoload": {
 762 |                 "files": [
 763 |                     "src/functions_include.php"
 764 |                 ],
 765 |                 "psr-4": {
 766 |                     "GuzzleHttp\\": "src/"
 767 |                 }
 768 |             },
 769 |             "notification-url": "https://packagist.org/downloads/",
 770 |             "license": [
 771 |                 "MIT"
 772 |             ],
 773 |             "authors": [
 774 |                 {
 775 |                     "name": "Michael Dowling",
 776 |                     "email": "mtdowling@gmail.com",
 777 |                     "homepage": "https://github.com/mtdowling"
 778 |                 }
 779 |             ],
 780 |             "description": "Guzzle is a PHP HTTP client library",
 781 |             "homepage": "http://guzzlephp.org/",
 782 |             "keywords": [
 783 |                 "client",
 784 |                 "curl",
 785 |                 "framework",
 786 |                 "http",
 787 |                 "http client",
 788 |                 "rest",
 789 |                 "web service"
 790 |             ],
 791 |             "time": "2016-10-08 15:01:37"
 792 |         },
 793 |         {
 794 |             "name": "guzzlehttp/promises",
 795 |             "version": "1.2.0",
 796 |             "source": {
 797 |                 "type": "git",
 798 |                 "url": "https://github.com/guzzle/promises.git",
 799 |                 "reference": "c10d860e2a9595f8883527fa0021c7da9e65f579"
 800 |             },
 801 |             "dist": {
 802 |                 "type": "zip",
 803 |                 "url": "https://api.github.com/repos/guzzle/promises/zipball/c10d860e2a9595f8883527fa0021c7da9e65f579",
 804 |                 "reference": "c10d860e2a9595f8883527fa0021c7da9e65f579",
 805 |                 "shasum": ""
 806 |             },
 807 |             "require": {
 808 |                 "php": ">=5.5.0"
 809 |             },
 810 |             "require-dev": {
 811 |                 "phpunit/phpunit": "~4.0"
 812 |             },
 813 |             "type": "library",
 814 |             "extra": {
 815 |                 "branch-alias": {
 816 |                     "dev-master": "1.0-dev"
 817 |                 }
 818 |             },
 819 |             "autoload": {
 820 |                 "psr-4": {
 821 |                     "GuzzleHttp\\Promise\\": "src/"
 822 |                 },
 823 |                 "files": [
 824 |                     "src/functions_include.php"
 825 |                 ]
 826 |             },
 827 |             "notification-url": "https://packagist.org/downloads/",
 828 |             "license": [
 829 |                 "MIT"
 830 |             ],
 831 |             "authors": [
 832 |                 {
 833 |                     "name": "Michael Dowling",
 834 |                     "email": "mtdowling@gmail.com",
 835 |                     "homepage": "https://github.com/mtdowling"
 836 |                 }
 837 |             ],
 838 |             "description": "Guzzle promises library",
 839 |             "keywords": [
 840 |                 "promise"
 841 |             ],
 842 |             "time": "2016-05-18 16:56:05"
 843 |         },
 844 |         {
 845 |             "name": "guzzlehttp/psr7",
 846 |             "version": "1.3.1",
 847 |             "source": {
 848 |                 "type": "git",
 849 |                 "url": "https://github.com/guzzle/psr7.git",
 850 |                 "reference": "5c6447c9df362e8f8093bda8f5d8873fe5c7f65b"
 851 |             },
 852 |             "dist": {
 853 |                 "type": "zip",
 854 |                 "url": "https://api.github.com/repos/guzzle/psr7/zipball/5c6447c9df362e8f8093bda8f5d8873fe5c7f65b",
 855 |                 "reference": "5c6447c9df362e8f8093bda8f5d8873fe5c7f65b",
 856 |                 "shasum": ""
 857 |             },
 858 |             "require": {
 859 |                 "php": ">=5.4.0",
 860 |                 "psr/http-message": "~1.0"
 861 |             },
 862 |             "provide": {
 863 |                 "psr/http-message-implementation": "1.0"
 864 |             },
 865 |             "require-dev": {
 866 |                 "phpunit/phpunit": "~4.0"
 867 |             },
 868 |             "type": "library",
 869 |             "extra": {
 870 |                 "branch-alias": {
 871 |                     "dev-master": "1.4-dev"
 872 |                 }
 873 |             },
 874 |             "autoload": {
 875 |                 "psr-4": {
 876 |                     "GuzzleHttp\\Psr7\\": "src/"
 877 |                 },
 878 |                 "files": [
 879 |                     "src/functions_include.php"
 880 |                 ]
 881 |             },
 882 |             "notification-url": "https://packagist.org/downloads/",
 883 |             "license": [
 884 |                 "MIT"
 885 |             ],
 886 |             "authors": [
 887 |                 {
 888 |                     "name": "Michael Dowling",
 889 |                     "email": "mtdowling@gmail.com",
 890 |                     "homepage": "https://github.com/mtdowling"
 891 |                 }
 892 |             ],
 893 |             "description": "PSR-7 message implementation",
 894 |             "keywords": [
 895 |                 "http",
 896 |                 "message",
 897 |                 "stream",
 898 |                 "uri"
 899 |             ],
 900 |             "time": "2016-06-24 23:00:38"
 901 |         },
 902 |         {
 903 |             "name": "monolog/monolog",
 904 |             "version": "1.21.0",
 905 |             "source": {
 906 |                 "type": "git",
 907 |                 "url": "https://github.com/Seldaek/monolog.git",
 908 |                 "reference": "f42fbdfd53e306bda545845e4dbfd3e72edb4952"
 909 |             },
 910 |             "dist": {
 911 |                 "type": "zip",
 912 |                 "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f42fbdfd53e306bda545845e4dbfd3e72edb4952",
 913 |                 "reference": "f42fbdfd53e306bda545845e4dbfd3e72edb4952",
 914 |                 "shasum": ""
 915 |             },
 916 |             "require": {
 917 |                 "php": ">=5.3.0",
 918 |                 "psr/log": "~1.0"
 919 |             },
 920 |             "provide": {
 921 |                 "psr/log-implementation": "1.0.0"
 922 |             },
 923 |             "require-dev": {
 924 |                 "aws/aws-sdk-php": "^2.4.9",
 925 |                 "doctrine/couchdb": "~1.0@dev",
 926 |                 "graylog2/gelf-php": "~1.0",
 927 |                 "jakub-onderka/php-parallel-lint": "0.9",
 928 |                 "php-amqplib/php-amqplib": "~2.4",
 929 |                 "php-console/php-console": "^3.1.3",
 930 |                 "phpunit/phpunit": "~4.5",
 931 |                 "phpunit/phpunit-mock-objects": "2.3.0",
 932 |                 "ruflin/elastica": ">=0.90 <3.0",
 933 |                 "sentry/sentry": "^0.13",
 934 |                 "swiftmailer/swiftmailer": "~5.3"
 935 |             },
 936 |             "suggest": {
 937 |                 "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB",
 938 |                 "doctrine/couchdb": "Allow sending log messages to a CouchDB server",
 939 |                 "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)",
 940 |                 "ext-mongo": "Allow sending log messages to a MongoDB server",
 941 |                 "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server",
 942 |                 "mongodb/mongodb": "Allow sending log messages to a MongoDB server via PHP Driver",
 943 |                 "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib",
 944 |                 "php-console/php-console": "Allow sending log messages to Google Chrome",
 945 |                 "rollbar/rollbar": "Allow sending log messages to Rollbar",
 946 |                 "ruflin/elastica": "Allow sending log messages to an Elastic Search server",
 947 |                 "sentry/sentry": "Allow sending log messages to a Sentry server"
 948 |             },
 949 |             "type": "library",
 950 |             "extra": {
 951 |                 "branch-alias": {
 952 |                     "dev-master": "2.0.x-dev"
 953 |                 }
 954 |             },
 955 |             "autoload": {
 956 |                 "psr-4": {
 957 |                     "Monolog\\": "src/Monolog"
 958 |                 }
 959 |             },
 960 |             "notification-url": "https://packagist.org/downloads/",
 961 |             "license": [
 962 |                 "MIT"
 963 |             ],
 964 |             "authors": [
 965 |                 {
 966 |                     "name": "Jordi Boggiano",
 967 |                     "email": "j.boggiano@seld.be",
 968 |                     "homepage": "http://seld.be"
 969 |                 }
 970 |             ],
 971 |             "description": "Sends your logs to files, sockets, inboxes, databases and various web services",
 972 |             "homepage": "http://github.com/Seldaek/monolog",
 973 |             "keywords": [
 974 |                 "log",
 975 |                 "logging",
 976 |                 "psr-3"
 977 |             ],
 978 |             "time": "2016-07-29 03:23:52"
 979 |         },
 980 |         {
 981 |             "name": "php-di/invoker",
 982 |             "version": "1.3.3",
 983 |             "source": {
 984 |                 "type": "git",
 985 |                 "url": "https://github.com/PHP-DI/Invoker.git",
 986 |                 "reference": "1f4ca63b9abc66109e53b255e465d0ddb5c2e3f7"
 987 |             },
 988 |             "dist": {
 989 |                 "type": "zip",
 990 |                 "url": "https://api.github.com/repos/PHP-DI/Invoker/zipball/1f4ca63b9abc66109e53b255e465d0ddb5c2e3f7",
 991 |                 "reference": "1f4ca63b9abc66109e53b255e465d0ddb5c2e3f7",
 992 |                 "shasum": ""
 993 |             },
 994 |             "require": {
 995 |                 "container-interop/container-interop": "~1.1"
 996 |             },
 997 |             "require-dev": {
 998 |                 "athletic/athletic": "~0.1.8",
 999 |                 "phpunit/phpunit": "~4.5"
1000 |             },
1001 |             "type": "library",
1002 |             "autoload": {
1003 |                 "psr-4": {
1004 |                     "Invoker\\": "src/"
1005 |                 }
1006 |             },
1007 |             "notification-url": "https://packagist.org/downloads/",
1008 |             "license": [
1009 |                 "MIT"
1010 |             ],
1011 |             "description": "Generic and extensible callable invoker",
1012 |             "homepage": "https://github.com/PHP-DI/Invoker",
1013 |             "keywords": [
1014 |                 "callable",
1015 |                 "dependency",
1016 |                 "dependency-injection",
1017 |                 "injection",
1018 |                 "invoke",
1019 |                 "invoker"
1020 |             ],
1021 |             "time": "2016-07-14 13:09:58"
1022 |         },
1023 |         {
1024 |             "name": "php-di/php-di",
1025 |             "version": "5.4.0",
1026 |             "source": {
1027 |                 "type": "git",
1028 |                 "url": "https://github.com/PHP-DI/PHP-DI.git",
1029 |                 "reference": "e348393488fa909e4bc0707ba5c9c44cd602a1cb"
1030 |             },
1031 |             "dist": {
1032 |                 "type": "zip",
1033 |                 "url": "https://api.github.com/repos/PHP-DI/PHP-DI/zipball/e348393488fa909e4bc0707ba5c9c44cd602a1cb",
1034 |                 "reference": "e348393488fa909e4bc0707ba5c9c44cd602a1cb",
1035 |                 "shasum": ""
1036 |             },
1037 |             "require": {
1038 |                 "container-interop/container-interop": "~1.0",
1039 |                 "php": ">=5.5.0",
1040 |                 "php-di/invoker": "^1.3.2",
1041 |                 "php-di/phpdoc-reader": "^2.0.1"
1042 |             },
1043 |             "provide": {
1044 |                 "container-interop/container-interop-implementation": "^1.0"
1045 |             },
1046 |             "replace": {
1047 |                 "mnapoli/php-di": "*"
1048 |             },
1049 |             "require-dev": {
1050 |                 "doctrine/annotations": "~1.2",
1051 |                 "doctrine/cache": "~1.4",
1052 |                 "mnapoli/phpunit-easymock": "~0.2.0",
1053 |                 "ocramius/proxy-manager": "~1.0|~2.0",
1054 |                 "phpunit/phpunit": "~4.5"
1055 |             },
1056 |             "suggest": {
1057 |                 "doctrine/annotations": "Install it if you want to use annotations (version ~1.2)",
1058 |                 "doctrine/cache": "Install it if you want to use the cache (version ~1.4)",
1059 |                 "ocramius/proxy-manager": "Install it if you want to use lazy injection (version ~1.0 or ~2.0)"
1060 |             },
1061 |             "type": "library",
1062 |             "autoload": {
1063 |                 "psr-4": {
1064 |                     "DI\\": "src/DI/"
1065 |                 },
1066 |                 "files": [
1067 |                     "src/DI/functions.php"
1068 |                 ]
1069 |             },
1070 |             "notification-url": "https://packagist.org/downloads/",
1071 |             "license": [
1072 |                 "MIT"
1073 |             ],
1074 |             "description": "The dependency injection container for humans",
1075 |             "homepage": "http://php-di.org/",
1076 |             "keywords": [
1077 |                 "container",
1078 |                 "dependency injection",
1079 |                 "di"
1080 |             ],
1081 |             "time": "2016-08-23 20:18:00"
1082 |         },
1083 |         {
1084 |             "name": "php-di/phpdoc-reader",
1085 |             "version": "2.0.1",
1086 |             "source": {
1087 |                 "type": "git",
1088 |                 "url": "https://github.com/PHP-DI/PhpDocReader.git",
1089 |                 "reference": "83f5ead159defccfa8e7092e5b6c1c533b326d68"
1090 |             },
1091 |             "dist": {
1092 |                 "type": "zip",
1093 |                 "url": "https://api.github.com/repos/PHP-DI/PhpDocReader/zipball/83f5ead159defccfa8e7092e5b6c1c533b326d68",
1094 |                 "reference": "83f5ead159defccfa8e7092e5b6c1c533b326d68",
1095 |                 "shasum": ""
1096 |             },
1097 |             "require": {
1098 |                 "php": ">=5.3.0"
1099 |             },
1100 |             "require-dev": {
1101 |                 "phpunit/phpunit": "~4.6"
1102 |             },
1103 |             "type": "library",
1104 |             "autoload": {
1105 |                 "psr-4": {
1106 |                     "PhpDocReader\\": "src/PhpDocReader"
1107 |                 }
1108 |             },
1109 |             "notification-url": "https://packagist.org/downloads/",
1110 |             "license": [
1111 |                 "MIT"
1112 |             ],
1113 |             "description": "PhpDocReader parses @var and @param values in PHP docblocks (supports namespaced class names with the same resolution rules as PHP)",
1114 |             "keywords": [
1115 |                 "phpdoc",
1116 |                 "reflection"
1117 |             ],
1118 |             "time": "2015-11-29 10:34:25"
1119 |         },
1120 |         {
1121 |             "name": "psr/cache",
1122 |             "version": "1.0.1",
1123 |             "source": {
1124 |                 "type": "git",
1125 |                 "url": "https://github.com/php-fig/cache.git",
1126 |                 "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8"
1127 |             },
1128 |             "dist": {
1129 |                 "type": "zip",
1130 |                 "url": "https://api.github.com/repos/php-fig/cache/zipball/d11b50ad223250cf17b86e38383413f5a6764bf8",
1131 |                 "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8",
1132 |                 "shasum": ""
1133 |             },
1134 |             "require": {
1135 |                 "php": ">=5.3.0"
1136 |             },
1137 |             "type": "library",
1138 |             "extra": {
1139 |                 "branch-alias": {
1140 |                     "dev-master": "1.0.x-dev"
1141 |                 }
1142 |             },
1143 |             "autoload": {
1144 |                 "psr-4": {
1145 |                     "Psr\\Cache\\": "src/"
1146 |                 }
1147 |             },
1148 |             "notification-url": "https://packagist.org/downloads/",
1149 |             "license": [
1150 |                 "MIT"
1151 |             ],
1152 |             "authors": [
1153 |                 {
1154 |                     "name": "PHP-FIG",
1155 |                     "homepage": "http://www.php-fig.org/"
1156 |                 }
1157 |             ],
1158 |             "description": "Common interface for caching libraries",
1159 |             "keywords": [
1160 |                 "cache",
1161 |                 "psr",
1162 |                 "psr-6"
1163 |             ],
1164 |             "time": "2016-08-06 20:24:11"
1165 |         },
1166 |         {
1167 |             "name": "psr/http-message",
1168 |             "version": "1.0.1",
1169 |             "source": {
1170 |                 "type": "git",
1171 |                 "url": "https://github.com/php-fig/http-message.git",
1172 |                 "reference": "f6561bf28d520154e4b0ec72be95418abe6d9363"
1173 |             },
1174 |             "dist": {
1175 |                 "type": "zip",
1176 |                 "url": "https://api.github.com/repos/php-fig/http-message/zipball/f6561bf28d520154e4b0ec72be95418abe6d9363",
1177 |                 "reference": "f6561bf28d520154e4b0ec72be95418abe6d9363",
1178 |                 "shasum": ""
1179 |             },
1180 |             "require": {
1181 |                 "php": ">=5.3.0"
1182 |             },
1183 |             "type": "library",
1184 |             "extra": {
1185 |                 "branch-alias": {
1186 |                     "dev-master": "1.0.x-dev"
1187 |                 }
1188 |             },
1189 |             "autoload": {
1190 |                 "psr-4": {
1191 |                     "Psr\\Http\\Message\\": "src/"
1192 |                 }
1193 |             },
1194 |             "notification-url": "https://packagist.org/downloads/",
1195 |             "license": [
1196 |                 "MIT"
1197 |             ],
1198 |             "authors": [
1199 |                 {
1200 |                     "name": "PHP-FIG",
1201 |                     "homepage": "http://www.php-fig.org/"
1202 |                 }
1203 |             ],
1204 |             "description": "Common interface for HTTP messages",
1205 |             "homepage": "https://github.com/php-fig/http-message",
1206 |             "keywords": [
1207 |                 "http",
1208 |                 "http-message",
1209 |                 "psr",
1210 |                 "psr-7",
1211 |                 "request",
1212 |                 "response"
1213 |             ],
1214 |             "time": "2016-08-06 14:39:51"
1215 |         },
1216 |         {
1217 |             "name": "psr/log",
1218 |             "version": "1.0.2",
1219 |             "source": {
1220 |                 "type": "git",
1221 |                 "url": "https://github.com/php-fig/log.git",
1222 |                 "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d"
1223 |             },
1224 |             "dist": {
1225 |                 "type": "zip",
1226 |                 "url": "https://api.github.com/repos/php-fig/log/zipball/4ebe3a8bf773a19edfe0a84b6585ba3d401b724d",
1227 |                 "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d",
1228 |                 "shasum": ""
1229 |             },
1230 |             "require": {
1231 |                 "php": ">=5.3.0"
1232 |             },
1233 |             "type": "library",
1234 |             "extra": {
1235 |                 "branch-alias": {
1236 |                     "dev-master": "1.0.x-dev"
1237 |                 }
1238 |             },
1239 |             "autoload": {
1240 |                 "psr-4": {
1241 |                     "Psr\\Log\\": "Psr/Log/"
1242 |                 }
1243 |             },
1244 |             "notification-url": "https://packagist.org/downloads/",
1245 |             "license": [
1246 |                 "MIT"
1247 |             ],
1248 |             "authors": [
1249 |                 {
1250 |                     "name": "PHP-FIG",
1251 |                     "homepage": "http://www.php-fig.org/"
1252 |                 }
1253 |             ],
1254 |             "description": "Common interface for logging libraries",
1255 |             "homepage": "https://github.com/php-fig/log",
1256 |             "keywords": [
1257 |                 "log",
1258 |                 "psr",
1259 |                 "psr-3"
1260 |             ],
1261 |             "time": "2016-10-10 12:19:37"
1262 |         },
1263 |         {
1264 |             "name": "rize/uri-template",
1265 |             "version": "0.3.0",
1266 |             "source": {
1267 |                 "type": "git",
1268 |                 "url": "https://github.com/rize/UriTemplate.git",
1269 |                 "reference": "2496aa674438f1c48fce122ffc44291ad7014717"
1270 |             },
1271 |             "dist": {
1272 |                 "type": "zip",
1273 |                 "url": "https://api.github.com/repos/rize/UriTemplate/zipball/2496aa674438f1c48fce122ffc44291ad7014717",
1274 |                 "reference": "2496aa674438f1c48fce122ffc44291ad7014717",
1275 |                 "shasum": ""
1276 |             },
1277 |             "require": {
1278 |                 "php": ">=5.3.0"
1279 |             },
1280 |             "require-dev": {
1281 |                 "phpunit/phpunit": "~4.0.0"
1282 |             },
1283 |             "type": "library",
1284 |             "autoload": {
1285 |                 "psr-0": {
1286 |                     "Rize\\UriTemplate": "src/"
1287 |                 }
1288 |             },
1289 |             "notification-url": "https://packagist.org/downloads/",
1290 |             "license": [
1291 |                 "MIT"
1292 |             ],
1293 |             "authors": [
1294 |                 {
1295 |                     "name": "Marut K",
1296 |                     "homepage": "http://twitter.com/rezigned"
1297 |                 }
1298 |             ],
1299 |             "description": "PHP URI Template (RFC 6570) supports both expansion & extraction",
1300 |             "keywords": [
1301 |                 "RFC 6570",
1302 |                 "template",
1303 |                 "uri"
1304 |             ],
1305 |             "time": "2015-04-17 16:12:22"
1306 |         },
1307 |         {
1308 |             "name": "symfony/console",
1309 |             "version": "v3.1.6",
1310 |             "source": {
1311 |                 "type": "git",
1312 |                 "url": "https://github.com/symfony/console.git",
1313 |                 "reference": "c99da1119ae61e15de0e4829196b9fba6f73d065"
1314 |             },
1315 |             "dist": {
1316 |                 "type": "zip",
1317 |                 "url": "https://api.github.com/repos/symfony/console/zipball/c99da1119ae61e15de0e4829196b9fba6f73d065",
1318 |                 "reference": "c99da1119ae61e15de0e4829196b9fba6f73d065",
1319 |                 "shasum": ""
1320 |             },
1321 |             "require": {
1322 |                 "php": ">=5.5.9",
1323 |                 "symfony/debug": "~2.8|~3.0",
1324 |                 "symfony/polyfill-mbstring": "~1.0"
1325 |             },
1326 |             "require-dev": {
1327 |                 "psr/log": "~1.0",
1328 |                 "symfony/event-dispatcher": "~2.8|~3.0",
1329 |                 "symfony/process": "~2.8|~3.0"
1330 |             },
1331 |             "suggest": {
1332 |                 "psr/log": "For using the console logger",
1333 |                 "symfony/event-dispatcher": "",
1334 |                 "symfony/process": ""
1335 |             },
1336 |             "type": "library",
1337 |             "extra": {
1338 |                 "branch-alias": {
1339 |                     "dev-master": "3.1-dev"
1340 |                 }
1341 |             },
1342 |             "autoload": {
1343 |                 "psr-4": {
1344 |                     "Symfony\\Component\\Console\\": ""
1345 |                 },
1346 |                 "exclude-from-classmap": [
1347 |                     "/Tests/"
1348 |                 ]
1349 |             },
1350 |             "notification-url": "https://packagist.org/downloads/",
1351 |             "license": [
1352 |                 "MIT"
1353 |             ],
1354 |             "authors": [
1355 |                 {
1356 |                     "name": "Fabien Potencier",
1357 |                     "email": "fabien@symfony.com"
1358 |                 },
1359 |                 {
1360 |                     "name": "Symfony Community",
1361 |                     "homepage": "https://symfony.com/contributors"
1362 |                 }
1363 |             ],
1364 |             "description": "Symfony Console Component",
1365 |             "homepage": "https://symfony.com",
1366 |             "time": "2016-10-06 01:44:51"
1367 |         },
1368 |         {
1369 |             "name": "symfony/debug",
1370 |             "version": "v3.1.6",
1371 |             "source": {
1372 |                 "type": "git",
1373 |                 "url": "https://github.com/symfony/debug.git",
1374 |                 "reference": "e2b3f74a67fc928adc3c1b9027f73e1bc01190a8"
1375 |             },
1376 |             "dist": {
1377 |                 "type": "zip",
1378 |                 "url": "https://api.github.com/repos/symfony/debug/zipball/e2b3f74a67fc928adc3c1b9027f73e1bc01190a8",
1379 |                 "reference": "e2b3f74a67fc928adc3c1b9027f73e1bc01190a8",
1380 |                 "shasum": ""
1381 |             },
1382 |             "require": {
1383 |                 "php": ">=5.5.9",
1384 |                 "psr/log": "~1.0"
1385 |             },
1386 |             "conflict": {
1387 |                 "symfony/http-kernel": ">=2.3,<2.3.24|~2.4.0|>=2.5,<2.5.9|>=2.6,<2.6.2"
1388 |             },
1389 |             "require-dev": {
1390 |                 "symfony/class-loader": "~2.8|~3.0",
1391 |                 "symfony/http-kernel": "~2.8|~3.0"
1392 |             },
1393 |             "type": "library",
1394 |             "extra": {
1395 |                 "branch-alias": {
1396 |                     "dev-master": "3.1-dev"
1397 |                 }
1398 |             },
1399 |             "autoload": {
1400 |                 "psr-4": {
1401 |                     "Symfony\\Component\\Debug\\": ""
1402 |                 },
1403 |                 "exclude-from-classmap": [
1404 |                     "/Tests/"
1405 |                 ]
1406 |             },
1407 |             "notification-url": "https://packagist.org/downloads/",
1408 |             "license": [
1409 |                 "MIT"
1410 |             ],
1411 |             "authors": [
1412 |                 {
1413 |                     "name": "Fabien Potencier",
1414 |                     "email": "fabien@symfony.com"
1415 |                 },
1416 |                 {
1417 |                     "name": "Symfony Community",
1418 |                     "homepage": "https://symfony.com/contributors"
1419 |                 }
1420 |             ],
1421 |             "description": "Symfony Debug Component",
1422 |             "homepage": "https://symfony.com",
1423 |             "time": "2016-09-06 11:02:40"
1424 |         },
1425 |         {
1426 |             "name": "symfony/polyfill-mbstring",
1427 |             "version": "v1.2.0",
1428 |             "source": {
1429 |                 "type": "git",
1430 |                 "url": "https://github.com/symfony/polyfill-mbstring.git",
1431 |                 "reference": "dff51f72b0706335131b00a7f49606168c582594"
1432 |             },
1433 |             "dist": {
1434 |                 "type": "zip",
1435 |                 "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/dff51f72b0706335131b00a7f49606168c582594",
1436 |                 "reference": "dff51f72b0706335131b00a7f49606168c582594",
1437 |                 "shasum": ""
1438 |             },
1439 |             "require": {
1440 |                 "php": ">=5.3.3"
1441 |             },
1442 |             "suggest": {
1443 |                 "ext-mbstring": "For best performance"
1444 |             },
1445 |             "type": "library",
1446 |             "extra": {
1447 |                 "branch-alias": {
1448 |                     "dev-master": "1.2-dev"
1449 |                 }
1450 |             },
1451 |             "autoload": {
1452 |                 "psr-4": {
1453 |                     "Symfony\\Polyfill\\Mbstring\\": ""
1454 |                 },
1455 |                 "files": [
1456 |                     "bootstrap.php"
1457 |                 ]
1458 |             },
1459 |             "notification-url": "https://packagist.org/downloads/",
1460 |             "license": [
1461 |                 "MIT"
1462 |             ],
1463 |             "authors": [
1464 |                 {
1465 |                     "name": "Nicolas Grekas",
1466 |                     "email": "p@tchwork.com"
1467 |                 },
1468 |                 {
1469 |                     "name": "Symfony Community",
1470 |                     "homepage": "https://symfony.com/contributors"
1471 |                 }
1472 |             ],
1473 |             "description": "Symfony polyfill for the Mbstring extension",
1474 |             "homepage": "https://symfony.com",
1475 |             "keywords": [
1476 |                 "compatibility",
1477 |                 "mbstring",
1478 |                 "polyfill",
1479 |                 "portable",
1480 |                 "shim"
1481 |             ],
1482 |             "time": "2016-05-18 14:26:46"
1483 |         },
1484 |         {
1485 |             "name": "vlucas/phpdotenv",
1486 |             "version": "v2.4.0",
1487 |             "source": {
1488 |                 "type": "git",
1489 |                 "url": "https://github.com/vlucas/phpdotenv.git",
1490 |                 "reference": "3cc116adbe4b11be5ec557bf1d24dc5e3a21d18c"
1491 |             },
1492 |             "dist": {
1493 |                 "type": "zip",
1494 |                 "url": "https://api.github.com/repos/vlucas/phpdotenv/zipball/3cc116adbe4b11be5ec557bf1d24dc5e3a21d18c",
1495 |                 "reference": "3cc116adbe4b11be5ec557bf1d24dc5e3a21d18c",
1496 |                 "shasum": ""
1497 |             },
1498 |             "require": {
1499 |                 "php": ">=5.3.9"
1500 |             },
1501 |             "require-dev": {
1502 |                 "phpunit/phpunit": "^4.8 || ^5.0"
1503 |             },
1504 |             "type": "library",
1505 |             "extra": {
1506 |                 "branch-alias": {
1507 |                     "dev-master": "2.4-dev"
1508 |                 }
1509 |             },
1510 |             "autoload": {
1511 |                 "psr-4": {
1512 |                     "Dotenv\\": "src/"
1513 |                 }
1514 |             },
1515 |             "notification-url": "https://packagist.org/downloads/",
1516 |             "license": [
1517 |                 "BSD-3-Clause-Attribution"
1518 |             ],
1519 |             "authors": [
1520 |                 {
1521 |                     "name": "Vance Lucas",
1522 |                     "email": "vance@vancelucas.com",
1523 |                     "homepage": "http://www.vancelucas.com"
1524 |                 }
1525 |             ],
1526 |             "description": "Loads environment variables from `.env` to `getenv()`, `$_ENV` and `$_SERVER` automagically.",
1527 |             "keywords": [
1528 |                 "dotenv",
1529 |                 "env",
1530 |                 "environment"
1531 |             ],
1532 |             "time": "2016-09-01 10:05:43"
1533 |         }
1534 |     ],
1535 |     "packages-dev": [],
1536 |     "aliases": [],
1537 |     "minimum-stability": "stable",
1538 |     "stability-flags": [],
1539 |     "prefer-stable": false,
1540 |     "prefer-lowest": false,
1541 |     "platform": [],
1542 |     "platform-dev": []
1543 | }
1544 | 


--------------------------------------------------------------------------------
/src/Console/Commands/SyncCommand.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace MysqlToGoogleBigQuery\Console\Commands;
 3 | 
 4 | use MysqlToGoogleBigQuery\Services\SyncService;
 5 | use Symfony\Component\Console\Command\Command;
 6 | use Symfony\Component\Console\Input\InputArgument;
 7 | use Symfony\Component\Console\Input\InputInterface;
 8 | use Symfony\Component\Console\Input\InputOption;
 9 | use Symfony\Component\Console\Output\OutputInterface;
10 | 
11 | class SyncCommand extends Command
12 | {
13 |     /**
14 |      * {@inheritdoc}
15 |      */
16 |     protected function configure()
17 |     {
18 |         $this
19 |             ->setName('sync')
20 |             ->setDescription('Sync a MySQL table to BigQuery')
21 |             ->setHelp('This commands syncs data between a MySQL table and BigQuery')
22 |             ->addArgument('table-name', InputArgument::REQUIRED, 'The name of the table you want to sync')
23 |             ->addOption('create-table', 'c', InputOption::VALUE_NONE, 'If BigQuery table doesn\'t exist, create it')
24 |             ->addOption('delete-table', 'd', InputOption::VALUE_NONE, 'Delete the BigQuery table before syncing')
25 |             ->addOption(
26 |                 'order-column',
27 |                 'o',
28 |                 InputOption::VALUE_OPTIONAL,
29 |                 'Column to order the results by. This column is also used to determine if new rows have to be synced.'
30 |             )
31 |             ->addOption('no-data', 'no', InputOption::VALUE_NONE, 'If specified do not copy data')
32 |             ->addOption('un-buffer', 'un', InputOption::VALUE_NONE, 'If specified use unbuffered json transfers')
33 |             ->addOption(
34 |                 'ignore-column',
35 |                 'i',
36 |                 InputOption::VALUE_OPTIONAL | InputOption::VALUE_IS_ARRAY,
37 |                 'Ignore a column from syncing. You can use this option multiple times'
38 |             )
39 |             ->addOption('database-name', null, InputOption::VALUE_OPTIONAL, 'MySQL database name')
40 |             ->addOption('bigquery-table-name', null, InputOption::VALUE_OPTIONAL, 'BigQuery table name');
41 |     }
42 | 
43 |     /**
44 |      * {@inheritdoc}
45 |      */
46 |     protected function execute(InputInterface $input, OutputInterface $output)
47 |     {
48 |         $container = \DI\ContainerBuilder::buildDevContainer();
49 | 
50 |         $ignoreColumns = $input->getOption('ignore-column');
51 |         $noData= $input->getOption('no-data') ? true : false;  
52 |         $unbuffered =  $input->getOption('un-buffer') ? true : false;  
53 |         if (empty($ignoreColumns) && isset($_ENV['IGNORE_COLUMNS'])) {
54 |             $ignoreColumns = explode(',', $_ENV['IGNORE_COLUMNS']);
55 |         }
56 | 
57 |         $orderColumn = $input->getOption('order-column');
58 | 
59 |         if (empty($orderColumn) && isset($_ENV['ORDER_COLUMN'])) {
60 |             $orderColumn = $_ENV['ORDER_COLUMN'];
61 |         }
62 | 
63 |         $databaseName = $input->getOption('database-name');
64 | 
65 |         if (empty($databaseName)) {
66 |             $databaseName = $_ENV['DB_DATABASE_NAME'];
67 |         }
68 | 
69 |         $bigQueryTableName = $input->getOption('bigquery-table-name');
70 | 
71 |         if (empty($bigQueryTableName)) {
72 |             $bigQueryTableName = $input->getArgument('table-name');
73 |         }
74 | 
75 |         $service = $container->get('MysqlToGoogleBigQuery\Services\SyncService');
76 |         $service->execute(
77 |             $databaseName,
78 |             $input->getArgument('table-name'),
79 |             $bigQueryTableName,
80 |             $input->getOption('create-table'),
81 |             $input->getOption('delete-table'),
82 |             $orderColumn,
83 |             $ignoreColumns,
84 |             $output, 
85 |             $noData, 
86 |             $unbuffered
87 |         );
88 |     }
89 | }
90 | 


--------------------------------------------------------------------------------
/src/Database/BigQuery.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace MysqlToGoogleBigQuery\Database;
  3 | 
  4 | use Doctrine\DBAL\Types\Type;
  5 | use Google\Cloud\BigQuery\BigQueryClient;
  6 | 
  7 | class BigQuery
  8 | {
  9 |     protected $client;
 10 |     protected $tablesMetadata = [];
 11 | 
 12 |     /**
 13 |      * Create a BigQuery Table based on MySQL Table columns
 14 |      * @param  string $tableName           Table Name
 15 |      * @param  array  $mysqlTableColumns   Array of Doctrine\DBAL\Schema\Column
 16 |      * @return Google\Cloud\BigQuery\Table Table object
 17 |      */
 18 |     public function createTable($tableName, $mysqlTableColumns)
 19 |     {
 20 |         $bigQueryColumns = [];
 21 | 
 22 |         // Valid types for BigQuery are:
 23 |         // STRING, BYTES, INTEGER, FLOAT, BOOLEAN,
 24 |         // TIMESTAMP, DATE, TIME, DATETIME
 25 |         foreach ($mysqlTableColumns as $name => $column) {
 26 |             switch ($column->getType()->getName()) {
 27 |                 case 'bigquerydate':
 28 |                     $type = 'DATE';
 29 |                     break;
 30 | 
 31 |                 case 'bigquerydatetime':
 32 |                     $type = 'DATETIME';
 33 |                     break;
 34 | 
 35 |                 case Type::BIGINT:
 36 |                     $type = 'INTEGER';
 37 |                     break;
 38 | 
 39 |                 case Type::BOOLEAN:
 40 |                     $type = 'BOOLEAN';
 41 |                     break;
 42 | 
 43 |                 case Type::DATE:
 44 |                     $type = 'DATE';
 45 |                     break;
 46 | 
 47 |                 case Type::DATETIME:
 48 |                     $type = 'DATETIME';
 49 |                     break;
 50 | 
 51 |                 case Type::DECIMAL:
 52 |                     $type = 'FLOAT';
 53 |                     break;
 54 | 
 55 |                 case Type::FLOAT:
 56 |                     $type = 'FLOAT';
 57 |                     break;
 58 | 
 59 |                 case Type::INTEGER:
 60 |                     $type = 'INTEGER';
 61 |                     break;
 62 | 
 63 |                 case Type::SMALLINT:
 64 |                     $type = 'INTEGER';
 65 |                     break;
 66 | 
 67 |                 case Type::TIME:
 68 |                     $type = 'TIME';
 69 |                     break;
 70 | 
 71 |                 default:
 72 |                     $type = 'STRING';
 73 |                     break;
 74 |             }
 75 | 
 76 |             $bigQueryColumns[] = [
 77 |                 'name' => $name,
 78 |                 'type' => $type
 79 |             ];
 80 |         }
 81 | 
 82 |         $client = $this->getClient();
 83 |         $dataset = $client->dataset($_ENV['BQ_DATASET']);
 84 | 
 85 |         return $dataset->createTable($tableName, [
 86 |             'schema' => [
 87 |                 'fields' => $bigQueryColumns
 88 |             ],
 89 |         ]);
 90 |     }
 91 | 
 92 |     /**
 93 |      * Delete a BigQuery Table
 94 |      * @param  string $tableName Table Name
 95 |      */
 96 |     public function deleteTable(string $tableName)
 97 |     {
 98 |         $client = $this->getClient();
 99 |         $dataset = $client->dataset($_ENV['BQ_DATASET']);
100 |         $dataset->table($tableName)->delete();
101 |     }
102 | 
103 |     /**
104 |      * Get the number of rows on a table
105 |      * @param  string $tableName Table name
106 |      * @return int|bool          false if table doesn't exists, or the number of rows
107 |      */
108 |     public function getCountTableRows(string $tableName)
109 |     {
110 |         $this->getTablesMetadata();
111 | 
112 |         if (! array_key_exists($tableName, $this->tablesMetadata)) {
113 |             return false;
114 |         }
115 | 
116 |         return $this->tablesMetadata[$tableName]['row_count'];
117 |     }
118 | 
119 |     /**
120 |      * Get the maximum value of a column
121 |      * @param  string $tableName Table name
122 |      * @param  string $columnName   Column name
123 |      * @return string               Max value
124 |      */
125 |     public function getMaxColumnValue(string $tableName, string $columnName)
126 |     {
127 |         $client = $this->getClient();
128 | 
129 |         $result = $client->runQuery(
130 |             'SELECT MAX([' . $columnName . ']) AS columnMax FROM [' . $_ENV['BQ_DATASET'] . '.' .  $tableName . ']'
131 |         );
132 | 
133 |         $isComplete = $result->isComplete();
134 |         while (!$isComplete) {
135 |             sleep(1);
136 |             $result->reload();
137 |             $isComplete = $result->isComplete();
138 |         }
139 | 
140 |         foreach ($result->rows() as $row) {
141 |             return $row['columnMax'];
142 |         }
143 | 
144 |         return false;
145 |     }
146 | 
147 |     /**
148 |      * Delete all values of a column
149 |      * @param  string $tableName Table name
150 |      * @param  string $columnName   Column name
151 |      * @param  string $columnValue  Value to be deleted
152 |      * @return string               Result
153 |      */
154 |     public function deleteColumnValue(string $tableName, string $columnName, string $columnValue)
155 |     {
156 |         $client = $this->getClient();
157 | 
158 |         // Non numeric values needs ""
159 |         if (! is_numeric($columnValue)) {
160 |             $columnValue = '"' . $columnValue . '"';
161 |         }
162 | 
163 |         $result = $client->runQuery(
164 |             'DELETE FROM `' . $_ENV['BQ_DATASET'] . '.' .  $tableName . '`' .
165 |             ' WHERE `' . $columnName .'` = ' . $columnValue,
166 |             ['useLegacySql' => false]
167 |         );
168 | 
169 |         $isComplete = $result->isComplete();
170 |         while (!$isComplete) {
171 |             sleep(1);
172 |             $result->reload();
173 |             $isComplete = $result->isComplete();
174 |         }
175 | 
176 |         return $result;
177 |     }
178 | 
179 |     /**
180 |      * Get BigQuery API Client
181 |      * @return BigQueryClient BigQuery API Client
182 |      */
183 |     public function getClient()
184 |     {
185 |         if ($this->client) {
186 |             return $this->client;
187 |         }
188 | 
189 |         $keyFilePath = $_ENV['BQ_KEY_FILE'];
190 | 
191 |         // Support relative and absolute path
192 |         if ($keyFilePath[0] !== '/') {
193 |             $keyFilePath = getcwd() . '/' . $keyFilePath;
194 |         }
195 | 
196 |         if (! file_exists($keyFilePath)) {
197 |             throw new \Exception('Google Service Account JSON Key File not found', 1);
198 |         }
199 | 
200 |         return $this->client = new BigQueryClient([
201 |             'projectId' => $_ENV['BQ_PROJECT_ID'],
202 |             'keyFile' => json_decode(file_get_contents($keyFilePath), true),
203 |             'scopes' => [BigQueryClient::SCOPE]
204 |         ]);
205 |     }
206 | 
207 |     /**
208 |      * Get table metadata
209 |      * See https://cloud.google.com/bigquery/querying-data#metadata_about_tables_in_a_dataset
210 |      *
211 |      * @return array Array with all dataset tables information
212 |      */
213 |     public function getTablesMetadata()
214 |     {
215 |         $client = $this->getClient();
216 |         $queryResults = $client->runQuery('SELECT * FROM ' . $_ENV['BQ_DATASET'] . '.__TABLES__;', [
217 |             'useQueryCache' => false
218 |         ]);
219 | 
220 |         foreach ($queryResults->rows() as $row) {
221 |             $this->tablesMetadata[$row['table_id']] = $row;
222 |         }
223 | 
224 |         return $this->tablesMetadata;
225 |     }
226 | 
227 |     /**
228 |      * Load data to BigQuery reading it from JSON NEWLINE DELIMITED File
229 |      * @param  resource|string $file                Resource or String (path) of JSON file
230 |      * @param  string          $tableName           Table Name
231 |      * @return Google\Cloud\BigQuery\Job            BigQuery Data Load Job
232 |      */
233 |     public function loadFromJson($file, $tableName)
234 |     {
235 |         $client = $this->getClient();
236 |         $dataset = $client->dataset($_ENV['BQ_DATASET']);
237 |         $table = $dataset->table($tableName);
238 | 
239 |         $job = $table->load(
240 |             $file,
241 |             [
242 |                 'jobConfig' => [
243 |                     'sourceFormat' => 'NEWLINE_DELIMITED_JSON'
244 |                 ]
245 |             ]
246 |         );
247 | 
248 |         return $job;
249 |     }
250 | 
251 |     /**
252 |      * Check if a BigQuery table exists
253 |      * @param  string $tableName Table name
254 |      * @return bool              True if table exists
255 |      */
256 |     public function tableExists(string $tableName)
257 |     {
258 |         $client = $this->getClient();
259 |         $dataset = $client->dataset($_ENV['BQ_DATASET']);
260 | 
261 |         return $dataset->table($tableName)->exists();
262 |     }
263 | }
264 | 


--------------------------------------------------------------------------------
/src/Database/Mysql.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace MysqlToGoogleBigQuery\Database;
  3 | 
  4 | use Doctrine\DBAL\Types\Type;
  5 | 
  6 | class Mysql
  7 | {
  8 |     protected $conn;
  9 | 
 10 |     /**
 11 |      * Configure and connect to MySQL Database
 12 |      * @param  string $databaseName      Database name
 13 |      * @return Doctrine\DBAL\Connection  Doctrine DBAL Connection
 14 |      */
 15 |     public function getConnection(string $databaseName)
 16 |     {
 17 |         // If we are connected, just return the last connection
 18 |         if ($this->conn) {
 19 |             return $this->conn;
 20 |         }
 21 | 
 22 |         $config = new \Doctrine\DBAL\Configuration();
 23 | 
 24 |         $connParams = array(
 25 |             'dbname' => $databaseName,
 26 |             'user' => $_ENV['DB_USERNAME'],
 27 |             'password' => $_ENV['DB_PASSWORD'],
 28 |             'host' => $_ENV['DB_HOST'],
 29 |             'port' => $_ENV['DB_PORT'],
 30 |             'charset'  => 'utf8',
 31 |             // Special doctrine driver, with reconnect attempts support
 32 |             'wrapperClass' => 'Facile\DoctrineMySQLComeBack\Doctrine\DBAL\Connection',
 33 |             'driverClass' => 'Facile\DoctrineMySQLComeBack\Doctrine\DBAL\Driver\PDOMySql\Driver',
 34 |             'driverOptions' => [
 35 |                 'x_reconnect_attempts' => 9
 36 |             ]
 37 |         );
 38 | 
 39 |         $this->conn = \Doctrine\DBAL\DriverManager::getConnection($connParams, $config);
 40 | 
 41 |         // Replace the DateTime conversion
 42 |         if (Type::hasType('bigquerydatetime') === false) {
 43 |             Type::addType('bigquerydatetime', 'MysqlToGoogleBigQuery\Doctrine\BigQueryDateTimeType');
 44 |         }
 45 |         if (Type::hasType('bigquerydate') === false) {
 46 |             Type::addType('bigquerydate', 'MysqlToGoogleBigQuery\Doctrine\BigQueryDateType');
 47 |         }
 48 | 
 49 |         // Map types to classes
 50 |         $this->conn->getDatabasePlatform()->registerDoctrineTypeMapping('date', 'bigquerydate');
 51 |         $this->conn->getDatabasePlatform()->registerDoctrineTypeMapping('datetime', 'bigquerydatetime');
 52 |         $this->conn->getDatabasePlatform()->registerDoctrineTypeMapping('timestamp', 'bigquerydatetime');
 53 | 
 54 |         // Add support for MySQL 5.7 JSON type
 55 |         $this->conn->getDatabasePlatform()->registerDoctrineTypeMapping('json', 'text');
 56 |         $this->conn->getDatabasePlatform()->registerDoctrineTypeMapping('enum', 'text');
 57 | 
 58 | 		// Add support for tinyint.
 59 | 		$this->conn->getDatabasePlatform()->registerDoctrineTypeMapping('tinyint', 'integer');
 60 | 
 61 |         return $this->conn;
 62 |     }
 63 | 
 64 |     /**
 65 |      * Get the number of rows on a table
 66 |      * @param  string $databaseName Database name
 67 |      * @param  string $tableName    Table name
 68 |      * @param  string $columnName   Column name
 69 |      * @param  string $columnValue  Column value
 70 |      * @return int                  Number of rows
 71 |      */
 72 |     public function getCountTableRows(string $databaseName, string $tableName, $columnName = null, $columnValue = null)
 73 |     {
 74 |         if ($columnName && $columnValue) {
 75 |             $mysqlQueryResult = $this->getConnection($databaseName)->query(
 76 |                 'SELECT COUNT(*) AS count FROM `' . $tableName . '` WHERE ' . $columnName . ' >= "' . $columnValue . '"'
 77 |             );
 78 |         } else {
 79 |             $mysqlQueryResult = $this->getConnection($databaseName)->query('SELECT COUNT(*) AS count FROM `' . $tableName . '`');
 80 |         }
 81 | 
 82 |         while ($row = $mysqlQueryResult->fetch()) {
 83 |             return (int) $row['count'];
 84 |         }
 85 | 
 86 |         throw new \Exception('Mysql table ' . $tableName . ' not found');
 87 |     }
 88 | 
 89 |     /**
 90 |      * Get the maximum value of a column of a table
 91 |      * @param  string $databaseName Database name
 92 |      * @param  string $tableName    Table name
 93 |      * @param  string $columnName   Column name
 94 |      * @return string               Max value
 95 |      */
 96 |     public function getMaxColumnValue(string $databaseName, string $tableName, string $columnName)
 97 |     {
 98 |         $mysqlQueryResult = $this->getConnection($databaseName)->query('SELECT MAX(' . $columnName . ') AS columnMax FROM `' . $tableName . '`');
 99 | 
100 |         while ($row = $mysqlQueryResult->fetch()) {
101 |             return $row['columnMax'];
102 |         }
103 | 
104 |         throw new \Exception('Mysql table ' . $tableName . ' not found');
105 |     }
106 | 
107 |     /**
108 |      * Return the table columns
109 |      * @param  string $databaseName Database name
110 |      * @param  string $tableName    Table name
111 |      * @return array                Array of Doctrine\DBAL\Schema\Column
112 |      */
113 |     public function getTableColumns($databaseName, $tableName)
114 |     {
115 |         $mysqlConnection = $this->getConnection($databaseName);
116 |         $mysqlPlatform = $mysqlConnection->getDatabasePlatform();
117 |         $mysqlSchemaManager = $mysqlConnection->getSchemaManager();
118 | 
119 |         $mysqlTableDetails = $mysqlSchemaManager->listTableDetails($tableName);
120 |         return $mysqlTableDetails->getColumns();
121 |     }
122 | }
123 | 


--------------------------------------------------------------------------------
/src/Doctrine/BigQueryDateTimeType.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace MysqlToGoogleBigQuery\Doctrine;
 3 | 
 4 | use Doctrine\DBAL\Types\Type;
 5 | use Doctrine\DBAL\Platforms\AbstractPlatform;
 6 | 
 7 | class BigQueryDateTimeType extends Type
 8 | {
 9 |     /**
10 |      * {@inheritdoc}
11 |      */
12 |     public function getSQLDeclaration(array $fieldDeclaration, AbstractPlatform $platform)
13 |     {
14 |         return 'datetime';
15 |     }
16 | 
17 |     /**
18 |      * {@inheritdoc}
19 |      */
20 |     public function convertToPHPValue($value, AbstractPlatform $platform)
21 |     {
22 |         if ($value === '0000-00-00 00:00:00') {
23 |             return null;
24 |         }
25 | 
26 |         return str_replace(' ', 'T', $value);
27 |     }
28 | 
29 |     /**
30 |      * {@inheritdoc}
31 |      */
32 |     public function convertToDatabaseValue($value, AbstractPlatform $platform)
33 |     {
34 |         return str_replace('T', ' ', $value);
35 |     }
36 | 
37 |     /**
38 |      * {@inheritdoc}
39 |      */
40 |     public function getName()
41 |     {
42 |         return 'bigquerydatetime';
43 |     }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/Doctrine/BigQueryDateType.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace MysqlToGoogleBigQuery\Doctrine;
 3 | 
 4 | use Doctrine\DBAL\Types\Type;
 5 | use Doctrine\DBAL\Platforms\AbstractPlatform;
 6 | 
 7 | class BigQueryDateType extends Type
 8 | {
 9 |     /**
10 |      * {@inheritdoc}
11 |      */
12 |     public function getSQLDeclaration(array $fieldDeclaration, AbstractPlatform $platform)
13 |     {
14 |         return 'date';
15 |     }
16 | 
17 |     /**
18 |      * {@inheritdoc}
19 |      */
20 |     public function convertToPHPValue($value, AbstractPlatform $platform)
21 |     {
22 |         if ($value === '0000-00-00') {
23 |             return null;
24 |         }
25 | 
26 |         return $value;
27 |     }
28 | 
29 |     /**
30 |      * {@inheritdoc}
31 |      */
32 |     public function convertToDatabaseValue($value, AbstractPlatform $platform)
33 |     {
34 |         return $value;
35 |     }
36 | 
37 |     /**
38 |      * {@inheritdoc}
39 |      */
40 |     public function getName()
41 |     {
42 |         return 'bigquerydate';
43 |     }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/Services/SyncService.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace MysqlToGoogleBigQuery\Services;
  3 | 
  4 | use Doctrine\DBAL\Types\Type;
  5 | use MysqlToGoogleBigQuery\Database\BigQuery;
  6 | use MysqlToGoogleBigQuery\Database\Mysql;
  7 | use Symfony\Component\Console\Helper\ProgressBar;
  8 | use Symfony\Component\Console\Input\InputInterface;
  9 | use Symfony\Component\Console\Output\OutputInterface;
 10 | 
 11 | class SyncService
 12 | {
 13 |     protected $bigQuery;
 14 |     protected $currentJob;
 15 |     protected $mysql;
 16 | 
 17 |     public function __construct(BigQuery $bigQuery, Mysql $mysql)
 18 |     {
 19 |         $this->bigQuery = $bigQuery;
 20 |         $this->mysql = $mysql;
 21 |     }
 22 | 
 23 |     /**
 24 |      * Create a BigQuery table using MySQL table schema
 25 |      * @param  string $databaseName         Database name
 26 |      * @param  string $tableName            Table name
 27 |      * @param  string $bigQueryTableName    BigQuery Table name
 28 |      */
 29 |     protected function createTable(string $databaseName, string $tableName, string $bigQueryTableName)
 30 |     {
 31 |         $mysqlTableColumns = $this->mysql->getTableColumns($databaseName, $tableName);
 32 |         $this->bigQuery->createTable($bigQueryTableName, $mysqlTableColumns);
 33 |     }
 34 | 
 35 |     /**
 36 |      * Execute the service, syncing MySQL and BigQuery table
 37 |      * @param  string          $databaseName          Database Name
 38 |      * @param  string          $tableName             Table Name
 39 |      * @param  string          $bigQueryTableName     Table Name
 40 |      * @param  bool            $createTable           If BigQuery table doesn't exists, create
 41 |      * @param  bool            $deleteTable           If BigQuery table exists, delete and recreate
 42 |      * @param  string          $orderColumn           Column to sort and compare result sets
 43 |      * @param  array           $ignoreColumns         Ignore columns from syncing
 44 |      * @param  OutputInterface $output                Output
 45 |      */
 46 |     public function execute(
 47 |         string $databaseName,
 48 |         string $tableName,
 49 |         string $bigQueryTableName,
 50 |         bool $createTable,
 51 |         bool $deleteTable,
 52 |         $orderColumn,
 53 |         array $ignoreColumns,
 54 |         OutputInterface $output, 
 55 |         bool $noData, 
 56 |         bool $unbuffered
 57 |     ) {
 58 |         if ($deleteTable) {
 59 |             // Delete the BigQuery Table before any operation
 60 |             if ($this->bigQuery->tableExists($bigQueryTableName)) {
 61 |                 $this->bigQuery->deleteTable($bigQueryTableName);
 62 |             }
 63 | 
 64 |             // Create the table after deleting
 65 |             $createTable = true;
 66 |         }
 67 | 
 68 |         if (!$this->bigQuery->tableExists($bigQueryTableName)) {
 69 |             if (!$createTable) {
 70 |                 throw new \Exception('BigQuery table ' . $bigQueryTableName . ' not found');
 71 |             }
 72 |             $output->writeln("<fg=green>Creating table: " . $tableName."</>"); 
 73 |             $this->createTable($databaseName, $tableName, $bigQueryTableName);
 74 |         }
 75 |         else 
 76 |         {
 77 |             $output->writeln("<fg=green>We will not create a table</>"); 
 78 |         }
 79 | 
 80 |         if ( $noData )
 81 |         {
 82 |           $output->writeln("<fg=green>No data specified, will not sync data.</>");
 83 |           exit;
 84 |         }
 85 | 
 86 |         if (!$unbuffered && $orderColumn) {
 87 |             $output->writeln('<fg=green>Using order column "' . $orderColumn . '"</>');
 88 | 
 89 |             $mysqlMaxColumnValue = $this->mysql->getMaxColumnValue($databaseName, $tableName, $orderColumn);
 90 |             $bigQueryMaxColumnValue = $this->bigQuery->getMaxColumnValue($bigQueryTableName, $orderColumn);
 91 | 
 92 |             if (strcmp($mysqlMaxColumnValue, $bigQueryMaxColumnValue) === 0) {
 93 |                 $output->writeln('<fg=green>Already synced!</>');
 94 |                 return;
 95 |             }
 96 | 
 97 |             /**
 98 |              * Nothing to delete on a empty table
 99 |              */
100 |             if ($bigQueryMaxColumnValue) {
101 |                 /**
102 |                  * Delete latest values, there are no primary keys in bigQuery so we miss some values
103 |                  */
104 |                 $output->writeln(
105 |                     '<fg=yellow>Cleaning "' . $bigQueryTableName . '" for "' .
106 |                     $orderColumn . '" = "' . $bigQueryMaxColumnValue . '"</>'
107 |                 );
108 |                 $this->bigQuery->deleteColumnValue($bigQueryTableName, $orderColumn, $bigQueryMaxColumnValue);
109 | 
110 |                 /**
111 |                  * Now get the latest "real" value
112 |                  */
113 |                 $bigQueryMaxColumnValue = $this->bigQuery->getMaxColumnValue($bigQueryTableName, $orderColumn);
114 |                 $output->writeln('<fg=green>Syncing from "' . $bigQueryMaxColumnValue . '"</>');
115 |             } else {
116 |                 $bigQueryMaxColumnValue = false;
117 |             }
118 |         } else {
119 |             $bigQueryMaxColumnValue = false;
120 |         }
121 | 
122 |         if ( !$unbuffered ) 
123 |         {
124 |             	$mysqlCountTableRows = $this->mysql->getCountTableRows($databaseName, $tableName, $orderColumn, $bigQueryMaxColumnValue);
125 |         }
126 |         $bigQueryCountTableRows = $orderColumn ? 0 : $this->bigQuery->getCountTableRows($bigQueryTableName, $orderColumn);
127 |         
128 |         if ( !$unbuffered ) 
129 |         {
130 |             $rowsDiff = $mysqlCountTableRows - $bigQueryCountTableRows;
131 |         
132 |             // We don't need to sync
133 |             if ($rowsDiff <= 0) {
134 |                 $output->writeln('<fg=green>Already synced!</>');
135 |                 return;
136 |             } else {
137 |                 $output->writeln('<fg=green>Syncing ' . $rowsDiff . ' rows</>');
138 |             }
139 |         }
140 | 
141 |         $maxRowsPerBatch = (isset($_ENV['MAX_ROWS_PER_BATCH'])) ? $_ENV['MAX_ROWS_PER_BATCH'] : 600000;
142 |         
143 |         if ( $unbuffered ) 
144 |         {
145 |             $output->writeln('<fg=green>Starting unbuffered copy..</>');
146 |             $this->sendBatchUnbuffered(
147 |                     $databaseName,
148 |                     $tableName,
149 |                     $bigQueryTableName,
150 |                     $ignoreColumns
151 |                     );
152 |                     $output->writeln('<fg=green>Synced!</>');
153 |         }
154 |         else 
155 |         {
156 |             $batches = ceil($rowsDiff / $maxRowsPerBatch);
157 |             $output->writeln('<info>Sending ' . $batches . ' batches of ' . $maxRowsPerBatch . ' rows/batch</info>');
158 |             $progress = new ProgressBar($output, $batches);
159 |     
160 |             for ($i = 0; $i < $batches; $i++) {
161 |                 $offset = $bigQueryCountTableRows + ($i * $maxRowsPerBatch);
162 |                 
163 |                 $this->sendBatch(
164 |                     $databaseName,
165 |                     $tableName,
166 |                     $bigQueryTableName,
167 |                     $orderColumn,
168 |                     $ignoreColumns,
169 |                     $offset,
170 |                     $maxRowsPerBatch,
171 |                     $bigQueryMaxColumnValue
172 |                 );
173 |                 $progress->advance();
174 |             }
175 |             $output->writeln('<fg=green>Synced!</>');
176 |             $progress->finish();
177 |         }
178 |     }
179 | 
180 |     /**
181 |      * Send a batch of data
182 |      * @param  string $databaseName          Database name
183 |      * @param  string $tableName             Table name
184 |      * @param  string $bigQueryTableName     BigQuery Table name
185 |      * @param  array  $ignoreColumns         Ignore columns from syncing
186 |      * @param  int    $offset                Initial MySQL rows offset
187 |      * @param  int    $limit                 MySQL rows limit, per batch
188 |      */
189 |     protected function sendBatch(
190 |         string $databaseName,
191 |         string $tableName,
192 |         string $bigQueryTableName,
193 |         $orderColumn = null,
194 |         array $ignoreColumns,
195 |         int $offset,
196 |         int $limit,
197 |         $orderColumnOffset
198 |     ) {
199 |         $mysqlConnection = $this->mysql->getConnection($databaseName);
200 |         $mysqlPlatform = $mysqlConnection->getDatabasePlatform();
201 |         $mysqlTableColumns = $this->mysql->getTableColumns($databaseName, $tableName);
202 | 
203 |         $jsonFilePath = ((isset($_ENV['CACHE_DIR'])) ? $_ENV['CACHE_DIR'] : __DIR__ . '/../../cache/') . $tableName;
204 | 
205 |         if (file_exists($jsonFilePath)) {
206 |             unlink($jsonFilePath);
207 |         }
208 | 
209 |         $json = fopen($jsonFilePath, 'a+');
210 | 
211 |         if ($orderColumn) {
212 |             if ($orderColumnOffset) {
213 |                 $mysqlQueryResult = $mysqlConnection->query(
214 |                     'SELECT * FROM `' . $tableName . '`' .
215 |                     ' WHERE ' . $orderColumn . ' > "' . $orderColumnOffset . '"' .
216 |                     ' ORDER BY ' . $orderColumn .
217 |                     ' LIMIT '. $offset . ', ' . $limit
218 |                 );
219 |             } else {
220 |                 $mysqlQueryResult = $mysqlConnection->query(
221 |                     'SELECT * FROM `' . $tableName . '` ORDER BY ' . $orderColumn . ' LIMIT '. $offset . ', ' . $limit
222 |                 );
223 |             }
224 |         } else {
225 |             $mysqlQueryResult = $mysqlConnection->query('SELECT * FROM `' . $tableName . '` LIMIT ' . $offset . ', ' . $limit);
226 |         }
227 | 
228 |         while ($row = $mysqlQueryResult->fetch()) {
229 |             $row = $this->processRow($mysqlTableColumns, $mysqlPlatform, $ignoreColumns, $row);
230 |             $string = json_encode($row);
231 | 
232 |             // Google BigQuery needs JSON new line delimited file
233 |             // Each line of the file will be each MySQL row converted to JSON
234 |             fwrite($json, json_encode($row) . PHP_EOL);
235 |         }
236 | 
237 |         // Rewind to the beginning of the JSON file
238 |         rewind($json);
239 | 
240 |         // We have a job running, waits to send the next
241 |         if ($this->currentJob) {
242 |             $this->waitJob($this->currentJob);
243 |         }
244 | 
245 |         // Send JSON to BigQuery
246 |         $job = $this->bigQuery->loadFromJson($json, $bigQueryTableName);
247 | 
248 |         // This is the first job, waits for a first success to continue
249 |         if (! $this->currentJob) {
250 |             $this->waitJob($job);
251 |         }
252 | 
253 |         $this->currentJob = $job;
254 | 
255 |         unlink($jsonFilePath);
256 |     }
257 | 
258 |     /**
259 |      * Process one record of data
260 |      *
261 |      * @param  string $mysqlTableColumns     The columns in the mysql record
262 |      * @param  string $mysqlPlatform         The platform from the database connection
263 |      * @param  array  $ignoreColumns         Ignore columns from syncing
264 |      * @param  array $row                    The record pulled from mysql
265 |      */    
266 |     protected function processRow($mysqlTableColumns, $mysqlPlatform, $ignoreColumns, $row)
267 |     {
268 |         foreach ($row as $key => $value) {
269 |             // Ignore the column
270 |             if (in_array($key, $ignoreColumns)) {
271 |                 unset($row[$key]);
272 |                 continue;
273 |             }
274 |         
275 |             // Convert to PHP values, BigQuery requires the correct types on JSON, uppercase is not supported by
276 |             // BigQuery - make keys lowercase
277 |             $type = $mysqlTableColumns[strtolower($key)]->getType();
278 |         
279 |             if ($type->getName() !== Type::STRING
280 |                 && $type->getName() !== Type::TEXT
281 |             ) {
282 |                 $row[$key] = $type->convertToPhpValue($value, $mysqlPlatform);
283 |             }
284 |         
285 |             if (is_string($row[$key])) {
286 |                 $row[$key] = mb_convert_encoding($row[$key], 'UTF-8', mb_detect_encoding($value));
287 |             }
288 |         }
289 |         return $row; 
290 |     }
291 | 
292 | /**
293 |  * Send a batch of data UNBUFFERED
294 |  * 
295 |  * https://www.php.net/manual/en/mysqlinfo.concepts.buffering.php
296 |  * 
297 |  * @param  string $databaseName          Database name
298 |  * @param  string $tableName             Table name
299 |  * @param  string $bigQueryTableName     BigQuery Table name
300 |  * @param  array  $ignoreColumns         Ignore columns from syncing
301 |  */
302 | protected function sendBatchUnbuffered(
303 |         string $databaseName,
304 |         string $tableName,
305 |         string $bigQueryTableName,
306 |         array $ignoreColumns
307 |     ) {
308 |         $mysqlConnection = $this->mysql->getConnection($databaseName); 
309 |         $mysqlConnection->getWrappedConnection()->setAttribute(\PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
310 |         
311 |         $mysqlPlatform = $mysqlConnection->getDatabasePlatform();
312 |         $mysqlTableColumns = $this->mysql->getTableColumns($databaseName, $tableName);
313 | 
314 |         $jsonFilePath = ((isset($_ENV['CACHE_DIR'])) ? $_ENV['CACHE_DIR'] : __DIR__ . '/../../cache/') . $tableName;
315 | 
316 |         if (file_exists($jsonFilePath)) {
317 |             unlink($jsonFilePath);
318 |         }
319 | 
320 |         $json = fopen($jsonFilePath, 'a+');
321 | 
322 |         $mysqlQueryResult = $mysqlConnection->query('SELECT * FROM `' . $tableName . '`', MYSQLI_USE_RESULT);
323 | 
324 |         while ($row = $mysqlQueryResult->fetch()) {
325 |             $row = $this->processRow($mysqlTableColumns, $mysqlPlatform, $ignoreColumns, $row);
326 |             $string = json_encode($row);
327 | 
328 |             // Google BigQuery needs JSON new line delimited file
329 |             // Each line of the file will be each MySQL row converted to JSON
330 |             fwrite($json, json_encode($row) . PHP_EOL);
331 |         }
332 | 
333 |         // Rewind to the beginning of the JSON file
334 |         rewind($json);
335 | 
336 |         // We have a job running, waits to send the next
337 |         if ($this->currentJob) {
338 |             $this->waitJob($this->currentJob);
339 |         }
340 | 
341 |         // Send JSON to BigQuery
342 |         $job = $this->bigQuery->loadFromJson($json, $bigQueryTableName);
343 | 
344 |         // This is the first job, waits for a first success to continue
345 |         if (! $this->currentJob) {
346 |             $this->waitJob($job);
347 |         }
348 | 
349 |         $this->currentJob = $job;
350 |         unlink($jsonFilePath);
351 |     }
352 | 
353 | 
354 |     /**
355 |      * Wait for a BigQuery Job
356 |      * @param  Google\Cloud\BigQuery\Job $job BigQuery Job
357 |      */
358 |     protected function waitJob($job)
359 |     {
360 |         $errors = [];
361 |         $jobInfo = $job->info();
362 | 
363 |         while ($jobInfo['status']['state'] === 'RUNNING') {
364 |             echo '.';
365 | 	    $jobInfo = $job->reload();
366 |             // Wait a second to retry
367 |             sleep(1);
368 |         }
369 | 
370 |         if (array_key_exists('errors', $jobInfo['status'])
371 |             && is_array($jobInfo['status']['errors'])
372 |             && count($jobInfo['status']['errors']) > 0
373 |         ) {
374 |             foreach ($jobInfo['status']['errors'] as $error) {
375 |                 $errors[] = $error['message'];
376 |             }
377 | 
378 |             throw new \Exception('BigQuery replied with errors: ' . PHP_EOL . implode(PHP_EOL, $errors));
379 |         }
380 |     }
381 | }
382 | 


--------------------------------------------------------------------------------