├── .eslintrc.json ├── .gitignore ├── .prettierrc ├── .travis.yml ├── CHANGELOG.md ├── README.md ├── gatsby-node.js ├── index.js ├── package.json ├── src ├── __snapshots__ │ └── create-mysql-nodes.spec.js.snap ├── create-mysql-nodes.js ├── create-mysql-nodes.spec.js └── db.js └── yarn.lock /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["eslint:recommended"], 3 | "parserOptions": { 4 | "ecmaVersion": 2017 5 | }, 6 | "env": { 7 | "node": true, 8 | "es6": true, 9 | "jest": true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "singleQuote": true, 4 | "printWidth": 90 5 | } 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '12' 4 | cache: 5 | yarn: true 6 | directories: 7 | - '~/.cache' 8 | install: 9 | - yarn 10 | script: 11 | - yarn lint 12 | - yarn test 13 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## [2.2.3] 4 | 5 | - fix error reporting 6 | - bump version to update dependencies 7 | 8 | ## [2.2.2] 9 | 10 | - chunk mysql requests to avoid timeout when downloading many images 11 | - bump version to update dependencies 12 | 13 | ## [2.2.1] 14 | 15 | Note: bump version to update dependencies 16 | 17 | ## [2.2.0] 18 | 19 | ### Features 20 | 21 | - add `remoteImageFieldNames` to plugin options' query object to allow image optimization. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gatsby-source-mysql 2 | 3 | [![version](https://img.shields.io/npm/v/gatsby-source-mysql.svg)](https://www.npmjs.com/package/gatsby-source-mysql) ![license](https://img.shields.io/npm/l/gatsby-source-mysql.svg) 4 | 5 | Source plugin for pulling data into Gatsby from MySQL database. 6 | 7 | ## How to use 8 | 9 | ```javascript 10 | // In your gatsby-config.js 11 | module.exports = { 12 | plugins: [ 13 | { 14 | resolve: `gatsby-source-mysql`, 15 | options: { 16 | connectionDetails: { 17 | host: 'localhost', 18 | user: 'db-username', 19 | password: 'db-password', 20 | database: 'world' 21 | }, 22 | queries: [ 23 | { 24 | statement: 'SELECT * FROM country', 25 | idFieldName: 'Code', 26 | name: 'country' 27 | } 28 | ] 29 | } 30 | } 31 | // ... other plugins 32 | ] 33 | }; 34 | ``` 35 | 36 | And then you can query via GraphQL with the type `allMysql` where `` is the `name` for your query. 37 | 38 | Below is a sample query, however, it is probably different from yours as it would dependent on your configuration and your SQL query results. 39 | 40 | Use [GraphiQL](https://www.gatsbyjs.org/docs/introducing-graphiql/) to explore the available fields. 41 | 42 | ```graphql 43 | query { 44 | allMysqlCountry { 45 | edges { 46 | node { 47 | Code 48 | Name 49 | Population 50 | } 51 | } 52 | } 53 | } 54 | ``` 55 | 56 | ### Multiple Queries 57 | 58 | When you have multiple queries, add another item in the `queries` option with different `name`. 59 | 60 | ```javascript 61 | // In your gatsby-config.js 62 | module.exports = { 63 | plugins: [ 64 | { 65 | resolve: `gatsby-source-mysql`, 66 | options: { 67 | connectionDetails: { 68 | host: 'localhost', 69 | user: 'db-username', 70 | password: 'db-password', 71 | database: 'world' 72 | }, 73 | queries: [ 74 | { 75 | statement: 'SELECT * FROM country', 76 | idFieldName: 'Code', 77 | name: 'country' 78 | }, 79 | { 80 | statement: 'SELECT * FROM city', 81 | idFieldName: 'ID', 82 | name: 'city' 83 | } 84 | ] 85 | } 86 | } 87 | // ... other plugins 88 | ] 89 | }; 90 | ``` 91 | 92 | ### Joining Queries 93 | 94 | It's possible to join the results of the queries by providing `parentName`, `foreignKey`, and `cardinality` to the query object. 95 | 96 | > Currently only one-to-one and one-to-many relationship are supported. If you have a use case for many-to-many relationship, [raise an issue][raise-issue], and I'll look into it. 97 | 98 | ```javascript 99 | // In your gatsby-config.js 100 | module.exports = { 101 | plugins: [ 102 | { 103 | resolve: `gatsby-source-mysql`, 104 | options: { 105 | connectionDetails: { 106 | host: 'localhost', 107 | user: 'db-username', 108 | password: 'db-password', 109 | database: 'world' 110 | }, 111 | queries: [ 112 | { 113 | statement: 'SELECT * FROM country', 114 | idFieldName: 'Code', 115 | name: 'country' 116 | }, 117 | { 118 | statement: 'SELECT * FROM city', 119 | idFieldName: 'ID', 120 | name: 'city', 121 | parentName: 'country', 122 | foreignKey: 'CountryCode', 123 | cardinality: 'OneToMany' 124 | } 125 | ] 126 | } 127 | } 128 | // ... other plugins 129 | ] 130 | }; 131 | ``` 132 | 133 | In the example above, `country` and `city` is one-to-many relationship (one country to multiple cities), and they are joined with `country.Code = city.CountryCode`. 134 | 135 | With the configuration above, you can query a country joined with all the related cities with 136 | 137 | ```graphql 138 | query { 139 | allMysqlCountry { 140 | edges { 141 | node { 142 | Code 143 | Name 144 | Population 145 | cities { 146 | Name 147 | } 148 | } 149 | } 150 | } 151 | } 152 | ``` 153 | 154 | It also works the other way, i.e. you can query the country when getting the city 155 | 156 | ```graphql 157 | query { 158 | allMysqlCity { 159 | edges { 160 | node { 161 | Name 162 | country { 163 | Name 164 | } 165 | } 166 | } 167 | } 168 | } 169 | ``` 170 | 171 | ### Download Image for Image Processing 172 | 173 | If your queries stores the remote url for image and you would like to utilize image processing of Gatsby, provide `remoteImageFieldNames` to the query object. 174 | 175 | > Make sure you've installed both [`gatsby-plugin-sharp`][gatsby-plugin-sharp] and [`gatsby-transform-sharp`][gatsby-transformer-sharp] packages and add them to your `gatsby-config.js`. 176 | 177 | For example, assuming you have a `actor` table where the `profile_url` column stores the remote image url, e.g. `'https://cdn.pixabay.com/photo/2014/07/10/11/15/balloons-388973_1280.jpg'`. 178 | 179 | ```javascript 180 | // In your gatsby-config.js 181 | module.exports = { 182 | plugins: [ 183 | `gatsby-plugin-sharp`, 184 | `gatsby-transformer-sharp`, 185 | { 186 | resolve: `gatsby-source-mysql`, 187 | options: { 188 | connectionDetails: { 189 | host: 'localhost', 190 | user: 'db-username', 191 | password: 'db-password', 192 | database: 'world' 193 | }, 194 | queries: [ 195 | { 196 | statement: 'SELECT * FROM actor', 197 | idFieldName: 'actor_id', 198 | name: 'actor', 199 | remoteImageFieldNames: ['profile_url'] 200 | } 201 | ] 202 | } 203 | } 204 | // ... other plugins 205 | ] 206 | }; 207 | ``` 208 | 209 | Then you can query all the images like below. (Note that you have to filter `null` value for the records whose `profile_url` is empty). 210 | 211 | ```jsx 212 | import React from 'react'; 213 | import { useStaticQuery, graphql } from 'gatsby'; 214 | import Img from 'gatsby-image'; 215 | 216 | export const SqlImage = () => { 217 | const data = useStaticQuery(graphql` 218 | { 219 | allMysqlActor { 220 | edges { 221 | node { 222 | mysqlImage { 223 | childImageSharp { 224 | fluid(maxWidth: 300) { 225 | ...GatsbyImageSharpFluid 226 | } 227 | } 228 | } 229 | } 230 | } 231 | } 232 | } 233 | `); 234 | 235 | const images = data.allMysqlActor.edges 236 | .filter(edge => !!edge.node.mysqlImage) 237 | .map(edge => edge.node.mysqlImage.childImageSharp.fluid); 238 | 239 | return ( 240 |
241 | {images.map((img, index) => ( 242 | 243 | ))} 244 |
245 | ); 246 | }; 247 | ``` 248 | 249 | If you have multiple columns with image url, pass down multiple values to `remoteImageFieldNames` and use `mysqlImages` in your graphql query, which will be an array of images. 250 | 251 | ## Plugin options 252 | 253 | - **connectionDetails** (required): options when establishing the connection. Refer to [`mysql` connection options](https://www.npmjs.com/package/mysql#connection-options) 254 | - **queries** (required): an array of object for your query. Each object could have the following fields: 255 | 256 | | Field | Required? | Description | 257 | | ----------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | 258 | | `statement` | Required | the SQL query statement to be executed. Stored procedures are supported, e.g. `'CALL myProcedureThatReturnsResult(1, 1)'` | 259 | | `idFieldName` | Required | column that is unique for each record. This column must be returned by the `statement`. | 260 | | `name` | Required | name for the query. Will impact the value for the graphql type | 261 | | `parentName` | Optional | name for the parent entity. In a one-to-many relationship, this field should be specified on the child entity (entity with many records). | 262 | | `foreignKey` | Optional | foreign key to join the parent entity. | 263 | | `cardinality` | Optional | the relationship between the parent and this entity. Possible values: `"OneToMany"`, `"OneToOne"`. Default to `"OneToMany"`. (Note: many-to-many relationship is currently not supported.) | 264 | | `remoteImageFieldNames` | Optional | columns that contain image url which you want to download and utilize Gatsby image processing capability. | 265 | 266 | [raise-issue]: https://github.com/malcolm-kee/gatsby-source-mysql/issues/new 267 | [gatsby-plugin-sharp]: https://www.gatsbyjs.org/packages/gatsby-plugin-sharp/ 268 | [gatsby-transformer-sharp]: https://www.gatsbyjs.org/packages/gatsby-transformer-sharp/ 269 | -------------------------------------------------------------------------------- /gatsby-node.js: -------------------------------------------------------------------------------- 1 | const queryDb = require('./src/db'); 2 | const createMysqlNodes = require('./src/create-mysql-nodes'); 3 | 4 | exports.sourceNodes = async ( 5 | { actions, store, createNodeId, cache, reporter }, 6 | configOptions 7 | ) => { 8 | const { createNode } = actions; 9 | const { connectionDetails, queries } = configOptions; 10 | 11 | const { db, queryResults } = await queryDb(connectionDetails, queries, reporter); 12 | 13 | try { 14 | const sqlData = queries.map((query, index) => 15 | Object.assign({}, query, { __sqlResult: queryResults[index] }) 16 | ); 17 | 18 | await Promise.all( 19 | sqlData.map((sqlResult, _, sqlResults) => 20 | createMysqlNodes(sqlResult, sqlResults, { 21 | createNode, 22 | store, 23 | createNodeId, 24 | cache, 25 | reporter 26 | }) 27 | ) 28 | ); 29 | 30 | db.end(); 31 | } catch (e) { 32 | reporter.error(`Error while sourcing data with gatsby-source-mysql`, e); 33 | db.end(); 34 | } 35 | }; 36 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // noop 2 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gatsby-source-mysql", 3 | "version": "2.2.3", 4 | "description": "Source plugin for pulling data into Gatsby from MySQL database.", 5 | "author": "Malcolm Kee ", 6 | "scripts": { 7 | "test": "jest", 8 | "test:watch": "jest --watch", 9 | "lint": "eslint src" 10 | }, 11 | "keywords": [ 12 | "gatsby", 13 | "gatsby-plugin", 14 | "mysql" 15 | ], 16 | "license": "MIT", 17 | "dependencies": { 18 | "gatsby-node-helpers": "^0.3.0", 19 | "lodash": "^4.17.15", 20 | "mysql": "^2.18.1", 21 | "pluralize": "^8.0.0" 22 | }, 23 | "peerDependencies": { 24 | "gatsby": ">2.0.0", 25 | "gatsby-source-filesystem": ">2.0.0" 26 | }, 27 | "main": "index.js", 28 | "repository": "https://github.com/malcolm-kee/gatsby-source-mysql", 29 | "homepage": "https://github.com/malcolm-kee/gatsby-source-mysql#readme", 30 | "bugs": { 31 | "url": "https://github.com/malcolm-kee/gatsby-source-mysql/issues" 32 | }, 33 | "devDependencies": { 34 | "@types/jest": "^26.0.3", 35 | "eslint": "^7.4.0", 36 | "gatsby-source-filesystem": "^2.3.18", 37 | "jest": "^26.1.0" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/__snapshots__/create-mysql-nodes.spec.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`createMysqlNodes creates correct node for child entity in one-to-one relationship 1`] = ` 4 | Array [ 5 | Array [ 6 | Object { 7 | "ID": 1, 8 | "children": Array [], 9 | "countryCode": "MY", 10 | "country___NODE": "mysql__Country__MY", 11 | "id": "mysql__Countrycapital__1", 12 | "internal": Object { 13 | "contentDigest": "becbdc82e333eea44e9bf616ff5a0254", 14 | "type": "MysqlCountrycapital", 15 | }, 16 | "mysqlId": 1, 17 | "mysqlImages___NODE": Array [], 18 | "name": "Kuala Lumpur", 19 | "parent": "__SOURCE__", 20 | }, 21 | ], 22 | Array [ 23 | Object { 24 | "ID": 2, 25 | "children": Array [], 26 | "countryCode": "US", 27 | "country___NODE": "mysql__Country__US", 28 | "id": "mysql__Countrycapital__2", 29 | "internal": Object { 30 | "contentDigest": "1e2e2cc195d672d267783f2084a904d1", 31 | "type": "MysqlCountrycapital", 32 | }, 33 | "mysqlId": 2, 34 | "mysqlImages___NODE": Array [], 35 | "name": "Washington, D.C.", 36 | "parent": "__SOURCE__", 37 | }, 38 | ], 39 | Array [ 40 | Object { 41 | "ID": 3, 42 | "children": Array [], 43 | "countryCode": "UK", 44 | "country___NODE": "mysql__Country__UK", 45 | "id": "mysql__Countrycapital__3", 46 | "internal": Object { 47 | "contentDigest": "ae53a02bb994429c6fff920c91e9155f", 48 | "type": "MysqlCountrycapital", 49 | }, 50 | "mysqlId": 3, 51 | "mysqlImages___NODE": Array [], 52 | "name": "London", 53 | "parent": "__SOURCE__", 54 | }, 55 | ], 56 | ] 57 | `; 58 | 59 | exports[`createMysqlNodes creates correct node for parent entity in one-to-one relationship 1`] = ` 60 | Array [ 61 | Array [ 62 | Object { 63 | "Code": "MY", 64 | "children": Array [], 65 | "countrycapital___NODE": "mysql__Countrycapital__1", 66 | "id": "mysql__Country__MY", 67 | "internal": Object { 68 | "contentDigest": "ddb98c674ff3819cc13340f9c401f824", 69 | "type": "MysqlCountry", 70 | }, 71 | "mysqlId": "MY", 72 | "mysqlImages___NODE": Array [], 73 | "name": "Malaysia", 74 | "parent": "__SOURCE__", 75 | }, 76 | ], 77 | Array [ 78 | Object { 79 | "Code": "US", 80 | "children": Array [], 81 | "countrycapital___NODE": "mysql__Countrycapital__2", 82 | "id": "mysql__Country__US", 83 | "internal": Object { 84 | "contentDigest": "ce9424bb485dbb8d027919fb216da0ad", 85 | "type": "MysqlCountry", 86 | }, 87 | "mysqlId": "US", 88 | "mysqlImages___NODE": Array [], 89 | "name": "United States", 90 | "parent": "__SOURCE__", 91 | }, 92 | ], 93 | Array [ 94 | Object { 95 | "Code": "UK", 96 | "children": Array [], 97 | "countrycapital___NODE": "mysql__Countrycapital__3", 98 | "id": "mysql__Country__UK", 99 | "internal": Object { 100 | "contentDigest": "bd7d6bb4933f9e9fc9bcc5b026a9ec0d", 101 | "type": "MysqlCountry", 102 | }, 103 | "mysqlId": "UK", 104 | "mysqlImages___NODE": Array [], 105 | "name": "United Kingdom", 106 | "parent": "__SOURCE__", 107 | }, 108 | ], 109 | ] 110 | `; 111 | 112 | exports[`createMysqlNodes will create correct nodes 1`] = ` 113 | Array [ 114 | Array [ 115 | Object { 116 | "ID": 1, 117 | "children": Array [], 118 | "id": "mysql__City__1", 119 | "internal": Object { 120 | "contentDigest": "69bb5879ba68b28191856fd59b5b2f45", 121 | "type": "MysqlCity", 122 | }, 123 | "mysqlId": 1, 124 | "mysqlImages___NODE": Array [], 125 | "name": "Kuala Lumpur", 126 | "parent": "__SOURCE__", 127 | }, 128 | ], 129 | Array [ 130 | Object { 131 | "ID": 2, 132 | "children": Array [], 133 | "id": "mysql__City__2", 134 | "internal": Object { 135 | "contentDigest": "ba914dca118e5b6f49409da3724c8cab", 136 | "type": "MysqlCity", 137 | }, 138 | "mysqlId": 2, 139 | "mysqlImages___NODE": Array [], 140 | "name": "New York", 141 | "parent": "__SOURCE__", 142 | }, 143 | ], 144 | Array [ 145 | Object { 146 | "ID": 3, 147 | "children": Array [], 148 | "id": "mysql__City__3", 149 | "internal": Object { 150 | "contentDigest": "22d13fc67d19aa249f22e303317b2969", 151 | "type": "MysqlCity", 152 | }, 153 | "mysqlId": 3, 154 | "mysqlImages___NODE": Array [], 155 | "name": "London", 156 | "parent": "__SOURCE__", 157 | }, 158 | ], 159 | ] 160 | `; 161 | 162 | exports[`createMysqlNodes will create correct nodes for child entity in one-to-many relationship 1`] = ` 163 | Array [ 164 | Array [ 165 | Object { 166 | "ID": 1, 167 | "children": Array [], 168 | "countryCode": "MY", 169 | "country___NODE": "mysql__Country__MY", 170 | "id": "mysql__City__1", 171 | "internal": Object { 172 | "contentDigest": "2bdef45cb788bbaaba6594f5fb5919f7", 173 | "type": "MysqlCity", 174 | }, 175 | "mysqlId": 1, 176 | "mysqlImages___NODE": Array [], 177 | "name": "Kuala Lumpur", 178 | "parent": "__SOURCE__", 179 | }, 180 | ], 181 | Array [ 182 | Object { 183 | "ID": 2, 184 | "children": Array [], 185 | "countryCode": "US", 186 | "country___NODE": "mysql__Country__US", 187 | "id": "mysql__City__2", 188 | "internal": Object { 189 | "contentDigest": "b52d5b2e631824222a6db6d5685ced27", 190 | "type": "MysqlCity", 191 | }, 192 | "mysqlId": 2, 193 | "mysqlImages___NODE": Array [], 194 | "name": "New York", 195 | "parent": "__SOURCE__", 196 | }, 197 | ], 198 | Array [ 199 | Object { 200 | "ID": 3, 201 | "children": Array [], 202 | "countryCode": "UK", 203 | "country___NODE": "mysql__Country__UK", 204 | "id": "mysql__City__3", 205 | "internal": Object { 206 | "contentDigest": "4c3762009897d9228b933992b27865a6", 207 | "type": "MysqlCity", 208 | }, 209 | "mysqlId": 3, 210 | "mysqlImages___NODE": Array [], 211 | "name": "London", 212 | "parent": "__SOURCE__", 213 | }, 214 | ], 215 | Array [ 216 | Object { 217 | "ID": 4, 218 | "children": Array [], 219 | "countryCode": "US", 220 | "country___NODE": "mysql__Country__US", 221 | "id": "mysql__City__4", 222 | "internal": Object { 223 | "contentDigest": "2d45fd733b09bc5206f7e3124a26b51a", 224 | "type": "MysqlCity", 225 | }, 226 | "mysqlId": 4, 227 | "mysqlImages___NODE": Array [], 228 | "name": "Chicago", 229 | "parent": "__SOURCE__", 230 | }, 231 | ], 232 | Array [ 233 | Object { 234 | "ID": 5, 235 | "children": Array [], 236 | "countryCode": "MY", 237 | "country___NODE": "mysql__Country__MY", 238 | "id": "mysql__City__5", 239 | "internal": Object { 240 | "contentDigest": "200941caf1a3f0e76b3acf017bf56a25", 241 | "type": "MysqlCity", 242 | }, 243 | "mysqlId": 5, 244 | "mysqlImages___NODE": Array [], 245 | "name": "Penang", 246 | "parent": "__SOURCE__", 247 | }, 248 | ], 249 | ] 250 | `; 251 | 252 | exports[`createMysqlNodes will create correct nodes for parent entity in one-to-many relationship 1`] = ` 253 | Array [ 254 | Array [ 255 | Object { 256 | "Code": "MY", 257 | "children": Array [], 258 | "cities___NODE": Array [ 259 | "mysql__City__1", 260 | "mysql__City__5", 261 | ], 262 | "id": "mysql__Country__MY", 263 | "internal": Object { 264 | "contentDigest": "92e287518025f05313e860e6bf669522", 265 | "type": "MysqlCountry", 266 | }, 267 | "mysqlId": "MY", 268 | "mysqlImages___NODE": Array [], 269 | "name": "Malaysia", 270 | "parent": "__SOURCE__", 271 | }, 272 | ], 273 | Array [ 274 | Object { 275 | "Code": "US", 276 | "children": Array [], 277 | "cities___NODE": Array [ 278 | "mysql__City__2", 279 | "mysql__City__4", 280 | ], 281 | "id": "mysql__Country__US", 282 | "internal": Object { 283 | "contentDigest": "2709c7859a37db004e8940a6444bbedd", 284 | "type": "MysqlCountry", 285 | }, 286 | "mysqlId": "US", 287 | "mysqlImages___NODE": Array [], 288 | "name": "United States", 289 | "parent": "__SOURCE__", 290 | }, 291 | ], 292 | Array [ 293 | Object { 294 | "Code": "UK", 295 | "children": Array [], 296 | "cities___NODE": Array [ 297 | "mysql__City__3", 298 | ], 299 | "id": "mysql__Country__UK", 300 | "internal": Object { 301 | "contentDigest": "bd79688fda1ea624e781724aea99430a", 302 | "type": "MysqlCountry", 303 | }, 304 | "mysqlId": "UK", 305 | "mysqlImages___NODE": Array [], 306 | "name": "United Kingdom", 307 | "parent": "__SOURCE__", 308 | }, 309 | ], 310 | ] 311 | `; 312 | -------------------------------------------------------------------------------- /src/create-mysql-nodes.js: -------------------------------------------------------------------------------- 1 | const createNodeHelpers = require('gatsby-node-helpers').default; 2 | const { createRemoteFileNode } = require('gatsby-source-filesystem'); 3 | const pluralize = require('pluralize'); 4 | const { chunk } = require('lodash'); 5 | 6 | const BATCH_SIZE = 3; 7 | 8 | const { createNodeFactory, generateNodeId } = createNodeHelpers({ 9 | typePrefix: 'mysql' 10 | }); 11 | 12 | function reduceChildFields(childEntities, nodeId) { 13 | let childFields = {}; 14 | 15 | childEntities.forEach( 16 | ({ 17 | name: childName, 18 | idFieldName: childIdFieldName, 19 | foreignKey, 20 | cardinality = 'OneToMany', 21 | __sqlResult 22 | }) => { 23 | const childIds = __sqlResult 24 | .filter(child => child[foreignKey] === nodeId) 25 | .map(child => generateNodeId(childName, child[childIdFieldName])); 26 | 27 | if (cardinality === 'OneToMany') { 28 | childFields[`${pluralize.plural(childName)}___NODE`] = childIds; 29 | } else { 30 | childFields[`${pluralize.singular(childName)}___NODE`] = childIds[0]; 31 | } 32 | } 33 | ); 34 | 35 | return childFields; 36 | } 37 | 38 | function mapSqlResults( 39 | __sqlResult, 40 | { parentName, foreignKey, childEntities, idFieldName } 41 | ) { 42 | return __sqlResult.map(result => { 43 | const nodeId = result[idFieldName]; 44 | const parentField = 45 | parentName && foreignKey 46 | ? { 47 | [`${parentName}___NODE`]: generateNodeId(parentName, result[foreignKey]) 48 | } 49 | : {}; 50 | 51 | const childFields = reduceChildFields(childEntities, nodeId); 52 | 53 | return Object.assign( 54 | {}, 55 | result, 56 | { 57 | id: nodeId 58 | }, 59 | parentField, 60 | childFields 61 | ); 62 | }); 63 | } 64 | 65 | async function createMysqlNode( 66 | node, 67 | { name, remoteImageFieldNames }, 68 | { createNode, store, createNodeId, cache, reporter } 69 | ) { 70 | const MySqlNode = createNodeFactory(name); 71 | const sqlNode = MySqlNode(node); 72 | 73 | const remoteNodes = await Promise.all( 74 | remoteImageFieldNames 75 | .filter(field => !!node[field]) 76 | .map(async field => { 77 | try { 78 | return await createRemoteFileNode({ 79 | url: node[field], 80 | parentNodeId: sqlNode.id, 81 | store, 82 | createNode, 83 | createNodeId, 84 | cache 85 | }); 86 | } catch (e) { 87 | if (typeof e === 'string') { 88 | reporter.error(`Error when getting image ${node[field]}: ${e.toString()}`); 89 | } else { 90 | reporter.error(`Error when getting image ${node[field]}`, e); 91 | } 92 | } 93 | }) 94 | ); 95 | 96 | // filter out nodes which fail 97 | const imageNodes = remoteNodes.filter(Boolean); 98 | 99 | if (remoteImageFieldNames.length === 1) { 100 | if (imageNodes.length > 0) { 101 | sqlNode.mysqlImage___NODE = imageNodes[0].id; 102 | } 103 | } 104 | 105 | sqlNode.mysqlImages___NODE = imageNodes.map(imageNode => imageNode.id); 106 | 107 | await createNode(sqlNode); 108 | } 109 | 110 | async function createMysqlNodes( 111 | { name, __sqlResult, idFieldName, parentName, foreignKey, remoteImageFieldNames = [] }, 112 | allSqlResults, 113 | { createNode, store, createNodeId, cache, reporter } 114 | ) { 115 | const childEntities = allSqlResults.filter( 116 | ({ parentName }) => !!parentName && parentName === name 117 | ); 118 | 119 | if (Array.isArray(__sqlResult)) { 120 | const sqlNodesChunks = chunk(mapSqlResults( 121 | __sqlResult, 122 | { foreignKey, parentName, childEntities, idFieldName }, 123 | childEntities 124 | ), BATCH_SIZE); 125 | 126 | for (let sqlNodes of sqlNodesChunks) { 127 | await Promise.all( 128 | sqlNodes.map(node => 129 | createMysqlNode( 130 | node, 131 | { name, remoteImageFieldNames }, 132 | { createNode, store, createNodeId, cache, reporter } 133 | ) 134 | ) 135 | ); 136 | } 137 | } 138 | } 139 | 140 | module.exports = createMysqlNodes; 141 | -------------------------------------------------------------------------------- /src/create-mysql-nodes.spec.js: -------------------------------------------------------------------------------- 1 | const createMysqlNodes = require('./create-mysql-nodes'); 2 | 3 | describe('createMysqlNodes', () => { 4 | it('is a function', () => { 5 | expect(typeof createMysqlNodes).toBe('function'); 6 | }); 7 | 8 | it('will create correct nodes', async () => { 9 | const createNode = jest.fn(); 10 | const allSqlResults = [ 11 | { 12 | name: 'city', 13 | idFieldName: 'ID', 14 | statement: 'SELECT * FROM city', 15 | __sqlResult: [ 16 | { ID: 1, name: 'Kuala Lumpur' }, 17 | { ID: 2, name: 'New York' }, 18 | { ID: 3, name: 'London' } 19 | ] 20 | } 21 | ]; 22 | 23 | await createMysqlNodes(allSqlResults[0], allSqlResults, { createNode }); 24 | 25 | expect(createNode.mock.calls).toMatchSnapshot(); 26 | }); 27 | 28 | it('will create correct nodes for child entity in one-to-many relationship', async () => { 29 | const createNode = jest.fn(); 30 | const allSqlResults = [ 31 | { 32 | name: 'city', 33 | idFieldName: 'ID', 34 | statement: 'SELECT * FROM city', 35 | __sqlResult: [ 36 | { ID: 1, name: 'Kuala Lumpur', countryCode: 'MY' }, 37 | { ID: 2, name: 'New York', countryCode: 'US' }, 38 | { ID: 3, name: 'London', countryCode: 'UK' }, 39 | { ID: 4, name: 'Chicago', countryCode: 'US' }, 40 | { ID: 5, name: 'Penang', countryCode: 'MY' } 41 | ], 42 | parentName: 'country', 43 | foreignKey: 'countryCode', 44 | cardinality: 'OneToMany' 45 | }, 46 | { 47 | name: 'country', 48 | idFieldName: 'Code', 49 | statement: 'SELECT * FROM country', 50 | __sqlResult: [ 51 | { Code: 'MY', name: 'Malaysia' }, 52 | { Code: 'US', name: 'United States' }, 53 | { Code: 'UK', name: 'United Kingdom' } 54 | ] 55 | } 56 | ]; 57 | 58 | await createMysqlNodes(allSqlResults[0], allSqlResults, { createNode }); 59 | 60 | expect(createNode.mock.calls).toMatchSnapshot(); 61 | }); 62 | 63 | it('will create correct nodes for parent entity in one-to-many relationship', async () => { 64 | const createNode = jest.fn(); 65 | const allSqlResults = [ 66 | { 67 | name: 'city', 68 | idFieldName: 'ID', 69 | statement: 'SELECT * FROM city', 70 | __sqlResult: [ 71 | { ID: 1, name: 'Kuala Lumpur', countryCode: 'MY' }, 72 | { ID: 2, name: 'New York', countryCode: 'US' }, 73 | { ID: 3, name: 'London', countryCode: 'UK' }, 74 | { ID: 4, name: 'Chicago', countryCode: 'US' }, 75 | { ID: 5, name: 'Penang', countryCode: 'MY' } 76 | ], 77 | parentName: 'country', 78 | foreignKey: 'countryCode', 79 | cardinality: 'OneToMany' 80 | }, 81 | { 82 | name: 'country', 83 | idFieldName: 'Code', 84 | statement: 'SELECT * FROM country', 85 | __sqlResult: [ 86 | { Code: 'MY', name: 'Malaysia' }, 87 | { Code: 'US', name: 'United States' }, 88 | { Code: 'UK', name: 'United Kingdom' } 89 | ] 90 | } 91 | ]; 92 | 93 | await createMysqlNodes(allSqlResults[1], allSqlResults, { createNode }); 94 | 95 | expect(createNode.mock.calls).toMatchSnapshot(); 96 | }); 97 | 98 | it('creates correct node for child entity in one-to-one relationship', async () => { 99 | const createNode = jest.fn(); 100 | const allSqlResults = [ 101 | { 102 | name: 'countrycapital', 103 | idFieldName: 'ID', 104 | statement: 'SELECT * FROM countrycapital', 105 | __sqlResult: [ 106 | { ID: 1, name: 'Kuala Lumpur', countryCode: 'MY' }, 107 | { ID: 2, name: 'Washington, D.C.', countryCode: 'US' }, 108 | { ID: 3, name: 'London', countryCode: 'UK' } 109 | ], 110 | parentName: 'country', 111 | foreignKey: 'countryCode', 112 | cardinality: 'OneToOne' 113 | }, 114 | { 115 | name: 'country', 116 | idFieldName: 'Code', 117 | statement: 'SELECT * FROM country', 118 | __sqlResult: [ 119 | { Code: 'MY', name: 'Malaysia' }, 120 | { Code: 'US', name: 'United States' }, 121 | { Code: 'UK', name: 'United Kingdom' } 122 | ] 123 | } 124 | ]; 125 | 126 | await createMysqlNodes(allSqlResults[0], allSqlResults, { createNode }); 127 | 128 | expect(createNode.mock.calls).toMatchSnapshot(); 129 | }); 130 | 131 | it('creates correct node for parent entity in one-to-one relationship', async () => { 132 | const createNode = jest.fn(); 133 | const allSqlResults = [ 134 | { 135 | name: 'countrycapital', 136 | idFieldName: 'ID', 137 | statement: 'SELECT * FROM countrycapital', 138 | __sqlResult: [ 139 | { ID: 1, name: 'Kuala Lumpur', countryCode: 'MY' }, 140 | { ID: 2, name: 'Washington, D.C.', countryCode: 'US' }, 141 | { ID: 3, name: 'London', countryCode: 'UK' } 142 | ], 143 | parentName: 'country', 144 | foreignKey: 'countryCode', 145 | cardinality: 'OneToOne' 146 | }, 147 | { 148 | name: 'country', 149 | idFieldName: 'Code', 150 | statement: 'SELECT * FROM country', 151 | __sqlResult: [ 152 | { Code: 'MY', name: 'Malaysia' }, 153 | { Code: 'US', name: 'United States' }, 154 | { Code: 'UK', name: 'United Kingdom' } 155 | ] 156 | } 157 | ]; 158 | 159 | await createMysqlNodes(allSqlResults[1], allSqlResults, { createNode }); 160 | 161 | expect(createNode.mock.calls).toMatchSnapshot(); 162 | }); 163 | }); 164 | -------------------------------------------------------------------------------- /src/db.js: -------------------------------------------------------------------------------- 1 | const mysql = require('mysql'); 2 | 3 | function query(dbConnection, statement, reporter) { 4 | return new Promise((fulfill, reject) => { 5 | dbConnection.query(statement, (error, results) => { 6 | if (error) return reject(error); 7 | 8 | if (/^call/i.test(statement)) { 9 | reporter.info(`stored procedure statments: '${statement}'`); 10 | return fulfill(results[0]); 11 | } 12 | 13 | fulfill(results); 14 | }); 15 | }); 16 | } 17 | 18 | function queryDb(connectionDetails, queries, reporter) { 19 | const db = mysql.createConnection(connectionDetails); 20 | 21 | db.connect(err => { 22 | if (err) { 23 | reporter.panic(`Error establishing db connection`, err); 24 | } 25 | }); 26 | 27 | return Promise.all(queries.map(({ statement }) => query(db, statement, reporter))) 28 | .then(queryResults => ({ 29 | queryResults, 30 | db 31 | })) 32 | .catch(err => { 33 | db.end(); 34 | reporter.error(`Error making queries`, err); 35 | }); 36 | } 37 | 38 | module.exports = queryDb; 39 | --------------------------------------------------------------------------------