├── .editorconfig
├── .eslintrc.json
├── .github
├── dependabot.yml
└── workflows
│ ├── codeql-analysis.yml
│ └── main.yml
├── .gitignore
├── .npmignore
├── .nvmrc
├── .prettierignore
├── .prettierrc.json
├── .project
├── CHANGELOG.md
├── LICENSE
├── README.md
├── bin
└── jsbq
├── package-lock.json
├── package.json
├── src
├── converter.js
├── errors.js
├── gbq.js
├── log.js
└── utils.js
├── tablediff
├── README.md
├── create_tables.sh
└── tablediff.sh
└── test
├── integration
├── continueOnError
│ └── additionalProperties
│ │ ├── expected.json
│ │ └── input.json
├── continueOnErrorAndPreventAdditionalObjectProperties
│ └── additionalProperties
│ │ ├── expected.json
│ │ └── input.json
├── converter.js
├── preventAdditionalObjectProperties
│ ├── additionalPropertiesFalse
│ │ ├── expected.json
│ │ └── input.json
│ └── unevaluatedPropertiesFalse
│ │ ├── expected.json
│ │ └── input.json
└── samples
│ ├── allOf
│ ├── expected.json
│ └── input.json
│ ├── allOf_nested
│ ├── expected.json
│ └── input.json
│ ├── anyOfMultipleTypes
│ ├── expected.json
│ └── input.json
│ ├── complex
│ ├── expected.json
│ └── input.json
│ ├── date
│ ├── expected.json
│ └── input.json
│ ├── fieldDescription
│ ├── expected.json
│ └── input.json
│ ├── json
│ ├── expected.json
│ └── input.json
│ ├── nestedDescription
│ ├── expected.json
│ └── input.json
│ ├── nestedOneOf
│ ├── expected.json
│ └── input.json
│ ├── nestedRepeated
│ ├── expected.json
│ └── input.json
│ ├── nullable
│ ├── expected.json
│ └── input.json
│ ├── numeric
│ ├── expected.json
│ └── input.json
│ ├── objectDescription
│ ├── expected.json
│ └── input.json
│ ├── oneOfWithDateFormat
│ ├── expected.json
│ └── input.json
│ ├── oneOfWithDuplicateDescriptions
│ ├── expected.json
│ └── input.json
│ ├── oneOfWithNull
│ ├── expected.json
│ └── input.json
│ ├── oneof
│ ├── expected.json
│ └── input.json
│ ├── repeated
│ ├── expected.json
│ └── input.json
│ ├── simple
│ ├── expected.json
│ └── input.json
│ ├── time
│ ├── expected.json
│ └── input.json
│ ├── timestamp
│ ├── expected.json
│ └── input.json
│ └── typeArray
│ ├── expected.json
│ └── input.json
└── unit
├── converter.js
└── utils.js
/.editorconfig:
--------------------------------------------------------------------------------
1 | root = true
2 |
3 | [*]
4 | indent_style = space
5 | indent_size = 2
6 |
--------------------------------------------------------------------------------
/.eslintrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "env": {
3 | "node": true,
4 | "es2022": true,
5 | "mocha": true
6 | },
7 | "extends": ["eslint:recommended", "plugin:mocha/recommended"],
8 | "parserOptions": {
9 | "ecmaVersion": "latest"
10 | },
11 | "rules": {
12 | "mocha/no-mocha-arrows": 0,
13 | "comma-dangle": 0,
14 | "space-before-function-paren": 0,
15 | "camelcase": 0
16 | },
17 | "plugins": ["mocha"]
18 | }
19 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | # To get started with Dependabot version updates, you'll need to specify which
2 | # package ecosystems to update and where the package manifests are located.
3 | # Please see the documentation for all configuration options:
4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
5 |
6 | version: 2
7 | updates:
8 | - package-ecosystem: 'npm' # See documentation for possible values
9 | directory: '/' # Location of package manifests
10 | schedule:
11 | interval: 'daily'
12 |
--------------------------------------------------------------------------------
/.github/workflows/codeql-analysis.yml:
--------------------------------------------------------------------------------
1 | # For most projects, this workflow file will not need changing; you simply need
2 | # to commit it to your repository.
3 | #
4 | # You may wish to alter this file to override the set of languages analyzed,
5 | # or to provide custom queries or build logic.
6 | #
7 | # ******** NOTE ********
8 | # We have attempted to detect the languages in your repository. Please check
9 | # the `language` matrix defined below to confirm you have the correct set of
10 | # supported CodeQL languages.
11 | #
12 | name: 'CodeQL'
13 |
14 | on:
15 | push:
16 | branches: [master]
17 | pull_request:
18 | # The branches below must be a subset of the branches above
19 | branches: [master]
20 | schedule:
21 | - cron: '43 1 * * 6'
22 |
23 | jobs:
24 | analyze:
25 | name: Analyze
26 | runs-on: ubuntu-latest
27 | permissions:
28 | actions: read
29 | contents: read
30 | security-events: write
31 |
32 | strategy:
33 | fail-fast: false
34 | matrix:
35 | language: ['javascript']
36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support
38 |
39 | steps:
40 | - name: Checkout repository
41 | uses: actions/checkout@v2
42 |
43 | # Initializes the CodeQL tools for scanning.
44 | - name: Initialize CodeQL
45 | uses: github/codeql-action/init@v1
46 | with:
47 | languages: ${{ matrix.language }}
48 | # If you wish to specify custom queries, you can do so here or in a config file.
49 | # By default, queries listed here will override any specified in a config file.
50 | # Prefix the list here with "+" to use these queries and those in the config file.
51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main
52 |
53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
54 | # If this step fails, then you should remove it and run the build manually (see below)
55 | - name: Autobuild
56 | uses: github/codeql-action/autobuild@v1
57 |
58 | # ℹ️ Command-line programs to run using the OS shell.
59 | # 📚 https://git.io/JvXDl
60 |
61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
62 | # and modify them (or add more) to build your code if your project
63 | # uses a compiled language
64 |
65 | #- run: |
66 | # make bootstrap
67 | # make release
68 |
69 | - name: Perform CodeQL Analysis
70 | uses: github/codeql-action/analyze@v1
71 |
--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
3 |
4 | name: Node.js CI
5 |
6 | on:
7 | push:
8 | branches: [master]
9 | pull_request:
10 | branches: [master]
11 |
12 | jobs:
13 | build:
14 | environment: ci
15 | runs-on: ubuntu-latest
16 |
17 | strategy:
18 | matrix:
19 | node-version: [20.x]
20 |
21 | # For auto merging
22 | permissions:
23 | pull-requests: write
24 | contents: write
25 |
26 | steps:
27 | - uses: actions/checkout@v3
28 | - name: Use Node.js ${{ matrix.node-version }}
29 | uses: actions/setup-node@v3
30 | with:
31 | node-version: ${{ matrix.node-version }}
32 | cache: 'npm'
33 | - run: npm ci
34 | - run: npm run ci
35 | - uses: fastify/github-action-merge-dependabot@v3
36 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /node_modules/
2 | /.nyc_output/
3 | /.settings/
4 | .DS_Store
5 |
--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | .npmignore
2 | .nvmrc
3 | .settings
4 | .project
5 | .nyc_output
6 | .editorconfig
7 | .prettierignore
8 | .prettierrc.json
9 | .eslintrc.json
10 | coverage
11 | .github
12 |
--------------------------------------------------------------------------------
/.nvmrc:
--------------------------------------------------------------------------------
1 | 20
2 |
--------------------------------------------------------------------------------
/.prettierignore:
--------------------------------------------------------------------------------
1 | .nyc_output
2 | coverage
3 |
--------------------------------------------------------------------------------
/.prettierrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "semi": false,
3 | "singleQuote": true
4 | }
5 |
--------------------------------------------------------------------------------
/.project:
--------------------------------------------------------------------------------
1 |
2 |
3 | jsonschema-bigquery
4 |
5 |
6 |
7 |
8 |
9 | org.eclipse.wst.validation.validationbuilder
10 |
11 |
12 |
13 |
14 |
15 | org.eclipse.wst.jsdt.core.jsNature
16 |
17 |
18 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Changelog
2 |
3 | ## v7.0.0 (03/04/2023)
4 |
5 | - Updated converter to generate big numeric instead of numeric for number type.
6 | - Updated dependencies.
7 |
8 | ## v6.1.0 (27/01/2023)
9 |
10 | - unevaluatedProperties now supported when checking for additional properties.
11 |
12 | ## v6.0.0 (24/01/2023)
13 |
14 | - Objects with no properties convert to JSON fields.
15 | - Updated dependencies.
16 |
17 | ## v5.1.0 (11/08/2022)
18 |
19 | - JSBQ CLI output now compatible with Google's bq CLI.
20 | - Logging code reduced.
21 | - Updated dependencies.
22 |
23 | ## v5.0.1 (04/07/2022)
24 |
25 | - JSBQ CLI now supports draft-7+ schemas.
26 | - Updated dependencies.
27 |
28 | ## v5.0.0 (30/12/2021)
29 |
30 | - Updated converter to generate numeric instead of float for number type.
31 | - Fixed field name validation.
32 | - Now supports TIME columns.
33 | - Updated to node 16, and updated dependencies.
34 |
35 | ## v4.1.0 (27/09/2021)
36 |
37 | - Now checks for field name format.
38 |
39 | ## v4.0.1 (12/02/2021)
40 |
41 | - Support more formats for table ID utility.
42 |
43 | ## v4.0.0 (24/12/2020)
44 |
45 | - Dropped support for node 8.
46 | - Updated dependencies.
47 |
48 | ## v3.4.0 (02/06/2020)
49 |
50 | - Date formats now supported in `OneOf` or `AnyOf`.
51 | - Repeated descriptions in `OneOf` or `AnyOf` are no longer duplicated.
52 |
53 | ## v3.3.0 (23/05/2020)
54 |
55 | - Schema problems can now be ignored, with warnings shown.
56 |
57 | ## v3.2.1 (23/05/2020)
58 |
59 | - Updated to node 12, and updated dependencies.
60 |
61 | ## v3.2.0 (27/02/2020)
62 |
63 | - Now supports DATE columns.
64 | - Simplified custom error class.
65 |
66 | ## v3.1.1 (07/08/2019)
67 |
68 | - Logging now sent to STDERR.
69 |
70 | ## v3.1.0 (30/06/2019)
71 |
72 | - Now throws an error if properties of an object has not been given or does not contain any fields.
73 | - Object descriptions are now processed correctly
74 |
75 | ## v3.0.0 (08/05/2019)
76 |
77 | - `additionalProperties` check is now optional.
78 |
79 | ## v2.0.3 (07/05/2019)
80 |
81 | - Set requirePartitionFilter to true when creating GBQ tables
82 |
83 | ## v2.0.2 (09/04/2019)
84 |
85 | - Supports nested combined schemas with null types.
86 | - `jsbq` can convert and print a schema without touching BigQuery.
87 |
88 | ## v2.0.1 (07/04/2019)
89 |
90 | - Now throws an error if no type is given for a property.
91 |
92 | ## v2.0.0 (01/04/2019)
93 |
94 | - Make `"additionalProperties": false` required for all `object` type properties in input JSON Schema.
95 |
96 | ## v1.0.0 (30/01/2019)
97 |
98 | - Support for nested descriptions.
99 |
100 | ## v0.1.2 (11/11/2018)
101 |
102 | - Improved error message when dealing with union types.
103 |
104 | ## v0.1.1 (01/11/2018)
105 |
106 | - Bug fix for jsbq command.
107 |
108 | ## v0.1.0 (23/10/2018)
109 |
110 | - Rewrite with improved support for `allOf`, `anyOf` and `oneOf` combined schemas.
111 |
112 | ## v0.0.7 (13/08/2018)
113 |
114 | - Support fields with nullable types using `anyOf` method.
115 |
116 | ## v0.0.6 (11/08/2018)
117 |
118 | - Supports `allOf`, `anyOf` and `oneOf` combined schemas.
119 |
120 | ## v0.0.5 (29/07/2018)
121 |
122 | - Supports timestamp fields.
123 |
124 | ## v0.0.4 (27/07/2018)
125 |
126 | - Supports repeated fields.
127 |
128 | ## v0.0.3 (25/07/2018)
129 |
130 | - Now supporting fields with type(s) as an array.
131 | - Field descriptions now supported.
132 |
133 | ## v0.0.2 (23/07/2018)
134 |
135 | - Required fields supported.
136 |
137 | ## v0.0.1 (21/07/2018)
138 |
139 | - Initial release.
140 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 The Dumb Terminal
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # jsonschema-bigquery
2 |
3 | [](https://www.npmjs.com/package/jsonschema-bigquery)
4 | [](https://github.com/thedumbterminal/jsonschema-bigquery/actions/workflows/main.yml)
5 |
6 | Convert JSON schema to Google BigQuery schema
7 |
8 | This includes the ability to:
9 |
10 | 1. Create tables
11 | 1. Patch tables
12 |
13 | Further enhancements are planned: delete tables (dev only), create dataset, set data ACLs
14 |
15 | Note that some features involve bespoke interpretation of schema details suited to our environment.
16 |
17 | ## Install
18 |
19 | npm install jsonschema-bigquery
20 |
21 | ## Consume
22 |
23 | jsbq -p -d -j --preventAdditionalObjectProperties --continueOnError
24 |
25 | For embedded usage the following will allow and support runtime schema conversion and table maintenance:
26 |
27 | const jsonSchemaBigquery = require('jsonschema-bigquery')
28 | const bigquerySchema = jsonSchemaBigquery.run(jsonSchemaObject, options)
29 |
30 | Please ensure that the input JSON schema is dereferenced so that all external references have been resolved. [json-schema-ref-parser](https://www.npmjs.com/package/json-schema-ref-parser) can do this, prior to using this module.
31 |
32 | ### Options
33 |
34 | ```
35 | {
36 | preventAdditionalObjectProperties: true,
37 | continueOnError: false
38 | }
39 | ```
40 |
41 | - `preventAdditionalObjectProperties` - boolean, check for additional object properties in schemas.
42 | - `continueOnError` - boolean, continues conversion if problem JSON is encountered. Problems will be excluded from resulting schema.
43 |
44 | ### Usage with bq CLI tool
45 |
46 | Google maintains a CLI tool called `bq` which allows the management of BigQuery tables:
47 |
48 | https://cloud.google.com/bigquery/docs/reference/bq-cli-reference
49 |
50 | To create a table from a JSON schema using other options that our `jsbq` does not support, use the following commands:
51 |
52 | First, output the generated GBQ schema to a file:
53 |
54 | ```
55 | npx jsbq -j test/integration/samples/complex/input.json > /tmp/schema.json
56 | ```
57 |
58 | Then run the `bq` command to create a table:
59 |
60 | ```
61 | bq mk --schema=/tmp/schema.json test_dataset.test_table
62 | ```
63 |
64 | Please see the bq reference for table creation options:
65 |
66 | https://cloud.google.com/bigquery/docs/reference/bq-cli-reference#bq_mk
67 |
68 | ## Test
69 |
70 | npm test
71 |
--------------------------------------------------------------------------------
/bin/jsbq:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 |
3 | const jsbq = (module.exports = {})
4 |
5 | const gbq = require('../src/gbq')
6 | const { get_table_id } = require('../src/utils')
7 | const converter = require('../src/converter')
8 | const { logger } = require('../src/log')
9 | const fs = require('fs')
10 | const { promisify } = require('util')
11 |
12 | jsbq.process = async (project, datasetName, jsonSchema, options) => {
13 | logger.info('Processing JSON schema...')
14 | const tableOptions = converter.run(jsonSchema, options)
15 | logger.info('Generated BigQuery schema:')
16 | console.log(JSON.stringify(tableOptions.schema.fields))
17 |
18 | if (!project && !datasetName) {
19 | return logger.info('Skipping table operations')
20 | }
21 |
22 | const schemaId = jsonSchema.$id || jsonSchema.id
23 | logger.info('Extracting table name from schema ID:', schemaId)
24 | const tableName = get_table_id(schemaId)
25 | logger.info('Table name:', tableName)
26 |
27 | logger.info('Setting table options...')
28 | tableOptions.friendly_name = jsonSchema.title
29 | tableOptions.description = jsonSchema.description || jsonSchema.title
30 | tableOptions.timePartitioning = {
31 | type: 'DAY',
32 | requirePartitionFilter: true,
33 | }
34 |
35 | logger.info('Checking if table exists...')
36 | const tableExists = await gbq.tableExists(project, datasetName, tableName)
37 | if (tableExists) {
38 | logger.info('Patching table:', tableName)
39 | await gbq.patchTable(project, datasetName, tableName, tableOptions)
40 | } else {
41 | logger.info('Creating table:', tableName)
42 | await gbq.createTable(project, datasetName, tableName, tableOptions)
43 | }
44 | logger.info('Finished')
45 | }
46 |
47 | jsbq.run = async () => {
48 | const readFile = promisify(fs.readFile)
49 |
50 | const argv = require('yargs')
51 | .usage(
52 | 'Usage: $0 -p [project] -d [dataset] -j --preventAdditionalObjectProperties --continueOnError',
53 | )
54 | .options({
55 | j: {
56 | describe: 'JSON schema file',
57 | demandOption: true,
58 | },
59 | p: {
60 | describe: 'GCP project name',
61 | },
62 | d: {
63 | describe: 'BigQuery dataset name',
64 | },
65 | preventAdditionalObjectProperties: {
66 | describe: 'boolean, check for additional object properties in schemas.',
67 | type: 'boolean',
68 | default: false,
69 | },
70 | continueOnError: {
71 | describe:
72 | 'boolean, if error in json schema, skip element in big query schema and continue.',
73 | type: 'boolean',
74 | default: false,
75 | },
76 | debug: {
77 | describe: 'Enable debug logging',
78 | type: 'boolean',
79 | default: false,
80 | },
81 | }).argv
82 |
83 | if (argv.debug) {
84 | logger.level = 'debug'
85 | logger.debug('Debug mode enabled')
86 | }
87 |
88 | const schemaData = await readFile(argv.j)
89 | const options = {
90 | preventAdditionalObjectProperties: argv.preventAdditionalObjectProperties,
91 | continueOnError: argv.continueOnError,
92 | }
93 | const jsonSchema = JSON.parse(schemaData)
94 | logger.debug('Input schema: ', jsonSchema)
95 | return jsbq.process(argv.p, argv.d, jsonSchema, options)
96 | }
97 |
98 | jsbq.run().catch((e) => {
99 | logger.error(e)
100 | process.exit(1)
101 | })
102 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "jsonschema-bigquery",
3 | "version": "7.0.0",
4 | "description": "Convert JSON schema to Google BigQuery schema",
5 | "main": "src/converter.js",
6 | "scripts": {
7 | "ci": "npm run lint && npm run coverage",
8 | "test": "mocha test/unit test/integration",
9 | "coverage": "nyc npm run test",
10 | "lint:eslint": "eslint .",
11 | "lint:prettier": "prettier -c .",
12 | "lint": "npm run lint:prettier && npm run lint:eslint",
13 | "format": "prettier --write ."
14 | },
15 | "bin": {
16 | "jsbq": "./bin/jsbq"
17 | },
18 | "engines": {
19 | "node": ">=12"
20 | },
21 | "repository": {
22 | "type": "git",
23 | "url": "git+https://github.com/thedumbterminal/jsonschema-bigquery.git"
24 | },
25 | "keywords": [
26 | "json",
27 | "schema",
28 | "google",
29 | "bigquery",
30 | "convert"
31 | ],
32 | "author": {
33 | "name": "thedumbterminal",
34 | "email": "github@thedumbterminal.co.uk",
35 | "url": "http://www.thedumbterminal.co.uk"
36 | },
37 | "license": "MIT",
38 | "bugs": {
39 | "url": "https://github.com/thedumbterminal/jsonschema-bigquery/issues"
40 | },
41 | "homepage": "https://github.com/thedumbterminal/jsonschema-bigquery#readme",
42 | "devDependencies": {
43 | "eslint": "^8.5.0",
44 | "eslint-plugin-import": "^2.29.1",
45 | "eslint-plugin-mocha": "^10.1.0",
46 | "eslint-plugin-n": "^17.10.1",
47 | "eslint-plugin-promise": "^7.0.0",
48 | "mocha": "^11.0.1",
49 | "nyc": "^17.0.0",
50 | "prettier": "^3.3.3"
51 | },
52 | "dependencies": {
53 | "@google-cloud/bigquery": "^8.0.0",
54 | "lodash": "^4.17.20",
55 | "log4js": "^6.3.0",
56 | "yargs": "^18.0.0"
57 | },
58 | "nyc": {
59 | "check-coverage": true,
60 | "branches": 88,
61 | "lines": 98,
62 | "functions": 100,
63 | "statements": 97
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/src/converter.js:
--------------------------------------------------------------------------------
1 | const converter = (module.exports = {})
2 | const _ = require('lodash')
3 | const { SchemaError } = require('./errors')
4 | const { logger } = require('./log')
5 |
6 | const JSON_SCHEMA_TO_BIGQUERY_TYPE_DICT = {
7 | boolean: 'BOOLEAN',
8 | 'date-time': 'TIMESTAMP',
9 | integer: 'INTEGER',
10 | number: 'BIGNUMERIC',
11 | string: 'STRING',
12 | date: 'DATE',
13 | time: 'TIME',
14 | }
15 |
16 | const OFS = ['allOf', 'anyOf', 'oneOf']
17 |
18 | const BIGQUERY_FIELD_NAME_REGEXP = /^[a-z_]([a-z0-9_]+|)$/i
19 |
20 | converter._copy = (o) => _.clone(o, false)
21 | converter._deepCopy = (o) => _.clone(o, true)
22 |
23 | converter._merge_property = (
24 | merge_type,
25 | property_name,
26 | destination_value,
27 | source_value,
28 | ) => {
29 | // Merges two properties.
30 | if (destination_value === undefined && source_value === undefined) {
31 | return undefined
32 | }
33 | if (destination_value === undefined) return source_value
34 | if (source_value === undefined) return destination_value
35 | if (
36 | typeof destination_value === 'boolean' &&
37 | typeof source_value === 'boolean'
38 | ) {
39 | return destination_value && source_value
40 | }
41 | if (_.isPlainObject(destination_value) && _.isPlainObject(source_value)) {
42 | return converter._merge_dicts(merge_type, destination_value, source_value)
43 | }
44 | let destination_list
45 | if (Array.isArray(destination_value)) {
46 | destination_list = converter._copy(destination_value)
47 | } else {
48 | destination_list = [destination_value]
49 | }
50 | let source_list
51 | if (Array.isArray(source_value)) {
52 | source_list = source_value
53 | } else {
54 | source_list = [source_value]
55 | }
56 | if (property_name === 'description') {
57 | destination_list.push(source_value)
58 | return _.uniq(destination_list).join(' ')
59 | }
60 | if (property_name === 'required' && ['anyOf', 'oneOf'].includes(merge_type)) {
61 | return destination_list.filter((v) => source_list.includes(v))
62 | }
63 | if (property_name === 'format') {
64 | // if we have multiple formats we have to remove them all
65 | return _.uniq(destination_list).length === 1 ? destination_list.shift() : ''
66 | }
67 | destination_list.push(
68 | ...source_list.filter((v) => !destination_list.includes(v)),
69 | )
70 | return destination_list
71 | }
72 |
73 | /**
74 | * Merges multiple sources given as an Array was *dicts,
75 | *
76 | * Cloned from the original merge_dicts from python-based bigjson This method had variable
77 | * expansion, varags and two slightly different usages which appeared to be incompatible.
78 | *
79 | * Could be returned to a single method
80 | *
81 | * See merge_dicts for the alternate call pattern
82 | */
83 | converter._merge_dicts_array = (merge_type, dest_dict, source_dicts) => {
84 | const result = converter._deepCopy(dest_dict)
85 | for (let source_dict of source_dicts) {
86 | // First check if we need to recurse and merge deeper results first
87 | for (const x_of of OFS) {
88 | if (_.has(source_dict, x_of)) {
89 | source_dict = converter._merge_dicts_array(
90 | x_of,
91 | source_dict,
92 | source_dict[x_of],
93 | )
94 | delete source_dict[x_of]
95 | }
96 | }
97 |
98 | const keys = Object.keys(source_dict)
99 | for (const name of keys) {
100 | const merged_property = converter._merge_property(
101 | merge_type,
102 | name,
103 | result[name],
104 | source_dict[name],
105 | )
106 | if (merged_property !== undefined) {
107 | result[name] = merged_property
108 | }
109 | }
110 | }
111 | return result
112 | }
113 |
114 | /**
115 | * Merges a single object
116 | *
117 | * The original merge_dicts from python-based bigjson This method had variable expansion, varags and
118 | * two slightly different usages which appeared to be incompatible.
119 | *
120 | * Could be returned to a single method
121 | *
122 | * See merge_dicts_array above for the alternate call pattern
123 | */
124 | converter._merge_dicts = (merge_type, dest_dict, source_dict) => {
125 | const result = converter._deepCopy(dest_dict)
126 | const keys = Object.keys(source_dict)
127 | for (const name of keys) {
128 | const merged_property = converter._merge_property(
129 | merge_type,
130 | name,
131 | result[name],
132 | source_dict[name],
133 | )
134 | if (merged_property !== undefined) {
135 | result[name] = merged_property
136 | }
137 | }
138 | return result
139 | }
140 |
141 | converter._scalar = (name, type, mode, description) => {
142 | if (!name.match(BIGQUERY_FIELD_NAME_REGEXP)) {
143 | throw new SchemaError(`Invalid field name: ${name}`)
144 | }
145 |
146 | const result = {
147 | name,
148 | type,
149 | mode,
150 | }
151 |
152 | if (description) {
153 | result.description = description
154 | }
155 |
156 | return result
157 | }
158 |
159 | converter._array = (name, node) => {
160 | const items_with_description = converter._deepCopy(node.items)
161 | if (_.has(items_with_description, 'description')) {
162 | items_with_description.description = node.description
163 | }
164 | return converter._visit(name, items_with_description, 'REPEATED')
165 | }
166 |
167 | converter._allowsAdditionalProperties = (node) =>
168 | node.additionalProperties !== false && node.unevaluatedProperties !== false
169 |
170 | converter._object = (name, node, mode) => {
171 | let result = {
172 | fields: [],
173 | }
174 | let fieldType = 'RECORD'
175 | try {
176 | if (
177 | converter._allowsAdditionalProperties(node) &&
178 | converter._options.preventAdditionalObjectProperties
179 | ) {
180 | throw new SchemaError(
181 | 'Objects must not have additional or unevaluated properties',
182 | node,
183 | )
184 | }
185 | if (
186 | !_.isPlainObject(node.properties) ||
187 | Object.keys(node.properties).length === 0
188 | ) {
189 | // Big Query can handle semi-structured data :
190 | // https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#loading_semi-structured_json_data
191 | fieldType = 'JSON'
192 | return converter._scalar(name, fieldType, mode, node.description)
193 | }
194 |
195 | result = converter._scalar(name, fieldType, mode, node.description)
196 |
197 | const required_properties = node.required || []
198 | const properties = node.properties
199 |
200 | let fields = Object.keys(properties).map((key) => {
201 | const required = required_properties.includes(key)
202 | ? 'REQUIRED'
203 | : 'NULLABLE'
204 | return converter._visit(key, properties[key], required)
205 | })
206 |
207 | // remove empty fields
208 | fields = fields.filter(function (field) {
209 | return field != null
210 | })
211 | result.fields = fields
212 | } catch (e) {
213 | if (!converter._options.continueOnError) {
214 | throw e
215 | }
216 | logger.warn(e)
217 | }
218 |
219 | return result
220 | }
221 |
222 | converter._bigQueryType = (node, type) => {
223 | // handle string formats
224 | let actualType = type
225 | const format = node.format
226 | if (type === 'string' && ['date-time', 'date', 'time'].includes(format)) {
227 | actualType = format
228 | }
229 | const bqType = JSON_SCHEMA_TO_BIGQUERY_TYPE_DICT[actualType]
230 | if (!bqType) {
231 | throw new SchemaError(`Invalid type given: ${type}`, node)
232 | }
233 | return bqType
234 | }
235 |
236 | converter._simple = (name, type, node, mode) => {
237 | if (type === 'array') {
238 | return converter._array(name, node)
239 | }
240 | if (type === 'object') {
241 | return converter._object(name, node, mode)
242 | }
243 |
244 | const bqType = converter._bigQueryType(node, type)
245 | return converter._scalar(name, bqType, mode, node.description)
246 | }
247 |
248 | converter._visit = (name, node, mode = 'NULLABLE') => {
249 | let merged_node = node
250 | for (const x_of of OFS) {
251 | if (_.has(node, x_of)) {
252 | merged_node = converter._merge_dicts_array(x_of, node, _.get(node, x_of))
253 | delete merged_node[x_of]
254 | }
255 | }
256 | let type_ = merged_node.type
257 | let actual_mode = mode
258 | if (Array.isArray(type_)) {
259 | const non_null_types = type_.filter((scalar_type) => scalar_type !== 'null')
260 | if (non_null_types.length > 1) {
261 | throw new SchemaError('Union type not supported', node)
262 | }
263 | // When mode is REPEATED, we want to leave it, even if it is NULLABLE
264 | if (type_.includes('null') && actual_mode !== 'REPEATED') {
265 | actual_mode = 'NULLABLE'
266 | }
267 | type_ = non_null_types[0]
268 | }
269 | return converter._simple(name, type_, merged_node, actual_mode)
270 | }
271 |
272 | converter.run = (input_schema, options = {}) => {
273 | converter._options = options
274 | return {
275 | schema: {
276 | fields: converter._visit('root', input_schema).fields,
277 | },
278 | }
279 | }
280 |
--------------------------------------------------------------------------------
/src/errors.js:
--------------------------------------------------------------------------------
1 | const utils = require('./utils')
2 |
3 | class SchemaError extends Error {
4 | constructor(message, node) {
5 | super(message)
6 |
7 | // Maintains proper stack trace for where our error was thrown (only available on V8)
8 | if (Error.captureStackTrace) {
9 | Error.captureStackTrace(this, SchemaError)
10 | }
11 |
12 | this.name = 'SchemaError'
13 | if (node) {
14 | this.message += ':\n' + utils.inspector(node)
15 | }
16 | }
17 | }
18 |
19 | module.exports = {
20 | SchemaError,
21 | }
22 |
--------------------------------------------------------------------------------
/src/gbq.js:
--------------------------------------------------------------------------------
1 | const gbq = (module.exports = {})
2 | const { BigQuery } = require('@google-cloud/bigquery')
3 | const { logger } = require('./log')
4 |
5 | gbq.createTable = async (projectId, datasetId, tableId, schema) => {
6 | const bigquery = new BigQuery({ projectId })
7 |
8 | const [table] = await bigquery.dataset(datasetId).createTable(tableId, schema)
9 |
10 | logger.info(`Table ${table.id} created.`)
11 | }
12 |
13 | gbq.patchTable = async (projectId, datasetId, tableId, schema) => {
14 | const bigquery = new BigQuery({ projectId })
15 |
16 | const res = await bigquery
17 | .dataset(datasetId)
18 | .table(tableId)
19 | .setMetadata(schema)
20 |
21 | logger.info('Patch result:', res)
22 | }
23 |
24 | gbq.tableExists = async (projectId, datasetId, tableId) => {
25 | const bigquery = new BigQuery({ projectId })
26 |
27 | const table = await bigquery.dataset(datasetId).table(tableId)
28 | const res = await table.exists()
29 | logger.debug('Table check:', res)
30 | return res[0]
31 | }
32 |
--------------------------------------------------------------------------------
/src/log.js:
--------------------------------------------------------------------------------
1 | const log = (module.exports = {})
2 | const log4js = require('log4js')
3 | const utils = require('./utils')
4 |
5 | log4js.configure({
6 | appenders: {
7 | err: {
8 | type: 'stderr',
9 | layout: {
10 | type: 'pattern',
11 | pattern: '%[%d %p %c%] %x{plain}%x{schema}',
12 | tokens: {
13 | plain: (logEvent) => logEvent.data[0],
14 | schema: (logEvent) => {
15 | if (!logEvent.data[1]) {
16 | return ''
17 | }
18 | return '\n' + utils.inspector(logEvent.data[1])
19 | },
20 | },
21 | },
22 | },
23 | },
24 | categories: {
25 | default: {
26 | appenders: ['err'],
27 | level: 'ERROR',
28 | },
29 | },
30 | })
31 |
32 | const logger = log4js.getLogger()
33 |
34 | // Allow log level to be changed in development
35 | logger.level = process.env.LOG_LEVEL || 'info'
36 |
37 | log.logger = logger
38 |
--------------------------------------------------------------------------------
/src/utils.js:
--------------------------------------------------------------------------------
1 | const util = require('util')
2 | const utils = (module.exports = {})
3 |
4 | // Expected format is "id": "/events/server/ecommerce/v5.1.5/schema.json"
5 | utils.get_table_id = (id) => {
6 | const idArr = id.split('/').slice(2, -1)
7 | const version = idArr.pop()
8 | return idArr.join('_') + '_' + version.split('.')[0]
9 | }
10 |
11 | utils.inspector = (obj) =>
12 | util.inspect(obj, {
13 | depth: null,
14 | colors: process.stdout.isTTY, // Add colours if in an interactive terminal
15 | compact: false,
16 | breakLength: Infinity,
17 | })
18 |
--------------------------------------------------------------------------------
/tablediff/README.md:
--------------------------------------------------------------------------------
1 | Useful for QA comparison of generated tables.
2 |
3 | Review and run create_tables.sh. For example it can be useful to recreate tables in a new dataset.
4 |
5 | Once run the tablediff.sh script (again review for config), can generate actual vs expected gbq schema files (from gbq itself).
6 | These can then be used in combination with a diff tool to highlight any differences with the expected outputs.
7 |
--------------------------------------------------------------------------------
/tablediff/create_tables.sh:
--------------------------------------------------------------------------------
1 | #Environment dependencies
2 | #PROJECT=
3 | #DATASET=
4 | #SCHEMA_PATH=???/dist/dereferenced/events/
5 |
6 | for EVENT_VERSION in `cat events.list`
7 | do
8 | echo Starting $schema
9 | cp $SCHEMA_PATH/$EVENT_VERSION/schema.json ../src
10 | bin/jsbq -p $PROJECT -d $DATASET -j schema.json
11 | done
12 |
13 | echo Finished
14 |
--------------------------------------------------------------------------------
/tablediff/tablediff.sh:
--------------------------------------------------------------------------------
1 | #Environment dependencies
2 | #SOURCE_DATASET=
3 | #TARGET_DATASET=
4 |
5 | for TABLE in `bq ls $SOURCE_DATASET | tail +3 | cut -c 1-35| sed 's; ;;g'`
6 | do
7 | echo $TABLE
8 | # Ignore descriptions for now, as there are some non-critical changes that have occurred slowly over time
9 | bq show --schema $SOURCE_DATASET.$TABLE | python -m json.tool | sed 's; (Dereferenced);;g' | fgrep -v '"description":' > SOURCE/$TABLE
10 | bq show --schema $TARGET_DATASET.$TABLE | python -m json.tool | sed 's; (Dereferenced);;g' | fgrep -v '"description":' > TARGET/$TABLE
11 | done
12 |
13 | echo "Generated pretty schemas in SOURCE and TARGET, use diff or an IDE to review differences"
14 |
--------------------------------------------------------------------------------
/test/integration/continueOnError/additionalProperties/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "address",
6 | "type": "RECORD",
7 | "mode": "NULLABLE",
8 | "fields": [
9 | {
10 | "name": "street_address",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | },
14 | {
15 | "name": "country",
16 | "type": "JSON",
17 | "mode": "NULLABLE"
18 | }
19 | ]
20 | }
21 | ]
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/test/integration/continueOnError/additionalProperties/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Complex example empty Object",
4 | "type": "object",
5 | "properties": {
6 | "address": {
7 | "type": "object",
8 | "properties": {
9 | "street_address": {
10 | "type": "string"
11 | },
12 | "country": {
13 | "type": "object",
14 | "properties": {}
15 | }
16 | },
17 | "additionalProperties": true
18 | }
19 | },
20 | "additionalProperties": false
21 | }
22 |
--------------------------------------------------------------------------------
/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": []
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Complex example empty Object",
4 | "type": "object",
5 | "properties": {
6 | "address": {
7 | "type": "object",
8 | "properties": {
9 | "street_address": {
10 | "type": "string"
11 | },
12 | "country": {
13 | "type": "object",
14 | "properties": {}
15 | }
16 | },
17 | "additionalProperties": true
18 | }
19 | },
20 | "additionalProperties": true
21 | }
22 |
--------------------------------------------------------------------------------
/test/integration/converter.js:
--------------------------------------------------------------------------------
1 | const converter = require('../../src/converter')
2 | const assert = require('assert')
3 | const fs = require('fs')
4 |
5 | describe('converter integration', () => {
6 | describe('default options', () => {
7 | const sampleDir = './test/integration/samples'
8 | // eslint-disable-next-line mocha/no-setup-in-describe
9 | const testDirs = fs.readdirSync(sampleDir)
10 |
11 | // eslint-disable-next-line mocha/no-setup-in-describe
12 | testDirs.forEach((dir) => {
13 | describe(dir, () => {
14 | let expected
15 | let result
16 |
17 | before(() => {
18 | const inJson = require(`../../${sampleDir}/${dir}/input.json`)
19 | expected = require(`../../${sampleDir}/${dir}/expected.json`)
20 | result = converter.run(inJson, 'p', 't')
21 | })
22 |
23 | it('converts to big query', () => {
24 | assert.deepStrictEqual(result, expected)
25 | })
26 | })
27 | })
28 | })
29 |
30 | describe('continueOnError option', () => {
31 | const sampleDir = './test/integration/continueOnError'
32 | // eslint-disable-next-line mocha/no-setup-in-describe
33 | const testDirs = fs.readdirSync(sampleDir)
34 |
35 | // eslint-disable-next-line mocha/no-setup-in-describe
36 | testDirs.forEach((dir) => {
37 | describe(dir, () => {
38 | let expected
39 | let result
40 |
41 | before(() => {
42 | const inJson = require(`../../${sampleDir}/${dir}/input.json`)
43 | expected = require(`../../${sampleDir}/${dir}/expected.json`)
44 | const options = {
45 | continueOnError: true,
46 | }
47 | result = converter.run(inJson, options, 'p', 't')
48 | })
49 |
50 | it('converts to big query', () => {
51 | assert.deepStrictEqual(result, expected)
52 | })
53 | })
54 | })
55 | })
56 |
57 | describe('preventAdditionalObjectProperties option', () => {
58 | const sampleDir = './test/integration/preventAdditionalObjectProperties'
59 | // eslint-disable-next-line mocha/no-setup-in-describe
60 | const testDirs = fs.readdirSync(sampleDir)
61 |
62 | // eslint-disable-next-line mocha/no-setup-in-describe
63 | testDirs.forEach((dir) => {
64 | describe(dir, () => {
65 | let expected
66 | let result
67 |
68 | before(() => {
69 | const inJson = require(`../../${sampleDir}/${dir}/input.json`)
70 | expected = require(`../../${sampleDir}/${dir}/expected.json`)
71 | const options = {
72 | preventAdditionalObjectProperties: true,
73 | }
74 | result = converter.run(inJson, options, 'p', 't')
75 | })
76 |
77 | it('converts to big query', () => {
78 | assert.deepStrictEqual(result, expected)
79 | })
80 | })
81 | })
82 | })
83 |
84 | describe('continueOnError and preventAdditionalObjectProperties options', () => {
85 | const sampleDir =
86 | './test/integration/continueOnErrorAndPreventAdditionalObjectProperties'
87 | // eslint-disable-next-line mocha/no-setup-in-describe
88 | const testDirs = fs.readdirSync(sampleDir)
89 |
90 | // eslint-disable-next-line mocha/no-setup-in-describe
91 | testDirs.forEach((dir) => {
92 | describe(dir, () => {
93 | let expected
94 | let result
95 |
96 | before(() => {
97 | const inJson = require(`../../${sampleDir}/${dir}/input.json`)
98 | expected = require(`../../${sampleDir}/${dir}/expected.json`)
99 | const options = {
100 | continueOnError: true,
101 | preventAdditionalObjectProperties: true,
102 | }
103 | result = converter.run(inJson, options, 'p', 't')
104 | })
105 |
106 | it('converts to big query', () => {
107 | assert.deepStrictEqual(result, expected)
108 | })
109 | })
110 | })
111 | })
112 | })
113 |
--------------------------------------------------------------------------------
/test/integration/preventAdditionalObjectProperties/additionalPropertiesFalse/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "first_name",
6 | "type": "STRING",
7 | "mode": "NULLABLE"
8 | },
9 | {
10 | "name": "last_name",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | }
14 | ]
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/test/integration/preventAdditionalObjectProperties/additionalPropertiesFalse/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "first_name": {
7 | "type": "string"
8 | },
9 | "last_name": {
10 | "type": "string"
11 | }
12 | },
13 | "additionalProperties": false
14 | }
15 |
--------------------------------------------------------------------------------
/test/integration/preventAdditionalObjectProperties/unevaluatedPropertiesFalse/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "first_name",
6 | "type": "STRING",
7 | "mode": "NULLABLE"
8 | },
9 | {
10 | "name": "last_name",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | }
14 | ]
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/test/integration/preventAdditionalObjectProperties/unevaluatedPropertiesFalse/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "first_name": {
7 | "type": "string"
8 | },
9 | "last_name": {
10 | "type": "string"
11 | }
12 | },
13 | "unevaluatedProperties": false
14 | }
15 |
--------------------------------------------------------------------------------
/test/integration/samples/allOf/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "names",
6 | "type": "RECORD",
7 | "mode": "NULLABLE",
8 | "fields": [
9 | {
10 | "name": "first_name",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | },
14 | {
15 | "name": "last_name",
16 | "type": "STRING",
17 | "mode": "NULLABLE"
18 | }
19 | ]
20 | }
21 | ]
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/test/integration/samples/allOf/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "names": {
7 | "allOf": [
8 | {
9 | "type": "object",
10 | "properties": {
11 | "first_name": {
12 | "type": "string"
13 | }
14 | },
15 | "additionalProperties": false
16 | },
17 | {
18 | "type": "object",
19 | "properties": {
20 | "last_name": {
21 | "type": "string"
22 | }
23 | },
24 | "additionalProperties": false
25 | }
26 | ]
27 | }
28 | },
29 | "additionalProperties": false
30 | }
31 |
--------------------------------------------------------------------------------
/test/integration/samples/allOf_nested/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "addresses",
6 | "type": "RECORD",
7 | "mode": "NULLABLE",
8 | "fields": [
9 | {
10 | "name": "address",
11 | "type": "RECORD",
12 | "mode": "NULLABLE",
13 | "fields": [
14 | {
15 | "name": "street_address",
16 | "type": "STRING",
17 | "mode": "NULLABLE"
18 | },
19 | {
20 | "name": "country",
21 | "type": "STRING",
22 | "mode": "NULLABLE"
23 | },
24 | {
25 | "name": "town",
26 | "type": "STRING",
27 | "mode": "NULLABLE"
28 | }
29 | ]
30 | }
31 | ]
32 | }
33 | ]
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/test/integration/samples/allOf_nested/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "addresses": {
7 | "allOf": [
8 | {
9 | "type": "object",
10 | "properties": {
11 | "address": {
12 | "type": "object",
13 | "properties": {
14 | "street_address": {
15 | "type": "string"
16 | },
17 | "country": {
18 | "type": "string"
19 | }
20 | }
21 | }
22 | },
23 | "additionalProperties": false
24 | },
25 | {
26 | "type": "object",
27 | "properties": {
28 | "address": {
29 | "type": "object",
30 | "properties": {
31 | "town": {
32 | "type": "string"
33 | },
34 | "country": {
35 | "type": "string"
36 | }
37 | },
38 | "additionalProperties": false
39 | }
40 | },
41 | "additionalProperties": false
42 | }
43 | ]
44 | }
45 | },
46 | "additionalProperties": false
47 | }
48 |
--------------------------------------------------------------------------------
/test/integration/samples/anyOfMultipleTypes/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "first_name",
6 | "type": "STRING",
7 | "mode": "NULLABLE"
8 | }
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/anyOfMultipleTypes/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "first_name": {
7 | "anyOf": [
8 | {
9 | "type": "string"
10 | },
11 | {
12 | "type": "null"
13 | }
14 | ]
15 | }
16 | },
17 | "required": ["first_name"],
18 | "additionalProperties": false
19 | }
20 |
--------------------------------------------------------------------------------
/test/integration/samples/complex/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "address",
6 | "type": "RECORD",
7 | "mode": "NULLABLE",
8 | "fields": [
9 | {
10 | "name": "street_address",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | },
14 | {
15 | "name": "country",
16 | "type": "STRING",
17 | "mode": "NULLABLE"
18 | }
19 | ]
20 | }
21 | ]
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/test/integration/samples/complex/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Complex example",
4 | "type": "object",
5 | "properties": {
6 | "address": {
7 | "type": "object",
8 | "properties": {
9 | "street_address": {
10 | "type": "string"
11 | },
12 | "country": {
13 | "type": "string"
14 | }
15 | },
16 | "additionalProperties": false
17 | }
18 | },
19 | "additionalProperties": false
20 | }
21 |
--------------------------------------------------------------------------------
/test/integration/samples/date/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "modified",
6 | "type": "DATE",
7 | "mode": "NULLABLE"
8 | }
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/date/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "modified": {
7 | "type": "string",
8 | "format": "date"
9 | }
10 | },
11 | "additionalProperties": false
12 | }
13 |
--------------------------------------------------------------------------------
/test/integration/samples/fieldDescription/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "description": "the first name",
6 | "name": "first_name",
7 | "type": "STRING",
8 | "mode": "NULLABLE"
9 | }
10 | ]
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/test/integration/samples/fieldDescription/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "first_name": {
7 | "description": "the first name",
8 | "type": "string"
9 | }
10 | },
11 | "additionalProperties": false
12 | }
13 |
--------------------------------------------------------------------------------
/test/integration/samples/json/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "description": "Some json data without specific schema.",
6 | "name": "semi_structured",
7 | "type": "JSON",
8 | "mode": "NULLABLE"
9 | }
10 | ]
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/test/integration/samples/json/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "semi_structured": {
7 | "description": "Some json data without specific schema.",
8 | "type": "object"
9 | }
10 | },
11 | "additionalProperties": false
12 | }
13 |
--------------------------------------------------------------------------------
/test/integration/samples/nestedDescription/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "description": "Any first name A type of first name Another type of first name",
6 | "name": "first_name",
7 | "type": "STRING",
8 | "mode": "NULLABLE"
9 | }
10 | ]
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/test/integration/samples/nestedDescription/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "first_name": {
7 | "description": "Any first name",
8 | "anyOf": [
9 | {
10 | "description": "A type of first name",
11 | "type": "string",
12 | "enum": ["Bob"]
13 | },
14 | {
15 | "description": "Another type of first name",
16 | "type": "string",
17 | "enum": ["Alice"]
18 | }
19 | ]
20 | }
21 | },
22 | "additionalProperties": false
23 | }
24 |
--------------------------------------------------------------------------------
/test/integration/samples/nestedOneOf/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "mode": "NULLABLE",
6 | "name": "product",
7 | "type": "STRING"
8 | }
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/nestedOneOf/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "product.json",
3 | "type": "object",
4 | "properties": {
5 | "product": {
6 | "oneOf": [
7 | {
8 | "oneOf": [
9 | {
10 | "type": "string",
11 | "enum": ["sweets"]
12 | },
13 | {
14 | "type": "string",
15 | "enum": ["shoes"]
16 | }
17 | ]
18 | },
19 | {
20 | "type": "null"
21 | }
22 | ]
23 | }
24 | },
25 | "additionalProperties": false
26 | }
27 |
--------------------------------------------------------------------------------
/test/integration/samples/nestedRepeated/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "names",
6 | "type": "RECORD",
7 | "mode": "REPEATED",
8 | "fields": [
9 | {
10 | "name": "first",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | }
14 | ]
15 | }
16 | ]
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/test/integration/samples/nestedRepeated/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "names": {
7 | "type": "array",
8 | "items": {
9 | "type": "object",
10 | "properties": {
11 | "first": {
12 | "type": "string"
13 | }
14 | },
15 | "additionalProperties": false
16 | }
17 | }
18 | },
19 | "additionalProperties": false
20 | }
21 |
--------------------------------------------------------------------------------
/test/integration/samples/nullable/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "first_name",
6 | "type": "STRING",
7 | "mode": "REQUIRED"
8 | },
9 | {
10 | "name": "last_name",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | }
14 | ]
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/test/integration/samples/nullable/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "first_name": {
7 | "type": "string"
8 | },
9 | "last_name": {
10 | "type": "string"
11 | }
12 | },
13 | "required": ["first_name"],
14 | "additionalProperties": false
15 | }
16 |
--------------------------------------------------------------------------------
/test/integration/samples/numeric/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "modified",
6 | "type": "BIGNUMERIC",
7 | "mode": "NULLABLE"
8 | }
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/numeric/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "modified": {
7 | "type": "number"
8 | }
9 | },
10 | "additionalProperties": false
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/objectDescription/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "address",
6 | "type": "RECORD",
7 | "mode": "NULLABLE",
8 | "description": "An address",
9 | "fields": [
10 | {
11 | "name": "street_address",
12 | "type": "STRING",
13 | "mode": "NULLABLE"
14 | }
15 | ]
16 | }
17 | ]
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/test/integration/samples/objectDescription/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Object description",
4 | "type": "object",
5 | "properties": {
6 | "address": {
7 | "description": "An address",
8 | "type": "object",
9 | "properties": {
10 | "street_address": {
11 | "type": "string"
12 | }
13 | },
14 | "additionalProperties": false
15 | }
16 | },
17 | "additionalProperties": false
18 | }
19 |
--------------------------------------------------------------------------------
/test/integration/samples/oneOfWithDateFormat/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "fields": [
6 | {
7 | "mode": "NULLABLE",
8 | "name": "first_name",
9 | "type": "STRING"
10 | },
11 | {
12 | "mode": "NULLABLE",
13 | "name": "created",
14 | "type": "TIMESTAMP"
15 | },
16 | {
17 | "mode": "NULLABLE",
18 | "name": "last_name",
19 | "type": "STRING"
20 | }
21 | ],
22 | "mode": "NULLABLE",
23 | "name": "name",
24 | "type": "RECORD"
25 | }
26 | ]
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/test/integration/samples/oneOfWithDateFormat/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "type": "object",
4 | "properties": {
5 | "name": {
6 | "oneOf": [
7 | {
8 | "type": "object",
9 | "properties": {
10 | "first_name": {
11 | "type": "string"
12 | },
13 | "created": {
14 | "type": "string",
15 | "format": "date-time"
16 | }
17 | }
18 | },
19 | {
20 | "type": "object",
21 | "properties": {
22 | "last_name": {
23 | "type": "string"
24 | },
25 | "created": {
26 | "type": "string",
27 | "format": "date-time"
28 | }
29 | }
30 | }
31 | ]
32 | }
33 | },
34 | "additionalProperties": false
35 | }
36 |
--------------------------------------------------------------------------------
/test/integration/samples/oneOfWithDuplicateDescriptions/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "fields": [
6 | {
7 | "mode": "NULLABLE",
8 | "name": "first_name",
9 | "type": "STRING"
10 | },
11 | {
12 | "description": "Created date",
13 | "mode": "NULLABLE",
14 | "name": "created",
15 | "type": "STRING"
16 | },
17 | {
18 | "mode": "NULLABLE",
19 | "name": "last_name",
20 | "type": "STRING"
21 | }
22 | ],
23 | "mode": "NULLABLE",
24 | "name": "name",
25 | "type": "RECORD"
26 | }
27 | ]
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/test/integration/samples/oneOfWithDuplicateDescriptions/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "type": "object",
4 | "properties": {
5 | "name": {
6 | "oneOf": [
7 | {
8 | "type": "object",
9 | "properties": {
10 | "first_name": {
11 | "type": "string"
12 | },
13 | "created": {
14 | "type": "string",
15 | "description": "Created date"
16 | }
17 | }
18 | },
19 | {
20 | "type": "object",
21 | "properties": {
22 | "last_name": {
23 | "type": "string"
24 | },
25 | "created": {
26 | "type": "string",
27 | "description": "Created date"
28 | }
29 | }
30 | }
31 | ]
32 | }
33 | },
34 | "additionalProperties": false
35 | }
36 |
--------------------------------------------------------------------------------
/test/integration/samples/oneOfWithNull/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "mode": "NULLABLE",
6 | "name": "name",
7 | "type": "STRING"
8 | }
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/oneOfWithNull/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "type": "object",
4 | "properties": {
5 | "name": {
6 | "oneOf": [
7 | {
8 | "type": "string"
9 | },
10 | {
11 | "type": "null"
12 | }
13 | ]
14 | }
15 | },
16 | "additionalProperties": false
17 | }
18 |
--------------------------------------------------------------------------------
/test/integration/samples/oneof/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "tags",
6 | "mode": "REPEATED",
7 | "type": "RECORD",
8 | "fields": [
9 | {
10 | "name": "name",
11 | "description": "The name of the tag that you want to include",
12 | "type": "STRING",
13 | "mode": "REQUIRED"
14 | },
15 | {
16 | "name": "string",
17 | "description": "A string type data point for the tag",
18 | "type": "STRING",
19 | "mode": "NULLABLE"
20 | },
21 | {
22 | "name": "number",
23 | "description": "A integer type data point for the tag",
24 | "type": "INTEGER",
25 | "mode": "NULLABLE"
26 | }
27 | ]
28 | }
29 | ]
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/test/integration/samples/oneof/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "allOf": [
5 | {
6 | "type": "object",
7 | "properties": {
8 | "tags": {
9 | "description": "The users additional information",
10 | "$schema": "http://json-schema.org/draft-04/schema#",
11 | "id": "/models/tags/v2.0.0/schema.json",
12 | "title": "Tag Array",
13 | "oneOf": [
14 | {
15 | "type": "array",
16 | "items": {
17 | "type": "object",
18 | "redact": {
19 | "type": "key_value",
20 | "items": [
21 | {
22 | "key_property": "name",
23 | "value_property": "string",
24 | "key_value": "email",
25 | "redact": {
26 | "type": "email"
27 | }
28 | }
29 | ]
30 | },
31 | "properties": {
32 | "name": {
33 | "type": "string",
34 | "description": "The name of the tag that you want to include"
35 | },
36 | "string": {
37 | "type": "string",
38 | "description": "A string type data point for the tag"
39 | },
40 | "number": {
41 | "type": "integer",
42 | "description": "A integer type data point for the tag"
43 | }
44 | },
45 | "anyOf": [
46 | {
47 | "required": ["name", "string"]
48 | },
49 | {
50 | "required": ["name", "number"]
51 | }
52 | ],
53 | "additionalProperties": false
54 | }
55 | },
56 | {
57 | "type": "null"
58 | }
59 | ]
60 | }
61 | }
62 | }
63 | ],
64 | "additionalProperties": false
65 | }
66 |
--------------------------------------------------------------------------------
/test/integration/samples/repeated/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "names",
6 | "type": "STRING",
7 | "mode": "REPEATED"
8 | }
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/repeated/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "names": {
7 | "type": "array",
8 | "items": {
9 | "type": "string"
10 | }
11 | }
12 | },
13 | "additionalProperties": false
14 | }
15 |
--------------------------------------------------------------------------------
/test/integration/samples/simple/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "first_name",
6 | "type": "STRING",
7 | "mode": "NULLABLE"
8 | },
9 | {
10 | "name": "last_name",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | }
14 | ]
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/test/integration/samples/simple/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "first_name": {
7 | "type": "string"
8 | },
9 | "last_name": {
10 | "type": "string"
11 | }
12 | },
13 | "additionalProperties": false
14 | }
15 |
--------------------------------------------------------------------------------
/test/integration/samples/time/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "modified",
6 | "type": "TIME",
7 | "mode": "NULLABLE"
8 | }
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/time/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "modified": {
7 | "type": "string",
8 | "format": "time"
9 | }
10 | },
11 | "additionalProperties": false
12 | }
13 |
--------------------------------------------------------------------------------
/test/integration/samples/timestamp/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "modified",
6 | "type": "TIMESTAMP",
7 | "mode": "NULLABLE"
8 | }
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/integration/samples/timestamp/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "modified": {
7 | "type": "string",
8 | "format": "date-time"
9 | }
10 | },
11 | "additionalProperties": false
12 | }
13 |
--------------------------------------------------------------------------------
/test/integration/samples/typeArray/expected.json:
--------------------------------------------------------------------------------
1 | {
2 | "schema": {
3 | "fields": [
4 | {
5 | "name": "first_name",
6 | "type": "STRING",
7 | "mode": "NULLABLE"
8 | },
9 | {
10 | "name": "last_name",
11 | "type": "STRING",
12 | "mode": "NULLABLE"
13 | }
14 | ]
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/test/integration/samples/typeArray/input.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "http://yourdomain.com/schemas/myschema.json",
3 | "description": "Example description",
4 | "type": "object",
5 | "properties": {
6 | "first_name": {
7 | "type": ["string", "null"]
8 | },
9 | "last_name": {
10 | "type": ["string", "null"]
11 | }
12 | },
13 | "required": ["first_name"],
14 | "additionalProperties": false
15 | }
16 |
--------------------------------------------------------------------------------
/test/unit/converter.js:
--------------------------------------------------------------------------------
1 | const converter = require('../../src/converter')
2 | const assert = require('assert')
3 |
4 | describe('converter unit', () => {
5 | describe('_visit()', () => {
6 | context('when multiple types are given', () => {
7 | it('throws an error', () => {
8 | assert.throws(() => {
9 | const node = {
10 | type: ['string', 'boolean'],
11 | }
12 | converter._visit('test', node)
13 | }, /Union type not supported/)
14 | })
15 | })
16 | })
17 |
18 | describe('_bigQueryType()', () => {
19 | context('with an unknown type', () => {
20 | it('throws an error', () => {
21 | assert.throws(() => {
22 | const node = {
23 | type: 'foo',
24 | }
25 | converter._bigQueryType(node, 'foo')
26 | }, /SchemaError: Invalid type given: foo/)
27 | })
28 | })
29 | })
30 |
31 | describe('_merge_dicts_array()', () => {
32 | context('with two string enums', () => {
33 | it('returns the correct structure', () => {
34 | const source = [
35 | {
36 | type: 'string',
37 | enum: ['sweets'],
38 | },
39 | {
40 | type: 'string',
41 | enum: ['shoes'],
42 | },
43 | ]
44 | const result = converter._merge_dicts_array('oneOf', {}, source)
45 | const expected = {
46 | enum: ['sweets', 'shoes'],
47 | type: ['string'],
48 | }
49 | assert.deepStrictEqual(result, expected)
50 | })
51 | })
52 |
53 | context('with a nested oneOf containing two string enums', () => {
54 | it('returns the correct structure', () => {
55 | const source = [
56 | {
57 | oneOf: [
58 | {
59 | type: 'string',
60 | enum: ['sweets'],
61 | },
62 | {
63 | type: 'string',
64 | enum: ['shoes'],
65 | },
66 | ],
67 | },
68 | {
69 | type: 'string',
70 | enum: ['bag'],
71 | },
72 | ]
73 | const result = converter._merge_dicts_array('oneOf', {}, source)
74 | const expected = {
75 | enum: ['sweets', 'shoes', 'bag'],
76 | type: ['string'],
77 | }
78 | assert.deepStrictEqual(result, expected)
79 | })
80 | })
81 | })
82 |
83 | describe('_object()', () => {
84 | beforeEach(() => {
85 | converter._options = {}
86 | })
87 |
88 | context('without the "preventAdditionalObjectProperties" option', () => {
89 | it('allows additional properties', () => {
90 | const node = {
91 | properties: {
92 | name: {
93 | type: 'string',
94 | },
95 | },
96 | }
97 | assert.doesNotThrow(() => {
98 | converter._object('test', node, 'NULLABLE')
99 | }, /Objects must not have additional or unevaluated properties/)
100 | })
101 | })
102 |
103 | context('with the "preventAdditionalObjectProperties" option', () => {
104 | beforeEach(() => {
105 | converter._options = {
106 | preventAdditionalObjectProperties: true,
107 | }
108 | })
109 |
110 | it('does not allow additional properties', () => {
111 | const node = {
112 | properties: {},
113 | }
114 | assert.throws(() => {
115 | converter._object('test', node, 'NULLABLE')
116 | }, /Objects must not have additional or unevaluated properties/)
117 | })
118 | })
119 |
120 | context(
121 | 'with the "preventAdditionalObjectProperties" and "continueOnError" options',
122 | () => {
123 | let result
124 |
125 | beforeEach(() => {
126 | converter._options = {
127 | preventAdditionalObjectProperties: true,
128 | continueOnError: true,
129 | }
130 | const node = {
131 | properties: {},
132 | }
133 | result = converter._object('test', node, 'NULLABLE')
134 | })
135 |
136 | it('skips the field', () => {
137 | const expected = { fields: [] }
138 | assert.deepStrictEqual(result, expected)
139 | })
140 | },
141 | )
142 |
143 | context('with no properties', () => {
144 | it('converts to JSON when properties not defined', () => {
145 | const expected = {
146 | mode: 'NULLABLE',
147 | name: 'test',
148 | type: 'JSON',
149 | }
150 | const result = converter._object('test', {}, 'NULLABLE')
151 |
152 | assert.deepStrictEqual(result, expected)
153 | })
154 | })
155 | })
156 |
157 | describe('run()', () => {
158 | beforeEach(() => {
159 | converter.run(
160 | {
161 | type: 'boolean',
162 | },
163 | {
164 | option: true,
165 | },
166 | )
167 | })
168 |
169 | it('sets given options', () => {
170 | assert.deepStrictEqual(converter._options, {
171 | option: true,
172 | })
173 | })
174 | })
175 |
176 | describe('_scalar()', () => {
177 | context('with a field beginning with a number', () => {
178 | it('throws an error', () => {
179 | assert.throws(() => {
180 | converter._scalar('123test', 'STRING', 'NULLABLE')
181 | }, /Invalid field name: 123test/)
182 | })
183 | })
184 |
185 | context('with a field containing non alphanumeric characters', () => {
186 | it('throws an error', () => {
187 | assert.throws(() => {
188 | converter._scalar('test!', 'STRING', 'NULLABLE')
189 | }, /Invalid field name: test!/)
190 | })
191 | })
192 |
193 | context('with an single character field', () => {
194 | it('returns a bigquery field object', () => {
195 | assert.deepStrictEqual(converter._scalar('t', 'STRING', 'NULLABLE'), {
196 | mode: 'NULLABLE',
197 | name: 't',
198 | type: 'STRING',
199 | })
200 | })
201 | })
202 |
203 | context('with a valid field', () => {
204 | it('returns a bigquery field object', () => {
205 | assert.deepStrictEqual(
206 | converter._scalar('test123', 'STRING', 'NULLABLE'),
207 | {
208 | mode: 'NULLABLE',
209 | name: 'test123',
210 | type: 'STRING',
211 | },
212 | )
213 | })
214 | })
215 | })
216 | })
217 |
--------------------------------------------------------------------------------
/test/unit/utils.js:
--------------------------------------------------------------------------------
1 | const utils = require('../../src/utils')
2 | const assert = require('assert')
3 |
4 | describe('utils', () => {
5 | describe('get_table_id()', () => {
6 | it('it supports a short path', () => {
7 | assert.equal(
8 | utils.get_table_id('/events/one/two/v1.2.3/schema.json'),
9 | 'one_two_v1',
10 | )
11 | })
12 |
13 | it('it supports a long path', () => {
14 | assert.equal(
15 | utils.get_table_id('/events/one/two/three/four/v1.2.3/schema.json'),
16 | 'one_two_three_four_v1',
17 | )
18 | })
19 | })
20 | })
21 |
--------------------------------------------------------------------------------