├── .editorconfig ├── .eslintrc.json ├── .github ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ └── main.yml ├── .gitignore ├── .npmignore ├── .nvmrc ├── .prettierignore ├── .prettierrc.json ├── .project ├── CHANGELOG.md ├── LICENSE ├── README.md ├── bin └── jsbq ├── package-lock.json ├── package.json ├── src ├── converter.js ├── errors.js ├── gbq.js ├── log.js └── utils.js ├── tablediff ├── README.md ├── create_tables.sh └── tablediff.sh └── test ├── integration ├── continueOnError │ └── additionalProperties │ │ ├── expected.json │ │ └── input.json ├── continueOnErrorAndPreventAdditionalObjectProperties │ └── additionalProperties │ │ ├── expected.json │ │ └── input.json ├── converter.js ├── preventAdditionalObjectProperties │ ├── additionalPropertiesFalse │ │ ├── expected.json │ │ └── input.json │ └── unevaluatedPropertiesFalse │ │ ├── expected.json │ │ └── input.json └── samples │ ├── allOf │ ├── expected.json │ └── input.json │ ├── allOf_nested │ ├── expected.json │ └── input.json │ ├── anyOfMultipleTypes │ ├── expected.json │ └── input.json │ ├── complex │ ├── expected.json │ └── input.json │ ├── date │ ├── expected.json │ └── input.json │ ├── fieldDescription │ ├── expected.json │ └── input.json │ ├── json │ ├── expected.json │ └── input.json │ ├── nestedDescription │ ├── expected.json │ └── input.json │ ├── nestedOneOf │ ├── expected.json │ └── input.json │ ├── nestedRepeated │ ├── expected.json │ └── input.json │ ├── nullable │ ├── expected.json │ └── input.json │ ├── numeric │ ├── expected.json │ └── input.json │ ├── objectDescription │ ├── expected.json │ └── input.json │ ├── oneOfWithDateFormat │ ├── expected.json │ └── input.json │ ├── oneOfWithDuplicateDescriptions │ ├── expected.json │ └── input.json │ ├── oneOfWithNull │ ├── expected.json │ └── input.json │ ├── oneof │ ├── expected.json │ └── input.json │ ├── repeated │ ├── expected.json │ └── input.json │ ├── simple │ ├── expected.json │ └── input.json │ ├── time │ ├── expected.json │ └── input.json │ ├── timestamp │ ├── expected.json │ └── input.json │ └── typeArray │ ├── expected.json │ └── input.json └── unit ├── converter.js └── utils.js /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true, 4 | "es2022": true, 5 | "mocha": true 6 | }, 7 | "extends": ["eslint:recommended", "plugin:mocha/recommended"], 8 | "parserOptions": { 9 | "ecmaVersion": "latest" 10 | }, 11 | "rules": { 12 | "mocha/no-mocha-arrows": 0, 13 | "comma-dangle": 0, 14 | "space-before-function-paren": 0, 15 | "camelcase": 0 16 | }, 17 | "plugins": ["mocha"] 18 | } 19 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: 'npm' # See documentation for possible values 9 | directory: '/' # Location of package manifests 10 | schedule: 11 | interval: 'daily' 12 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: 'CodeQL' 13 | 14 | on: 15 | push: 16 | branches: [master] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [master] 20 | schedule: 21 | - cron: '43 1 * * 6' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: ['javascript'] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v2 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v1 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v1 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v1 71 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: 7 | push: 8 | branches: [master] 9 | pull_request: 10 | branches: [master] 11 | 12 | jobs: 13 | build: 14 | environment: ci 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | node-version: [20.x] 20 | 21 | # For auto merging 22 | permissions: 23 | pull-requests: write 24 | contents: write 25 | 26 | steps: 27 | - uses: actions/checkout@v3 28 | - name: Use Node.js ${{ matrix.node-version }} 29 | uses: actions/setup-node@v3 30 | with: 31 | node-version: ${{ matrix.node-version }} 32 | cache: 'npm' 33 | - run: npm ci 34 | - run: npm run ci 35 | - uses: fastify/github-action-merge-dependabot@v3 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /.nyc_output/ 3 | /.settings/ 4 | .DS_Store 5 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .npmignore 2 | .nvmrc 3 | .settings 4 | .project 5 | .nyc_output 6 | .editorconfig 7 | .prettierignore 8 | .prettierrc.json 9 | .eslintrc.json 10 | coverage 11 | .github 12 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 20 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | .nyc_output 2 | coverage 3 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true 4 | } 5 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | jsonschema-bigquery 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.wst.validation.validationbuilder 10 | 11 | 12 | 13 | 14 | 15 | org.eclipse.wst.jsdt.core.jsNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v7.0.0 (03/04/2023) 4 | 5 | - Updated converter to generate big numeric instead of numeric for number type. 6 | - Updated dependencies. 7 | 8 | ## v6.1.0 (27/01/2023) 9 | 10 | - unevaluatedProperties now supported when checking for additional properties. 11 | 12 | ## v6.0.0 (24/01/2023) 13 | 14 | - Objects with no properties convert to JSON fields. 15 | - Updated dependencies. 16 | 17 | ## v5.1.0 (11/08/2022) 18 | 19 | - JSBQ CLI output now compatible with Google's bq CLI. 20 | - Logging code reduced. 21 | - Updated dependencies. 22 | 23 | ## v5.0.1 (04/07/2022) 24 | 25 | - JSBQ CLI now supports draft-7+ schemas. 26 | - Updated dependencies. 27 | 28 | ## v5.0.0 (30/12/2021) 29 | 30 | - Updated converter to generate numeric instead of float for number type. 31 | - Fixed field name validation. 32 | - Now supports TIME columns. 33 | - Updated to node 16, and updated dependencies. 34 | 35 | ## v4.1.0 (27/09/2021) 36 | 37 | - Now checks for field name format. 38 | 39 | ## v4.0.1 (12/02/2021) 40 | 41 | - Support more formats for table ID utility. 42 | 43 | ## v4.0.0 (24/12/2020) 44 | 45 | - Dropped support for node 8. 46 | - Updated dependencies. 47 | 48 | ## v3.4.0 (02/06/2020) 49 | 50 | - Date formats now supported in `OneOf` or `AnyOf`. 51 | - Repeated descriptions in `OneOf` or `AnyOf` are no longer duplicated. 52 | 53 | ## v3.3.0 (23/05/2020) 54 | 55 | - Schema problems can now be ignored, with warnings shown. 56 | 57 | ## v3.2.1 (23/05/2020) 58 | 59 | - Updated to node 12, and updated dependencies. 60 | 61 | ## v3.2.0 (27/02/2020) 62 | 63 | - Now supports DATE columns. 64 | - Simplified custom error class. 65 | 66 | ## v3.1.1 (07/08/2019) 67 | 68 | - Logging now sent to STDERR. 69 | 70 | ## v3.1.0 (30/06/2019) 71 | 72 | - Now throws an error if properties of an object has not been given or does not contain any fields. 73 | - Object descriptions are now processed correctly 74 | 75 | ## v3.0.0 (08/05/2019) 76 | 77 | - `additionalProperties` check is now optional. 78 | 79 | ## v2.0.3 (07/05/2019) 80 | 81 | - Set requirePartitionFilter to true when creating GBQ tables 82 | 83 | ## v2.0.2 (09/04/2019) 84 | 85 | - Supports nested combined schemas with null types. 86 | - `jsbq` can convert and print a schema without touching BigQuery. 87 | 88 | ## v2.0.1 (07/04/2019) 89 | 90 | - Now throws an error if no type is given for a property. 91 | 92 | ## v2.0.0 (01/04/2019) 93 | 94 | - Make `"additionalProperties": false` required for all `object` type properties in input JSON Schema. 95 | 96 | ## v1.0.0 (30/01/2019) 97 | 98 | - Support for nested descriptions. 99 | 100 | ## v0.1.2 (11/11/2018) 101 | 102 | - Improved error message when dealing with union types. 103 | 104 | ## v0.1.1 (01/11/2018) 105 | 106 | - Bug fix for jsbq command. 107 | 108 | ## v0.1.0 (23/10/2018) 109 | 110 | - Rewrite with improved support for `allOf`, `anyOf` and `oneOf` combined schemas. 111 | 112 | ## v0.0.7 (13/08/2018) 113 | 114 | - Support fields with nullable types using `anyOf` method. 115 | 116 | ## v0.0.6 (11/08/2018) 117 | 118 | - Supports `allOf`, `anyOf` and `oneOf` combined schemas. 119 | 120 | ## v0.0.5 (29/07/2018) 121 | 122 | - Supports timestamp fields. 123 | 124 | ## v0.0.4 (27/07/2018) 125 | 126 | - Supports repeated fields. 127 | 128 | ## v0.0.3 (25/07/2018) 129 | 130 | - Now supporting fields with type(s) as an array. 131 | - Field descriptions now supported. 132 | 133 | ## v0.0.2 (23/07/2018) 134 | 135 | - Required fields supported. 136 | 137 | ## v0.0.1 (21/07/2018) 138 | 139 | - Initial release. 140 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 The Dumb Terminal 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jsonschema-bigquery 2 | 3 | [![npm](https://img.shields.io/npm/v/jsonschema-bigquery.svg)](https://www.npmjs.com/package/jsonschema-bigquery) 4 | [![Node.js CI](https://github.com/thedumbterminal/jsonschema-bigquery/actions/workflows/main.yml/badge.svg)](https://github.com/thedumbterminal/jsonschema-bigquery/actions/workflows/main.yml) 5 | 6 | Convert JSON schema to Google BigQuery schema 7 | 8 | This includes the ability to: 9 | 10 | 1. Create tables 11 | 1. Patch tables 12 | 13 | Further enhancements are planned: delete tables (dev only), create dataset, set data ACLs 14 | 15 | Note that some features involve bespoke interpretation of schema details suited to our environment. 16 | 17 | ## Install 18 | 19 | npm install jsonschema-bigquery 20 | 21 | ## Consume 22 | 23 | jsbq -p -d -j --preventAdditionalObjectProperties --continueOnError 24 | 25 | For embedded usage the following will allow and support runtime schema conversion and table maintenance: 26 | 27 | const jsonSchemaBigquery = require('jsonschema-bigquery') 28 | const bigquerySchema = jsonSchemaBigquery.run(jsonSchemaObject, options) 29 | 30 | Please ensure that the input JSON schema is dereferenced so that all external references have been resolved. [json-schema-ref-parser](https://www.npmjs.com/package/json-schema-ref-parser) can do this, prior to using this module. 31 | 32 | ### Options 33 | 34 | ``` 35 | { 36 | preventAdditionalObjectProperties: true, 37 | continueOnError: false 38 | } 39 | ``` 40 | 41 | - `preventAdditionalObjectProperties` - boolean, check for additional object properties in schemas. 42 | - `continueOnError` - boolean, continues conversion if problem JSON is encountered. Problems will be excluded from resulting schema. 43 | 44 | ### Usage with bq CLI tool 45 | 46 | Google maintains a CLI tool called `bq` which allows the management of BigQuery tables: 47 | 48 | https://cloud.google.com/bigquery/docs/reference/bq-cli-reference 49 | 50 | To create a table from a JSON schema using other options that our `jsbq` does not support, use the following commands: 51 | 52 | First, output the generated GBQ schema to a file: 53 | 54 | ``` 55 | npx jsbq -j test/integration/samples/complex/input.json > /tmp/schema.json 56 | ``` 57 | 58 | Then run the `bq` command to create a table: 59 | 60 | ``` 61 | bq mk --schema=/tmp/schema.json test_dataset.test_table 62 | ``` 63 | 64 | Please see the bq reference for table creation options: 65 | 66 | https://cloud.google.com/bigquery/docs/reference/bq-cli-reference#bq_mk 67 | 68 | ## Test 69 | 70 | npm test 71 | -------------------------------------------------------------------------------- /bin/jsbq: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const jsbq = (module.exports = {}) 4 | 5 | const gbq = require('../src/gbq') 6 | const { get_table_id } = require('../src/utils') 7 | const converter = require('../src/converter') 8 | const { logger } = require('../src/log') 9 | const fs = require('fs') 10 | const { promisify } = require('util') 11 | 12 | jsbq.process = async (project, datasetName, jsonSchema, options) => { 13 | logger.info('Processing JSON schema...') 14 | const tableOptions = converter.run(jsonSchema, options) 15 | logger.info('Generated BigQuery schema:') 16 | console.log(JSON.stringify(tableOptions.schema.fields)) 17 | 18 | if (!project && !datasetName) { 19 | return logger.info('Skipping table operations') 20 | } 21 | 22 | const schemaId = jsonSchema.$id || jsonSchema.id 23 | logger.info('Extracting table name from schema ID:', schemaId) 24 | const tableName = get_table_id(schemaId) 25 | logger.info('Table name:', tableName) 26 | 27 | logger.info('Setting table options...') 28 | tableOptions.friendly_name = jsonSchema.title 29 | tableOptions.description = jsonSchema.description || jsonSchema.title 30 | tableOptions.timePartitioning = { 31 | type: 'DAY', 32 | requirePartitionFilter: true, 33 | } 34 | 35 | logger.info('Checking if table exists...') 36 | const tableExists = await gbq.tableExists(project, datasetName, tableName) 37 | if (tableExists) { 38 | logger.info('Patching table:', tableName) 39 | await gbq.patchTable(project, datasetName, tableName, tableOptions) 40 | } else { 41 | logger.info('Creating table:', tableName) 42 | await gbq.createTable(project, datasetName, tableName, tableOptions) 43 | } 44 | logger.info('Finished') 45 | } 46 | 47 | jsbq.run = async () => { 48 | const readFile = promisify(fs.readFile) 49 | 50 | const argv = require('yargs') 51 | .usage( 52 | 'Usage: $0 -p [project] -d [dataset] -j --preventAdditionalObjectProperties --continueOnError', 53 | ) 54 | .options({ 55 | j: { 56 | describe: 'JSON schema file', 57 | demandOption: true, 58 | }, 59 | p: { 60 | describe: 'GCP project name', 61 | }, 62 | d: { 63 | describe: 'BigQuery dataset name', 64 | }, 65 | preventAdditionalObjectProperties: { 66 | describe: 'boolean, check for additional object properties in schemas.', 67 | type: 'boolean', 68 | default: false, 69 | }, 70 | continueOnError: { 71 | describe: 72 | 'boolean, if error in json schema, skip element in big query schema and continue.', 73 | type: 'boolean', 74 | default: false, 75 | }, 76 | debug: { 77 | describe: 'Enable debug logging', 78 | type: 'boolean', 79 | default: false, 80 | }, 81 | }).argv 82 | 83 | if (argv.debug) { 84 | logger.level = 'debug' 85 | logger.debug('Debug mode enabled') 86 | } 87 | 88 | const schemaData = await readFile(argv.j) 89 | const options = { 90 | preventAdditionalObjectProperties: argv.preventAdditionalObjectProperties, 91 | continueOnError: argv.continueOnError, 92 | } 93 | const jsonSchema = JSON.parse(schemaData) 94 | logger.debug('Input schema: ', jsonSchema) 95 | return jsbq.process(argv.p, argv.d, jsonSchema, options) 96 | } 97 | 98 | jsbq.run().catch((e) => { 99 | logger.error(e) 100 | process.exit(1) 101 | }) 102 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jsonschema-bigquery", 3 | "version": "7.0.0", 4 | "description": "Convert JSON schema to Google BigQuery schema", 5 | "main": "src/converter.js", 6 | "scripts": { 7 | "ci": "npm run lint && npm run coverage", 8 | "test": "mocha test/unit test/integration", 9 | "coverage": "nyc npm run test", 10 | "lint:eslint": "eslint .", 11 | "lint:prettier": "prettier -c .", 12 | "lint": "npm run lint:prettier && npm run lint:eslint", 13 | "format": "prettier --write ." 14 | }, 15 | "bin": { 16 | "jsbq": "./bin/jsbq" 17 | }, 18 | "engines": { 19 | "node": ">=12" 20 | }, 21 | "repository": { 22 | "type": "git", 23 | "url": "git+https://github.com/thedumbterminal/jsonschema-bigquery.git" 24 | }, 25 | "keywords": [ 26 | "json", 27 | "schema", 28 | "google", 29 | "bigquery", 30 | "convert" 31 | ], 32 | "author": { 33 | "name": "thedumbterminal", 34 | "email": "github@thedumbterminal.co.uk", 35 | "url": "http://www.thedumbterminal.co.uk" 36 | }, 37 | "license": "MIT", 38 | "bugs": { 39 | "url": "https://github.com/thedumbterminal/jsonschema-bigquery/issues" 40 | }, 41 | "homepage": "https://github.com/thedumbterminal/jsonschema-bigquery#readme", 42 | "devDependencies": { 43 | "eslint": "^8.5.0", 44 | "eslint-plugin-import": "^2.29.1", 45 | "eslint-plugin-mocha": "^10.1.0", 46 | "eslint-plugin-n": "^17.10.1", 47 | "eslint-plugin-promise": "^7.0.0", 48 | "mocha": "^11.0.1", 49 | "nyc": "^17.0.0", 50 | "prettier": "^3.3.3" 51 | }, 52 | "dependencies": { 53 | "@google-cloud/bigquery": "^8.0.0", 54 | "lodash": "^4.17.20", 55 | "log4js": "^6.3.0", 56 | "yargs": "^18.0.0" 57 | }, 58 | "nyc": { 59 | "check-coverage": true, 60 | "branches": 88, 61 | "lines": 98, 62 | "functions": 100, 63 | "statements": 97 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/converter.js: -------------------------------------------------------------------------------- 1 | const converter = (module.exports = {}) 2 | const _ = require('lodash') 3 | const { SchemaError } = require('./errors') 4 | const { logger } = require('./log') 5 | 6 | const JSON_SCHEMA_TO_BIGQUERY_TYPE_DICT = { 7 | boolean: 'BOOLEAN', 8 | 'date-time': 'TIMESTAMP', 9 | integer: 'INTEGER', 10 | number: 'BIGNUMERIC', 11 | string: 'STRING', 12 | date: 'DATE', 13 | time: 'TIME', 14 | } 15 | 16 | const OFS = ['allOf', 'anyOf', 'oneOf'] 17 | 18 | const BIGQUERY_FIELD_NAME_REGEXP = /^[a-z_]([a-z0-9_]+|)$/i 19 | 20 | converter._copy = (o) => _.clone(o, false) 21 | converter._deepCopy = (o) => _.clone(o, true) 22 | 23 | converter._merge_property = ( 24 | merge_type, 25 | property_name, 26 | destination_value, 27 | source_value, 28 | ) => { 29 | // Merges two properties. 30 | if (destination_value === undefined && source_value === undefined) { 31 | return undefined 32 | } 33 | if (destination_value === undefined) return source_value 34 | if (source_value === undefined) return destination_value 35 | if ( 36 | typeof destination_value === 'boolean' && 37 | typeof source_value === 'boolean' 38 | ) { 39 | return destination_value && source_value 40 | } 41 | if (_.isPlainObject(destination_value) && _.isPlainObject(source_value)) { 42 | return converter._merge_dicts(merge_type, destination_value, source_value) 43 | } 44 | let destination_list 45 | if (Array.isArray(destination_value)) { 46 | destination_list = converter._copy(destination_value) 47 | } else { 48 | destination_list = [destination_value] 49 | } 50 | let source_list 51 | if (Array.isArray(source_value)) { 52 | source_list = source_value 53 | } else { 54 | source_list = [source_value] 55 | } 56 | if (property_name === 'description') { 57 | destination_list.push(source_value) 58 | return _.uniq(destination_list).join(' ') 59 | } 60 | if (property_name === 'required' && ['anyOf', 'oneOf'].includes(merge_type)) { 61 | return destination_list.filter((v) => source_list.includes(v)) 62 | } 63 | if (property_name === 'format') { 64 | // if we have multiple formats we have to remove them all 65 | return _.uniq(destination_list).length === 1 ? destination_list.shift() : '' 66 | } 67 | destination_list.push( 68 | ...source_list.filter((v) => !destination_list.includes(v)), 69 | ) 70 | return destination_list 71 | } 72 | 73 | /** 74 | * Merges multiple sources given as an Array was *dicts, 75 | * 76 | * Cloned from the original merge_dicts from python-based bigjson This method had variable 77 | * expansion, varags and two slightly different usages which appeared to be incompatible. 78 | * 79 | * Could be returned to a single method 80 | * 81 | * See merge_dicts for the alternate call pattern 82 | */ 83 | converter._merge_dicts_array = (merge_type, dest_dict, source_dicts) => { 84 | const result = converter._deepCopy(dest_dict) 85 | for (let source_dict of source_dicts) { 86 | // First check if we need to recurse and merge deeper results first 87 | for (const x_of of OFS) { 88 | if (_.has(source_dict, x_of)) { 89 | source_dict = converter._merge_dicts_array( 90 | x_of, 91 | source_dict, 92 | source_dict[x_of], 93 | ) 94 | delete source_dict[x_of] 95 | } 96 | } 97 | 98 | const keys = Object.keys(source_dict) 99 | for (const name of keys) { 100 | const merged_property = converter._merge_property( 101 | merge_type, 102 | name, 103 | result[name], 104 | source_dict[name], 105 | ) 106 | if (merged_property !== undefined) { 107 | result[name] = merged_property 108 | } 109 | } 110 | } 111 | return result 112 | } 113 | 114 | /** 115 | * Merges a single object 116 | * 117 | * The original merge_dicts from python-based bigjson This method had variable expansion, varags and 118 | * two slightly different usages which appeared to be incompatible. 119 | * 120 | * Could be returned to a single method 121 | * 122 | * See merge_dicts_array above for the alternate call pattern 123 | */ 124 | converter._merge_dicts = (merge_type, dest_dict, source_dict) => { 125 | const result = converter._deepCopy(dest_dict) 126 | const keys = Object.keys(source_dict) 127 | for (const name of keys) { 128 | const merged_property = converter._merge_property( 129 | merge_type, 130 | name, 131 | result[name], 132 | source_dict[name], 133 | ) 134 | if (merged_property !== undefined) { 135 | result[name] = merged_property 136 | } 137 | } 138 | return result 139 | } 140 | 141 | converter._scalar = (name, type, mode, description) => { 142 | if (!name.match(BIGQUERY_FIELD_NAME_REGEXP)) { 143 | throw new SchemaError(`Invalid field name: ${name}`) 144 | } 145 | 146 | const result = { 147 | name, 148 | type, 149 | mode, 150 | } 151 | 152 | if (description) { 153 | result.description = description 154 | } 155 | 156 | return result 157 | } 158 | 159 | converter._array = (name, node) => { 160 | const items_with_description = converter._deepCopy(node.items) 161 | if (_.has(items_with_description, 'description')) { 162 | items_with_description.description = node.description 163 | } 164 | return converter._visit(name, items_with_description, 'REPEATED') 165 | } 166 | 167 | converter._allowsAdditionalProperties = (node) => 168 | node.additionalProperties !== false && node.unevaluatedProperties !== false 169 | 170 | converter._object = (name, node, mode) => { 171 | let result = { 172 | fields: [], 173 | } 174 | let fieldType = 'RECORD' 175 | try { 176 | if ( 177 | converter._allowsAdditionalProperties(node) && 178 | converter._options.preventAdditionalObjectProperties 179 | ) { 180 | throw new SchemaError( 181 | 'Objects must not have additional or unevaluated properties', 182 | node, 183 | ) 184 | } 185 | if ( 186 | !_.isPlainObject(node.properties) || 187 | Object.keys(node.properties).length === 0 188 | ) { 189 | // Big Query can handle semi-structured data : 190 | // https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#loading_semi-structured_json_data 191 | fieldType = 'JSON' 192 | return converter._scalar(name, fieldType, mode, node.description) 193 | } 194 | 195 | result = converter._scalar(name, fieldType, mode, node.description) 196 | 197 | const required_properties = node.required || [] 198 | const properties = node.properties 199 | 200 | let fields = Object.keys(properties).map((key) => { 201 | const required = required_properties.includes(key) 202 | ? 'REQUIRED' 203 | : 'NULLABLE' 204 | return converter._visit(key, properties[key], required) 205 | }) 206 | 207 | // remove empty fields 208 | fields = fields.filter(function (field) { 209 | return field != null 210 | }) 211 | result.fields = fields 212 | } catch (e) { 213 | if (!converter._options.continueOnError) { 214 | throw e 215 | } 216 | logger.warn(e) 217 | } 218 | 219 | return result 220 | } 221 | 222 | converter._bigQueryType = (node, type) => { 223 | // handle string formats 224 | let actualType = type 225 | const format = node.format 226 | if (type === 'string' && ['date-time', 'date', 'time'].includes(format)) { 227 | actualType = format 228 | } 229 | const bqType = JSON_SCHEMA_TO_BIGQUERY_TYPE_DICT[actualType] 230 | if (!bqType) { 231 | throw new SchemaError(`Invalid type given: ${type}`, node) 232 | } 233 | return bqType 234 | } 235 | 236 | converter._simple = (name, type, node, mode) => { 237 | if (type === 'array') { 238 | return converter._array(name, node) 239 | } 240 | if (type === 'object') { 241 | return converter._object(name, node, mode) 242 | } 243 | 244 | const bqType = converter._bigQueryType(node, type) 245 | return converter._scalar(name, bqType, mode, node.description) 246 | } 247 | 248 | converter._visit = (name, node, mode = 'NULLABLE') => { 249 | let merged_node = node 250 | for (const x_of of OFS) { 251 | if (_.has(node, x_of)) { 252 | merged_node = converter._merge_dicts_array(x_of, node, _.get(node, x_of)) 253 | delete merged_node[x_of] 254 | } 255 | } 256 | let type_ = merged_node.type 257 | let actual_mode = mode 258 | if (Array.isArray(type_)) { 259 | const non_null_types = type_.filter((scalar_type) => scalar_type !== 'null') 260 | if (non_null_types.length > 1) { 261 | throw new SchemaError('Union type not supported', node) 262 | } 263 | // When mode is REPEATED, we want to leave it, even if it is NULLABLE 264 | if (type_.includes('null') && actual_mode !== 'REPEATED') { 265 | actual_mode = 'NULLABLE' 266 | } 267 | type_ = non_null_types[0] 268 | } 269 | return converter._simple(name, type_, merged_node, actual_mode) 270 | } 271 | 272 | converter.run = (input_schema, options = {}) => { 273 | converter._options = options 274 | return { 275 | schema: { 276 | fields: converter._visit('root', input_schema).fields, 277 | }, 278 | } 279 | } 280 | -------------------------------------------------------------------------------- /src/errors.js: -------------------------------------------------------------------------------- 1 | const utils = require('./utils') 2 | 3 | class SchemaError extends Error { 4 | constructor(message, node) { 5 | super(message) 6 | 7 | // Maintains proper stack trace for where our error was thrown (only available on V8) 8 | if (Error.captureStackTrace) { 9 | Error.captureStackTrace(this, SchemaError) 10 | } 11 | 12 | this.name = 'SchemaError' 13 | if (node) { 14 | this.message += ':\n' + utils.inspector(node) 15 | } 16 | } 17 | } 18 | 19 | module.exports = { 20 | SchemaError, 21 | } 22 | -------------------------------------------------------------------------------- /src/gbq.js: -------------------------------------------------------------------------------- 1 | const gbq = (module.exports = {}) 2 | const { BigQuery } = require('@google-cloud/bigquery') 3 | const { logger } = require('./log') 4 | 5 | gbq.createTable = async (projectId, datasetId, tableId, schema) => { 6 | const bigquery = new BigQuery({ projectId }) 7 | 8 | const [table] = await bigquery.dataset(datasetId).createTable(tableId, schema) 9 | 10 | logger.info(`Table ${table.id} created.`) 11 | } 12 | 13 | gbq.patchTable = async (projectId, datasetId, tableId, schema) => { 14 | const bigquery = new BigQuery({ projectId }) 15 | 16 | const res = await bigquery 17 | .dataset(datasetId) 18 | .table(tableId) 19 | .setMetadata(schema) 20 | 21 | logger.info('Patch result:', res) 22 | } 23 | 24 | gbq.tableExists = async (projectId, datasetId, tableId) => { 25 | const bigquery = new BigQuery({ projectId }) 26 | 27 | const table = await bigquery.dataset(datasetId).table(tableId) 28 | const res = await table.exists() 29 | logger.debug('Table check:', res) 30 | return res[0] 31 | } 32 | -------------------------------------------------------------------------------- /src/log.js: -------------------------------------------------------------------------------- 1 | const log = (module.exports = {}) 2 | const log4js = require('log4js') 3 | const utils = require('./utils') 4 | 5 | log4js.configure({ 6 | appenders: { 7 | err: { 8 | type: 'stderr', 9 | layout: { 10 | type: 'pattern', 11 | pattern: '%[%d %p %c%] %x{plain}%x{schema}', 12 | tokens: { 13 | plain: (logEvent) => logEvent.data[0], 14 | schema: (logEvent) => { 15 | if (!logEvent.data[1]) { 16 | return '' 17 | } 18 | return '\n' + utils.inspector(logEvent.data[1]) 19 | }, 20 | }, 21 | }, 22 | }, 23 | }, 24 | categories: { 25 | default: { 26 | appenders: ['err'], 27 | level: 'ERROR', 28 | }, 29 | }, 30 | }) 31 | 32 | const logger = log4js.getLogger() 33 | 34 | // Allow log level to be changed in development 35 | logger.level = process.env.LOG_LEVEL || 'info' 36 | 37 | log.logger = logger 38 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const utils = (module.exports = {}) 3 | 4 | // Expected format is "id": "/events/server/ecommerce/v5.1.5/schema.json" 5 | utils.get_table_id = (id) => { 6 | const idArr = id.split('/').slice(2, -1) 7 | const version = idArr.pop() 8 | return idArr.join('_') + '_' + version.split('.')[0] 9 | } 10 | 11 | utils.inspector = (obj) => 12 | util.inspect(obj, { 13 | depth: null, 14 | colors: process.stdout.isTTY, // Add colours if in an interactive terminal 15 | compact: false, 16 | breakLength: Infinity, 17 | }) 18 | -------------------------------------------------------------------------------- /tablediff/README.md: -------------------------------------------------------------------------------- 1 | Useful for QA comparison of generated tables. 2 | 3 | Review and run create_tables.sh. For example it can be useful to recreate tables in a new dataset. 4 | 5 | Once run the tablediff.sh script (again review for config), can generate actual vs expected gbq schema files (from gbq itself). 6 | These can then be used in combination with a diff tool to highlight any differences with the expected outputs. 7 | -------------------------------------------------------------------------------- /tablediff/create_tables.sh: -------------------------------------------------------------------------------- 1 | #Environment dependencies 2 | #PROJECT= 3 | #DATASET= 4 | #SCHEMA_PATH=???/dist/dereferenced/events/ 5 | 6 | for EVENT_VERSION in `cat events.list` 7 | do 8 | echo Starting $schema 9 | cp $SCHEMA_PATH/$EVENT_VERSION/schema.json ../src 10 | bin/jsbq -p $PROJECT -d $DATASET -j schema.json 11 | done 12 | 13 | echo Finished 14 | -------------------------------------------------------------------------------- /tablediff/tablediff.sh: -------------------------------------------------------------------------------- 1 | #Environment dependencies 2 | #SOURCE_DATASET= 3 | #TARGET_DATASET= 4 | 5 | for TABLE in `bq ls $SOURCE_DATASET | tail +3 | cut -c 1-35| sed 's; ;;g'` 6 | do 7 | echo $TABLE 8 | # Ignore descriptions for now, as there are some non-critical changes that have occurred slowly over time 9 | bq show --schema $SOURCE_DATASET.$TABLE | python -m json.tool | sed 's; (Dereferenced);;g' | fgrep -v '"description":' > SOURCE/$TABLE 10 | bq show --schema $TARGET_DATASET.$TABLE | python -m json.tool | sed 's; (Dereferenced);;g' | fgrep -v '"description":' > TARGET/$TABLE 11 | done 12 | 13 | echo "Generated pretty schemas in SOURCE and TARGET, use diff or an IDE to review differences" 14 | -------------------------------------------------------------------------------- /test/integration/continueOnError/additionalProperties/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "address", 6 | "type": "RECORD", 7 | "mode": "NULLABLE", 8 | "fields": [ 9 | { 10 | "name": "street_address", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | }, 14 | { 15 | "name": "country", 16 | "type": "JSON", 17 | "mode": "NULLABLE" 18 | } 19 | ] 20 | } 21 | ] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /test/integration/continueOnError/additionalProperties/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Complex example empty Object", 4 | "type": "object", 5 | "properties": { 6 | "address": { 7 | "type": "object", 8 | "properties": { 9 | "street_address": { 10 | "type": "string" 11 | }, 12 | "country": { 13 | "type": "object", 14 | "properties": {} 15 | } 16 | }, 17 | "additionalProperties": true 18 | } 19 | }, 20 | "additionalProperties": false 21 | } 22 | -------------------------------------------------------------------------------- /test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [] 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Complex example empty Object", 4 | "type": "object", 5 | "properties": { 6 | "address": { 7 | "type": "object", 8 | "properties": { 9 | "street_address": { 10 | "type": "string" 11 | }, 12 | "country": { 13 | "type": "object", 14 | "properties": {} 15 | } 16 | }, 17 | "additionalProperties": true 18 | } 19 | }, 20 | "additionalProperties": true 21 | } 22 | -------------------------------------------------------------------------------- /test/integration/converter.js: -------------------------------------------------------------------------------- 1 | const converter = require('../../src/converter') 2 | const assert = require('assert') 3 | const fs = require('fs') 4 | 5 | describe('converter integration', () => { 6 | describe('default options', () => { 7 | const sampleDir = './test/integration/samples' 8 | // eslint-disable-next-line mocha/no-setup-in-describe 9 | const testDirs = fs.readdirSync(sampleDir) 10 | 11 | // eslint-disable-next-line mocha/no-setup-in-describe 12 | testDirs.forEach((dir) => { 13 | describe(dir, () => { 14 | let expected 15 | let result 16 | 17 | before(() => { 18 | const inJson = require(`../../${sampleDir}/${dir}/input.json`) 19 | expected = require(`../../${sampleDir}/${dir}/expected.json`) 20 | result = converter.run(inJson, 'p', 't') 21 | }) 22 | 23 | it('converts to big query', () => { 24 | assert.deepStrictEqual(result, expected) 25 | }) 26 | }) 27 | }) 28 | }) 29 | 30 | describe('continueOnError option', () => { 31 | const sampleDir = './test/integration/continueOnError' 32 | // eslint-disable-next-line mocha/no-setup-in-describe 33 | const testDirs = fs.readdirSync(sampleDir) 34 | 35 | // eslint-disable-next-line mocha/no-setup-in-describe 36 | testDirs.forEach((dir) => { 37 | describe(dir, () => { 38 | let expected 39 | let result 40 | 41 | before(() => { 42 | const inJson = require(`../../${sampleDir}/${dir}/input.json`) 43 | expected = require(`../../${sampleDir}/${dir}/expected.json`) 44 | const options = { 45 | continueOnError: true, 46 | } 47 | result = converter.run(inJson, options, 'p', 't') 48 | }) 49 | 50 | it('converts to big query', () => { 51 | assert.deepStrictEqual(result, expected) 52 | }) 53 | }) 54 | }) 55 | }) 56 | 57 | describe('preventAdditionalObjectProperties option', () => { 58 | const sampleDir = './test/integration/preventAdditionalObjectProperties' 59 | // eslint-disable-next-line mocha/no-setup-in-describe 60 | const testDirs = fs.readdirSync(sampleDir) 61 | 62 | // eslint-disable-next-line mocha/no-setup-in-describe 63 | testDirs.forEach((dir) => { 64 | describe(dir, () => { 65 | let expected 66 | let result 67 | 68 | before(() => { 69 | const inJson = require(`../../${sampleDir}/${dir}/input.json`) 70 | expected = require(`../../${sampleDir}/${dir}/expected.json`) 71 | const options = { 72 | preventAdditionalObjectProperties: true, 73 | } 74 | result = converter.run(inJson, options, 'p', 't') 75 | }) 76 | 77 | it('converts to big query', () => { 78 | assert.deepStrictEqual(result, expected) 79 | }) 80 | }) 81 | }) 82 | }) 83 | 84 | describe('continueOnError and preventAdditionalObjectProperties options', () => { 85 | const sampleDir = 86 | './test/integration/continueOnErrorAndPreventAdditionalObjectProperties' 87 | // eslint-disable-next-line mocha/no-setup-in-describe 88 | const testDirs = fs.readdirSync(sampleDir) 89 | 90 | // eslint-disable-next-line mocha/no-setup-in-describe 91 | testDirs.forEach((dir) => { 92 | describe(dir, () => { 93 | let expected 94 | let result 95 | 96 | before(() => { 97 | const inJson = require(`../../${sampleDir}/${dir}/input.json`) 98 | expected = require(`../../${sampleDir}/${dir}/expected.json`) 99 | const options = { 100 | continueOnError: true, 101 | preventAdditionalObjectProperties: true, 102 | } 103 | result = converter.run(inJson, options, 'p', 't') 104 | }) 105 | 106 | it('converts to big query', () => { 107 | assert.deepStrictEqual(result, expected) 108 | }) 109 | }) 110 | }) 111 | }) 112 | }) 113 | -------------------------------------------------------------------------------- /test/integration/preventAdditionalObjectProperties/additionalPropertiesFalse/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "first_name", 6 | "type": "STRING", 7 | "mode": "NULLABLE" 8 | }, 9 | { 10 | "name": "last_name", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | } 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/integration/preventAdditionalObjectProperties/additionalPropertiesFalse/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "first_name": { 7 | "type": "string" 8 | }, 9 | "last_name": { 10 | "type": "string" 11 | } 12 | }, 13 | "additionalProperties": false 14 | } 15 | -------------------------------------------------------------------------------- /test/integration/preventAdditionalObjectProperties/unevaluatedPropertiesFalse/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "first_name", 6 | "type": "STRING", 7 | "mode": "NULLABLE" 8 | }, 9 | { 10 | "name": "last_name", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | } 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/integration/preventAdditionalObjectProperties/unevaluatedPropertiesFalse/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "first_name": { 7 | "type": "string" 8 | }, 9 | "last_name": { 10 | "type": "string" 11 | } 12 | }, 13 | "unevaluatedProperties": false 14 | } 15 | -------------------------------------------------------------------------------- /test/integration/samples/allOf/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "names", 6 | "type": "RECORD", 7 | "mode": "NULLABLE", 8 | "fields": [ 9 | { 10 | "name": "first_name", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | }, 14 | { 15 | "name": "last_name", 16 | "type": "STRING", 17 | "mode": "NULLABLE" 18 | } 19 | ] 20 | } 21 | ] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /test/integration/samples/allOf/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "names": { 7 | "allOf": [ 8 | { 9 | "type": "object", 10 | "properties": { 11 | "first_name": { 12 | "type": "string" 13 | } 14 | }, 15 | "additionalProperties": false 16 | }, 17 | { 18 | "type": "object", 19 | "properties": { 20 | "last_name": { 21 | "type": "string" 22 | } 23 | }, 24 | "additionalProperties": false 25 | } 26 | ] 27 | } 28 | }, 29 | "additionalProperties": false 30 | } 31 | -------------------------------------------------------------------------------- /test/integration/samples/allOf_nested/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "addresses", 6 | "type": "RECORD", 7 | "mode": "NULLABLE", 8 | "fields": [ 9 | { 10 | "name": "address", 11 | "type": "RECORD", 12 | "mode": "NULLABLE", 13 | "fields": [ 14 | { 15 | "name": "street_address", 16 | "type": "STRING", 17 | "mode": "NULLABLE" 18 | }, 19 | { 20 | "name": "country", 21 | "type": "STRING", 22 | "mode": "NULLABLE" 23 | }, 24 | { 25 | "name": "town", 26 | "type": "STRING", 27 | "mode": "NULLABLE" 28 | } 29 | ] 30 | } 31 | ] 32 | } 33 | ] 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /test/integration/samples/allOf_nested/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "addresses": { 7 | "allOf": [ 8 | { 9 | "type": "object", 10 | "properties": { 11 | "address": { 12 | "type": "object", 13 | "properties": { 14 | "street_address": { 15 | "type": "string" 16 | }, 17 | "country": { 18 | "type": "string" 19 | } 20 | } 21 | } 22 | }, 23 | "additionalProperties": false 24 | }, 25 | { 26 | "type": "object", 27 | "properties": { 28 | "address": { 29 | "type": "object", 30 | "properties": { 31 | "town": { 32 | "type": "string" 33 | }, 34 | "country": { 35 | "type": "string" 36 | } 37 | }, 38 | "additionalProperties": false 39 | } 40 | }, 41 | "additionalProperties": false 42 | } 43 | ] 44 | } 45 | }, 46 | "additionalProperties": false 47 | } 48 | -------------------------------------------------------------------------------- /test/integration/samples/anyOfMultipleTypes/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "first_name", 6 | "type": "STRING", 7 | "mode": "NULLABLE" 8 | } 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/anyOfMultipleTypes/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "first_name": { 7 | "anyOf": [ 8 | { 9 | "type": "string" 10 | }, 11 | { 12 | "type": "null" 13 | } 14 | ] 15 | } 16 | }, 17 | "required": ["first_name"], 18 | "additionalProperties": false 19 | } 20 | -------------------------------------------------------------------------------- /test/integration/samples/complex/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "address", 6 | "type": "RECORD", 7 | "mode": "NULLABLE", 8 | "fields": [ 9 | { 10 | "name": "street_address", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | }, 14 | { 15 | "name": "country", 16 | "type": "STRING", 17 | "mode": "NULLABLE" 18 | } 19 | ] 20 | } 21 | ] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /test/integration/samples/complex/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Complex example", 4 | "type": "object", 5 | "properties": { 6 | "address": { 7 | "type": "object", 8 | "properties": { 9 | "street_address": { 10 | "type": "string" 11 | }, 12 | "country": { 13 | "type": "string" 14 | } 15 | }, 16 | "additionalProperties": false 17 | } 18 | }, 19 | "additionalProperties": false 20 | } 21 | -------------------------------------------------------------------------------- /test/integration/samples/date/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "modified", 6 | "type": "DATE", 7 | "mode": "NULLABLE" 8 | } 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/date/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "modified": { 7 | "type": "string", 8 | "format": "date" 9 | } 10 | }, 11 | "additionalProperties": false 12 | } 13 | -------------------------------------------------------------------------------- /test/integration/samples/fieldDescription/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "description": "the first name", 6 | "name": "first_name", 7 | "type": "STRING", 8 | "mode": "NULLABLE" 9 | } 10 | ] 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /test/integration/samples/fieldDescription/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "first_name": { 7 | "description": "the first name", 8 | "type": "string" 9 | } 10 | }, 11 | "additionalProperties": false 12 | } 13 | -------------------------------------------------------------------------------- /test/integration/samples/json/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "description": "Some json data without specific schema.", 6 | "name": "semi_structured", 7 | "type": "JSON", 8 | "mode": "NULLABLE" 9 | } 10 | ] 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /test/integration/samples/json/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "semi_structured": { 7 | "description": "Some json data without specific schema.", 8 | "type": "object" 9 | } 10 | }, 11 | "additionalProperties": false 12 | } 13 | -------------------------------------------------------------------------------- /test/integration/samples/nestedDescription/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "description": "Any first name A type of first name Another type of first name", 6 | "name": "first_name", 7 | "type": "STRING", 8 | "mode": "NULLABLE" 9 | } 10 | ] 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /test/integration/samples/nestedDescription/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "first_name": { 7 | "description": "Any first name", 8 | "anyOf": [ 9 | { 10 | "description": "A type of first name", 11 | "type": "string", 12 | "enum": ["Bob"] 13 | }, 14 | { 15 | "description": "Another type of first name", 16 | "type": "string", 17 | "enum": ["Alice"] 18 | } 19 | ] 20 | } 21 | }, 22 | "additionalProperties": false 23 | } 24 | -------------------------------------------------------------------------------- /test/integration/samples/nestedOneOf/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "mode": "NULLABLE", 6 | "name": "product", 7 | "type": "STRING" 8 | } 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/nestedOneOf/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "product.json", 3 | "type": "object", 4 | "properties": { 5 | "product": { 6 | "oneOf": [ 7 | { 8 | "oneOf": [ 9 | { 10 | "type": "string", 11 | "enum": ["sweets"] 12 | }, 13 | { 14 | "type": "string", 15 | "enum": ["shoes"] 16 | } 17 | ] 18 | }, 19 | { 20 | "type": "null" 21 | } 22 | ] 23 | } 24 | }, 25 | "additionalProperties": false 26 | } 27 | -------------------------------------------------------------------------------- /test/integration/samples/nestedRepeated/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "names", 6 | "type": "RECORD", 7 | "mode": "REPEATED", 8 | "fields": [ 9 | { 10 | "name": "first", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | } 14 | ] 15 | } 16 | ] 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /test/integration/samples/nestedRepeated/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "names": { 7 | "type": "array", 8 | "items": { 9 | "type": "object", 10 | "properties": { 11 | "first": { 12 | "type": "string" 13 | } 14 | }, 15 | "additionalProperties": false 16 | } 17 | } 18 | }, 19 | "additionalProperties": false 20 | } 21 | -------------------------------------------------------------------------------- /test/integration/samples/nullable/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "first_name", 6 | "type": "STRING", 7 | "mode": "REQUIRED" 8 | }, 9 | { 10 | "name": "last_name", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | } 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/integration/samples/nullable/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "first_name": { 7 | "type": "string" 8 | }, 9 | "last_name": { 10 | "type": "string" 11 | } 12 | }, 13 | "required": ["first_name"], 14 | "additionalProperties": false 15 | } 16 | -------------------------------------------------------------------------------- /test/integration/samples/numeric/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "modified", 6 | "type": "BIGNUMERIC", 7 | "mode": "NULLABLE" 8 | } 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/numeric/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "modified": { 7 | "type": "number" 8 | } 9 | }, 10 | "additionalProperties": false 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/objectDescription/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "address", 6 | "type": "RECORD", 7 | "mode": "NULLABLE", 8 | "description": "An address", 9 | "fields": [ 10 | { 11 | "name": "street_address", 12 | "type": "STRING", 13 | "mode": "NULLABLE" 14 | } 15 | ] 16 | } 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test/integration/samples/objectDescription/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Object description", 4 | "type": "object", 5 | "properties": { 6 | "address": { 7 | "description": "An address", 8 | "type": "object", 9 | "properties": { 10 | "street_address": { 11 | "type": "string" 12 | } 13 | }, 14 | "additionalProperties": false 15 | } 16 | }, 17 | "additionalProperties": false 18 | } 19 | -------------------------------------------------------------------------------- /test/integration/samples/oneOfWithDateFormat/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "fields": [ 6 | { 7 | "mode": "NULLABLE", 8 | "name": "first_name", 9 | "type": "STRING" 10 | }, 11 | { 12 | "mode": "NULLABLE", 13 | "name": "created", 14 | "type": "TIMESTAMP" 15 | }, 16 | { 17 | "mode": "NULLABLE", 18 | "name": "last_name", 19 | "type": "STRING" 20 | } 21 | ], 22 | "mode": "NULLABLE", 23 | "name": "name", 24 | "type": "RECORD" 25 | } 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /test/integration/samples/oneOfWithDateFormat/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "type": "object", 4 | "properties": { 5 | "name": { 6 | "oneOf": [ 7 | { 8 | "type": "object", 9 | "properties": { 10 | "first_name": { 11 | "type": "string" 12 | }, 13 | "created": { 14 | "type": "string", 15 | "format": "date-time" 16 | } 17 | } 18 | }, 19 | { 20 | "type": "object", 21 | "properties": { 22 | "last_name": { 23 | "type": "string" 24 | }, 25 | "created": { 26 | "type": "string", 27 | "format": "date-time" 28 | } 29 | } 30 | } 31 | ] 32 | } 33 | }, 34 | "additionalProperties": false 35 | } 36 | -------------------------------------------------------------------------------- /test/integration/samples/oneOfWithDuplicateDescriptions/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "fields": [ 6 | { 7 | "mode": "NULLABLE", 8 | "name": "first_name", 9 | "type": "STRING" 10 | }, 11 | { 12 | "description": "Created date", 13 | "mode": "NULLABLE", 14 | "name": "created", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "last_name", 20 | "type": "STRING" 21 | } 22 | ], 23 | "mode": "NULLABLE", 24 | "name": "name", 25 | "type": "RECORD" 26 | } 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /test/integration/samples/oneOfWithDuplicateDescriptions/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "type": "object", 4 | "properties": { 5 | "name": { 6 | "oneOf": [ 7 | { 8 | "type": "object", 9 | "properties": { 10 | "first_name": { 11 | "type": "string" 12 | }, 13 | "created": { 14 | "type": "string", 15 | "description": "Created date" 16 | } 17 | } 18 | }, 19 | { 20 | "type": "object", 21 | "properties": { 22 | "last_name": { 23 | "type": "string" 24 | }, 25 | "created": { 26 | "type": "string", 27 | "description": "Created date" 28 | } 29 | } 30 | } 31 | ] 32 | } 33 | }, 34 | "additionalProperties": false 35 | } 36 | -------------------------------------------------------------------------------- /test/integration/samples/oneOfWithNull/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "mode": "NULLABLE", 6 | "name": "name", 7 | "type": "STRING" 8 | } 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/oneOfWithNull/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "type": "object", 4 | "properties": { 5 | "name": { 6 | "oneOf": [ 7 | { 8 | "type": "string" 9 | }, 10 | { 11 | "type": "null" 12 | } 13 | ] 14 | } 15 | }, 16 | "additionalProperties": false 17 | } 18 | -------------------------------------------------------------------------------- /test/integration/samples/oneof/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "tags", 6 | "mode": "REPEATED", 7 | "type": "RECORD", 8 | "fields": [ 9 | { 10 | "name": "name", 11 | "description": "The name of the tag that you want to include", 12 | "type": "STRING", 13 | "mode": "REQUIRED" 14 | }, 15 | { 16 | "name": "string", 17 | "description": "A string type data point for the tag", 18 | "type": "STRING", 19 | "mode": "NULLABLE" 20 | }, 21 | { 22 | "name": "number", 23 | "description": "A integer type data point for the tag", 24 | "type": "INTEGER", 25 | "mode": "NULLABLE" 26 | } 27 | ] 28 | } 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/integration/samples/oneof/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "allOf": [ 5 | { 6 | "type": "object", 7 | "properties": { 8 | "tags": { 9 | "description": "The users additional information", 10 | "$schema": "http://json-schema.org/draft-04/schema#", 11 | "id": "/models/tags/v2.0.0/schema.json", 12 | "title": "Tag Array", 13 | "oneOf": [ 14 | { 15 | "type": "array", 16 | "items": { 17 | "type": "object", 18 | "redact": { 19 | "type": "key_value", 20 | "items": [ 21 | { 22 | "key_property": "name", 23 | "value_property": "string", 24 | "key_value": "email", 25 | "redact": { 26 | "type": "email" 27 | } 28 | } 29 | ] 30 | }, 31 | "properties": { 32 | "name": { 33 | "type": "string", 34 | "description": "The name of the tag that you want to include" 35 | }, 36 | "string": { 37 | "type": "string", 38 | "description": "A string type data point for the tag" 39 | }, 40 | "number": { 41 | "type": "integer", 42 | "description": "A integer type data point for the tag" 43 | } 44 | }, 45 | "anyOf": [ 46 | { 47 | "required": ["name", "string"] 48 | }, 49 | { 50 | "required": ["name", "number"] 51 | } 52 | ], 53 | "additionalProperties": false 54 | } 55 | }, 56 | { 57 | "type": "null" 58 | } 59 | ] 60 | } 61 | } 62 | } 63 | ], 64 | "additionalProperties": false 65 | } 66 | -------------------------------------------------------------------------------- /test/integration/samples/repeated/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "names", 6 | "type": "STRING", 7 | "mode": "REPEATED" 8 | } 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/repeated/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "names": { 7 | "type": "array", 8 | "items": { 9 | "type": "string" 10 | } 11 | } 12 | }, 13 | "additionalProperties": false 14 | } 15 | -------------------------------------------------------------------------------- /test/integration/samples/simple/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "first_name", 6 | "type": "STRING", 7 | "mode": "NULLABLE" 8 | }, 9 | { 10 | "name": "last_name", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | } 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/integration/samples/simple/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "first_name": { 7 | "type": "string" 8 | }, 9 | "last_name": { 10 | "type": "string" 11 | } 12 | }, 13 | "additionalProperties": false 14 | } 15 | -------------------------------------------------------------------------------- /test/integration/samples/time/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "modified", 6 | "type": "TIME", 7 | "mode": "NULLABLE" 8 | } 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/time/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "modified": { 7 | "type": "string", 8 | "format": "time" 9 | } 10 | }, 11 | "additionalProperties": false 12 | } 13 | -------------------------------------------------------------------------------- /test/integration/samples/timestamp/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "modified", 6 | "type": "TIMESTAMP", 7 | "mode": "NULLABLE" 8 | } 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/integration/samples/timestamp/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "modified": { 7 | "type": "string", 8 | "format": "date-time" 9 | } 10 | }, 11 | "additionalProperties": false 12 | } 13 | -------------------------------------------------------------------------------- /test/integration/samples/typeArray/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": { 3 | "fields": [ 4 | { 5 | "name": "first_name", 6 | "type": "STRING", 7 | "mode": "NULLABLE" 8 | }, 9 | { 10 | "name": "last_name", 11 | "type": "STRING", 12 | "mode": "NULLABLE" 13 | } 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/integration/samples/typeArray/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "http://yourdomain.com/schemas/myschema.json", 3 | "description": "Example description", 4 | "type": "object", 5 | "properties": { 6 | "first_name": { 7 | "type": ["string", "null"] 8 | }, 9 | "last_name": { 10 | "type": ["string", "null"] 11 | } 12 | }, 13 | "required": ["first_name"], 14 | "additionalProperties": false 15 | } 16 | -------------------------------------------------------------------------------- /test/unit/converter.js: -------------------------------------------------------------------------------- 1 | const converter = require('../../src/converter') 2 | const assert = require('assert') 3 | 4 | describe('converter unit', () => { 5 | describe('_visit()', () => { 6 | context('when multiple types are given', () => { 7 | it('throws an error', () => { 8 | assert.throws(() => { 9 | const node = { 10 | type: ['string', 'boolean'], 11 | } 12 | converter._visit('test', node) 13 | }, /Union type not supported/) 14 | }) 15 | }) 16 | }) 17 | 18 | describe('_bigQueryType()', () => { 19 | context('with an unknown type', () => { 20 | it('throws an error', () => { 21 | assert.throws(() => { 22 | const node = { 23 | type: 'foo', 24 | } 25 | converter._bigQueryType(node, 'foo') 26 | }, /SchemaError: Invalid type given: foo/) 27 | }) 28 | }) 29 | }) 30 | 31 | describe('_merge_dicts_array()', () => { 32 | context('with two string enums', () => { 33 | it('returns the correct structure', () => { 34 | const source = [ 35 | { 36 | type: 'string', 37 | enum: ['sweets'], 38 | }, 39 | { 40 | type: 'string', 41 | enum: ['shoes'], 42 | }, 43 | ] 44 | const result = converter._merge_dicts_array('oneOf', {}, source) 45 | const expected = { 46 | enum: ['sweets', 'shoes'], 47 | type: ['string'], 48 | } 49 | assert.deepStrictEqual(result, expected) 50 | }) 51 | }) 52 | 53 | context('with a nested oneOf containing two string enums', () => { 54 | it('returns the correct structure', () => { 55 | const source = [ 56 | { 57 | oneOf: [ 58 | { 59 | type: 'string', 60 | enum: ['sweets'], 61 | }, 62 | { 63 | type: 'string', 64 | enum: ['shoes'], 65 | }, 66 | ], 67 | }, 68 | { 69 | type: 'string', 70 | enum: ['bag'], 71 | }, 72 | ] 73 | const result = converter._merge_dicts_array('oneOf', {}, source) 74 | const expected = { 75 | enum: ['sweets', 'shoes', 'bag'], 76 | type: ['string'], 77 | } 78 | assert.deepStrictEqual(result, expected) 79 | }) 80 | }) 81 | }) 82 | 83 | describe('_object()', () => { 84 | beforeEach(() => { 85 | converter._options = {} 86 | }) 87 | 88 | context('without the "preventAdditionalObjectProperties" option', () => { 89 | it('allows additional properties', () => { 90 | const node = { 91 | properties: { 92 | name: { 93 | type: 'string', 94 | }, 95 | }, 96 | } 97 | assert.doesNotThrow(() => { 98 | converter._object('test', node, 'NULLABLE') 99 | }, /Objects must not have additional or unevaluated properties/) 100 | }) 101 | }) 102 | 103 | context('with the "preventAdditionalObjectProperties" option', () => { 104 | beforeEach(() => { 105 | converter._options = { 106 | preventAdditionalObjectProperties: true, 107 | } 108 | }) 109 | 110 | it('does not allow additional properties', () => { 111 | const node = { 112 | properties: {}, 113 | } 114 | assert.throws(() => { 115 | converter._object('test', node, 'NULLABLE') 116 | }, /Objects must not have additional or unevaluated properties/) 117 | }) 118 | }) 119 | 120 | context( 121 | 'with the "preventAdditionalObjectProperties" and "continueOnError" options', 122 | () => { 123 | let result 124 | 125 | beforeEach(() => { 126 | converter._options = { 127 | preventAdditionalObjectProperties: true, 128 | continueOnError: true, 129 | } 130 | const node = { 131 | properties: {}, 132 | } 133 | result = converter._object('test', node, 'NULLABLE') 134 | }) 135 | 136 | it('skips the field', () => { 137 | const expected = { fields: [] } 138 | assert.deepStrictEqual(result, expected) 139 | }) 140 | }, 141 | ) 142 | 143 | context('with no properties', () => { 144 | it('converts to JSON when properties not defined', () => { 145 | const expected = { 146 | mode: 'NULLABLE', 147 | name: 'test', 148 | type: 'JSON', 149 | } 150 | const result = converter._object('test', {}, 'NULLABLE') 151 | 152 | assert.deepStrictEqual(result, expected) 153 | }) 154 | }) 155 | }) 156 | 157 | describe('run()', () => { 158 | beforeEach(() => { 159 | converter.run( 160 | { 161 | type: 'boolean', 162 | }, 163 | { 164 | option: true, 165 | }, 166 | ) 167 | }) 168 | 169 | it('sets given options', () => { 170 | assert.deepStrictEqual(converter._options, { 171 | option: true, 172 | }) 173 | }) 174 | }) 175 | 176 | describe('_scalar()', () => { 177 | context('with a field beginning with a number', () => { 178 | it('throws an error', () => { 179 | assert.throws(() => { 180 | converter._scalar('123test', 'STRING', 'NULLABLE') 181 | }, /Invalid field name: 123test/) 182 | }) 183 | }) 184 | 185 | context('with a field containing non alphanumeric characters', () => { 186 | it('throws an error', () => { 187 | assert.throws(() => { 188 | converter._scalar('test!', 'STRING', 'NULLABLE') 189 | }, /Invalid field name: test!/) 190 | }) 191 | }) 192 | 193 | context('with an single character field', () => { 194 | it('returns a bigquery field object', () => { 195 | assert.deepStrictEqual(converter._scalar('t', 'STRING', 'NULLABLE'), { 196 | mode: 'NULLABLE', 197 | name: 't', 198 | type: 'STRING', 199 | }) 200 | }) 201 | }) 202 | 203 | context('with a valid field', () => { 204 | it('returns a bigquery field object', () => { 205 | assert.deepStrictEqual( 206 | converter._scalar('test123', 'STRING', 'NULLABLE'), 207 | { 208 | mode: 'NULLABLE', 209 | name: 'test123', 210 | type: 'STRING', 211 | }, 212 | ) 213 | }) 214 | }) 215 | }) 216 | }) 217 | -------------------------------------------------------------------------------- /test/unit/utils.js: -------------------------------------------------------------------------------- 1 | const utils = require('../../src/utils') 2 | const assert = require('assert') 3 | 4 | describe('utils', () => { 5 | describe('get_table_id()', () => { 6 | it('it supports a short path', () => { 7 | assert.equal( 8 | utils.get_table_id('/events/one/two/v1.2.3/schema.json'), 9 | 'one_two_v1', 10 | ) 11 | }) 12 | 13 | it('it supports a long path', () => { 14 | assert.equal( 15 | utils.get_table_id('/events/one/two/three/four/v1.2.3/schema.json'), 16 | 'one_two_three_four_v1', 17 | ) 18 | }) 19 | }) 20 | }) 21 | --------------------------------------------------------------------------------