├── VERSION.md ├── test ├── fixtures │ ├── moreData.json │ ├── simpleData.json │ ├── allWrong.zip │ ├── multiZip.zip │ ├── simpleData.gz │ ├── simpleZip.zip │ ├── dataWithoutVersion.json │ ├── dataNonStringVersion.json │ ├── dataWithVersion.json │ └── sampleSchema.json ├── DockerManager.test.ts ├── commands.test.ts ├── SchemaManager.test.ts └── DownloadManager.test.ts ├── .gitignore ├── .prettierrc ├── .gitmodules ├── DEPENDENCY-NOTES.md ├── jest.config.js ├── Dockerfile ├── .github └── workflows │ ├── publish.yml │ └── main.yml ├── tsconfig.json ├── src ├── logger.ts ├── index.ts ├── DockerManager.ts ├── DownloadManager.ts ├── commands.ts ├── SchemaManager.ts └── utils.ts ├── eslint.config.mjs ├── test-files ├── allowed-amounts.json ├── allowed-amounts-error.json └── in-network-rates-fee-for-service-sample.json ├── package.json ├── README.md ├── LICENSE └── schemavalidator.cpp /VERSION.md: -------------------------------------------------------------------------------- 1 | 1.2.0 2 | -------------------------------------------------------------------------------- /test/fixtures/moreData.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": "This is a simple JSON file.", 3 | "cookie": true 4 | } -------------------------------------------------------------------------------- /test/fixtures/simpleData.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": "This is a simple JSON file.", 3 | "cookie": true 4 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/** 2 | out/** 3 | schema-repo/ 4 | schema-repo/** 5 | coverage 6 | validator 7 | .vscode 8 | .DS_Store 9 | -------------------------------------------------------------------------------- /test/fixtures/allWrong.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/price-transparency-guide-validator/HEAD/test/fixtures/allWrong.zip -------------------------------------------------------------------------------- /test/fixtures/multiZip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/price-transparency-guide-validator/HEAD/test/fixtures/multiZip.zip -------------------------------------------------------------------------------- /test/fixtures/simpleData.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/price-transparency-guide-validator/HEAD/test/fixtures/simpleData.gz -------------------------------------------------------------------------------- /test/fixtures/simpleZip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/price-transparency-guide-validator/HEAD/test/fixtures/simpleZip.zip -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "none", 3 | "arrowParens": "avoid", 4 | "endOfLine": "auto", 5 | "printWidth": 100, 6 | "tabWidth": 2, 7 | "singleQuote": true 8 | } -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "rapidjson"] 2 | path = rapidjson 3 | url = https://github.com/Tencent/rapidjson.git 4 | branch = master 5 | [submodule "tclap"] 6 | path = tclap 7 | url = https://git.code.sf.net/p/tclap/code 8 | branch = 1.4 9 | -------------------------------------------------------------------------------- /DEPENDENCY-NOTES.md: -------------------------------------------------------------------------------- 1 | The `npm outdated` command lists some dependencies as outdated. The reasons for this are given below: 2 | 3 | - `@types/node`: Node's newest LTS version is 20. Keep types at 20 until Node 22 is marked as LTS. 4 | - `chalk`: Version 5 is an esmodule. Keep updated to newest 4.x.x release. 5 | -------------------------------------------------------------------------------- /test/fixtures/dataWithoutVersion.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporting_entity_name": "medicare", 3 | "reporting_entity_type": "medicare", 4 | "plan_name": "medicaid", 5 | "plan_id_type": "hios", 6 | "plan_id": "1111111111", 7 | "plan_market_type": "individual", 8 | "last_updated_on": "2020-08-27" 9 | } 10 | -------------------------------------------------------------------------------- /test/fixtures/dataNonStringVersion.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporting_entity_name": "medicare", 3 | "reporting_entity_type": "medicare", 4 | "plan_name": "medicaid", 5 | "plan_id_type": "hios", 6 | "plan_id": "1111111111", 7 | "plan_market_type": "individual", 8 | "last_updated_on": "2020-08-27", 9 | "version": 1.5 10 | } 11 | -------------------------------------------------------------------------------- /test/fixtures/dataWithVersion.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporting_entity_name": "medicare", 3 | "reporting_entity_type": "medicare", 4 | "plan_name": "medicaid", 5 | "plan_id_type": "hios", 6 | "plan_id": "1111111111", 7 | "plan_market_type": "individual", 8 | "last_updated_on": "2020-08-27", 9 | "version": "1.2.0" 10 | } 11 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | transform: { 3 | '^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }] 4 | }, 5 | moduleFileExtensions: ['js', 'ts', 'json'], 6 | testMatch: ['**/test/**/*.test.(ts|js)'], 7 | testEnvironment: 'node', 8 | setupFilesAfterEnv: ['jest-extended/all'], 9 | preset: 'ts-jest' 10 | }; -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu as build 2 | 3 | ARG VERSION=v1.0.0 4 | RUN apt-get update 5 | RUN apt-get install -y g++ cmake doxygen valgrind wget 6 | COPY ./schemavalidator.cpp / 7 | COPY ./rapidjson /rapidjson 8 | COPY ./tclap /tclap 9 | RUN g++ -O3 --std=c++17 -I /rapidjson/include -I /tclap/include/ schemavalidator.cpp -o validator -lstdc++fs 10 | 11 | FROM ubuntu 12 | COPY --from=build /validator /validator 13 | ENTRYPOINT ["/validator"] 14 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Package to npmjs 2 | on: 3 | release: 4 | types: [published] 5 | jobs: 6 | build-and-test: 7 | uses: ./.github/workflows/main.yml # test-run workflow 8 | publish-npm: 9 | needs: build-and-test 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | # Setup .npmrc file to publish to npm 14 | - uses: actions/setup-node@v3 15 | with: 16 | node-version: '16.x' 17 | registry-url: 'https://registry.npmjs.org' 18 | - run: npm ci 19 | - run: npm publish 20 | env: 21 | NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_KEY }} 22 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Autotest 2 | 3 | on: [push, pull_request, workflow_call] 4 | 5 | jobs: 6 | test: 7 | name: Test on node ${{ matrix.node-version }} and ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest, windows-latest, macos-latest] 12 | node-version: [16, 18] 13 | 14 | steps: 15 | - uses: actions/checkout@v1 16 | - name: Use Node.js ${{ matrix.node-version }} 17 | uses: actions/setup-node@v1 18 | with: 19 | node-version: ${{ matrix.node-version }} 20 | - run: npm install 21 | - run: npm test 22 | env: 23 | CI: true 24 | -------------------------------------------------------------------------------- /test/fixtures/sampleSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "definitions": { 4 | "food": { 5 | "enum": [ 6 | "orange", 7 | "apple", 8 | "banana", 9 | "pear" 10 | ] 11 | }, 12 | "garment": { 13 | "type": "object", 14 | "properties": { 15 | "color": { 16 | "type": "string" 17 | }, 18 | "material": { 19 | "type": "string" 20 | } 21 | } 22 | } 23 | }, 24 | "type": "object", 25 | "properties": { 26 | "name": { 27 | "type": "string" 28 | }, 29 | "snacks": { 30 | "type": "array", 31 | "items": { 32 | "$ref": "#/definitions/food" 33 | } 34 | }, 35 | "clothes": { 36 | "type": "array", 37 | "items": { 38 | "$ref": "#/definitions/garment" 39 | } 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, 4 | "module": "commonjs" /* Specify what module code is generated. */, 5 | "baseUrl": "./" /* Specify the base directory to resolve non-relative module names. */, 6 | "paths": { 7 | "*": ["node_modules/*"] 8 | } /* Specify a set of entries that re-map imports to additional lookup locations. */, 9 | "outDir": "out" /* Specify an output folder for all emitted files. */, 10 | "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, 11 | "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, 12 | "noImplicitAny": true, 13 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 14 | }, 15 | "include": ["src/**/*"] 16 | } 17 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import { createLogger, format, transports, Logger, LeveledLogMethod } from 'winston'; 2 | import chalk from 'chalk'; 3 | 4 | const { printf } = format; 5 | 6 | const customLevels = { 7 | error: 0, 8 | warn: 1, 9 | menu: 2, 10 | info: 2, 11 | debug: 3 12 | }; 13 | 14 | const printer = printf(info => { 15 | let level: string; 16 | switch (info.level) { 17 | case 'debug': 18 | level = chalk.whiteBright.bgBlue(`${info.level}`); 19 | break; 20 | case 'info': 21 | level = chalk.whiteBright.bgGreen(`${info.level} `); 22 | break; 23 | case 'warn': 24 | level = chalk.whiteBright.bgRgb(195, 105, 0)(`${info.level} `); 25 | break; 26 | case 'error': 27 | level = chalk.whiteBright.bgRed(`${info.level}`); 28 | break; 29 | default: 30 | break; 31 | } 32 | return `${level} ${info.message}`; 33 | }); 34 | 35 | export const logger = createLogger({ 36 | levels: customLevels, 37 | format: printer, 38 | transports: [new transports.Console()] 39 | }) as Logger & Record; 40 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import tsParser from '@typescript-eslint/parser'; 2 | import path from 'node:path'; 3 | import { fileURLToPath } from 'node:url'; 4 | import js from '@eslint/js'; 5 | import { FlatCompat } from '@eslint/eslintrc'; 6 | 7 | const __filename = fileURLToPath(import.meta.url); 8 | const __dirname = path.dirname(__filename); 9 | const compat = new FlatCompat({ 10 | baseDirectory: __dirname, 11 | recommendedConfig: js.configs.recommended, 12 | allConfig: js.configs.all 13 | }); 14 | 15 | export default [ 16 | { 17 | ignores: [ 18 | 'out/*', 19 | 'coverage/*', 20 | 'schema-repo/*', 21 | 'rapidjson/*', 22 | 'tclap/*', 23 | '**/*.d.ts' 24 | ] 25 | }, 26 | ...compat.extends('plugin:@typescript-eslint/recommended', 'prettier'), 27 | { 28 | languageOptions: { 29 | parser: tsParser, 30 | ecmaVersion: 2018, 31 | sourceType: 'module' 32 | }, 33 | 34 | rules: { 35 | semi: ['error', 'always'], 36 | 37 | quotes: [ 38 | 'error', 39 | 'single', 40 | { 41 | avoidEscape: true 42 | } 43 | ], 44 | 45 | '@typescript-eslint/ban-ts-comment': 'off', 46 | '@typescript-eslint/explicit-function-return-type': 'off', 47 | '@typescript-eslint/no-empty-function': 'off', 48 | '@typescript-eslint/no-explicit-any': 'off', 49 | '@typescript-eslint/no-unsafe-declaration-merging': 'off' 50 | } 51 | } 52 | ]; 53 | -------------------------------------------------------------------------------- /test-files/allowed-amounts.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporting_entity_name": "medicare", 3 | "reporting_entity_type": "medicare", 4 | "reporting_plans":[{ 5 | "plan_name": "medicare", 6 | "plan_id_type": "hios", 7 | "plan_id": "11111111111", 8 | "plan_market_type": "individual" 9 | }, { 10 | "plan_name": "medicaid", 11 | "plan_id_type": "hios", 12 | "plan_id": "0000000000", 13 | "plan_market_type": "individual" 14 | }], 15 | "last_updated_on": "2020-08-27", 16 | "version": "1.0.0", 17 | "out_of_network":[{ 18 | "name": "Established Patient Office or Other Outpatient Services", 19 | "billing_code_type": "CPT", 20 | "billing_code_type_version": "2020", 21 | "billing_code": "99214", 22 | "description": "office or other outpatient visits for the evaluation and management of an established patient, which requires at least two of these three key components: a detailed history, a detailed examination and medical decision making of moderate complexity", 23 | "allowed_amounts": [{ 24 | "tin": { 25 | "type": "ein", 26 | "value": "1234567890" 27 | }, 28 | "service_code": ["01", "02", "03"], 29 | "billing_class": "professional", 30 | "payments": [{ 31 | "allowed_amount": 25.00, 32 | "providers": [{ 33 | "billed_charge": 50.00, 34 | "npi": [1234567891,1234567892,1234567893] 35 | },{ 36 | "billed_charge": 60.00, 37 | "npi": [1111111111] 38 | },{ 39 | "billed_charge": 70.00, 40 | "npi": [2222222222,3333333333,4444444444,5555555555] 41 | }] 42 | }] 43 | }] 44 | }] 45 | } -------------------------------------------------------------------------------- /test-files/allowed-amounts-error.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporting_entity_name": "medicare", 3 | "reporting_entity_type": "medicare", 4 | "rporting_plans":[{ 5 | "plan_name": "medicare", 6 | "plan_id_type": "hios", 7 | "plan_id": "11111111111", 8 | "plan_market_type": "individual" 9 | },{ 10 | "plan_name": "medicaid", 11 | "plan_id_type": "hios", 12 | "plan_id": "0000000000", 13 | "plan_market_type": "individual" 14 | }], 15 | "last_updated_on": "Jan 2nd, 2020", 16 | "version": "1.0.0", 17 | "out_of_network":[{ 18 | "Name": "Established Patient Office or Other Outpatient Services", 19 | "billing_code_type": "CPT", 20 | "billing_code_type_version": "2020", 21 | "billing_code": "99214", 22 | "description": "office or other outpatient visits for the evaluation and management of an established patient, which requires at least two of these three key components: a detailed history, a detailed examination and medical decision making of moderate complexity", 23 | "allowed_amounts": [{ 24 | "tin": { 25 | "type": "ein", 26 | "value": "1234567890" 27 | }, 28 | "service_code": ["01", "02", "03", 4], 29 | "billing_class": "professional", 30 | "payments": [{ 31 | "allowed_amount": "25.00", 32 | "providers": [{ 33 | "billed_charge": 50.00, 34 | "npi": [1234567891,1234567892,1234567893] 35 | },{ 36 | "billed_charge": 60.00, 37 | "npi": [1111111111] 38 | },{ 39 | "billed_charge": 70.00, 40 | "npi": [2222222222,3333333333,4444444444,5555555555] 41 | }] 42 | }] 43 | }] 44 | }] 45 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cms-mrf-validator", 3 | "version": "2.2.0", 4 | "description": "Node-based entry point for machine-readable file validator", 5 | "main": "out/index.js", 6 | "bin": { 7 | "cms-mrf-validator": "out/index.js" 8 | }, 9 | "files": [ 10 | "out/**/*.js" 11 | ], 12 | "scripts": { 13 | "build": "del-cli out && tsc", 14 | "lint": "eslint \"**/*.{js,ts}\"", 15 | "lint:fix": "tsc --noEmit && eslint \"**/*.{js,ts}\" --quiet --fix", 16 | "prettier": "prettier --check \"src/*.ts\"", 17 | "prettier:fix": "prettier --write \"**/*.{js,ts}\"", 18 | "prepare": "npm run build", 19 | "test": "node --expose-gc ./node_modules/jest/bin/jest --runInBand --logHeapUsage" 20 | }, 21 | "repository": { 22 | "type": "git", 23 | "url": "git+https://github.com/CMSgov/price-transparency-guide-validator.git" 24 | }, 25 | "author": "", 26 | "license": "Apache-2.0", 27 | "bugs": { 28 | "url": "https://github.com/CMSgov/price-transparency-guide-validator/issues" 29 | }, 30 | "homepage": "https://github.com/CMSgov/price-transparency-guide-validator#readme", 31 | "devDependencies": { 32 | "@types/fs-extra": "^11.0.4", 33 | "@types/jest": "^29.5.13", 34 | "@types/node": "^20.16.5", 35 | "@types/readline-sync": "^1.4.8", 36 | "@types/temp": "^0.9.4", 37 | "@typescript-eslint/eslint-plugin": "^8.44.1", 38 | "@typescript-eslint/parser": "^8.44.1", 39 | "@types/yauzl": "^2.10.3", 40 | "del-cli": "^5.1.0", 41 | "eslint": "^9.10.0", 42 | "eslint-config-prettier": "^9.1.0", 43 | "jest": "^29.7.0", 44 | "jest-extended": "^4.0.2", 45 | "nock": "^13.5.5", 46 | "prettier": "^3.3.3", 47 | "ts-jest": "^29.2.5", 48 | "typescript": "^5.6.0" 49 | }, 50 | "dependencies": { 51 | "@streamparser/json": "^0.0.21", 52 | "@streamparser/json-node": "^0.0.21", 53 | "axios": "^1.2.1", 54 | "chalk": "^4.1.2", 55 | "commander": "^12.1.0", 56 | "fs-extra": "^11.2.0", 57 | "readline-sync": "^1.4.10", 58 | "temp": "^0.9.4", 59 | "winston": "^3.14.2", 60 | "yauzl": "^3.1.3" 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /test/DockerManager.test.ts: -------------------------------------------------------------------------------- 1 | import 'jest-extended'; 2 | import path from 'path'; 3 | import { DockerManager } from '../src/DockerManager'; 4 | 5 | const SEP = path.sep; 6 | const PROJECT_DIR = path.resolve(path.join(__dirname, '..')); 7 | 8 | describe('DockerManager', () => { 9 | describe('#buildRunCommand', () => { 10 | it('should build the command to run the validation container', () => { 11 | const dockerManager = new DockerManager(); 12 | dockerManager.containerId = 'dadb0d'; 13 | const result = dockerManager.buildRunCommand( 14 | path.join('some', 'useful', 'schema.json'), // this is a relative path 15 | SEP + path.join('other', 'data', 'data.json'), // this is an absolute path 16 | path.join('results', 'output'), // this is a relative path 17 | 'table-of-contents' 18 | ); 19 | const expectedCommand = `docker run --rm -v "${path.join( 20 | PROJECT_DIR, 21 | 'some', 22 | 'useful' 23 | )}":/schema/ -v "${path.join(path.resolve(SEP), 'other', 'data')}":/data/ -v "${path.join( 24 | PROJECT_DIR, 25 | 'results', 26 | 'output' 27 | )}":/output/ dadb0d "schema/schema.json" "data/data.json" -o "output/" -s table-of-contents -f`; 28 | expect(result).toBe(expectedCommand); 29 | }); 30 | 31 | it('should build the command to run the validation container when the options are set to display all errors', () => { 32 | const dockerManager = new DockerManager('./command-output.txt', true); 33 | dockerManager.containerId = 'dadb0d'; 34 | const result = dockerManager.buildRunCommand( 35 | path.join('some', 'useful', 'schema.json'), // this is a relative path 36 | SEP + path.join('other', 'data', 'data.json'), // this is an absolute path 37 | path.join('results', 'output'), // this is a relative path 38 | 'table-of-contents' 39 | ); 40 | const expectedCommand = `docker run --rm -v "${path.join( 41 | PROJECT_DIR, 42 | 'some', 43 | 'useful' 44 | )}":/schema/ -v "${path.join(path.resolve(SEP), 'other', 'data')}":/data/ -v "${path.join( 45 | PROJECT_DIR, 46 | 'results', 47 | 'output' 48 | )}":/output/ dadb0d "schema/schema.json" "data/data.json" -o "output/" -s table-of-contents`; 49 | expect(result).toBe(expectedCommand); 50 | }); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import { Command, Option } from 'commander'; 4 | import { validate, update, validateFromUrl } from './commands'; 5 | import { config } from './utils'; 6 | import { logger } from './logger'; 7 | 8 | main().catch(error => { 9 | logger.error(`Encountered an unexpected error: ${error}`); 10 | }); 11 | 12 | async function main() { 13 | const program = new Command() 14 | .name('cms-mrf-validator') 15 | .description('Tool for validating health coverage machine-readable files.') 16 | .option('-d, --debug', 'show debug output') 17 | .hook('preAction', thisCommand => { 18 | if (thisCommand.opts().debug) { 19 | logger.level = 'debug'; 20 | logger.debug(process.argv.join(' ')); 21 | } 22 | }); 23 | 24 | program 25 | .command('validate') 26 | .description('Validate a file against a specific published version of a CMS schema.') 27 | .usage(' [options]') 28 | .argument('', 'path to data file to validate') 29 | .option('--schema-version ', 'version of schema to use for validation') 30 | .option('-o, --out ', 'output path') 31 | .addOption( 32 | new Option('-t, --target ', 'name of schema to use') 33 | .choices(config.AVAILABLE_SCHEMAS) 34 | .default('in-network-rates') 35 | ) 36 | .option( 37 | '-s, --strict', 38 | 'enable strict checking, which prohibits additional properties in data file' 39 | ) 40 | .option('-y, --yes-all', 'automatically respond "yes" to confirmation prompts') 41 | .option('-a, --all-errors', 'continue validating after errors are found') 42 | .action(validate); 43 | 44 | program 45 | .command('from-url') 46 | .description( 47 | 'Validate the file retrieved from a URL against a specific published version of a CMS schema.' 48 | ) 49 | .usage(' [options]') 50 | .argument('', 'URL to data file to validate') 51 | .option('--schema-version ', 'version of schema to use for validation') 52 | .option('-o, --out ', 'output path') 53 | .addOption( 54 | new Option('-t, --target ', 'name of schema to use') 55 | .choices(config.AVAILABLE_SCHEMAS) 56 | .default('in-network-rates') 57 | ) 58 | .option( 59 | '-s, --strict', 60 | 'enable strict checking, which prohibits additional properties in data file' 61 | ) 62 | .option('-y, --yes-all', 'automatically respond "yes" to confirmation prompts') 63 | .option('-a, --all-errors', 'continue validating after errors are found') 64 | .action((dataUrl, options) => { 65 | validateFromUrl(dataUrl, options).then(result => { 66 | if (result) { 67 | process.exitCode = 0; 68 | } else { 69 | process.exitCode = 1; 70 | } 71 | }); 72 | }); 73 | 74 | program 75 | .command('update') 76 | .description('Update the available schemas from the CMS repository.') 77 | .action(update); 78 | 79 | program.parseAsync(process.argv); 80 | } 81 | -------------------------------------------------------------------------------- /test-files/in-network-rates-fee-for-service-sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporting_entity_name": "medicare", 3 | "reporting_entity_type": "medicare", 4 | "reporting_plans": [{ 5 | "plan_name": "medicaid", 6 | "plan_id_type": "hios", 7 | "plan_id": "11111111111", 8 | "plan_market_type": "individual" 9 | },{ 10 | "plan_name": "medicare", 11 | "plan_id_type": "hios", 12 | "plan_id": "0000000000", 13 | "plan_market_type": "individual" 14 | }], 15 | "last_updated_on": "2020-08-27", 16 | "version": "1.0.0", 17 | "in_network": [{ 18 | "negotiation_arrangement": "ffs", 19 | "name": "Knee Replacement", 20 | "billing_code_type": "CPT", 21 | "billing_code_type_version": "2020", 22 | "billing_code": "27447", 23 | "description": "Arthroplasty, knee condyle and plateau, medial and lateral compartments", 24 | "negotiated_rates": [{ 25 | "provider_groups": [{ 26 | "npi": [1111111111, 2222222222, 3333333333, 4444444444, 5555555555], 27 | "tin":{ 28 | "type": "ein", 29 | "value": "11-1111111" 30 | } 31 | },{ 32 | "npi": [1111111111, 2222222222, 3333333333, 4444444444, 5555555555], 33 | "tin":{ 34 | "type": "ein", 35 | "value": "22-2222222" 36 | } 37 | }], 38 | "negotiated_prices": [{ 39 | "negotiated_type": "negotiated", 40 | "negotiated_rate": 123.45, 41 | "expiration_date": "2022-01-01", 42 | "service_code": ["18", "19", "11"], 43 | "billing_class": "professional" 44 | },{ 45 | "negotiated_type": "negotiated", 46 | "negotiated_rate": 1230.45, 47 | "expiration_date": "2022-01-01", 48 | "billing_class": "institutional" 49 | }] 50 | },{ 51 | "provider_groups": [{ 52 | "npi": [6666666666, 7777777777, 8888888888, 9999999999], 53 | "tin":{ 54 | "type": "ein", 55 | "value": "22-2222222" 56 | } 57 | }], 58 | "negotiated_prices": [{ 59 | "negotiated_type": "negotiated", 60 | "negotiated_rate": 120.45, 61 | "expiration_date": "2022-01-01", 62 | "service_code": ["05", "06", "07"], 63 | "billing_class": "professional" 64 | }] 65 | }] 66 | },{ 67 | "negotiation_arrangement": "ffs", 68 | "name": "Femur and Knee Joint Repair", 69 | "billing_code_type": "CPT", 70 | "billing_code_type_version": "2020", 71 | "billing_code": "27448", 72 | "description": "Under Repair, Revision, and/or Reconstruction Procedures on the Femur (Thigh Region) and Knee Joint", 73 | "negotiated_rates": [{ 74 | "provider_groups": [{ 75 | "npi": [1111111111, 2222222222, 3333333333, 4444444444, 5555555555], 76 | "tin":{ 77 | "type": "ein", 78 | "value": "11-1111111" 79 | } 80 | },{ 81 | "npi": [1111111111, 2222222222, 3333333333, 4444444444, 5555555555], 82 | "tin":{ 83 | "type": "ein", 84 | "value": "22-2222222" 85 | } 86 | }], 87 | "negotiated_prices": [{ 88 | "negotiated_type": "negotiated", 89 | "negotiated_rate": 12003.45, 90 | "expiration_date": "2022-01-01", 91 | "service_code": ["18", "19", "11"], 92 | "billing_class": "professional" 93 | }] 94 | },{ 95 | "provider_groups": [{ 96 | "npi": [6666666666], 97 | "tin":{ 98 | "type": "npi", 99 | "value": "6666666666" 100 | } 101 | }], 102 | "negotiated_prices": [{ 103 | "negotiated_type": "negotiated", 104 | "negotiated_rate": 12.45, 105 | "expiration_date": "2022-01-01", 106 | "service_code": ["18", "19", "11"], 107 | "billing_class": "institutional" 108 | }] 109 | }] 110 | }] 111 | } -------------------------------------------------------------------------------- /src/DockerManager.ts: -------------------------------------------------------------------------------- 1 | import util from 'util'; 2 | import path from 'path'; 3 | import { exec } from 'child_process'; 4 | import fs from 'fs-extra'; 5 | import temp from 'temp'; 6 | import { logger } from './logger'; 7 | 8 | export class DockerManager { 9 | containerId = ''; 10 | 11 | constructor( 12 | public outputPath = '', 13 | public allErrors = false 14 | ) {} 15 | 16 | private async initContainerId(): Promise { 17 | this.containerId = await util 18 | .promisify(exec)('docker images validator:latest --format "{{.ID}}"') 19 | .then(result => result.stdout.trim()) 20 | .catch(reason => { 21 | logger.error(reason.stderr); 22 | return ''; 23 | }); 24 | } 25 | 26 | async runContainer( 27 | schemaPath: string, 28 | schemaName: string, 29 | dataPath: string, 30 | outputPath = this.outputPath 31 | ): Promise { 32 | try { 33 | if (this.containerId.length === 0) { 34 | await this.initContainerId(); 35 | } 36 | if (this.containerId.length > 0) { 37 | // make temp dir for output 38 | temp.track(); 39 | const outputDir = temp.mkdirSync('output'); 40 | const containerOutputPath = path.join(outputDir, 'output.txt'); 41 | const containerLocationPath = path.join(outputDir, 'locations.json'); 42 | // copy output files after it finishes 43 | const runCommand = this.buildRunCommand(schemaPath, dataPath, outputDir, schemaName); 44 | logger.info('Running validator container...'); 45 | logger.debug(runCommand); 46 | return util 47 | .promisify(exec)(runCommand) 48 | .then(() => { 49 | const containerResult: ContainerResult = { pass: true }; 50 | if (fs.existsSync(containerOutputPath)) { 51 | if (outputPath) { 52 | fs.copySync(containerOutputPath, outputPath); 53 | } else { 54 | const outputText = fs.readFileSync(containerOutputPath, 'utf-8'); 55 | logger.info(outputText); 56 | } 57 | } 58 | if (fs.existsSync(containerLocationPath)) { 59 | try { 60 | containerResult.locations = fs.readJsonSync(containerLocationPath); 61 | } catch { 62 | // something went wrong when reading the location file that the validator produced 63 | } 64 | } 65 | return containerResult; 66 | }) 67 | .catch(() => { 68 | if (fs.existsSync(containerOutputPath)) { 69 | if (outputPath) { 70 | fs.copySync(containerOutputPath, outputPath); 71 | } else { 72 | const outputText = fs.readFileSync(containerOutputPath, 'utf-8'); 73 | logger.info(outputText); 74 | } 75 | } 76 | process.exitCode = 1; 77 | return { pass: false }; 78 | }); 79 | } else { 80 | logger.error('Could not find a validator docker container.'); 81 | process.exitCode = 1; 82 | return { pass: false }; 83 | } 84 | } catch (error) { 85 | logger.error(`Error when running validator container: ${error}`); 86 | process.exitCode = 1; 87 | return { pass: false }; 88 | } 89 | } 90 | 91 | buildRunCommand( 92 | schemaPath: string, 93 | dataPath: string, 94 | outputDir: string, 95 | schemaName: string 96 | ): string { 97 | // figure out mount for schema file 98 | const absoluteSchemaPath = path.resolve(schemaPath); 99 | const schemaDir = path.dirname(absoluteSchemaPath); 100 | const schemaFile = path.basename(absoluteSchemaPath); 101 | // figure out mount for data file 102 | const absoluteDataPath = path.resolve(dataPath); 103 | const dataDir = path.dirname(absoluteDataPath); 104 | const dataFile = path.basename(absoluteDataPath); 105 | const failFastArg = this.allErrors ? '' : ' -f'; 106 | return `docker run --rm -v "${schemaDir}":/schema/ -v "${dataDir}":/data/ -v "${path.resolve( 107 | outputDir 108 | )}":/output/ ${ 109 | this.containerId 110 | } "schema/${schemaFile}" "data/${dataFile}" -o "output/" -s ${schemaName}${failFastArg}`; 111 | } 112 | } 113 | 114 | export type ContainerResult = { 115 | pass: boolean; 116 | text?: string; 117 | locations?: { 118 | inNetwork?: string[]; 119 | allowedAmount?: string[]; 120 | providerReference?: string[]; 121 | }; 122 | }; 123 | -------------------------------------------------------------------------------- /test/commands.test.ts: -------------------------------------------------------------------------------- 1 | import 'jest-extended'; 2 | import path from 'path'; 3 | import { validate, validateFromUrl } from '../src/commands'; 4 | import { SchemaManager } from '../src/SchemaManager'; 5 | import { DockerManager } from '../src/DockerManager'; 6 | import { DownloadManager } from '../src/DownloadManager'; 7 | 8 | describe('commands', () => { 9 | let checkDataUrlSpy: jest.SpyInstance; 10 | let ensureRepoSpy: jest.SpyInstance; 11 | let useVersionSpy: jest.SpyInstance; 12 | let useSchemaSpy: jest.SpyInstance; 13 | let downloadDataSpy: jest.SpyInstance; 14 | let runContainerSpy: jest.SpyInstance; 15 | 16 | beforeAll(() => { 17 | checkDataUrlSpy = jest.spyOn(DownloadManager.prototype, 'checkDataUrl'); 18 | ensureRepoSpy = jest 19 | .spyOn(SchemaManager.prototype, 'ensureRepo') 20 | .mockResolvedValue({ stdout: '', stderr: '' }); 21 | useVersionSpy = jest.spyOn(SchemaManager.prototype, 'useVersion').mockResolvedValue(true); 22 | useSchemaSpy = jest.spyOn(SchemaManager.prototype, 'useSchema').mockResolvedValue('schemaPath'); 23 | downloadDataSpy = jest 24 | .spyOn(DownloadManager.prototype, 'downloadDataFile') 25 | .mockResolvedValue('data.json'); 26 | runContainerSpy = jest 27 | .spyOn(DockerManager.prototype, 'runContainer') 28 | .mockResolvedValue({ pass: false }); 29 | }); 30 | 31 | beforeEach(() => { 32 | checkDataUrlSpy.mockClear(); 33 | ensureRepoSpy.mockClear(); 34 | useVersionSpy.mockClear(); 35 | useSchemaSpy.mockClear(); 36 | downloadDataSpy.mockClear(); 37 | runContainerSpy.mockClear(); 38 | }); 39 | 40 | afterAll(() => { 41 | checkDataUrlSpy.mockRestore(); 42 | ensureRepoSpy.mockRestore(); 43 | useVersionSpy.mockRestore(); 44 | useSchemaSpy.mockRestore(); 45 | downloadDataSpy.mockRestore(); 46 | runContainerSpy.mockRestore(); 47 | }); 48 | 49 | describe('#validate', () => { 50 | it('should continue processing when the data file exists', async () => { 51 | await validate(path.join(__dirname, '..', 'test-files', 'allowed-amounts.json'), { 52 | target: null 53 | }); 54 | expect(useVersionSpy).toHaveBeenCalledTimes(1); 55 | }); 56 | 57 | it('should not continue processing when the data file does not exist', async () => { 58 | await validate(path.join(__dirname, '..', 'test-files', 'not-real.json'), { target: null }); 59 | expect(useVersionSpy).toHaveBeenCalledTimes(0); 60 | }); 61 | 62 | it('should run the container when the requested schema is available', async () => { 63 | useSchemaSpy.mockResolvedValueOnce('good.json'); 64 | await validate(path.join(__dirname, '..', 'test-files', 'allowed-amounts.json'), { 65 | target: null 66 | }); 67 | expect(runContainerSpy).toHaveBeenCalledTimes(1); 68 | }); 69 | 70 | it('should not run the container when the requested schema is not available', async () => { 71 | await validate(path.join(__dirname, '..', 'test-files', 'not-real.json'), { target: null }); 72 | expect(runContainerSpy).toHaveBeenCalledTimes(0); 73 | }); 74 | }); 75 | 76 | describe('#validateFromUrl', () => { 77 | it('should continue processing when the data url is valid and content length is less than or equal to the size limit', async () => { 78 | checkDataUrlSpy.mockResolvedValueOnce(true); 79 | await validateFromUrl('http://example.org/data.json', { schemaVersion: 'v1.0.0' }); 80 | expect(useSchemaSpy).toHaveBeenCalledTimes(1); 81 | }); 82 | 83 | it('should not continue processing when the data url is invalid', async () => { 84 | checkDataUrlSpy.mockResolvedValueOnce(false); 85 | await validateFromUrl('http://example.org/data.json', { schemaVersion: 'v1.0.0' }); 86 | expect(useSchemaSpy).toHaveBeenCalledTimes(0); 87 | }); 88 | 89 | it('should download the data file when the data url is valid and the requested schema is available', async () => { 90 | checkDataUrlSpy.mockResolvedValueOnce(true); 91 | useSchemaSpy.mockResolvedValueOnce('schemapath.json'); 92 | downloadDataSpy.mockResolvedValueOnce('data.json'); 93 | await validateFromUrl('http://example.org/data.json', { 94 | target: 'in-network-rates', 95 | schemaVersion: 'v1.0.0' 96 | }); 97 | expect(downloadDataSpy).toHaveBeenCalledTimes(1); 98 | expect(downloadDataSpy).toHaveBeenCalledWith('http://example.org/data.json'); 99 | expect(runContainerSpy).toHaveBeenCalledTimes(1); 100 | expect(runContainerSpy).toHaveBeenCalledWith( 101 | 'schemapath.json', 102 | 'in-network-rates', 103 | 'data.json' 104 | ); 105 | }); 106 | 107 | it('should not download the data file when the requested schema is not available', async () => { 108 | checkDataUrlSpy.mockResolvedValueOnce(true); 109 | useSchemaSpy.mockResolvedValueOnce(null); 110 | await validateFromUrl('http://example.org/data.json', { schemaVersion: 'v1.0.0' }); 111 | expect(downloadDataSpy).toHaveBeenCalledTimes(0); 112 | }); 113 | }); 114 | }); 115 | -------------------------------------------------------------------------------- /src/DownloadManager.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import readlineSync from 'readline-sync'; 3 | import fs from 'fs-extra'; 4 | import temp from 'temp'; 5 | import path from 'path'; 6 | import yauzl from 'yauzl'; 7 | import { createGunzip } from 'zlib'; 8 | import { ZipContents, isGzip, isZip } from './utils'; 9 | import { logger } from './logger'; 10 | 11 | import { pipeline } from 'stream/promises'; 12 | 13 | const ONE_MEGABYTE = 1024 * 1024; 14 | const DATA_SIZE_WARNING_THRESHOLD = ONE_MEGABYTE * 1024; // 1 gigabyte 15 | 16 | export class DownloadManager { 17 | folder: string; 18 | 19 | constructor(public alwaysYes = false) { 20 | temp.track(); 21 | this.folder = temp.mkdirSync(); 22 | } 23 | 24 | async checkDataUrl(url: string): Promise { 25 | try { 26 | const response = await axios.head(url); 27 | if (response.status === 200) { 28 | let proceedToDownload: boolean; 29 | const contentLength = parseInt(response.headers['content-length']); 30 | if (this.alwaysYes) { 31 | proceedToDownload = true; 32 | } else if (isNaN(contentLength)) { 33 | proceedToDownload = readlineSync.keyInYNStrict( 34 | 'Data file size is unknown. Download this file?' 35 | ); 36 | } else if (contentLength > DATA_SIZE_WARNING_THRESHOLD) { 37 | proceedToDownload = readlineSync.keyInYNStrict( 38 | `Data file is ${(contentLength / ONE_MEGABYTE).toFixed( 39 | 2 40 | )} MB in size. Download this file?` 41 | ); 42 | } else { 43 | proceedToDownload = true; 44 | } 45 | return proceedToDownload; 46 | } else { 47 | logger.error( 48 | `Received unsuccessful status code ${response.status} when checking data file URL: ${url}` 49 | ); 50 | return false; 51 | } 52 | } catch (e) { 53 | logger.error(`Request failed when checking data file URL: ${url}`); 54 | logger.error(e.message); 55 | return false; 56 | } 57 | } 58 | 59 | async downloadDataFile(url: string, folder = this.folder): Promise { 60 | const filenameGuess = 'data.json'; 61 | const dataPath = path.join(folder, filenameGuess); 62 | return new Promise((resolve, reject) => { 63 | logger.info('Beginning download...\n'); 64 | axios({ 65 | method: 'get', 66 | url: url, 67 | responseType: 'stream', 68 | onDownloadProgress: progressEvent => { 69 | if (process.stdout.isTTY) { 70 | let progressText: string; 71 | if (progressEvent.progress != null) { 72 | progressText = `Downloaded ${Math.floor(progressEvent.progress * 100)}% of file (${ 73 | progressEvent.loaded 74 | } bytes)`; 75 | } else { 76 | progressText = `Downloaded ${progressEvent.loaded} bytes`; 77 | } 78 | process.stdout.clearLine(0, () => { 79 | process.stdout.cursorTo(0, () => { 80 | process.stdout.moveCursor(0, -1, () => { 81 | logger.info(progressText); 82 | }); 83 | }); 84 | }); 85 | } 86 | } 87 | }) 88 | .then(response => { 89 | const contentType = response.headers['content-type'] ?? 'application/octet-stream'; 90 | const finalUrl = response.request.path; 91 | if (isZip(contentType, finalUrl)) { 92 | // zips require additional work to find a JSON file inside 93 | const zipPath = path.join(folder, 'data.zip'); 94 | const zipOutputStream = fs.createWriteStream(zipPath); 95 | pipeline(response.data, zipOutputStream).then(() => { 96 | yauzl.open(zipPath, { lazyEntries: true, autoClose: false }, (err, zipFile) => { 97 | if (err != null) { 98 | reject(err); 99 | } 100 | const jsonEntries: yauzl.Entry[] = []; 101 | 102 | zipFile.on('entry', (entry: yauzl.Entry) => { 103 | if (entry.fileName.endsWith('.json')) { 104 | jsonEntries.push(entry); 105 | } 106 | zipFile.readEntry(); 107 | }); 108 | 109 | zipFile.on('end', () => { 110 | logger.info('Download complete.'); 111 | if (jsonEntries.length === 0) { 112 | reject('No JSON file present in zip.'); 113 | } else { 114 | let chosenEntry: yauzl.Entry; 115 | if (jsonEntries.length === 1) { 116 | chosenEntry = jsonEntries[0]; 117 | zipFile.openReadStream(chosenEntry, (err, readStream) => { 118 | const outputStream = fs.createWriteStream(dataPath); 119 | outputStream.on('finish', () => { 120 | zipFile.close(); 121 | resolve(dataPath); 122 | }); 123 | outputStream.on('error', () => { 124 | zipFile.close(); 125 | reject('Error writing downloaded file.'); 126 | }); 127 | readStream.pipe(outputStream); 128 | }); 129 | } else { 130 | jsonEntries.sort((a, b) => { 131 | return a.fileName.localeCompare(b.fileName); 132 | }); 133 | resolve({ zipFile, jsonEntries, dataPath }); 134 | } 135 | } 136 | }); 137 | zipFile.readEntry(); 138 | }); 139 | }); 140 | } else { 141 | const outputStream = fs.createWriteStream(dataPath); 142 | outputStream.on('finish', () => { 143 | logger.info('Download complete.'); 144 | resolve(dataPath); 145 | }); 146 | outputStream.on('error', () => { 147 | reject('Error writing downloaded file.'); 148 | }); 149 | 150 | if (isGzip(contentType, finalUrl)) { 151 | pipeline(response.data, createGunzip(), outputStream); 152 | } else { 153 | response.data.pipe(outputStream); 154 | } 155 | } 156 | }) 157 | .catch(() => { 158 | reject('Error downloading data file.'); 159 | }); 160 | }); 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/commands.ts: -------------------------------------------------------------------------------- 1 | import util from 'util'; 2 | import path from 'path'; 3 | import { exec } from 'child_process'; 4 | import fs from 'fs-extra'; 5 | import readlineSync from 'readline-sync'; 6 | import { OptionValues } from 'commander'; 7 | 8 | import { 9 | config, 10 | chooseJsonFile, 11 | getEntryFromZip, 12 | assessTocContents, 13 | assessReferencedProviders 14 | } from './utils'; 15 | import temp from 'temp'; 16 | import { SchemaManager } from './SchemaManager'; 17 | import { DockerManager } from './DockerManager'; 18 | import { logger } from './logger'; 19 | import { DownloadManager } from './DownloadManager'; 20 | 21 | export async function validate(dataFile: string, options: OptionValues) { 22 | // check to see if supplied json file exists 23 | if (!fs.existsSync(dataFile)) { 24 | logger.error(`Could not find data file: ${dataFile}`); 25 | process.exitCode = 1; 26 | return; 27 | } 28 | const schemaManager = new SchemaManager(); 29 | await schemaManager.ensureRepo(); 30 | schemaManager.strict = options.strict; 31 | schemaManager.shouldDetectVersion = options.schemaVersion == null; 32 | let versionToUse: string; 33 | try { 34 | const detectedVersion = await schemaManager.determineVersion(dataFile); 35 | if (!schemaManager.shouldDetectVersion && detectedVersion != options.schemaVersion) { 36 | logger.warn( 37 | `Schema version ${options.schemaVersion} was provided, but file indicates it conforms to schema version ${detectedVersion}. ${options.schemaVersion} will be used.` 38 | ); 39 | } 40 | versionToUse = schemaManager.shouldDetectVersion ? detectedVersion : options.schemaVersion; 41 | } catch { 42 | if (!schemaManager.shouldDetectVersion) { 43 | versionToUse = options.schemaVersion; 44 | } else { 45 | // or maybe use the minimum. 46 | logger.error( 47 | 'Data file does not contain version information. Please run again using the --schema-version option to specify a version.' 48 | ); 49 | process.exitCode = 1; 50 | return; 51 | } 52 | } 53 | return schemaManager 54 | .useVersion(versionToUse) 55 | .then(versionIsAvailable => { 56 | if (versionIsAvailable) { 57 | return schemaManager.useSchema(options.target); 58 | } 59 | }) 60 | .then(async schemaPath => { 61 | temp.track(); 62 | if (schemaPath != null) { 63 | const dockerManager = new DockerManager(options.out, options.allErrors); 64 | const downloadManager = new DownloadManager(options.yesAll); 65 | const containerResult = await dockerManager.runContainer( 66 | schemaPath, 67 | options.target, 68 | dataFile 69 | ); 70 | if (containerResult.pass) { 71 | if (options.target === 'table-of-contents') { 72 | const providerReferences = await assessTocContents( 73 | containerResult.locations, 74 | schemaManager, 75 | dockerManager, 76 | downloadManager 77 | ); 78 | await assessReferencedProviders( 79 | providerReferences, 80 | schemaManager, 81 | dockerManager, 82 | downloadManager 83 | ); 84 | } else if ( 85 | options.target === 'in-network-rates' && 86 | containerResult.locations?.providerReference?.length > 0 87 | ) { 88 | await assessReferencedProviders( 89 | containerResult.locations.providerReference, 90 | schemaManager, 91 | dockerManager, 92 | downloadManager 93 | ); 94 | } 95 | } 96 | } else { 97 | logger.error('No schema available - not validating.'); 98 | process.exitCode = 1; 99 | } 100 | temp.cleanupSync(); 101 | }) 102 | .catch(err => { 103 | logger.error(err.message); 104 | process.exitCode = 1; 105 | }); 106 | } 107 | 108 | export async function validateFromUrl(dataUrl: string, options: OptionValues) { 109 | temp.track(); 110 | const downloadManager = new DownloadManager(options.yesAll); 111 | if (await downloadManager.checkDataUrl(dataUrl)) { 112 | const schemaManager = new SchemaManager(); 113 | await schemaManager.ensureRepo(); 114 | schemaManager.strict = options.strict; 115 | return schemaManager 116 | .useVersion(options.schemaVersion) 117 | .then(versionIsAvailable => { 118 | if (versionIsAvailable) { 119 | return schemaManager.useSchema(options.target); 120 | } 121 | }) 122 | .then(async schemaPath => { 123 | if (schemaPath != null) { 124 | const dockerManager = new DockerManager(options.out, options.allErrors); 125 | const dataFile = await downloadManager.downloadDataFile(dataUrl); 126 | if (typeof dataFile === 'string') { 127 | const containerResult = await dockerManager.runContainer( 128 | schemaPath, 129 | options.target, 130 | dataFile 131 | ); 132 | if (containerResult.pass) { 133 | if (options.target === 'table-of-contents') { 134 | const providerReferences = await assessTocContents( 135 | containerResult.locations, 136 | schemaManager, 137 | dockerManager, 138 | downloadManager 139 | ); 140 | await assessReferencedProviders( 141 | providerReferences, 142 | schemaManager, 143 | dockerManager, 144 | downloadManager 145 | ); 146 | } else if ( 147 | options.target === 'in-network-rates' && 148 | containerResult.locations?.providerReference?.length > 0 149 | ) { 150 | await assessReferencedProviders( 151 | containerResult.locations.providerReference, 152 | schemaManager, 153 | dockerManager, 154 | downloadManager 155 | ); 156 | } 157 | } 158 | return containerResult; 159 | } else { 160 | let continuation = true; 161 | // we have multiple files, so let's choose as many as we want 162 | while (continuation === true) { 163 | const chosenEntry = chooseJsonFile(dataFile.jsonEntries); 164 | await getEntryFromZip(dataFile.zipFile, chosenEntry, dataFile.dataPath); 165 | await dockerManager.runContainer(schemaPath, options.target, dataFile.dataPath); 166 | continuation = readlineSync.keyInYNStrict( 167 | 'Would you like to validate another file in the ZIP?' 168 | ); 169 | } 170 | dataFile.zipFile.close(); 171 | } 172 | } else { 173 | logger.error('No schema available - not validating.'); 174 | process.exitCode = 1; 175 | } 176 | }) 177 | .catch(err => { 178 | logger.error(err.message); 179 | process.exitCode = 1; 180 | }); 181 | } else { 182 | logger.info('Exiting.'); 183 | process.exitCode = 1; 184 | } 185 | } 186 | 187 | export async function update() { 188 | try { 189 | // check if the repo exists. if not, clone it. if it exists, fetch updates. 190 | if (!fs.existsSync(path.join(config.SCHEMA_REPO_FOLDER, '.git'))) { 191 | await util.promisify(exec)( 192 | `git clone ${config.SCHEMA_REPO_URL} "${config.SCHEMA_REPO_FOLDER}"` 193 | ); 194 | logger.info('Retrieved schemas.'); 195 | } else { 196 | await util.promisify(exec)( 197 | `git -C "${config.SCHEMA_REPO_FOLDER}" checkout master && git -C "${config.SCHEMA_REPO_FOLDER}" pull --no-rebase -t` 198 | ); 199 | logger.info('Updated schemas.'); 200 | } 201 | } catch (error) { 202 | logger.error(`Error when updating available schemas: ${error}`); 203 | process.exitCode = 1; 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /src/SchemaManager.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs-extra'; 2 | import path from 'path'; 3 | import { exec } from 'child_process'; 4 | import util from 'util'; 5 | import { config } from './utils'; 6 | import temp from 'temp'; 7 | import { logger } from './logger'; 8 | import { JSONParser } from '@streamparser/json-node'; 9 | 10 | const VERSION_TIME_LIMIT = 3000; // three seconds 11 | const BACKWARDS_BYTES = 1000; // read the last 1000 bytes during backwards search 12 | 13 | export class SchemaManager { 14 | private _version: string; 15 | private storageDirectory: string; 16 | public strict: boolean; 17 | public shouldDetectVersion: boolean; 18 | 19 | constructor( 20 | private repoDirectory = config.SCHEMA_REPO_FOLDER, 21 | private repoUrl = config.SCHEMA_REPO_URL 22 | ) { 23 | temp.track(); 24 | this.storageDirectory = temp.mkdirSync('schemas'); 25 | } 26 | 27 | public get version() { 28 | return this._version; 29 | } 30 | 31 | async ensureRepo() { 32 | if (!fs.existsSync(path.join(this.repoDirectory, '.git'))) { 33 | return util.promisify(exec)(`git clone ${this.repoUrl} "${this.repoDirectory}"`); 34 | } 35 | } 36 | 37 | async useVersion(version: string): Promise { 38 | if (this._version === version) { 39 | return true; 40 | } 41 | const tagResult = await util.promisify(exec)( 42 | `git -C "${this.repoDirectory}" tag --list --sort=taggerdate` 43 | ); 44 | const tags = tagResult.stdout 45 | .split('\n') 46 | .map(tag => tag.trim()) 47 | .filter(tag => tag.length > 0) 48 | .reduce( 49 | (acc, tag) => { 50 | if (tag.startsWith('v')) { 51 | acc[tag] = tag; // Key with 'v' prefix 52 | acc[tag.slice(1)] = tag; // Key without 'v' prefix 53 | } else { 54 | acc[tag] = 'v' + tag; // If no 'v' prefix, add one as the value 55 | } 56 | return acc; 57 | }, 58 | {} as Record 59 | ); 60 | 61 | if (version in tags) { 62 | await util.promisify(exec)(`git -C "${this.repoDirectory}" checkout ${tags[version]}`); 63 | this._version = tags[version]; 64 | return true; 65 | } else { 66 | // we didn't find your tag. maybe you mistyped it, so show the available ones. 67 | throw new Error( 68 | `Could not find a schema version named "${version}". Available versions are:\n${Object.keys( 69 | tags 70 | ).join('\n')}` 71 | ); 72 | } 73 | } 74 | 75 | async useSchema(schemaName: string): Promise { 76 | const schemaPath = path.join( 77 | this.storageDirectory, 78 | `${schemaName}-${this._version}-${this.strict ? 'strict' : 'loose'}.json` 79 | ); 80 | if (fs.existsSync(schemaPath)) { 81 | logger.debug(`Using cached schema: ${schemaName} ${this._version}`); 82 | return schemaPath; 83 | } 84 | const contentPath = path.join(this.repoDirectory, 'schemas', schemaName, `${schemaName}.json`); 85 | if (!fs.existsSync(contentPath)) { 86 | return null; 87 | } 88 | let schemaContents = fs.readFileSync(contentPath, 'utf-8'); 89 | if (this.strict) { 90 | const modifiedSchema = JSON.parse(schemaContents); 91 | makeSchemaStrict(modifiedSchema); 92 | schemaContents = JSON.stringify(modifiedSchema); 93 | } 94 | 95 | fs.writeFileSync(schemaPath, schemaContents, { encoding: 'utf-8' }); 96 | return schemaPath; 97 | } 98 | 99 | async determineVersion(dataFile: string): Promise { 100 | return new Promise((resolve, reject) => { 101 | logger.debug(`Detecting version for ${dataFile}`); 102 | const parser = new JSONParser({ paths: ['$.version'], keepStack: false }); 103 | const dataStream = fs.createReadStream(dataFile); 104 | let foundVersion = ''; 105 | 106 | let forwardReject: (reason?: any) => void; 107 | const forwardSearch = new Promise((resolve, reject) => { 108 | forwardReject = reject; 109 | parser.on('data', data => { 110 | if (typeof data.value === 'string') { 111 | foundVersion = data.value; 112 | } 113 | dataStream.unpipe(); 114 | dataStream.destroy(); 115 | parser.end(); 116 | }); 117 | parser.on('close', () => { 118 | if (foundVersion) { 119 | logger.debug(`Found version: ${foundVersion}`); 120 | resolve(foundVersion); 121 | } else { 122 | reject('No version property available.'); 123 | } 124 | }); 125 | parser.on('error', () => { 126 | // an error gets thrown when closing the stream early, but that's not an actual problem. 127 | // it'll get handled in the close event 128 | if (!foundVersion) { 129 | reject('Parse error when detecting version.'); 130 | } 131 | }); 132 | dataStream.pipe(parser); 133 | }); 134 | 135 | let backwardReject: (reason?: any) => void; 136 | const backwardSearch = new Promise((resolve, reject) => { 137 | backwardReject = reject; 138 | fs.promises 139 | .open(dataFile, 'r') 140 | .then(async fileHandle => { 141 | try { 142 | const stats = await fileHandle.stat(); 143 | const lastStuff = await fileHandle.read({ 144 | position: Math.max(0, stats.size - BACKWARDS_BYTES), 145 | length: BACKWARDS_BYTES 146 | }); 147 | if (lastStuff.bytesRead > 0) { 148 | const lastText = lastStuff.buffer.toString('utf-8'); 149 | const versionRegex = /"version"\s*:\s*("(?:\\"|\\\\|[^"])*")/; 150 | const versionMatch = lastText.match(versionRegex); 151 | if (versionMatch) { 152 | const foundVersion = JSON.parse(versionMatch[1]); 153 | logger.debug(`Found version during backwards search: ${foundVersion}`); 154 | resolve(foundVersion); 155 | } else { 156 | reject('No version found during backwards search'); 157 | } 158 | } else { 159 | reject('No bytes read during backwards search'); 160 | } 161 | } finally { 162 | fileHandle.close(); 163 | } 164 | }) 165 | .catch(err => { 166 | logger.debug(`Something went wrong during backwards search: ${err}`); 167 | reject('Something went wrong during backwards search'); 168 | }); 169 | }); 170 | 171 | const timeLimit = setTimeout(() => { 172 | logger.debug('Could not find version within time limit.'); 173 | if (forwardReject) { 174 | forwardReject('Forward timeout cancellation'); 175 | } 176 | if (backwardReject) { 177 | backwardReject('Backward timeout cancellation'); 178 | } 179 | reject('Could not find version within time limit.'); 180 | }, VERSION_TIME_LIMIT); 181 | 182 | Promise.any([forwardSearch, backwardSearch]) 183 | .then(foundVersion => { 184 | resolve(foundVersion); 185 | }) 186 | .catch(() => { 187 | reject(); 188 | }) 189 | .finally(() => { 190 | logger.debug('Cleaning up from version search.'); 191 | clearTimeout(timeLimit); 192 | dataStream.unpipe(); 193 | dataStream.destroy(); 194 | parser.end(); 195 | }); 196 | }); 197 | } 198 | } 199 | 200 | // note that this only sets additionalProperties to false at the top level, and at the first level of definitions. 201 | // if there are nested definitions, those will not be modified. 202 | function makeSchemaStrict(schema: any): void { 203 | if (typeof schema === 'object') { 204 | schema.additionalProperties = false; 205 | if (schema.definitions != null && typeof schema.definitions === 'object') { 206 | for (const defKey of Object.keys(schema.definitions)) { 207 | schema.definitions[defKey].additionalProperties = false; 208 | } 209 | } 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Price Transparency Machine-readable File Validator 2 | 3 | This tool is used to validate [Transparency in Coverage](https://www.cms.gov/priorities/key-initiatives/healthplan-price-transparency) machine-readable files in JSON format against the [schemas published by CMS](https://github.com/CMSgov/price-transparency-guide). 4 | 5 | ## Installation 6 | 7 | ### Prerequisites 8 | 9 | - Node (version 16.x) 10 | - NPM (version 8.5.x) 11 | - Git (latest version recommended, tested using 2.27.0) 12 | - Docker (version 19.x) 13 | 14 | ### Included Libraries 15 | 16 | - [RapidJSON](https://rapidjson.org) 17 | - [TCLAP](https://tclap.sourceforge.net) 18 | 19 | ### Instructions 20 | 21 | Clone this repository using Git in the desired installation location: 22 | 23 | ```bash 24 | git clone --recurse-submodules https://github.com/CMSgov/price-transparency-guide-validator.git 25 | ``` 26 | 27 | > **Hint** 28 | > 29 | > This repository references 3rd-party C++ libraries using Git submodules. If you clone without the `--recurse-submodules` flag, just run inside the repo: 30 | > 31 | > ```bash 32 | > git submodule update --init 33 | > ``` 34 | 35 | Make sure that Docker is running: 36 | 37 | ```bash 38 | docker ps 39 | ``` 40 | 41 | If this shows a table of active containers and their resource usage, then Docker is active. 42 | 43 | From the directory containing the clone, build the validator Docker image: 44 | 45 | ```bash 46 | cd price-transparency-guide-validator 47 | docker build -t validator . 48 | ``` 49 | 50 | Install the Node script with the `-g` global flag so it can be run from any location: 51 | 52 | ``` 53 | npm install -g cms-mrf-validator 54 | ``` 55 | 56 | ## Usage 57 | 58 | The validator can be run from any directory. For basic usage instructions: 59 | 60 | ``` 61 | cms-mrf-validator help 62 | ``` 63 | 64 | ``` 65 | Tool for validating health coverage machine-readable files. 66 | 67 | Options: 68 | -d, --debug show debug output 69 | -h, --help display help for command 70 | 71 | Commands: 72 | validate [options] Validate a file against a specific published version of a CMS schema. 73 | from-url [options] Validate the file retrieved from a URL against a specific published version of a CMS schema. 74 | update Update the available schemas from the CMS repository. 75 | help [command] display help for command 76 | ``` 77 | 78 | ### Update available schemas 79 | 80 | In order to perform validation, schemas must be available to the validator tool. The latest schemas can be obtained using the update command. 81 | 82 | From the installed directory: 83 | 84 | ``` 85 | cms-mrf-validator update 86 | ``` 87 | 88 | ### Validate a file 89 | 90 | Validating a file against one of the provided schemas is the primary usage of this tool. Be sure that you have the latest schemas available by [running the update command](#update-available-schemas) first. 91 | 92 | From the installed directory: 93 | 94 | ``` 95 | cms-mrf-validator validate [options] 96 | ``` 97 | 98 | Example usages: 99 | 100 | ```bash 101 | # basic usage, printing output directly and using the default in-network-rates schema with the version specified in the file 102 | cms-mrf-validator validate my-data.json 103 | # output will be written to a file. validate using specific version of allowed-amounts schema 104 | cms-mrf-validator validate my-data.json --schema-version v1.0.0 -o results.txt -t allowed-amounts 105 | ``` 106 | 107 | Further details: 108 | 109 | ``` 110 | Validate a file against a specific published version of a CMS schema. 111 | 112 | Arguments: 113 | data-file path to data file to validate 114 | 115 | Options: 116 | --schema-version version of schema to use for validation 117 | -o, --out output path 118 | -t, --target name of schema to use (choices: "allowed-amounts", "in-network-rates", "provider-reference", "table-of-contents", 119 | default: "in-network-rates") 120 | -s, --strict enable strict checking, which prohibits additional properties in data file 121 | -y, --yes-all automatically respond "yes" to confirmation prompts 122 | -a, --all-errors continue validating after errors are found 123 | -h, --help display help for command 124 | ``` 125 | 126 | The purpose of the `strict` option is to help detect when an optional attribute has been spelled incorrectly. Because additional properties are allowed by the schema, a misspelled optional attribute does not normally cause a validation failure. 127 | 128 | ### Validate a file at a URL 129 | 130 | It is also possible to specify a URL to the file to validate. From the installed directory: 131 | 132 | ``` 133 | cms-mrf-validator from-url [options] 134 | ``` 135 | 136 | The only difference in arguments is that a URL should be provided instead of a path to a file. All options from the `validate` command still apply. The URL must return a file that is one of the following: 137 | 138 | - a JSON file 139 | - a GZ-compressed JSON file 140 | - a ZIP archive that contains a JSON file. If multiple JSON files are found within the ZIP archive, you can choose which one you want to validate. 141 | 142 | Further details: 143 | 144 | ``` 145 | Validate the file retrieved from a URL against a specific published version of a CMS schema. 146 | 147 | Arguments: 148 | data-url URL to data file to validate 149 | 150 | Options: 151 | --schema-version version of schema to use for validation 152 | -o, --out output path 153 | -t, --target name of schema to use (choices: "allowed-amounts", "in-network-rates", "provider-reference", "table-of-contents", 154 | default: "in-network-rates") 155 | -s, --strict enable strict checking, which prohibits additional properties in data file 156 | -y, --yes-all automatically respond "yes" to confirmation prompts 157 | -a, --all-errors continue validating after errors are found 158 | -h, --help display help for command 159 | ``` 160 | 161 | ### Test file validation 162 | 163 | This project contains sample JSON files that can be used to familiarize yourself with the validation tool. These examples can be found in the [`test-files`](https://github.com/CMSgov/price-transparency-guide-validator/tree/documentation/test-files) directory. 164 | 165 | Running the command from the root of the project: 166 | 167 | #### Running a valid file: 168 | 169 | ```bash 170 | cms-mrf-validator validate test-files/in-network-rates-fee-for-service-sample.json --schema-version v1.0.0 171 | ``` 172 | 173 | Output: 174 | 175 | ``` 176 | Input JSON is valid. 177 | ``` 178 | 179 | #### Running an invalid file: 180 | 181 | ```bash 182 | cms-mrf-validator validate test-files/allowed-amounts-error.json --schema-version v1.0.0 -t allowed-amounts 183 | ``` 184 | 185 | Output: 186 | 187 | ```bash 188 | Input JSON is invalid. 189 | Error Name: type 190 | Message: Property has a type 'integer' that is not in the following list: 'string'. 191 | Instance: #/out_of_network/0/allowed_amounts/0/service_code/3 192 | Schema: #/definitions/allowed_amounts/properties/service_code/items 193 | 194 | 195 | Invalid schema: #/definitions/allowed_amounts/properties/service_code/items 196 | Invalid keyword: type 197 | Invalid code: 20 198 | Invalid message: Property has a type '%actual' that is not in the following list: '%expected'. 199 | Invalid document: #/out_of_network/0/allowed_amounts/0/service_code/3 200 | Error report: 201 | { 202 | "type": { 203 | "expected": [ 204 | "string" 205 | ], 206 | "actual": "integer", 207 | "errorCode": 20, 208 | "instanceRef": "#/out_of_network/0/allowed_amounts/0/service_code/3", 209 | "schemaRef": "#/definitions/allowed_amounts/properties/service_code/items" 210 | } 211 | } 212 | ``` 213 | 214 | ### Performance Considerations 215 | 216 | This validation tool is based on [rapidjson](https://rapidjson.org/) which is a high performance C++ JSON parser. You can find various benchmarks on [rapidjson's site](https://rapidjson.org/md_doc_performance.html) that should give the user an idea on what to expect when using. 217 | 218 | The exact amount of time needed for the validator to run will vary based on input file size and the machine running the validator. On a sample system with a 2.60GHz CPU and 16GB of memory, a typical processing rate is approximately 25 megabytes of input per second. 219 | -------------------------------------------------------------------------------- /test/SchemaManager.test.ts: -------------------------------------------------------------------------------- 1 | import 'jest-extended'; 2 | import path from 'path'; 3 | import temp from 'temp'; 4 | import util from 'util'; 5 | import child_process from 'child_process'; 6 | import { ensureDirSync, readFileSync, writeFileSync } from 'fs-extra'; 7 | import { SchemaManager } from '../src/SchemaManager'; 8 | 9 | jest.mock('child_process'); 10 | const child_process_real = jest.requireActual('child_process'); 11 | const execP = util.promisify(child_process_real.exec); 12 | 13 | describe('SchemaManager', () => { 14 | let repoDirectory: string; 15 | 16 | beforeAll(async () => { 17 | // set up our test repo with a sample schema 18 | temp.track(); 19 | repoDirectory = temp.mkdirSync(); 20 | const sampleSchema = JSON.parse( 21 | readFileSync(path.join(__dirname, 'fixtures', 'sampleSchema.json'), 'utf-8') 22 | ); 23 | ensureDirSync(path.join(repoDirectory, 'schemas', 'something-good')); 24 | await execP(`git init "${repoDirectory}"`); 25 | await execP(`git -C "${repoDirectory}" config user.name "test-user"`); 26 | await execP(`git -C "${repoDirectory}" config user.email "test-user@example.org"`); 27 | // create a few commits and tag them 28 | const schemaPath = path.join('schemas', 'something-good', 'something-good.json'); 29 | sampleSchema.version = '0.3'; 30 | writeFileSync(path.join(repoDirectory, schemaPath), JSON.stringify(sampleSchema)); 31 | await execP(`git -C "${repoDirectory}" add -A`); 32 | await execP(`git -C "${repoDirectory}" commit -m "first commit"`); 33 | await execP(`git -C "${repoDirectory}" tag -a "v0.3" -m ""`); 34 | sampleSchema.version = '0.7'; 35 | writeFileSync(path.join(repoDirectory, schemaPath), JSON.stringify(sampleSchema)); 36 | await execP(`git -C "${repoDirectory}" commit -am "second commit"`); 37 | await execP(`git -C "${repoDirectory}" tag -a "v0.7" -m ""`); 38 | sampleSchema.version = '1.0'; 39 | writeFileSync(path.join(repoDirectory, schemaPath), JSON.stringify(sampleSchema)); 40 | await execP(`git -C "${repoDirectory}" commit -am "third commit"`); 41 | await execP(`git -C "${repoDirectory}" tag -a "v1.0" -m ""`); 42 | }); 43 | 44 | beforeEach(() => { 45 | jest.resetAllMocks(); 46 | }); 47 | 48 | describe('#ensureRepo', () => { 49 | let mockedExec: jest.Mock; 50 | 51 | beforeAll(() => { 52 | mockedExec = child_process.exec as jest.Mocked; 53 | }); 54 | 55 | it('should not clone anything when the repo already exists', async () => { 56 | // basic callback mock, since we don't need anything more complex than that for testing 57 | mockedExec.mockImplementationOnce((_command: string, callback: any) => { 58 | if (callback) { 59 | callback(null, { stdout: 'ok' }); 60 | } 61 | }); 62 | const manager = new SchemaManager(repoDirectory); 63 | await manager.ensureRepo(); 64 | expect(mockedExec).toHaveBeenCalledTimes(0); 65 | }); 66 | 67 | it('should clone the schema repo when it does not exist', async () => { 68 | // basic callback mock, since we don't need anything more complex than that for testing 69 | mockedExec.mockImplementationOnce((_command: string, callback: any) => { 70 | if (callback) { 71 | callback(null, { stdout: 'ok' }); 72 | } 73 | }); 74 | const differentFolder = temp.mkdirSync(); 75 | const repoUrl = 'http://very.fake.url_goes_here'; 76 | const manager = new SchemaManager(differentFolder, repoUrl); 77 | await manager.ensureRepo(); 78 | expect(mockedExec).toHaveBeenCalledTimes(1); 79 | expect(mockedExec).toHaveBeenCalledWith( 80 | `git clone http://very.fake.url_goes_here "${differentFolder}"`, 81 | expect.anything() 82 | ); 83 | }); 84 | }); 85 | 86 | describe('#useVersion', () => { 87 | beforeEach(() => { 88 | (child_process.exec as jest.Mocked).mockImplementation( 89 | (command: string, callback: any) => { 90 | child_process_real.exec( 91 | command, 92 | (err: Error, stdout: string | Buffer, stderr: string | Buffer) => { 93 | if (callback) { 94 | callback(err, { stdout, stderr }); 95 | } 96 | } 97 | ); 98 | } 99 | ); 100 | }); 101 | 102 | afterEach(() => { 103 | (child_process.exec as jest.Mocked).mockRestore(); 104 | }); 105 | 106 | it('should resolve to true when the version exists in the repo', async () => { 107 | const schemaManager = new SchemaManager(repoDirectory); 108 | await expect(schemaManager.useVersion('v0.7')).resolves.toBe(true); 109 | }); 110 | 111 | it('should resolve to true when the version exists in the repo with a prefixed v', async () => { 112 | const schemaManager = new SchemaManager(repoDirectory); 113 | await expect(schemaManager.useVersion('0.7')).resolves.toBe(true); 114 | }); 115 | 116 | it('should reject when the version does not exist in the repo', async () => { 117 | const schemaManager = new SchemaManager(repoDirectory); 118 | await expect(schemaManager.useVersion('v0.6')).toReject(); 119 | }); 120 | 121 | it('should reject when the version does not exist in the repo with a prefixed v', async () => { 122 | const schemaManager = new SchemaManager(repoDirectory); 123 | await expect(schemaManager.useVersion('0.6')).toReject(); 124 | }); 125 | }); 126 | 127 | describe('#useSchema', () => { 128 | beforeEach(() => { 129 | (child_process.exec as jest.Mocked).mockImplementation( 130 | (command: string, callback: any) => { 131 | child_process_real.exec( 132 | command, 133 | (err: Error, stdout: string | Buffer, stderr: string | Buffer) => { 134 | if (callback) { 135 | callback(err, { stdout, stderr }); 136 | } 137 | } 138 | ); 139 | } 140 | ); 141 | }); 142 | 143 | afterEach(() => { 144 | (child_process.exec as jest.Mocked).mockRestore(); 145 | }); 146 | 147 | it('should return a path to the schema when the schema is available', async () => { 148 | const schemaManager = new SchemaManager(repoDirectory); 149 | await schemaManager.useVersion('v0.7'); 150 | const result = await schemaManager.useSchema('something-good'); 151 | const contents = JSON.parse(readFileSync(result, { encoding: 'utf-8' })); 152 | expect(contents.version).toBe('0.7'); 153 | }); 154 | 155 | it('should return a path to the schema in strict mode', async () => { 156 | const schemaManager = new SchemaManager(repoDirectory); 157 | schemaManager.strict = true; 158 | await schemaManager.useVersion('v1.0'); 159 | const result = await schemaManager.useSchema('something-good'); 160 | const contents = JSON.parse(readFileSync(result, { encoding: 'utf-8' })); 161 | expect(contents.version).toBe('1.0'); 162 | expect(contents.additionalProperties).toBeFalse(); 163 | expect(contents.definitions.food.additionalProperties).toBeFalse(); 164 | expect(contents.definitions.garment.additionalProperties).toBeFalse(); 165 | }); 166 | 167 | it('should return null when the schema is not available', async () => { 168 | const schemaManager = new SchemaManager(repoDirectory); 169 | await schemaManager.useVersion('v0.7'); 170 | const result = await schemaManager.useSchema('something-else'); 171 | expect(result).toBeNull(); 172 | }); 173 | }); 174 | 175 | describe('#determineVersion', () => { 176 | it('should resolve to the value of the version property when it exists and is a string', async () => { 177 | const dataPath = path.join(__dirname, 'fixtures', 'dataWithVersion.json'); 178 | const schemaManager = new SchemaManager(); 179 | await expect(schemaManager.determineVersion(dataPath)).resolves.toBe('1.2.0'); 180 | }); 181 | 182 | it('should reject when the version property does not exist', async () => { 183 | const dataPath = path.join(__dirname, 'fixtures', 'dataWithoutVersion.json'); 184 | const schemaManager = new SchemaManager(); 185 | await expect(schemaManager.determineVersion(dataPath)).toReject(); 186 | }); 187 | 188 | it('should reject when the version property exists, but is not a string', async () => { 189 | const dataPath = path.join(__dirname, 'fixtures', 'dataNonStringVersion.json'); 190 | const schemaManager = new SchemaManager(); 191 | await expect(schemaManager.determineVersion(dataPath)).toReject(); 192 | }); 193 | }); 194 | }); 195 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /test/DownloadManager.test.ts: -------------------------------------------------------------------------------- 1 | import 'jest-extended'; 2 | import path from 'path'; 3 | import temp from 'temp'; 4 | import nock from 'nock'; 5 | import readlineSync from 'readline-sync'; 6 | import fs from 'fs-extra'; 7 | import yauzl from 'yauzl'; 8 | import { DownloadManager } from '../src/DownloadManager'; 9 | import { ZipContents } from '../src/utils'; 10 | 11 | describe('DownloadManager', () => { 12 | let downloadManager: DownloadManager; 13 | 14 | beforeAll(() => { 15 | temp.track(); 16 | downloadManager = new DownloadManager(); 17 | }); 18 | 19 | beforeEach(() => { 20 | downloadManager.alwaysYes = false; 21 | }); 22 | 23 | describe('#checkDataUrl', () => { 24 | let keyInYNStrictSpy: jest.SpyInstance; 25 | 26 | beforeAll(() => { 27 | keyInYNStrictSpy = jest.spyOn(readlineSync, 'keyInYNStrict'); 28 | }); 29 | 30 | afterEach(() => { 31 | nock.cleanAll(); 32 | }); 33 | 34 | afterAll(() => { 35 | keyInYNStrictSpy.mockRestore(); 36 | }); 37 | 38 | it('should return true when the url is valid and the content length is less than one GB', async () => { 39 | nock('http://example.org').head('/data.json').reply(200, '', { 'content-length': '500' }); 40 | const result = await downloadManager.checkDataUrl('http://example.org/data.json'); 41 | expect(result).toBeTrue(); 42 | }); 43 | 44 | it('should return true when the url is valid and the user approves a content length greater than one GB', async () => { 45 | nock('http://example.org') 46 | .head('/data.json') 47 | .reply(200, '', { 'content-length': (Math.pow(1024, 3) * 2).toString() }); 48 | keyInYNStrictSpy.mockReturnValueOnce(true); 49 | const result = await downloadManager.checkDataUrl('http://example.org/data.json'); 50 | expect(result).toBeTrue(); 51 | }); 52 | 53 | it('should return true when the url is valid and and alwaysYes is true with a content length greater than one GB', async () => { 54 | nock('http://example.org') 55 | .head('/data.json') 56 | .reply(200, '', { 'content-length': (Math.pow(1024, 3) * 2).toString() }); 57 | downloadManager.alwaysYes = true; 58 | const result = await downloadManager.checkDataUrl('http://example.org/data.json'); 59 | expect(result).toBeTrue(); 60 | }); 61 | 62 | it('should return false when the url is valid and the user rejects a content length greater than one GB', async () => { 63 | nock('http://example.org') 64 | .head('/data.json') 65 | .reply(200, '', { 'content-length': (Math.pow(1024, 3) * 2).toString() }); 66 | keyInYNStrictSpy.mockReturnValueOnce(false); 67 | const result = await downloadManager.checkDataUrl('http://example.org/data.json'); 68 | expect(result).toBeFalse(); 69 | }); 70 | 71 | it('should return true when the url is valid and the user approves an unknown content length', async () => { 72 | nock('http://example.org').head('/data.json').reply(200); 73 | keyInYNStrictSpy.mockReturnValueOnce(true); 74 | const result = await downloadManager.checkDataUrl('http://example.org/data.json'); 75 | expect(result).toBeTrue(); 76 | }); 77 | 78 | it('should return true when the url is valid and alwaysYes is true with an unknown content length', async () => { 79 | nock('http://example.org').head('/data.json').reply(200); 80 | downloadManager.alwaysYes = true; 81 | const result = await downloadManager.checkDataUrl('http://example.org/data.json'); 82 | expect(result).toBeTrue(); 83 | }); 84 | 85 | it('should return false when the url is valid and the user rejects an unknown content length', async () => { 86 | nock('http://example.org').head('/data.json').reply(200); 87 | keyInYNStrictSpy.mockReturnValueOnce(false); 88 | const result = await downloadManager.checkDataUrl('http://example.org/data.json'); 89 | expect(result).toBeFalse(); 90 | }); 91 | 92 | it('should return false when the url is not valid', async () => { 93 | nock('http://example.org').head('/data.json').reply(404); 94 | const result = await downloadManager.checkDataUrl('http://example.org/data.json'); 95 | expect(result).toBeFalse(); 96 | }); 97 | }); 98 | 99 | describe('#downloadDataFile', () => { 100 | afterEach(() => { 101 | nock.cleanAll(); 102 | }); 103 | 104 | it('should write a file to the default folder', async () => { 105 | const simpleData = fs.readJsonSync( 106 | path.join(__dirname, 'fixtures', 'simpleData.json'), 107 | 'utf-8' 108 | ); 109 | nock('http://example.org').get('/data.json').reply(200, simpleData); 110 | const dataPath = (await downloadManager.downloadDataFile( 111 | 'http://example.org/data.json' 112 | )) as string; 113 | expect(dataPath).toBeString(); 114 | expect(fs.existsSync(dataPath)); 115 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 116 | expect(downloadedData).toEqual(simpleData); 117 | }); 118 | 119 | it('should write a file to the specified folder', async () => { 120 | const simpleData = fs.readJsonSync( 121 | path.join(__dirname, 'fixtures', 'simpleData.json'), 122 | 'utf-8' 123 | ); 124 | nock('http://example.org').get('/data.json').reply(200, simpleData); 125 | const outputDir = temp.mkdirSync(); 126 | const dataPath = (await downloadManager.downloadDataFile( 127 | 'http://example.org/data.json', 128 | outputDir 129 | )) as string; 130 | expect(dataPath).toBeString(); 131 | expect(fs.existsSync(dataPath)); 132 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 133 | expect(downloadedData).toEqual(simpleData); 134 | }); 135 | 136 | it('should write a decompressed gz file when the response has content type application/gzip', async () => { 137 | const simpleData = fs.readJsonSync( 138 | path.join(__dirname, 'fixtures', 'simpleData.json'), 139 | 'utf-8' 140 | ); 141 | const simpleGz = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleData.gz')); 142 | nock('http://example.org') 143 | .get('/data.gz') 144 | .reply(200, simpleGz, { 'content-type': 'application/gzip' }); 145 | const dataPath = (await downloadManager.downloadDataFile( 146 | 'http://example.org/data.gz' 147 | )) as string; 148 | expect(dataPath).toBeString(); 149 | expect(fs.existsSync(dataPath)); 150 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 151 | expect(downloadedData).toEqual(simpleData); 152 | }); 153 | 154 | it('should write a decompressed gz file when the response has content type application/octet-stream and the url ends with .gz', async () => { 155 | const simpleData = fs.readJsonSync( 156 | path.join(__dirname, 'fixtures', 'simpleData.json'), 157 | 'utf-8' 158 | ); 159 | const simpleGz = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleData.gz')); 160 | nock('http://example.org') 161 | .get('/data.gz') 162 | .reply(200, simpleGz, { 'content-type': 'application/octet-stream' }); 163 | const dataPath = (await downloadManager.downloadDataFile( 164 | 'http://example.org/data.gz' 165 | )) as string; 166 | expect(dataPath).toBeString(); 167 | expect(fs.existsSync(dataPath)); 168 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 169 | expect(downloadedData).toEqual(simpleData); 170 | }); 171 | 172 | it('should write a decompressed gz file when the response has content type application/octet-stream and the url ends with .gz followed by a query string', async () => { 173 | const simpleData = fs.readJsonSync( 174 | path.join(__dirname, 'fixtures', 'simpleData.json'), 175 | 'utf-8' 176 | ); 177 | const simpleGz = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleData.gz')); 178 | nock('http://example.org') 179 | .get('/data.gz') 180 | .query(true) 181 | .reply(200, simpleGz, { 'content-type': 'application/octet-stream' }); 182 | const dataPath = (await downloadManager.downloadDataFile( 183 | 'http://example.org/data.gz?Expires=123456&mode=true' 184 | )) as string; 185 | expect(dataPath).toBeString(); 186 | expect(fs.existsSync(dataPath)); 187 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 188 | expect(downloadedData).toEqual(simpleData); 189 | }); 190 | 191 | it('should write a decompressed gz file when the response has content type application/octet-stream and the url after redirection ends with .gz', async () => { 192 | const simpleData = fs.readJsonSync( 193 | path.join(__dirname, 'fixtures', 'simpleData.json'), 194 | 'utf-8' 195 | ); 196 | const simpleGz = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleData.gz')); 197 | nock('http://example.org') 198 | .get('/some-data') 199 | .reply(302, '', { Location: 'http://example.org/data.gz' }); 200 | nock('http://example.org') 201 | .get('/data.gz') 202 | .reply(200, simpleGz, { 'content-type': 'application/octet-stream' }); 203 | const outputDir = temp.mkdirSync(); 204 | await downloadManager.downloadDataFile('http://example.org/some-data', outputDir); 205 | expect(fs.existsSync(path.join(outputDir, 'data.json'))); 206 | const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); 207 | expect(downloadedData).toEqual(simpleData); 208 | }); 209 | 210 | it('should write a decompressed gz file when the response has content type application/octet-stream and the url after redirection ends with .gz followed by a query string', async () => { 211 | const simpleData = fs.readJsonSync( 212 | path.join(__dirname, 'fixtures', 'simpleData.json'), 213 | 'utf-8' 214 | ); 215 | const simpleGz = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleData.gz')); 216 | nock('http://example.org') 217 | .get('/some-data') 218 | .reply(302, '', { Location: 'http://example.org/data.gz?Expires=123456&mode=true' }); 219 | nock('http://example.org') 220 | .get('/data.gz') 221 | .query(true) 222 | .reply(200, simpleGz, { 'content-type': 'application/octet-stream' }); 223 | const outputDir = temp.mkdirSync(); 224 | await downloadManager.downloadDataFile('http://example.org/some-data', outputDir); 225 | expect(fs.existsSync(path.join(outputDir, 'data.json'))); 226 | const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); 227 | expect(downloadedData).toEqual(simpleData); 228 | }); 229 | 230 | it('should write a json file within a zip to the specified folder', async () => { 231 | const simpleData = fs.readJsonSync( 232 | path.join(__dirname, 'fixtures', 'simpleData.json'), 233 | 'utf-8' 234 | ); 235 | const simpleZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleZip.zip')); 236 | nock('http://example.org') 237 | .get('/data.zip') 238 | .reply(200, simpleZip, { 'content-type': 'application/zip' }); 239 | const dataPath = (await downloadManager.downloadDataFile( 240 | 'http://example.org/data.zip' 241 | )) as string; 242 | expect(dataPath).toBeString(); 243 | expect(fs.existsSync(dataPath)); 244 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 245 | expect(downloadedData).toEqual(simpleData); 246 | }); 247 | 248 | it('should write a json file within a zip when the response has content type application/octet-stream and the url ends with .zip', async () => { 249 | const simpleData = fs.readJsonSync( 250 | path.join(__dirname, 'fixtures', 'simpleData.json'), 251 | 'utf-8' 252 | ); 253 | const simpleZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleZip.zip')); 254 | nock('http://example.org') 255 | .get('/data.zip') 256 | .reply(200, simpleZip, { 'content-type': 'application/octet-stream' }); 257 | const dataPath = (await downloadManager.downloadDataFile( 258 | 'http://example.org/data.zip' 259 | )) as string; 260 | expect(dataPath).toBeString(); 261 | expect(fs.existsSync(dataPath)); 262 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 263 | expect(downloadedData).toEqual(simpleData); 264 | }); 265 | 266 | it('should write a json file within a zip when the response has content type application/x-zip-compressed and the url ends with .zip', async () => { 267 | const simpleData = fs.readJsonSync( 268 | path.join(__dirname, 'fixtures', 'simpleData.json'), 269 | 'utf-8' 270 | ); 271 | const simpleZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleZip.zip')); 272 | nock('http://example.org') 273 | .get('/data.zip') 274 | .reply(200, simpleZip, { 'content-type': 'application/x-zip-compressed' }); 275 | const dataPath = (await downloadManager.downloadDataFile( 276 | 'http://example.org/data.zip' 277 | )) as string; 278 | expect(dataPath).toBeString(); 279 | expect(fs.existsSync(dataPath)); 280 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 281 | expect(downloadedData).toEqual(simpleData); 282 | }); 283 | 284 | it('should write a json file within a zip when the response has content type application/octet-stream and the url ends with .zip followed by a query string', async () => { 285 | const simpleData = fs.readJsonSync( 286 | path.join(__dirname, 'fixtures', 'simpleData.json'), 287 | 'utf-8' 288 | ); 289 | const simpleZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleZip.zip')); 290 | nock('http://example.org') 291 | .get('/data.zip') 292 | .query(true) 293 | .reply(200, simpleZip, { 'content-type': 'application/octet-stream' }); 294 | const dataPath = (await downloadManager.downloadDataFile( 295 | 'http://example.org/data.zip?mode=on&rate=7' 296 | )) as string; 297 | expect(dataPath).toBeString(); 298 | expect(fs.existsSync(dataPath)); 299 | const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); 300 | expect(downloadedData).toEqual(simpleData); 301 | }); 302 | 303 | it('should return information about the zip contents when the zip has more than one json file', async () => { 304 | const multiZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'multiZip.zip')); 305 | nock('http://example.org') 306 | .get('/multi.zip') 307 | .query(true) 308 | .reply(200, multiZip, { 'content-type': 'application/zip' }); 309 | const zipInfo = (await downloadManager.downloadDataFile( 310 | 'http://example.org/multi.zip?mode=more' 311 | )) as ZipContents; 312 | expect(zipInfo).toBeObject(); 313 | expect(zipInfo.zipFile).toBeInstanceOf(yauzl.ZipFile); 314 | expect(zipInfo.jsonEntries).toHaveLength(2); 315 | expect(zipInfo.jsonEntries[0].fileName).toBe('moreData.json'); 316 | expect(zipInfo.jsonEntries[1].fileName).toBe('simpleData.json'); 317 | }); 318 | it('should write a json file within a zip when the response has content type application/octet-stream and the url after redirection ends with .zip', async () => { 319 | const simpleData = fs.readJsonSync( 320 | path.join(__dirname, 'fixtures', 'simpleData.json'), 321 | 'utf-8' 322 | ); 323 | const simpleZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleZip.zip')); 324 | nock('http://example.org') 325 | .get('/data-please') 326 | .reply(302, '', { Location: 'http://example.org/data.zip' }); 327 | nock('http://example.org') 328 | .get('/data.zip') 329 | .reply(200, simpleZip, { 'content-type': 'application/octet-stream' }); 330 | const outputDir = temp.mkdirSync(); 331 | await downloadManager.downloadDataFile('http://example.org/data-please', outputDir); 332 | expect(fs.existsSync(path.join(outputDir, 'data.json'))); 333 | const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); 334 | expect(downloadedData).toEqual(simpleData); 335 | }); 336 | 337 | it('should write a json file within a zip when the response has content type application/octet-stream and the url after redirection ends with .zip followed by a query string', async () => { 338 | const simpleData = fs.readJsonSync( 339 | path.join(__dirname, 'fixtures', 'simpleData.json'), 340 | 'utf-8' 341 | ); 342 | const simpleZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'simpleZip.zip')); 343 | nock('http://example.org') 344 | .get('/data-please') 345 | .reply(302, '', { Location: 'http://example.org/data.zip?mode=on&rate=7' }); 346 | nock('http://example.org') 347 | .get('/data.zip') 348 | .query(true) 349 | .reply(200, simpleZip, { 'content-type': 'application/octet-stream' }); 350 | const outputDir = temp.mkdirSync(); 351 | await downloadManager.downloadDataFile('http://example.org/data-please', outputDir); 352 | expect(fs.existsSync(path.join(outputDir, 'data.json'))); 353 | const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); 354 | expect(downloadedData).toEqual(simpleData); 355 | }); 356 | 357 | it('should reject when a zip contains no json files', async () => { 358 | const wrongZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'allWrong.zip')); 359 | nock('http://example.org') 360 | .get('/data.zip') 361 | .reply(200, wrongZip, { 'content-type': 'application/zip' }); 362 | const outputDir = temp.mkdirSync(); 363 | await expect( 364 | downloadManager.downloadDataFile('http://example.org/data.zip', outputDir) 365 | ).toReject(); 366 | }); 367 | 368 | it('should reject when the url is not valid', async () => { 369 | nock('http://example.org').get('/data.json').reply(500); 370 | const outputDir = temp.mkdirSync(); 371 | await expect( 372 | downloadManager.downloadDataFile('http://example.org/data.json', outputDir) 373 | ).toReject(); 374 | }); 375 | }); 376 | }); 377 | -------------------------------------------------------------------------------- /schemavalidator.cpp: -------------------------------------------------------------------------------- 1 | // Schema Validator example 2 | 3 | // The example validates JSON text from stdin with a JSON schema specified in the argument. 4 | 5 | #define RAPIDJSON_HAS_STDSTRING 1 6 | 7 | #include "rapidjson/include/rapidjson/error/en.h" 8 | #include "rapidjson/include/rapidjson/filereadstream.h" 9 | #include "rapidjson/include/rapidjson/filewritestream.h" 10 | #include "rapidjson/include/rapidjson/schema.h" 11 | #include "rapidjson/include/rapidjson/reader.h" 12 | #include "rapidjson/include/rapidjson/stringbuffer.h" 13 | #include "rapidjson/include/rapidjson/prettywriter.h" 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | using namespace rapidjson; 23 | using namespace std; 24 | 25 | typedef GenericValue, CrtAllocator> ValueType; 26 | 27 | // Forward ref 28 | static void CreateErrorMessages(const ValueType &errors, FILE *outFile, size_t depth, const char *context); 29 | 30 | // Convert GenericValue to std::string 31 | static std::string GetString(const ValueType &val) 32 | { 33 | std::ostringstream s; 34 | if (val.IsString()) 35 | s << val.GetString(); 36 | else if (val.IsDouble()) 37 | s << val.GetDouble(); 38 | else if (val.IsUint()) 39 | s << val.GetUint(); 40 | else if (val.IsInt()) 41 | s << val.GetInt(); 42 | else if (val.IsUint64()) 43 | s << val.GetUint64(); 44 | else if (val.IsInt64()) 45 | s << val.GetInt64(); 46 | else if (val.IsBool() && val.GetBool()) 47 | s << "true"; 48 | else if (val.IsBool()) 49 | s << "false"; 50 | else if (val.IsFloat()) 51 | s << val.GetFloat(); 52 | return s.str(); 53 | } 54 | 55 | // Create the error message for a named error 56 | // The error object can either be empty or contain at least member properties: 57 | // {"errorCode": , "instanceRef": "", "schemaRef": "" } 58 | // Additional properties may be present for use as inserts. 59 | // An "errors" property may be present if there are child errors. 60 | static void HandleError(const char *errorName, const ValueType &error, FILE *outFile, size_t depth, const char *context) 61 | { 62 | if (!error.ObjectEmpty()) 63 | { 64 | // Get error code and look up error message text (English) 65 | int code = error["errorCode"].GetInt(); 66 | std::string message(GetValidateError_En(static_cast(code))); 67 | // For each member property in the error, see if its name exists as an insert in the error message and if so replace with the stringified property value 68 | // So for example - "Number '%actual' is not a multiple of the 'multipleOf' value '%expected'." - we would expect "actual" and "expected" members. 69 | for (ValueType::ConstMemberIterator insertsItr = error.MemberBegin(); 70 | insertsItr != error.MemberEnd(); ++insertsItr) 71 | { 72 | std::string insertName("%"); 73 | insertName += insertsItr->name.GetString(); // eg "%actual" 74 | size_t insertPos = message.find(insertName); 75 | if (insertPos != std::string::npos) 76 | { 77 | std::string insertString(""); 78 | const ValueType &insert = insertsItr->value; 79 | if (insert.IsArray()) 80 | { 81 | // Member is an array so create comma-separated list of items for the insert string 82 | for (ValueType::ConstValueIterator itemsItr = insert.Begin(); itemsItr != insert.End(); ++itemsItr) 83 | { 84 | if (itemsItr != insert.Begin()) 85 | insertString += ","; 86 | insertString += GetString(*itemsItr); 87 | } 88 | } 89 | else 90 | { 91 | insertString += GetString(insert); 92 | } 93 | message.replace(insertPos, insertName.length(), insertString); 94 | } 95 | } 96 | // Output error message, references, context 97 | std::string indentStr(depth * 2, ' '); 98 | const char *indent = indentStr.c_str(); 99 | fprintf(outFile, "%sError Name: %s\n", indent, errorName); 100 | fprintf(outFile, "%sMessage: %s\n", indent, message.c_str()); 101 | fprintf(outFile, "%sInstance: %s\n", indent, error["instanceRef"].GetString()); 102 | fprintf(outFile, "%sSchema: %s\n", indent, error["schemaRef"].GetString()); 103 | if (depth > 0) 104 | { 105 | fprintf(outFile, "%sContext: %s\n", indent, context); 106 | } 107 | fprintf(outFile, "\n"); 108 | 109 | // If child errors exist, apply the process recursively to each error structure. 110 | // This occurs for "oneOf", "allOf", "anyOf" and "dependencies" errors, so pass the error name as context. 111 | if (error.HasMember("errors")) 112 | { 113 | depth++; 114 | const ValueType &childErrors = error["errors"]; 115 | if (childErrors.IsArray()) 116 | { 117 | // Array - each item is an error structure - example 118 | // "anyOf": {"errorCode": ..., "errors":[{"pattern": {"errorCode\": ...\"}}, {"pattern": {"errorCode\": ...}}] 119 | for (ValueType::ConstValueIterator errorsItr = childErrors.Begin(); 120 | errorsItr != childErrors.End(); ++errorsItr) 121 | { 122 | CreateErrorMessages(*errorsItr, outFile, depth, errorName); 123 | } 124 | } 125 | else if (childErrors.IsObject()) 126 | { 127 | // Object - each member is an error structure - example 128 | // "dependencies": {"errorCode": ..., "errors": {"address": {"required": {"errorCode": ...}}, "name": {"required": {"errorCode": ...}}} 129 | for (ValueType::ConstMemberIterator propsItr = childErrors.MemberBegin(); 130 | propsItr != childErrors.MemberEnd(); ++propsItr) 131 | { 132 | CreateErrorMessages(propsItr->value, outFile, depth, errorName); 133 | } 134 | } 135 | } 136 | } 137 | } 138 | 139 | // Create error message for all errors in an error structure 140 | // Context is used to indicate whether the error structure has a parent 'dependencies', 'allOf', 'anyOf' or 'oneOf' error 141 | static void CreateErrorMessages(const ValueType &errors, FILE *outFile, size_t depth = 0, const char *context = 0) 142 | { 143 | // Each member property contains one or more errors of a given type 144 | for (ValueType::ConstMemberIterator errorTypeItr = errors.MemberBegin(); errorTypeItr != errors.MemberEnd(); ++errorTypeItr) 145 | { 146 | const char *errorName = errorTypeItr->name.GetString(); 147 | const ValueType &errorContent = errorTypeItr->value; 148 | if (errorContent.IsArray()) 149 | { 150 | // Member is an array where each item is an error - eg "type": [{"errorCode": ...}, {"errorCode": ...}] 151 | for (ValueType::ConstValueIterator contentItr = errorContent.Begin(); contentItr != errorContent.End(); ++contentItr) 152 | { 153 | HandleError(errorName, *contentItr, outFile, depth, context); 154 | } 155 | } 156 | else if (errorContent.IsObject()) 157 | { 158 | // Member is an object which is a single error - eg "type": {"errorCode": ... } 159 | HandleError(errorName, errorContent, outFile, depth, context); 160 | } 161 | } 162 | } 163 | 164 | struct MessageHandler : public BaseReaderHandler, MessageHandler> 165 | { 166 | static list providerReferencePath; 167 | static list tocInNetworkPath; 168 | static list tocAllowedAmountPath; 169 | 170 | enum State 171 | { 172 | traversingObject, 173 | expectLocationKey, 174 | expectLocationValue 175 | } state_; 176 | list objectPath; 177 | list inNetworkLocations; 178 | list additionalLocations; 179 | string lastKey; 180 | string schemaName; 181 | 182 | MessageHandler(string name) 183 | { 184 | // we should store a bit more context so we know when we're in various location areas 185 | // in-network-rates: provider_references[].location 186 | // table-of-contents: reporting_structure[].in_network_files[].location, reporting_structure[].allowed_amount_file.location 187 | inNetworkLocations = {}; 188 | additionalLocations = {}; 189 | objectPath = {}; 190 | state_ = traversingObject; 191 | schemaName = name; 192 | } 193 | 194 | bool Key(const Ch *str, SizeType len, bool copy) 195 | { 196 | if (strcmp(str, "location") == 0 && state_ == traversingObject) 197 | { 198 | state_ = expectLocationKey; 199 | } 200 | lastKey = string(str); 201 | return BaseReaderHandler::Key(str, len, copy); 202 | } 203 | 204 | bool String(const Ch *str, SizeType len, bool copy) 205 | { 206 | if (state_ == expectLocationKey && strcmp(str, "location") == 0) 207 | { 208 | state_ = expectLocationValue; 209 | lastKey = ""; 210 | } 211 | else if (state_ == expectLocationValue) 212 | { 213 | // check the object path to see what list we want to add to 214 | // if it's the in network locations, use that list 215 | // otherwise use additionalLocations 216 | if (schemaName == "table-of-contents") 217 | { 218 | if (objectPath == tocInNetworkPath) 219 | { 220 | inNetworkLocations.push_back(string(str)); 221 | } 222 | else if (objectPath == tocAllowedAmountPath) 223 | { 224 | additionalLocations.push_back(string(str)); 225 | } 226 | } 227 | else if (schemaName == "in-network-rates" && objectPath == providerReferencePath) 228 | { 229 | additionalLocations.push_back(string(str)); 230 | } 231 | 232 | state_ = traversingObject; 233 | } 234 | 235 | return BaseReaderHandler::String(str, len, copy); 236 | } 237 | 238 | bool StartObject() 239 | { 240 | if (lastKey.size() > 0) 241 | { 242 | objectPath.push_back(lastKey); 243 | } 244 | return BaseReaderHandler::StartObject(); 245 | } 246 | 247 | bool EndObject(SizeType len) 248 | { 249 | if (objectPath.size() > 0 && objectPath.back() != "[]") 250 | { 251 | objectPath.pop_back(); 252 | } 253 | lastKey = ""; 254 | return BaseReaderHandler::EndObject(len); 255 | } 256 | 257 | bool StartArray() 258 | { 259 | objectPath.push_back(lastKey); 260 | objectPath.push_back("[]"); 261 | lastKey = ""; 262 | return BaseReaderHandler::StartArray(); 263 | } 264 | 265 | bool EndArray(SizeType len) 266 | { 267 | objectPath.pop_back(); 268 | objectPath.pop_back(); 269 | lastKey = ""; 270 | return BaseReaderHandler::EndArray(len); 271 | } 272 | }; 273 | 274 | list MessageHandler::providerReferencePath = {"provider_references", "[]"}; 275 | list MessageHandler::tocInNetworkPath = {"reporting_structure", "[]", "in_network_files", "[]"}; 276 | list MessageHandler::tocAllowedAmountPath = {"reporting_structure", "[]", "allowed_amount_file"}; 277 | 278 | int main(int argc, char *argv[]) 279 | { 280 | string schemaPath; 281 | string dataPath; 282 | string outputPath; 283 | int bufferSize = 4069; 284 | string schemaName; 285 | bool failFast; 286 | 287 | try 288 | { 289 | TCLAP::CmdLine cmd("validator for machine-readable files", ' ', "0.1"); 290 | TCLAP::UnlabeledValueArg schemaArg("schema-path", "path to schema file", true, "", "path"); 291 | TCLAP::UnlabeledValueArg dataArg("data-path", "path to data file", true, "", "path"); 292 | TCLAP::ValueArg outputArg("o", "output-path", "path to output directory", false, "/output", "path"); 293 | TCLAP::ValueArg bufferArg("b", "buffer-size", "buffer size in bytes", false, 4069, "integer"); 294 | TCLAP::ValueArg schemaNameArg("s", "schema-name", "schema name", false, "", "string"); 295 | TCLAP::SwitchArg failFastArg("f", "fail-fast", "if set, stop validating after the first error"); 296 | cmd.add(schemaArg); 297 | cmd.add(dataArg); 298 | cmd.add(outputArg); 299 | cmd.add(bufferArg); 300 | cmd.add(schemaNameArg); 301 | cmd.add(failFastArg); 302 | cmd.parse(argc, argv); 303 | schemaPath = schemaArg.getValue(); 304 | dataPath = dataArg.getValue(); 305 | outputPath = outputArg.getValue(); 306 | bufferSize = bufferArg.getValue(); 307 | schemaName = schemaNameArg.getValue(); 308 | failFast = failFastArg.getValue(); 309 | } 310 | catch (TCLAP::ArgException &e) 311 | { 312 | fprintf(stderr, "%s", e.error().c_str()); 313 | return EXIT_FAILURE; 314 | } 315 | 316 | // if an output file is specified, try to open it for writing 317 | FILE *outFile; 318 | FILE *locationFile; 319 | FILE *errFile; 320 | bool fileOutput = false; 321 | bool locationOutput = false; 322 | if (outputPath.length() > 0 && filesystem::is_directory(outputPath)) 323 | { 324 | outFile = fopen((filesystem::path(outputPath) / "output.txt").c_str(), "w"); 325 | if (!outFile) 326 | { 327 | printf("Could not use directory '%s' for output\n", outputPath.c_str()); 328 | return -1; 329 | } 330 | locationFile = fopen((filesystem::path(outputPath) / "locations.json").c_str(), "w"); 331 | if (!locationFile) 332 | { 333 | printf("Could not create location output file. Location information will not be saved to file."); 334 | locationFile = stdout; 335 | } 336 | else 337 | { 338 | locationOutput = true; 339 | } 340 | errFile = outFile; 341 | fileOutput = true; 342 | } 343 | else 344 | { 345 | outFile = stdout; 346 | locationFile = stdout; 347 | errFile = stderr; 348 | } 349 | 350 | // Read a JSON schema from file into Document 351 | Document d; 352 | char buffer[bufferSize]; 353 | 354 | { 355 | FILE *fp = fopen(schemaPath.c_str(), "r"); 356 | if (!fp) 357 | { 358 | fprintf(outFile, "Schema file '%s' not found\n", schemaPath.c_str()); 359 | if (fileOutput) 360 | { 361 | fclose(outFile); 362 | } 363 | return -1; 364 | } 365 | FileReadStream fs(fp, buffer, sizeof(buffer)); 366 | d.ParseStream(fs); 367 | if (d.HasParseError()) 368 | { 369 | fprintf(errFile, "Schema file '%s' is not a valid JSON\n", schemaPath.c_str()); 370 | fprintf(errFile, "Error(offset %u): %s\n", 371 | static_cast(d.GetErrorOffset()), 372 | GetParseError_En(d.GetParseError())); 373 | fclose(fp); 374 | if (fileOutput) 375 | { 376 | fclose(outFile); 377 | } 378 | return EXIT_FAILURE; 379 | } 380 | fclose(fp); 381 | } 382 | 383 | // Then convert the Document into SchemaDocument 384 | SchemaDocument sd(d); 385 | 386 | // Use reader to parse the JSON in stdin, and forward SAX events to validator 387 | // SchemaValidator validator(sd); 388 | MessageHandler handler(schemaName); 389 | GenericSchemaValidator validator(sd, handler); 390 | // set validator flags 391 | if (!failFast) 392 | { 393 | validator.SetValidateFlags(kValidateContinueOnErrorFlag); 394 | } 395 | Reader reader; 396 | FILE *fp2 = fopen(dataPath.c_str(), "r"); 397 | if (!fp2) 398 | { 399 | fprintf(outFile, "JSON file '%s' not found\n", dataPath.c_str()); 400 | if (fileOutput) 401 | { 402 | fclose(outFile); 403 | } 404 | return -1; 405 | } 406 | FileReadStream is(fp2, buffer, sizeof(buffer)); 407 | if (!reader.Parse(is, validator) && reader.GetParseErrorCode() != kParseErrorTermination) 408 | { 409 | // Schema validator error would cause kParseErrorTermination, which will handle it in next step. 410 | fprintf(errFile, "Input is not a valid JSON\n"); 411 | fprintf(errFile, "Error(offset %u): %s\n", 412 | static_cast(reader.GetErrorOffset()), 413 | GetParseError_En(reader.GetParseErrorCode())); 414 | return EXIT_FAILURE; 415 | } 416 | 417 | // Check the validation result 418 | if (validator.IsValid()) 419 | { 420 | fprintf(outFile, "Input JSON is valid.\n"); 421 | char lb[1024]; 422 | FileWriteStream locationStream(locationFile, lb, 1024); 423 | PrettyWriter locationWriter(locationStream); 424 | locationWriter.StartObject(); 425 | if (handler.inNetworkLocations.size() > 0) 426 | { 427 | locationWriter.Key("inNetwork"); 428 | locationWriter.StartArray(); 429 | for (string loc : handler.inNetworkLocations) 430 | { 431 | locationWriter.String(loc); 432 | } 433 | locationWriter.EndArray(); 434 | } 435 | if (handler.additionalLocations.size() > 0) 436 | { 437 | if (schemaName == "in-network-rates") 438 | { 439 | locationWriter.Key("providerReference"); 440 | } 441 | else 442 | { 443 | locationWriter.Key("allowedAmount"); 444 | } 445 | locationWriter.StartArray(); 446 | for (string loc : handler.additionalLocations) 447 | { 448 | locationWriter.String(loc); 449 | } 450 | locationWriter.EndArray(); 451 | } 452 | locationWriter.EndObject(); 453 | locationWriter.Flush(); 454 | 455 | if (fileOutput) 456 | { 457 | fclose(outFile); 458 | } 459 | if (locationOutput) 460 | { 461 | fclose(locationFile); 462 | } 463 | return EXIT_SUCCESS; 464 | } 465 | else 466 | { 467 | fprintf(outFile, "Input JSON is invalid.\n"); 468 | StringBuffer sb; 469 | validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); 470 | fprintf(errFile, "Invalid schema: %s\n", sb.GetString()); 471 | fprintf(errFile, "Invalid keyword: %s\n", validator.GetInvalidSchemaKeyword()); 472 | fprintf(errFile, "Invalid code: %d\n", validator.GetInvalidSchemaCode()); 473 | fprintf(errFile, "Invalid message: %s\n", GetValidateError_En(validator.GetInvalidSchemaCode())); 474 | sb.Clear(); 475 | validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); 476 | fprintf(errFile, "Invalid document: %s\n", sb.GetString()); 477 | // Detailed violation report is available as a JSON value 478 | sb.Clear(); 479 | PrettyWriter w(sb); 480 | validator.GetError().Accept(w); 481 | fprintf(errFile, "Error report:\n%s\n", sb.GetString()); 482 | CreateErrorMessages(validator.GetError(), outFile); 483 | if (fileOutput) 484 | { 485 | fclose(outFile); 486 | } 487 | if (locationOutput) 488 | { 489 | fclose(locationFile); 490 | } 491 | return EXIT_FAILURE; 492 | } 493 | } 494 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | import readlineSync from 'readline-sync'; 2 | import path from 'path'; 3 | import fs from 'fs-extra'; 4 | import temp from 'temp'; 5 | import yauzl from 'yauzl'; 6 | import { EOL } from 'os'; 7 | import { DockerManager } from './DockerManager'; 8 | import { SchemaManager } from './SchemaManager'; 9 | import { logger } from './logger'; 10 | import { DownloadManager } from './DownloadManager'; 11 | 12 | export type ZipContents = { 13 | zipFile: yauzl.ZipFile; 14 | jsonEntries: yauzl.Entry[]; 15 | dataPath: string; 16 | }; 17 | 18 | export const config = { 19 | AVAILABLE_SCHEMAS: [ 20 | 'allowed-amounts', 21 | 'in-network-rates', 22 | 'provider-reference', 23 | 'table-of-contents' 24 | ], 25 | SCHEMA_REPO_URL: 'https://github.com/CMSgov/price-transparency-guide.git', 26 | SCHEMA_REPO_FOLDER: path.normalize(path.join(__dirname, '..', 'schema-repo')) 27 | }; 28 | 29 | type ContainerResult = { 30 | pass: boolean; 31 | locations?: { 32 | inNetwork?: string[]; 33 | allowedAmount?: string[]; 34 | providerReference?: string[]; 35 | }; 36 | }; 37 | 38 | export async function getEntryFromZip( 39 | zipFile: yauzl.ZipFile, 40 | entry: yauzl.Entry, 41 | dataPath: string 42 | ): Promise { 43 | return new Promise((resolve, reject) => { 44 | zipFile.openReadStream(entry, (err, readStream) => { 45 | if (err) { 46 | reject(err); 47 | } else { 48 | const outputStream = fs.createWriteStream(dataPath); 49 | outputStream.on('finish', () => { 50 | // keep the zipFile open for now, in case we want more entries 51 | resolve(); 52 | }); 53 | outputStream.on('error', () => { 54 | zipFile.close(); 55 | reject('Error writing chosen file.'); 56 | }); 57 | readStream.pipe(outputStream); 58 | } 59 | }); 60 | }); 61 | } 62 | 63 | export function isGzip(contentType: string, url: string): boolean { 64 | return ( 65 | contentType === 'application/gzip' || 66 | contentType === 'application/x-gzip' || 67 | (contentType === 'application/octet-stream' && /\.gz(\?|$)/.test(url)) 68 | ); 69 | } 70 | 71 | export function isZip(contentType: string, url: string): boolean { 72 | return contentType === 'application/zip' || /\.zip(\?|$)/.test(url); 73 | } 74 | 75 | export function chooseJsonFile(entries: yauzl.Entry[]): yauzl.Entry { 76 | // there might be a lot of entries. show ten per page of results 77 | logger.info(`${entries.length} JSON files found within ZIP archive.`); 78 | const maxPage = Math.floor((entries.length - 1) / 10); 79 | let currentPage = 0; 80 | let chosenIndex: number; 81 | 82 | showMenuOptions( 83 | currentPage, 84 | maxPage, 85 | entries.slice(currentPage * 10, currentPage * 10 + 10).map(ent => ent.fileName) 86 | ); 87 | readlineSync.promptCLLoop((command, ...extraArgs) => { 88 | if (/^n(ext)?$/i.test(command)) { 89 | if (currentPage < maxPage) { 90 | currentPage++; 91 | } else { 92 | logger.menu('Already at last page.'); 93 | } 94 | } else if (/^p(revious)?$/i.test(command)) { 95 | if (currentPage > 0) { 96 | currentPage--; 97 | } else { 98 | logger.menu('Already at first page.'); 99 | } 100 | } else if (/^go?$/i.test(command)) { 101 | const targetPage = parseInt(extraArgs[0]); 102 | if (targetPage > 0 && targetPage <= maxPage + 1) { 103 | currentPage = targetPage - 1; 104 | } else { 105 | logger.menu("Can't go to that page."); 106 | } 107 | } else if (/^\d$/.test(command)) { 108 | chosenIndex = currentPage * 10 + parseInt(command); 109 | logger.menu(`You selected: ${entries[chosenIndex].fileName}`); 110 | return true; 111 | } else { 112 | logger.menu('Unrecognized command.'); 113 | } 114 | showMenuOptions( 115 | currentPage, 116 | maxPage, 117 | entries.slice(currentPage * 10, (currentPage + 1) * 10).map(ent => ent.fileName) 118 | ); 119 | }); 120 | return entries[chosenIndex]; 121 | } 122 | 123 | function showMenuOptions(currentPage: number, maxPage: number, items: string[]) { 124 | logger.menu(`Showing page ${currentPage + 1} of ${maxPage + 1}`); 125 | items.forEach((item, idx) => { 126 | logger.menu(`(${idx}): ${item}`); 127 | }); 128 | const commandsToShow: string[] = []; 129 | if (currentPage > 0) { 130 | commandsToShow.push('(p)revious page'); 131 | } 132 | if (currentPage < maxPage) { 133 | commandsToShow.push('(n)ext page'); 134 | } 135 | if (maxPage > 0) { 136 | commandsToShow.push('"(g)o X" to jump to a page'); 137 | } 138 | logger.menu(commandsToShow.join(' | ')); 139 | } 140 | 141 | export function appendResults(source: string, destination: string, prefixData: string = '') { 142 | try { 143 | const sourceData = fs.readFileSync(source); 144 | fs.appendFileSync(destination, `${prefixData}${sourceData}`); 145 | } catch (err) { 146 | logger.error('Problem copying results to output file', err); 147 | } 148 | } 149 | 150 | export async function assessTocContents( 151 | locations: ContainerResult['locations'], 152 | schemaManager: SchemaManager, 153 | dockerManager: DockerManager, 154 | downloadManager: DownloadManager 155 | ): Promise { 156 | const totalFileCount = 157 | (locations?.inNetwork?.length ?? 0) + (locations?.allowedAmount?.length ?? 0); 158 | const fileText = totalFileCount === 1 ? 'this file' : 'these files'; 159 | if (totalFileCount > 0) { 160 | logger.info(`Table of contents refers to ${fileText}:`); 161 | if (locations.inNetwork?.length > 0) { 162 | logger.info('== In-Network Rates =='); 163 | locations.inNetwork.forEach(inf => logger.info(`* ${inf}`)); 164 | } 165 | if (locations.allowedAmount?.length > 0) { 166 | logger.info('== Allowed Amounts =='); 167 | locations.allowedAmount.forEach(aaf => logger.info(`* ${aaf}`)); 168 | } 169 | const wantToValidateContents = 170 | downloadManager.alwaysYes || 171 | readlineSync.keyInYNStrict(`Would you like to validate ${fileText}?`); 172 | if (wantToValidateContents) { 173 | const providerReferences = await validateTocContents( 174 | locations.inNetwork ?? [], 175 | locations.allowedAmount ?? [], 176 | schemaManager, 177 | dockerManager, 178 | downloadManager 179 | ); 180 | return providerReferences; 181 | } 182 | } 183 | return []; 184 | } 185 | 186 | export async function validateTocContents( 187 | inNetwork: string[], 188 | allowedAmount: string[], 189 | schemaManager: SchemaManager, 190 | dockerManager: DockerManager, 191 | downloadManager: DownloadManager 192 | ): Promise { 193 | temp.track(); 194 | let tempOutput = ''; 195 | if (dockerManager.outputPath?.length > 0) { 196 | tempOutput = path.join(temp.mkdirSync('contents'), 'contained-result'); 197 | } 198 | let providerReferences: Set; 199 | if (inNetwork.length > 0) { 200 | if (schemaManager.shouldDetectVersion) { 201 | providerReferences = await validateInNetworkDetectedVersion( 202 | inNetwork, 203 | schemaManager, 204 | dockerManager, 205 | downloadManager, 206 | tempOutput 207 | ); 208 | } else { 209 | providerReferences = await validateInNetworkFixedVersion( 210 | inNetwork, 211 | schemaManager, 212 | dockerManager, 213 | downloadManager, 214 | tempOutput 215 | ); 216 | } 217 | } 218 | if (allowedAmount.length > 0) { 219 | if (schemaManager.shouldDetectVersion) { 220 | await validateAllowedAmountsDetectedVersion( 221 | allowedAmount, 222 | schemaManager, 223 | dockerManager, 224 | downloadManager, 225 | tempOutput 226 | ); 227 | } else { 228 | await validateAllowedAmountsFixedVersion( 229 | allowedAmount, 230 | schemaManager, 231 | dockerManager, 232 | downloadManager, 233 | tempOutput 234 | ); 235 | } 236 | } 237 | return [...providerReferences.values()]; 238 | } 239 | 240 | async function validateInNetworkFixedVersion( 241 | inNetwork: string[], 242 | schemaManager: SchemaManager, 243 | dockerManager: DockerManager, 244 | downloadManager: DownloadManager, 245 | tempOutput: string 246 | ) { 247 | const providerReferences: Set = new Set(); 248 | await schemaManager.useSchema('in-network-rates').then(async schemaPath => { 249 | if (schemaPath != null) { 250 | for (const dataUrl of inNetwork) { 251 | try { 252 | if (await downloadManager.checkDataUrl(dataUrl)) { 253 | logger.info(`File: ${dataUrl}`); 254 | const dataPath = await downloadManager.downloadDataFile(dataUrl); 255 | if (typeof dataPath === 'string') { 256 | // check if detected version matches the provided version 257 | // if there's no version property, that's ok 258 | await schemaManager 259 | .determineVersion(dataPath) 260 | .then(detectedVersion => { 261 | if (detectedVersion != schemaManager.version) { 262 | logger.warn( 263 | `Schema version ${schemaManager.version} was provided, but file indicates it conforms to schema version ${detectedVersion}. ${schemaManager.version} will be used.` 264 | ); 265 | } 266 | }) 267 | .catch(() => {}); 268 | const containedResult = await dockerManager.runContainer( 269 | schemaPath, 270 | 'in-network-rates', 271 | dataPath, 272 | tempOutput 273 | ); 274 | if ( 275 | containedResult.pass && 276 | containedResult.locations?.providerReference?.length > 0 277 | ) { 278 | containedResult.locations.providerReference.forEach(prf => 279 | providerReferences.add(prf) 280 | ); 281 | } 282 | if (tempOutput.length > 0) { 283 | appendResults( 284 | tempOutput, 285 | dockerManager.outputPath, 286 | `${dataUrl} - in-network${EOL}` 287 | ); 288 | } 289 | } 290 | } else { 291 | logger.error(`Could not download file: ${dataUrl}`); 292 | } 293 | } catch (err) { 294 | logger.error('Problem validating referenced in-network file', err); 295 | } 296 | } 297 | } else { 298 | logger.error('No schema available - not validating.'); 299 | } 300 | }); 301 | return providerReferences; 302 | } 303 | 304 | async function validateInNetworkDetectedVersion( 305 | inNetwork: string[], 306 | schemaManager: SchemaManager, 307 | dockerManager: DockerManager, 308 | downloadManager: DownloadManager, 309 | tempOutput: string 310 | ) { 311 | const providerReferences: Set = new Set(); 312 | for (const dataUrl of inNetwork) { 313 | try { 314 | if (await downloadManager.checkDataUrl(dataUrl)) { 315 | logger.info(`File: ${dataUrl}`); 316 | const dataPath = await downloadManager.downloadDataFile(dataUrl); 317 | if (typeof dataPath === 'string') { 318 | const versionToUse = await schemaManager.determineVersion(dataPath); 319 | await schemaManager 320 | .useVersion(versionToUse) 321 | .then(versionIsAvailable => { 322 | if (versionIsAvailable) { 323 | return schemaManager.useSchema('in-network-rates'); 324 | } else { 325 | return null; 326 | } 327 | }) 328 | .then(schemaPath => { 329 | if (schemaPath != null) { 330 | return dockerManager.runContainer( 331 | schemaPath, 332 | 'in-network-rates', 333 | dataPath, 334 | tempOutput 335 | ); 336 | } 337 | }) 338 | .then(containedResult => { 339 | if ( 340 | containedResult.pass && 341 | containedResult.locations?.providerReference?.length > 0 342 | ) { 343 | containedResult.locations.providerReference.forEach(prf => 344 | providerReferences.add(prf) 345 | ); 346 | } 347 | if (tempOutput.length > 0) { 348 | appendResults( 349 | tempOutput, 350 | dockerManager.outputPath, 351 | `${dataUrl} - in-network${EOL}` 352 | ); 353 | } 354 | }); 355 | } 356 | } 357 | } catch (err) { 358 | logger.error('Problem validating referenced in-network file', err); 359 | } 360 | } 361 | return providerReferences; 362 | } 363 | 364 | async function validateAllowedAmountsFixedVersion( 365 | allowedAmount: string[], 366 | schemaManager: SchemaManager, 367 | dockerManager: DockerManager, 368 | downloadManager: DownloadManager, 369 | tempOutput: string 370 | ) { 371 | await schemaManager.useSchema('allowed-amounts').then(async schemaPath => { 372 | if (schemaPath != null) { 373 | for (const dataUrl of allowedAmount) { 374 | try { 375 | if (await downloadManager.checkDataUrl(dataUrl)) { 376 | logger.info(`File: ${dataUrl}`); 377 | const dataPath = await downloadManager.downloadDataFile(dataUrl); 378 | if (typeof dataPath === 'string') { 379 | // check if detected version matches the provided version 380 | // if there's no version property, that's ok 381 | await schemaManager 382 | .determineVersion(dataPath) 383 | .then(detectedVersion => { 384 | if (detectedVersion != schemaManager.version) { 385 | logger.warn( 386 | `Schema version ${schemaManager.version} was provided, but file indicates it conforms to schema version ${detectedVersion}. ${schemaManager.version} will be used.` 387 | ); 388 | } 389 | }) 390 | .catch(() => {}); 391 | await dockerManager.runContainer(schemaPath, 'allowed-amounts', dataPath, tempOutput); 392 | if (tempOutput.length > 0) { 393 | appendResults( 394 | tempOutput, 395 | dockerManager.outputPath, 396 | `${dataUrl} - allowed-amounts${EOL}` 397 | ); 398 | } 399 | } 400 | } else { 401 | logger.error(`Could not download file: ${dataUrl}`); 402 | } 403 | } catch (err) { 404 | logger.error('Problem validating referenced allowed-amounts file', err); 405 | } 406 | } 407 | } else { 408 | logger.error('No schema available - not validating.'); 409 | } 410 | }); 411 | } 412 | 413 | async function validateAllowedAmountsDetectedVersion( 414 | allowedAmount: string[], 415 | schemaManager: SchemaManager, 416 | dockerManager: DockerManager, 417 | downloadManager: DownloadManager, 418 | tempOutput: string 419 | ) { 420 | for (const dataUrl of allowedAmount) { 421 | try { 422 | if (await downloadManager.checkDataUrl(dataUrl)) { 423 | logger.info(`File: ${dataUrl}`); 424 | const dataPath = await downloadManager.downloadDataFile(dataUrl); 425 | if (typeof dataPath === 'string') { 426 | const versionToUse = await schemaManager.determineVersion(dataPath); 427 | await schemaManager 428 | .useVersion(versionToUse) 429 | .then(versionIsAvailable => { 430 | if (versionIsAvailable) { 431 | return schemaManager.useSchema('allowed-amounts'); 432 | } else { 433 | return null; 434 | } 435 | }) 436 | .then(schemaPath => { 437 | if (schemaPath != null) { 438 | return dockerManager.runContainer( 439 | schemaPath, 440 | 'allowed-amounts', 441 | dataPath, 442 | tempOutput 443 | ); 444 | } 445 | }) 446 | .then(() => { 447 | if (tempOutput.length > 0) { 448 | appendResults( 449 | tempOutput, 450 | dockerManager.outputPath, 451 | `${dataUrl} - allowed-amounts${EOL}` 452 | ); 453 | } 454 | }); 455 | } 456 | } 457 | } catch (err) { 458 | logger.error('Problem validating referenced allowed-amounts file', err); 459 | } 460 | } 461 | } 462 | 463 | export async function assessReferencedProviders( 464 | providerReferences: string[], 465 | schemaManager: SchemaManager, 466 | dockerManager: DockerManager, 467 | downloadManager: DownloadManager 468 | ) { 469 | if (providerReferences.length > 0) { 470 | const fileText = providerReferences.length === 1 ? 'this file' : 'these files'; 471 | if (providerReferences.length === 1) { 472 | logger.info(`In-network file(s) refer to ${fileText}:`); 473 | logger.info('== Provider Reference =='); 474 | providerReferences.forEach(prf => logger.info(`* ${prf}`)); 475 | const wantToValidateProviders = 476 | downloadManager.alwaysYes || 477 | readlineSync.keyInYNStrict(`Would you like to validate ${fileText}?`); 478 | if (wantToValidateProviders) { 479 | await validateReferencedProviders( 480 | providerReferences, 481 | schemaManager, 482 | dockerManager, 483 | downloadManager 484 | ); 485 | } 486 | } 487 | } 488 | } 489 | 490 | export async function validateReferencedProviders( 491 | providerReferences: string[], 492 | schemaManager: SchemaManager, 493 | dockerManager: DockerManager, 494 | downloadManager: DownloadManager 495 | ) { 496 | temp.track(); 497 | let tempOutput = ''; 498 | if (dockerManager.outputPath?.length > 0) { 499 | tempOutput = path.join(temp.mkdirSync('providers'), 'contained-result'); 500 | } 501 | if (providerReferences.length > 0) { 502 | schemaManager.useSchema('provider-reference').then(async schemaPath => { 503 | if (schemaPath != null) { 504 | for (const dataUrl of providerReferences) { 505 | try { 506 | if (await downloadManager.checkDataUrl(dataUrl)) { 507 | logger.info(`File: ${dataUrl}`); 508 | const dataPath = await downloadManager.downloadDataFile(dataUrl); 509 | if (typeof dataPath === 'string') { 510 | await dockerManager.runContainer( 511 | schemaPath, 512 | 'provider-reference', 513 | dataPath, 514 | tempOutput 515 | ); 516 | if (tempOutput.length > 0) { 517 | appendResults( 518 | tempOutput, 519 | dockerManager.outputPath, 520 | `${dataUrl} - provider-reference${EOL}` 521 | ); 522 | } 523 | } 524 | } else { 525 | logger.error(`Could not download file: ${dataUrl}`); 526 | } 527 | } catch (err) { 528 | logger.error('Problem validating referenced provider-reference file', err); 529 | } 530 | } 531 | } else { 532 | logger.error('No schema available - not validating.'); 533 | } 534 | }); 535 | } 536 | } 537 | --------------------------------------------------------------------------------