├── .eslintrc ├── .github ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── check.yml │ └── npm-publish.yml ├── .gitignore ├── .nycrc ├── .oclif.manifest.json ├── .prettierignore ├── LICENSE.txt ├── README.md ├── messages └── org.json ├── package.json ├── src ├── commands │ └── bourne │ │ ├── export.ts │ │ └── import.ts ├── helper │ ├── ApexResponse.ts │ ├── Helper.ts │ └── ImportResult.ts ├── index.ts └── model │ └── CPQDataImportRequest.ts ├── tsconfig.json └── tslint.json /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "parser": "@typescript-eslint/parser", 4 | "plugins": ["@typescript-eslint"], 5 | "extends": [ 6 | "eslint:recommended", 7 | "plugin:@typescript-eslint/eslint-recommended", 8 | "plugin:@typescript-eslint/recommended" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | #### Issue Description 2 | 3 | 4 | #### Issue Fix Ideas 5 | 6 | 7 | #### Helpful Links 8 | 9 | * [Example Link](www.url.com) -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | #### What changes are included in this PR? 2 | -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: check 2 | on: 3 | pull_request: 4 | branches: 5 | - master 6 | jobs: 7 | lint: 8 | name: prettier 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout repository 12 | uses: actions/checkout@master 13 | - name: Set up Node.js 14 | uses: actions/setup-node@master 15 | with: 16 | node-version: '10.x' 17 | - name: Install packages 18 | run: npm install prettier 19 | - name: Run Style Check 20 | run: npx prettier -c **.ts 21 | 22 | build: 23 | name: typescript-build 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: Checkout repository 27 | uses: actions/checkout@master 28 | - name: Set up Node.js 29 | uses: actions/setup-node@master 30 | with: 31 | node-version: '10.x' 32 | - name: Install packages 33 | run: npm i 34 | - name: Run TypeScript Build 35 | run: tsc --build 36 | -------------------------------------------------------------------------------- /.github/workflows/npm-publish.yml: -------------------------------------------------------------------------------- 1 | name: npm-publish 2 | on: 3 | push: 4 | branches: 5 | - master 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | # Setup .npmrc file to publish to npm 12 | - uses: actions/setup-node@v1 13 | with: 14 | node-version: '12.x' 15 | registry-url: 'https://registry.npmjs.org' 16 | - run: | 17 | npm install 18 | tsc --build 19 | - run: npm publish 20 | env: 21 | NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *-debug.log 2 | *-error.log 3 | /.nyc_output 4 | /dist 5 | /lib 6 | /package-lock.json 7 | /tmp 8 | node_modules 9 | data/ 10 | .DS_Store 11 | .envrc 12 | npm-shrinkwrap.json 13 | oclif.manifest.json 14 | yarn.lock 15 | .vscode/ 16 | 17 | .idea/ 18 | -------------------------------------------------------------------------------- /.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "extension": [ 3 | ".ts" 4 | ], 5 | "include": [ 6 | "src/**/*.ts" 7 | ], 8 | "exclude": [ 9 | "**/*.d.ts" 10 | ], 11 | "all": true 12 | } 13 | -------------------------------------------------------------------------------- /.oclif.manifest.json: -------------------------------------------------------------------------------- 1 | {"version":"0.1.1-217","commands":{"bourne:export":{"id":"bourne:export","description":"Exports records from the object specified.","usage":"<%= command.id %> [-o ] [-c ] [-a] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]","pluginName":"json-bourne-sfdx","pluginType":"core","aliases":[],"examples":["$ sfdx bourne:export -o Product2 -u myOrg -c config/cpq-cli-def.json\n Requesting data, please wait.... Request completed! Received X records.\n "],"flags":{"json":{"name":"json","type":"boolean","description":"format output as json","allowNo":false},"loglevel":{"name":"loglevel","type":"option","description":"logging level for this command invocation","required":false,"helpValue":"(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL)","options":["trace","debug","info","warn","error","fatal","TRACE","DEBUG","INFO","WARN","ERROR","FATAL"],"default":"warn"},"targetusername":{"name":"targetusername","type":"option","char":"u","description":"username or alias for the target org; overrides default target org"},"apiversion":{"name":"apiversion","type":"option","description":"override the api version used for api requests made by this command"},"object":{"name":"object","type":"option","char":"o","description":"The sobject that you wish to import/export reference data from."},"configfile":{"name":"configfile","type":"option","char":"c","description":"[REQUIRED] The configuration JSON file location."},"processall":{"name":"processall","type":"boolean","char":"a","description":"Exports records from all objects specified in the config file.","allowNo":false}},"args":[{"name":"file"}]},"bourne:import":{"id":"bourne:import","description":"Imports records from the object specified.","usage":"<%= command.id %> [-o ] [-c ] [-a] [-d ] [-r] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]","pluginName":"json-bourne-sfdx","pluginType":"core","aliases":[],"examples":["$ sfdx bourne:import -o Product2 -u myOrg -c config/cpq-cli-def.json\n Deploying data, please wait.... Deployment completed!\n "],"flags":{"json":{"name":"json","type":"boolean","description":"format output as json","allowNo":false},"loglevel":{"name":"loglevel","type":"option","description":"logging level for this command invocation","required":false,"helpValue":"(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL)","options":["trace","debug","info","warn","error","fatal","TRACE","DEBUG","INFO","WARN","ERROR","FATAL"],"default":"warn"},"targetusername":{"name":"targetusername","type":"option","char":"u","description":"username or alias for the target org; overrides default target org"},"apiversion":{"name":"apiversion","type":"option","description":"override the api version used for api requests made by this command"},"object":{"name":"object","type":"option","char":"o","description":"The sobject that you wish to import/export reference data from."},"configfile":{"name":"configfile","type":"option","char":"c","description":"[REQUIRED] The configuration JSON file location."},"processall":{"name":"processall","type":"boolean","char":"a","description":"Imports records from all objects specified in the config file.","allowNo":false},"datadir":{"name":"datadir","type":"option","char":"d","description":"The path where the reference data resides. The default is 'data'."},"remove":{"name":"remove","type":"boolean","char":"r","description":"Delete the record(s) from the target within the specified directory.","allowNo":false}},"args":[{"name":"file"}]}}} -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # Ignore everything: 2 | /* 3 | 4 | # Except myapp: 5 | !/src 6 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | License 2 | 3 | Copyright (C) 2012 REA Group Ltd. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier) 2 | 3 | # JSON Bourne 4 | 5 | This plugin allows you to migrate reference data between Salesforce environments. The plugin relies on External IDs being setup on each of the objects that you wish to migrate, but has a generator that can assist you in populating your External Ids. 6 | 7 | JSON Bourne consists of two parts: 8 | 9 | 1. A Salesforce managed package that largely consists of an API to receive the data being imported (which is also open source, and can be found [here](https://github.com/realestate-com-au/json-bourne-sfdx-pkg). 10 | 2. A Salesforce CLI plugin that allows you to orchestrate the import and export of data (which can be run by a developer or installed into a CI pipeline). 11 | 12 | # Salesforce Managed Package 13 | 14 | There is a server side component to this package. This has been packaged into a Managed Package and the links can be found below: 15 | 16 | ## Package Links 17 | 18 | | Version | Link | 19 | | ------- | -------------------------------------------------------------------------------------------- | 20 | | 1.2 | [Click Here](https://login.salesforce.com/packaging/installPackage.apexp?p0=04t2v0000007RE9) | 21 | | 1.1 | [Click Here](https://login.salesforce.com/packaging/installPackage.apexp?p0=04t2v000000CdUM) | 22 | | 1.0 | _deprecated_ | 23 | 24 | The package must be installed into all orgs that you can to _import_ data to, but the package is not required on an org to export. 25 | 26 | > This package is also available open source, so you can choose to add this metadata into your Salesforce project as unmanaged meta. If you do this, ensure you set the flag `useManagedPackage` in all configuration files to `false`. 27 | 28 | ## Object setup 29 | 30 | Each object that will be exported/imported by JSON Bourne must have the following setup: 31 | 32 | 1. An external Id 33 | 2. A trigger to populate the external id field when a record is created _(optional)_ 34 | 35 | > The use of the trigger is optional, but any record extracted and migrated via JSON Bourne _must_ have an External Id. 36 | 37 | ## Trigger Example 38 | 39 | The managed package comes with the capability to generate an external id on any object. You first need to create a new class called `MigrationIdAllocation` 40 | 41 | ```java 42 | public class MigrationIdAllocation{ 43 | 44 | protected String EXTERNAL_ID_FIELD{ 45 | get{ 46 | if(String.valueOf(objectType) == 'Product2'){ 47 | return 'Unique_Product_Code__c'; 48 | } 49 | return 'Migration_Id__c'; 50 | } 51 | } 52 | 53 | public MigrationIdAllocation() { 54 | if(Trigger.isBefore && (Trigger.isInsert || Trigger.isUpdate)){ 55 | JSON.MigrationIdService.addMigrationId(records, EXTERNAL_ID_FIELD); 56 | } 57 | } 58 | 59 | } 60 | ``` 61 | 62 | Next, this class can be invoked by any trigger to generate an external Id: 63 | 64 | ```java 65 | trigger errorConditionTrigger on SBQQ__ErrorCondition__c (before insert,before update) { 66 | MigrationIdAllocation handler = new MigrationIdAllocation(); 67 | } 68 | ``` 69 | 70 | In this example, `MigrationIdAllocation` is configured to check the object name. If it is `Product2` then it specifies to populate the `Unique_Product_Code__c`, else it expects the field `Migration_Id__c` to exist. In the same manner, you can add fields called `Migration_Id__c` to most objects, but explicitly specify any objects that have an external id field with a different API name. 71 | 72 | You only need to create the `MigrationIdAllocation` class once, but each object that requires an auto-generated external Id will need a trigger. 73 | 74 | # Salesforce CLI Plugin 75 | 76 | To install this plugin to Salesforce CLI, use the `plugins:install` command: 77 | 78 | sfdx plugins:install json-bourne-sfdx 79 | 80 | ## Configuration File 81 | 82 | The plugin requires a configuration file. Here is an example: 83 | 84 | ```json 85 | { 86 | "pollTimeout": 30, 87 | "pollBatchSize": 150, 88 | "maxPollCount": 60, 89 | "payloadLength": 2999800, 90 | "importRetries": 3, 91 | "useManagedPackage": true, 92 | "allObjects": ["ObjectOne__c", "ObjectTwo__c"], 93 | "objects": { 94 | "ObjectOne__c": { 95 | "query": "SELECT Name, FieldOne__c, FieldTwo__c, Migration_Id__c FROM ObjectOne__c", 96 | "externalid": "Migration_ID__c", 97 | "directory": "objectOne", 98 | "cleanupFields": [], 99 | "hasRecordTypes": false 100 | }, 101 | "ObjectTwo__c": { 102 | "query": "SELECT Name, ObjectOne__r.Migration_Id__c, Migration_Id__c, FieldOne__c, RecordType.DeveloperName FROM ObjectTwo__c", 103 | "externalid": "Migration_Id__c", 104 | "directory": "objectTwo", 105 | "cleanupFields": ["ObjectOne__r"], 106 | "hasRecordTypes": true, 107 | "enableMultiThreading": true 108 | } 109 | } 110 | } 111 | ``` 112 | 113 | | Parameter | Definition | 114 | | ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 115 | | `pollTimeout` | How long the query will wait on a response | 116 | | `pollBatchSize` | The batch size the data is polled in | 117 | | `maxPollCount` | The maximum number of queries performed on one object in one transaction | 118 | | `payloadLength` | The character size data will be broken into for `imports` | 119 | | `importRetries` | How many times the import method will attempt to retry failed imports before exiting | 120 | | `useManagedPackage` | If the server side components are in a managed package, this should be set to true | 121 | | `allObjects` | An array with all of the objects (the order they are specified is the order they are processed) | 122 | | `objects` | An array of configuration for each object (there should be one entry per entry in the `allObjects` array) | 123 | | `objects:query` | The query that is performed to `export` the data from Salesforce | 124 | | `objects:externalId` | The external id field for this object (API Name). This will also be used as the file name for the extracted json files. | 125 | | `objects:directory` | The name of the directory the extracted data will be stored in. It is a relevant path from where the export command is executed of `data/directory` | | 126 | | `objects:cleanupFields` | Any fields fetched from a parent lookup that may be blank should be specified here. For example, if you are retrieving the Migration Id of a parent (`ObjectOne__r.Migration_Id__c`) and this lookup _could_ be blank, then `ObjectOne__r` should be added to this array as the plugin cannot handle null values on parent references. | 127 | | `objects:hasRecordTypes` | Whether dynamic record type Ids need to be handled or not (if true, ensure the `RecordType.DeveloperName` is in the query) | 128 | | `objects:enableMultiThreading` | The payloads can be imported into this object in parallel (instead of one after the other). | 129 | 130 | The configuration file can be stored within a Salesforce DX project, for example within the `config/` directory alongside the project configuration file. This file is referenced during each `import` or `export` request with the `-c` parameter (see below for more details). As this file is referenced each time, you can create a different configuration file for each logical grouping of reference data, for example if you have both Salesforce CPQ and Adobe Sign installed which both have reference data, you could create one configuration file for Salesforce CPQ and one configuration file for Adobe Sign. 131 | 132 | > All data that is exported using JSON Bourne will be saved into a `data/` directory (and into subdirectories that are the names specified on each object configuration specified within the configuration file). Ensure the `data/` directory exists in your Salesforce DX project before exporting any reference data. 133 | 134 | ### Configuration Files Templates 135 | 136 | These configuration files relate to common Salesforce packages. Crafting the configuration file can be the most difficult part of setting up JSON Bourne, so this list is a way to kick start your implementation. If you have any configuration files crafted please add them to this list! 137 | 138 | - [Salesforce CPQ Configuration File](https://gist.github.com/ddawson235/c6e639691d0876c3b0b591faf66a4565) (David Dawson [@ddawson235](https://github.com/ddawson235)) 139 | 140 | # Plugin commands 141 | 142 | 143 | * [JSON Bourne](#json-bourne) 144 | * [Salesforce Managed Package](#salesforce-managed-package) 145 | * [Salesforce CLI Plugin](#salesforce-cli-plugin) 146 | * [Plugin commands](#plugin-commands) 147 | 148 | 149 | 150 | ```sh-session 151 | $ npm install -g json-bourne-sfdx 152 | $ json-bourne-sfdx COMMAND 153 | running command... 154 | $ json-bourne-sfdx (-v|--version|version) 155 | json-bourne-sfdx/0.1.1-217 darwin-x64 node-v14.15.1 156 | $ json-bourne-sfdx --help [COMMAND] 157 | USAGE 158 | $ json-bourne-sfdx COMMAND 159 | ... 160 | ``` 161 | 162 | 163 | * [`json-bourne-sfdx bourne:export [-o ] [-c ] [-a] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#json-bourne-sfdx-bourneexport--o-string--c-string--a--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal) 164 | * [`json-bourne-sfdx bourne:import [-o ] [-c ] [-a] [-d ] [-r] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#json-bourne-sfdx-bourneimport--o-string--c-string--a--d-string--r--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal) 165 | 166 | ## `json-bourne-sfdx bourne:export [-o ] [-c ] [-a] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]` 167 | 168 | Exports records from the object specified. 169 | 170 | ``` 171 | Exports records from the object specified. 172 | 173 | USAGE 174 | $ json-bourne-sfdx bourne:export [-o ] [-c ] [-a] [-u ] [--apiversion ] [--json] 175 | [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL] 176 | 177 | OPTIONS 178 | -a, --processall Exports records from all objects 179 | specified in the config file. 180 | 181 | -c, --configfile=configfile [REQUIRED] The configuration JSON 182 | file location. 183 | 184 | -o, --object=object The sobject that you wish to 185 | import/export reference data from. 186 | 187 | -u, --targetusername=targetusername username or alias for the target 188 | org; overrides default target org 189 | 190 | --apiversion=apiversion override the api version used for 191 | api requests made by this command 192 | 193 | --json format output as json 194 | 195 | --loglevel=(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL) [default: warn] logging level for 196 | this command invocation 197 | 198 | EXAMPLE 199 | $ sfdx bourne:export -o Product2 -u myOrg -c config/cpq-cli-def.json 200 | Requesting data, please wait.... Request completed! Received X records. 201 | ``` 202 | 203 | _See code: [src/commands/bourne/export.ts](https://github.com/realestate-com-au/json-bourne-sfdx-cli/blob/v0.1.1-217/src/commands/bourne/export.ts)_ 204 | 205 | ## `json-bourne-sfdx bourne:import [-o ] [-c ] [-a] [-d ] [-r] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]` 206 | 207 | Imports records from the object specified. 208 | 209 | ``` 210 | Imports records from the object specified. 211 | 212 | USAGE 213 | $ json-bourne-sfdx bourne:import [-o ] [-c ] [-a] [-d ] [-r] [-u ] [--apiversion 214 | ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL] 215 | 216 | OPTIONS 217 | -a, --processall Imports records from all objects 218 | specified in the config file. 219 | 220 | -c, --configfile=configfile [REQUIRED] The configuration JSON 221 | file location. 222 | 223 | -d, --datadir=datadir The path where the reference data 224 | resides. The default is 'data'. 225 | 226 | -o, --object=object The sobject that you wish to 227 | import/export reference data from. 228 | 229 | -r, --remove Delete the record(s) from the target 230 | within the specified directory. 231 | 232 | -u, --targetusername=targetusername username or alias for the target 233 | org; overrides default target org 234 | 235 | --apiversion=apiversion override the api version used for 236 | api requests made by this command 237 | 238 | --json format output as json 239 | 240 | --loglevel=(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL) [default: warn] logging level for 241 | this command invocation 242 | 243 | EXAMPLE 244 | $ sfdx bourne:import -o Product2 -u myOrg -c config/cpq-cli-def.json 245 | Deploying data, please wait.... Deployment completed! 246 | ``` 247 | 248 | _See code: [src/commands/bourne/import.ts](https://github.com/realestate-com-au/json-bourne-sfdx-cli/blob/v0.1.1-217/src/commands/bourne/import.ts)_ 249 | 250 | 251 | License 252 | 253 | --- 254 | 255 | Copyright (C) 2012 REA Group Ltd. 256 | 257 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 258 | 259 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 260 | 261 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 262 | -------------------------------------------------------------------------------- /messages/org.json: -------------------------------------------------------------------------------- 1 | { 2 | "commandDescription": "import/export reference data from Salesforce into your dx project files", 3 | "pullDescription": "Exports records from the object specified.", 4 | "pushDescription": "Imports records from the object specified.", 5 | "pushAllDescription": "Imports records from all objects specified in the config file.", 6 | "pullAllDescription": "Exports records from all objects specified in the config file.", 7 | "objectDescription": "The sobject that you wish to import/export reference data from.", 8 | "allObjectsDescription": "Import/export from all objects.", 9 | "configFileDescription": "[REQUIRED] The configuration JSON file location.", 10 | "pathToDataDir": "The path where the reference data resides. The default is 'data'.", 11 | "removeObjects": "Delete the record(s) from the target within the specified directory." 12 | } 13 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "json-bourne-sfdx", 3 | "description": "A CLI plugin for Salesforce CLI to push and pull CPQ data", 4 | "version": "0.1.1-217", 5 | "author": "REA Group Ltd", 6 | "bugs": "https://github.com/realestate-com-au/json-bourne-sfdx-cli/issues", 7 | "dependencies": { 8 | "@oclif/command": "1", 9 | "@oclif/config": "1", 10 | "@oclif/errors": "1", 11 | "@salesforce/command": "^2.0.0", 12 | "@types/jsforce": "^1.9.4", 13 | "colors": "^1.3.1", 14 | "console.table": "^0.10.0", 15 | "jsforce": "^1.9.2", 16 | "lodash": "^4.17.11", 17 | "tslib": "1", 18 | "uuid": "^3.3.2" 19 | }, 20 | "devDependencies": { 21 | "@oclif/dev-cli": "^1.15.4", 22 | "@oclif/plugin-help": "^2.2.0", 23 | "@oclif/test": "^1", 24 | "@salesforce/dev-config": "^1.4.4", 25 | "@types/chai": "4", 26 | "@types/mocha": "5", 27 | "@types/node": "^12.0.8", 28 | "@typescript-eslint/eslint-plugin": "^3.9.0", 29 | "@typescript-eslint/parser": "^3.9.0", 30 | "chai": "^4", 31 | "eslint": "^7.7.0", 32 | "globby": "^9.2.0", 33 | "mocha": "^6.1.4", 34 | "nyc": "^14.1.1", 35 | "prettier": "^2.0.5", 36 | "sinon": "^7.3.2", 37 | "ts-node": "^8.3.0", 38 | "typescript": "^3.5.2" 39 | }, 40 | "engines": { 41 | "node": ">=8.0.0" 42 | }, 43 | "files": [ 44 | "/lib", 45 | "/messages", 46 | "/npm-shrinkwrap.json", 47 | "/.oclif.manifest.json" 48 | ], 49 | "homepage": "https://github.com/realestate-com-au/json-bourne-sfdx-cli", 50 | "keywords": [ 51 | "sfdx-plugin", 52 | "json-bourne", 53 | "json-bourne-sfdx" 54 | ], 55 | "license": "MIT", 56 | "oclif": { 57 | "commands": "./lib/commands", 58 | "topics": { 59 | "hello": { 60 | "description": "Commands to say hello." 61 | } 62 | }, 63 | "devPlugins": [ 64 | "@oclif/plugin-help" 65 | ] 66 | }, 67 | "repository": "realestate-com-au/json-bourne-sfdx-cli", 68 | "scripts": { 69 | "lint": "eslint . --ext .ts", 70 | "postpack": "rm -f .oclif.manifest.json npm-shrinkwrap.json", 71 | "posttest": "tsc -p test --noEmit && tslint -p test -t stylish", 72 | "prepack": "rm -rf lib && tsc && oclif-dev manifest && oclif-dev readme && npm shrinkwrap", 73 | "prepare": "rm -rf lib && tsc && oclif-dev manifest && oclif-dev readme && npm shrinkwrap", 74 | "style": "prettier --write **.ts", 75 | "test": "nyc mocha --forbid-only \"test/**/*.test.ts\"", 76 | "version": "oclif-dev readme && git add README.md" 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/commands/bourne/export.ts: -------------------------------------------------------------------------------- 1 | import { flags, SfdxCommand, core } from "@salesforce/command"; 2 | import { Helper } from "../../helper/Helper"; 3 | import { AnyJson } from "@salesforce/ts-types"; 4 | import * as _ from "lodash"; 5 | 6 | export default class Export extends SfdxCommand { 7 | public static description = Helper.messages.getMessage("pullDescription"); 8 | 9 | public static examples = [ 10 | `$ sfdx bourne:export -o Product2 -u myOrg -c config/cpq-cli-def.json 11 | Requesting data, please wait.... Request completed! Received X records. 12 | `, 13 | ]; 14 | 15 | public static args = [{ name: "file" }]; 16 | 17 | protected static flagsConfig: any = { 18 | object: flags.string({ 19 | char: "o", 20 | description: Helper.messages.getMessage("objectDescription"), 21 | }), 22 | configfile: flags.string({ 23 | char: "c", 24 | description: Helper.messages.getMessage("configFileDescription"), 25 | }), 26 | processall: flags.boolean({ 27 | char: "a", 28 | description: Helper.messages.getMessage("pullAllDescription"), 29 | }), 30 | }; 31 | 32 | protected static requiresUsername = true; 33 | 34 | protected static config; 35 | 36 | protected connection; 37 | 38 | private objectsToProcess(): String[] { 39 | return _.uniq(Helper.getObjectsToProcess(this.flags, Export.config)); 40 | } 41 | 42 | private exportRecordsToDir(records, sObjectName, dirPath) { 43 | let externalIdField = Export.config.objects[sObjectName].externalid; 44 | if (records.length > 0 && !records[0].hasOwnProperty(externalIdField)) { 45 | throw new core.SfdxError( 46 | "The External Id provided on the configuration file does not exist on the extracted record(s). Please ensure it is included in the object's query." 47 | ); 48 | } 49 | 50 | records.forEach((record) => { 51 | Helper.removeField(record, "attributes"); 52 | this.removeNullFields(record, sObjectName); 53 | let fileName = record[externalIdField]; 54 | if (fileName == null) { 55 | throw new core.SfdxError( 56 | "There are records without External Ids. Ensure all records that are extracted have a value for the field specified as the External Id." 57 | ); 58 | } else { 59 | fileName = dirPath + "/" + fileName.replace(/\s+/g, "-") + ".json"; 60 | Helper.fs.writeFile( 61 | fileName, 62 | JSON.stringify(record, undefined, 2), 63 | function (err) { 64 | if (err) { 65 | throw err; 66 | } 67 | } 68 | ); 69 | } 70 | }); 71 | } 72 | 73 | private removeNullFields(record, sObjectName) { 74 | Export.config.objects[sObjectName].cleanupFields.forEach((fields) => { 75 | if (null === record[fields]) { 76 | delete record[fields]; 77 | let lookupField: string; 78 | if (fields.substr(fields.length - 3) == "__r") { 79 | lookupField = fields.substr(0, fields.length - 1) + "c"; 80 | } else { 81 | lookupField = fields + "Id"; 82 | } 83 | record[lookupField] = null; 84 | } 85 | }); 86 | } 87 | 88 | private async getExportRecords(sObject: any): Promise { 89 | return new Promise((resolve) => { 90 | var records = []; 91 | var query = this.connection 92 | .query(`${Export.config.objects[sObject].query}`) 93 | .on("record", (record) => { 94 | records.push(record); 95 | }) 96 | .on("end", () => { 97 | this.ux.log("total in database : " + query.totalSize); 98 | this.ux.log("total fetched : " + query.totalFetched); 99 | resolve(records); 100 | }) 101 | .on("error", (err) => { 102 | this.ux.error(err); 103 | }) 104 | .run({ autoFetch: true, maxFetch: 100000 }); 105 | }); 106 | } 107 | 108 | private clearDirectory(dirPath: string) { 109 | if (Helper.fs.existsSync(dirPath)) { 110 | Helper.fs.readdirSync(dirPath).forEach((file) => { 111 | Helper.fs.unlink(dirPath + "/" + file, (err) => { 112 | if (err) { 113 | throw err; 114 | } 115 | }); 116 | }); 117 | } else { 118 | Helper.fs.mkdirSync(dirPath); 119 | } 120 | } 121 | 122 | public async run(): Promise { 123 | this.connection = this.org.getConnection(); 124 | Export.config = Helper.initConfig(this.flags); 125 | 126 | let sObjects = this.objectsToProcess(); 127 | for (let i in sObjects) { 128 | let sObjectName = sObjects[i].toString(); 129 | 130 | this.ux.startSpinner( 131 | "Retrieving " + 132 | Helper.colors.blue(sObjectName) + 133 | " records, please wait..." 134 | ); 135 | 136 | let records: any[] = await this.getExportRecords(sObjectName); 137 | let dirPath = "data/" + Export.config.objects[sObjectName].directory; 138 | this.clearDirectory(dirPath); 139 | this.exportRecordsToDir(records, sObjectName, dirPath); 140 | 141 | this.ux.stopSpinner( 142 | "Request completed! Received " + records.length + " records." 143 | ); 144 | } 145 | return {}; 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/commands/bourne/import.ts: -------------------------------------------------------------------------------- 1 | import { core, flags, SfdxCommand } from "@salesforce/command"; 2 | import { Connection } from "jsforce"; 3 | import { Helper } from "../../helper/Helper"; 4 | import { ImportResult } from "../../helper/ImportResult"; 5 | import { CPQDataImportRequest } from "../../model/CPQDataImportRequest"; 6 | import { AnyJson } from "@salesforce/ts-types"; 7 | import * as _ from "lodash"; 8 | 9 | export default class Import extends SfdxCommand { 10 | public static description = Helper.messages.getMessage("pushDescription"); 11 | 12 | public static examples = [ 13 | `$ sfdx bourne:import -o Product2 -u myOrg -c config/cpq-cli-def.json 14 | Deploying data, please wait.... Deployment completed! 15 | `, 16 | ]; 17 | 18 | protected static flagsConfig: any = { 19 | object: flags.string({ 20 | char: "o", 21 | description: Helper.messages.getMessage("objectDescription"), 22 | }), 23 | configfile: flags.string({ 24 | char: "c", 25 | description: Helper.messages.getMessage("configFileDescription"), 26 | }), 27 | processall: flags.boolean({ 28 | char: "a", 29 | description: Helper.messages.getMessage("pushAllDescription"), 30 | }), 31 | datadir: flags.string({ 32 | char: "d", 33 | description: Helper.messages.getMessage("pathToDataDir"), 34 | }), 35 | remove: flags.boolean({ 36 | char: "r", 37 | description: Helper.messages.getMessage("removeObjects"), 38 | }), 39 | }; 40 | 41 | public static args = [{ name: "file" }]; 42 | 43 | protected static requiresUsername = true; 44 | 45 | protected static config; 46 | 47 | protected connection; 48 | 49 | private objectsToProcess() { 50 | let sObjects = Helper.getObjectsToProcess(this.flags, Import.config); 51 | return this.flags.remove === true ? _.uniq(sObjects.reverse()) : sObjects; 52 | } 53 | 54 | private getDataDir() { 55 | return this.flags.datadir ? this.flags.datadir : "data"; 56 | } 57 | 58 | private async getRecordTypeRef(sObject, configObject) { 59 | let recordTypeRef = {}; 60 | if (configObject.hasRecordTypes) { 61 | this.ux.log(Helper.colors.blue("Aligning RecordType IDs...")); 62 | this.ux.startSpinner("Processing"); 63 | let recordTypes: any = await this.connection.query( 64 | `SELECT Id, Name, DeveloperName FROM RecordType WHERE sObjectType ='${sObject}'` 65 | ); 66 | if (recordTypes && recordTypes.records.length > 0) { 67 | recordTypes.records.forEach((recordType) => { 68 | recordTypeRef[recordType.DeveloperName] = recordType.Id; 69 | }); 70 | } 71 | this.ux.stopSpinner("RecordType information retrieved"); 72 | } 73 | return recordTypeRef; 74 | } 75 | 76 | private readRecords(configObject: any) { 77 | let dirPath = this.getDataDir() + "/" + configObject.directory; 78 | if (Helper.fs.existsSync(dirPath)) { 79 | let files = Helper.fs.readdirSync(dirPath); 80 | return files.map((file) => { 81 | let filePath = dirPath + "/" + file; 82 | try { 83 | return Helper.fs.readFileSync(filePath, "utf8"); 84 | } catch (e) { 85 | console.error(Helper.colors.red("Could not load " + filePath)); 86 | } 87 | return; 88 | }); 89 | } 90 | return []; 91 | } 92 | 93 | private resolveToSObjects( 94 | recordTypeRef: any, 95 | configObject: any, 96 | originalRecords: any[] 97 | ) { 98 | return originalRecords.map((original) => { 99 | let record = JSON.parse(original); 100 | if (configObject.hasRecordTypes && recordTypeRef) { 101 | if (recordTypeRef.hasOwnProperty(record.RecordType.DeveloperName)) { 102 | record.RecordTypeId = recordTypeRef[record.RecordType.DeveloperName]; 103 | delete record.RecordType; 104 | } else if (record.RecordType) { 105 | this.ux.log( 106 | "This record does not contain a value for Record Type, skipping transformation." 107 | ); 108 | } else { 109 | throw new core.SfdxError( 110 | "Record Type not found for " + record.RecordType.DeveloperName 111 | ); 112 | } 113 | } 114 | return record; 115 | }); 116 | } 117 | 118 | private createPayload( 119 | records, 120 | configObject, 121 | sObject, 122 | payloads: Array 123 | ) { 124 | let payload = JSON.stringify(records, null, 0); 125 | if (payload.length > Import.config.payloadLength) { 126 | let splitRecords = Import.splitInHalf(records); 127 | this.createPayload(splitRecords[0], configObject, sObject, payloads); 128 | this.createPayload(splitRecords[1], configObject, sObject, payloads); 129 | } else { 130 | let operation = this.flags.remove === true ? "delete" : "upsert"; 131 | let dataImportObj: CPQDataImportRequest = new CPQDataImportRequest( 132 | sObject, 133 | operation, 134 | records, 135 | configObject.externalid 136 | ); 137 | payloads.push(dataImportObj); 138 | } 139 | } 140 | 141 | private static splitInHalf(records) { 142 | let halfSize = records.length / 2; 143 | let splitRecords = []; 144 | splitRecords.push(records.slice(0, halfSize)); 145 | splitRecords.push(records.slice(halfSize)); 146 | return splitRecords; 147 | } 148 | 149 | private getProcessPayload(isManagedPackage) { 150 | let restUrl = isManagedPackage == true ? "/JSON/bourne/v1" : "/bourne/v1"; 151 | return (function () { 152 | return function (payload, connection) { 153 | return new Promise((resolve) => { 154 | let resultPromise = connection.apex.post(restUrl, payload, (err) => { 155 | if (err) { 156 | return console.error(err); 157 | } 158 | }); 159 | 160 | if (typeof resultPromise === "undefined") { 161 | this.ux.log("Error: Undefined promise"); 162 | return; 163 | } 164 | 165 | resolve( 166 | resultPromise 167 | .then((result) => { 168 | return result; 169 | }) 170 | .catch((error) => { 171 | this.ux.log(error); 172 | }) 173 | ); 174 | }); 175 | }; 176 | })(); 177 | } 178 | 179 | private async importRecords( 180 | records: any[], 181 | configObject: any, 182 | sObject: any, 183 | connection: Connection, 184 | useManagedPackage: Boolean 185 | ) { 186 | let responses = []; 187 | if (records.length > 0) { 188 | let payloads = []; 189 | this.createPayload(records, configObject, sObject, payloads); 190 | let processPayload = this.getProcessPayload(useManagedPackage); 191 | if (configObject.enableMultiThreading) { 192 | let promises = []; 193 | payloads.forEach((payload) => 194 | promises.push(processPayload(payload, connection)) 195 | ); 196 | responses = await Promise.all(promises); 197 | } else { 198 | for (let i in payloads) { 199 | let promises = []; 200 | promises.push(processPayload(payloads[i], connection)); 201 | responses.push(await Promise.all(promises)); 202 | } 203 | } 204 | } 205 | return responses; 206 | } 207 | 208 | public async run(): Promise { 209 | this.connection = this.org.getConnection(); 210 | Import.config = Helper.initConfig(this.flags); 211 | let sObjects = this.objectsToProcess(); 212 | let allImportResults: ImportResult[] = []; 213 | 214 | for (let i in sObjects) { 215 | let success: boolean = false; 216 | let retries: number = 0; 217 | do { 218 | let sObject = sObjects[i]; 219 | let configObject = Import.config.objects[sObject]; 220 | let recordTypeRef = await this.getRecordTypeRef(sObject, configObject); 221 | let originalRecords = this.readRecords(configObject); 222 | let records = this.resolveToSObjects( 223 | recordTypeRef, 224 | configObject, 225 | originalRecords 226 | ); 227 | 228 | this.ux.log("Deploying " + Helper.colors.blue(sObject) + " records"); 229 | 230 | let responses = await this.importRecords( 231 | records, 232 | configObject, 233 | sObject, 234 | this.connection, 235 | Import.config.useManagedPackage 236 | ); 237 | let importResult = new ImportResult(responses); 238 | importResult.print(); 239 | allImportResults.push(importResult); 240 | 241 | if (importResult.failure == 0) { 242 | success = true; 243 | } else if (retries + 1 == Import.config.importRetries) { 244 | throw ( 245 | "Import was unsuccessful after " + 246 | Import.config.importRetries + 247 | " attempts." 248 | ); 249 | } else { 250 | this.ux.log("Retrying..."); 251 | retries++; 252 | } 253 | } while (success == false && retries < Import.config.importRetries); 254 | } 255 | return JSON.stringify(allImportResults); 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /src/helper/ApexResponse.ts: -------------------------------------------------------------------------------- 1 | export class ApexResponse { 2 | public result: String; 3 | public recordId: String; 4 | public message: String; 5 | public externalId: String; 6 | } 7 | -------------------------------------------------------------------------------- /src/helper/Helper.ts: -------------------------------------------------------------------------------- 1 | import { core } from "@salesforce/command"; 2 | 3 | // Initialize Messages with the current plugin directory 4 | core.Messages.importMessagesDirectory(__dirname); 5 | 6 | export class Helper { 7 | public static fs = require("fs"); 8 | public static cTable = require("console.table"); 9 | public static colors = require("colors"); 10 | public static messages = core.Messages.loadMessages( 11 | "json-bourne-sfdx", 12 | "org" 13 | ); 14 | 15 | public static removeField(record, fieldName) { 16 | delete record[fieldName]; 17 | for (let i in record) { 18 | if (record[i] != null && typeof record[i] === "object") { 19 | Helper.removeField(record[i], fieldName); 20 | } 21 | } 22 | } 23 | 24 | public static getObjectsToProcess(flags, config) { 25 | if (flags.processall === true && flags.object) { 26 | throw new core.SfdxError( 27 | "You cannot specify both process all flag and an object name" 28 | ); 29 | } 30 | return flags.processall === true 31 | ? this.getAllObjectsToProcess(config) 32 | : this.getSingleObjectToProcess(flags, config); 33 | } 34 | 35 | public static initConfig(flags) { 36 | if (Helper.fs.existsSync(flags.configfile)) { 37 | let configPath = process.cwd() + "/" + flags.configfile; 38 | console.log("Load config from " + configPath); 39 | return require(configPath); 40 | } else { 41 | throw new core.SfdxError( 42 | Helper.messages.getMessage( 43 | "No configuration file found at this location." 44 | ) 45 | ); 46 | } 47 | } 48 | 49 | private static getAllObjectsToProcess(config) { 50 | return config.allObjects; 51 | } 52 | 53 | private static getSingleObjectToProcess(flags, config) { 54 | if (flags.object in config.objects) { 55 | return [flags.object]; 56 | } 57 | throw new core.SfdxError( 58 | this.messages.getMessage("There is no configuration for this object.") 59 | ); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/helper/ImportResult.ts: -------------------------------------------------------------------------------- 1 | import { ApexResponse } from "./ApexResponse"; 2 | import { Helper } from "./Helper"; 3 | 4 | export class ImportResult { 5 | public results: Array; 6 | public requests: number; 7 | public responses: any[]; 8 | public total: number; 9 | public success: number; 10 | public failure: number; 11 | public failureResults: Array; 12 | 13 | constructor(responses: any[]) { 14 | this.responses = responses; 15 | this.requests = responses.length; 16 | this.results = this.getResult(); 17 | this.total = this.results.length; 18 | this.failureResults = this.getFailureResults(); 19 | this.failure = this.failureResults.length; 20 | this.success = this.total - this.failure; 21 | } 22 | 23 | private getFailureResults() { 24 | return this.results.filter((result) => { 25 | return result.result === "FAILED"; 26 | }); 27 | } 28 | 29 | private getResult() { 30 | let resolveResults = []; 31 | this.responses.forEach((res) => { 32 | resolveResults = resolveResults.concat(JSON.parse(res)); 33 | }); 34 | return resolveResults; 35 | } 36 | 37 | public print() { 38 | if (this.failure > 0 && this.success > 0) { 39 | console.log("Error deploying data"); 40 | console.log(Helper.colors.yellow("== DEPLOYED WITH ERRORS")); 41 | } else if (this.success > 0) { 42 | console.log(Helper.colors.green("== SUCCESS")); 43 | } else if (this.failure > 0) { 44 | console.log(Helper.colors.red("== ERROR")); 45 | } else { 46 | console.log(Helper.colors.magenta("== NO RECORDS TO PUSH")); 47 | } 48 | 49 | console.log("Successful deployments: ", this.success); 50 | console.log("Failed deployments: ", this.failure); 51 | console.log(Helper.cTable.getTable(this.failureResults)); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export default {}; 2 | 3 | exports.namespace = { 4 | name: "json-bourne-sfdx", 5 | description: 6 | "import/export reference data from Salesforce into your dx project files", 7 | }; 8 | -------------------------------------------------------------------------------- /src/model/CPQDataImportRequest.ts: -------------------------------------------------------------------------------- 1 | export class CPQDataImportRequest { 2 | public sObjectType: String; 3 | public operation: String; 4 | public payload: Array; 5 | public extIdField: String; 6 | 7 | constructor( 8 | sObjectType: String, 9 | operation: String, 10 | payload: Array, 11 | extIdField: String 12 | ) { 13 | this.sObjectType = sObjectType; 14 | this.operation = operation; 15 | this.payload = payload; 16 | this.extIdField = extIdField; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/@salesforce/dev-config/tsconfig", 3 | "compilerOptions": { 4 | "outDir": "./lib", 5 | "rootDir": "./src" 6 | }, 7 | "include": [ 8 | "./src/**/*" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@salesforce/dev-config/tslint" 3 | } 4 | --------------------------------------------------------------------------------