├── .circleci ├── config.yml └── greenkeeper ├── .editorconfig ├── .gitattributes ├── .gitignore ├── .images └── vscodeScreenshot.png ├── .nycrc ├── .travis.yml ├── .vscode └── launch.json ├── AccountData_GFx.json ├── README.md ├── Sample-dataMap.json ├── appveyor.yml ├── bin ├── run └── run.cmd ├── insertResult.json ├── messages └── examine.json ├── package.json ├── planMap.json ├── planMapOriginal.json ├── relationship_map.json ├── src ├── commands │ ├── djc │ │ ├── cleardata.ts │ │ ├── export.ts │ │ └── import.ts │ └── tohoom │ │ └── data │ │ ├── export.ts │ │ └── split.ts ├── dataApi.ts ├── index.ts └── tohoom.ts ├── test ├── commands │ └── djc │ │ └── data │ │ └── data.test.ts ├── helpers │ └── init.js ├── mocha.opts └── tsconfig.json ├── tsconfig.json └── yarn.lock /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | jobs: 4 | node-latest: &test 5 | docker: 6 | - image: node:latest 7 | working_directory: ~/cli 8 | steps: 9 | - checkout 10 | - restore_cache: &restore_cache 11 | keys: 12 | - v1-npm-{{checksum ".circleci/config.yml"}}-{{ checksum "yarn.lock"}} 13 | - v1-npm-{{checksum ".circleci/config.yml"}} 14 | - run: 15 | name: Install dependencies 16 | command: .circleci/greenkeeper 17 | - run: 18 | name: Testing 19 | command: yarn test 20 | - run: 21 | name: Submitting code coverage to codecov 22 | command: | 23 | ./node_modules/.bin/nyc report --reporter text-lcov > coverage.lcov 24 | curl -s https://codecov.io/bash | bash 25 | - save_cache: 26 | key: v1-yarn-{{checksum ".circleci/config.yml"}}-{{checksum "yarn.lock"}} 27 | paths: 28 | - ~/cli/node_modules 29 | - /usr/local/share/.cache/yarn 30 | - /usr/local/share/.config/yarn 31 | node-8: 32 | <<: *test 33 | docker: 34 | - image: node:8 35 | 36 | workflows: 37 | version: 2 38 | "datatree": 39 | jobs: 40 | - node-latest 41 | - node-8 42 | -------------------------------------------------------------------------------- /.circleci/greenkeeper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | PATH=/usr/local/share/.config/yarn/global/node_modules/.bin:$PATH 6 | 7 | if [[ "$CIRCLE_BRANCH" != greenkeeper/* ]]; then 8 | yarn 9 | # yarn check 10 | exit 0 11 | fi 12 | 13 | if [[ ! -z "$GIT_EMAIL" ]] & [[ ! -z "$GIT_USERNAME" ]]; then 14 | git config --global push.default simple 15 | git config --global user.email "$GIT_EMAIL" 16 | git config --global user.name "$GIT_USERNAME" 17 | fi 18 | 19 | if [[ ! -x "$(command -v greenkeeper-lockfile-update)" ]]; then 20 | yarn global add greenkeeper-lockfile@1 21 | fi 22 | 23 | greenkeeper-lockfile-update 24 | yarn 25 | greenkeeper-lockfile-upload 26 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.md] 11 | trim_trailing_whitespace = false 12 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | *.js text eol=lf 3 | *.ts text eol=lf 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *-debug.log 2 | *-error.log 3 | /.nyc_output 4 | /dist 5 | /lib 6 | /package-lock.json 7 | /tmp 8 | node_modules 9 | -------------------------------------------------------------------------------- /.images/vscodeScreenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarroll/sfdx-djc-plugin/664e6109bab9ffb54ae2fe9338496a3a63207f5d/.images/vscodeScreenshot.png -------------------------------------------------------------------------------- /.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "extension": [ 3 | ".ts" 4 | ], 5 | "include": [ 6 | "src/**/*.ts" 7 | ], 8 | "exclude": [ 9 | "**/*.d.ts" 10 | ], 11 | "all": true 12 | } 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "stable" -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "attach", 10 | "name": "Attach by Process ID", 11 | "processId": "${command:PickProcess}", 12 | "skipFiles": [ 13 | "/**" 14 | ] 15 | }, 16 | { 17 | "type": "node", 18 | "request": "launch", 19 | "name": "Launch Program", 20 | "program": "${workspaceFolder}/bin/run", 21 | "args": [ 22 | "djc:data:export", 23 | "-m", 24 | "100", 25 | "-o", 26 | "Account,Contact,Hoom__c,Target__c,Hoom_Team_Member__c,Influencer__c,Collaboration__c,HUB__c,Hub_Hoom__c", 27 | "-t", 28 | "data" 29 | ], 30 | "skipFiles": [ 31 | "/**" 32 | ] 33 | }, 34 | { 35 | "type": "node", 36 | "request": "attach", 37 | "name": "Attach to Remote", 38 | "address": "127.0.0.1", 39 | "port": 9229, 40 | "localRoot": "${workspaceFolder}", 41 | "remoteRoot": "${workspaceFolder}" 42 | }, 43 | { 44 | "name": "Unit Tests", 45 | "type": "node", 46 | "request": "launch", 47 | "protocol": "inspector", 48 | "program": "${workspaceRoot}/node_modules/.bin/_mocha", 49 | "args": [ 50 | "--require", "test/helpers/init.js", 51 | "--require", "ts-node/register", 52 | "--require", "source-map-support/register", 53 | "--recursive", 54 | "--reporter", "spec", 55 | "test/**/*.test.ts" 56 | ], 57 | "cwd": "${workspaceRoot}" 58 | } 59 | ] 60 | } -------------------------------------------------------------------------------- /AccountData_GFx.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "attributes": { 3 | "type": "Account", 4 | "url": "/services/data/v51.0/sobjects/Account/001B000001KJzZvIAL" 5 | }, 6 | "Description": "GBM is the worldwide leader in technology news and information on the Web and the producer of the longest-running and farthest-reaching television shows about technology. GBM's network of sites combines breakthrough interactive technology with engaging content and design and is consistently ranked as the Internet's leading content network in terms of both audience size and revenue, serving millions of users each day. The company's television programming is broadcast by the USA Network and the Sci-Fi Channel to more than 70 million households and is syndicated to broadcast television stations in the nation's top 120 markets, including the top 10 markets.", 7 | "Fax": null, 8 | "Id": "001B000001KJzZvIAL", 9 | "Industry": "Media", 10 | "Name": "Global Media", 11 | "NumberOfEmployees": 14668, 12 | "Phone": "(905) 555-1212", 13 | "PhotoUrl": "/services/images/photo/001B000001KJzZvIAL", 14 | "ShippingCity": "Toronto", 15 | "ShippingCountry": "Canada", 16 | "ShippingPostalCode": "L4B 1Y3", 17 | "ShippingState": "Ontario", 18 | "ShippingStreet": "150 Chestnut Street", 19 | "SicDesc": null, 20 | "Type": "Prospect", 21 | "Website": null 22 | }, 23 | { 24 | "attributes": { 25 | "type": "Account", 26 | "url": "/services/data/v51.0/sobjects/Account/001B000001KJzZwIAL" 27 | }, 28 | "Description": null, 29 | "Fax": "(212) 555-5555", 30 | "Id": "001B000001KJzZwIAL", 31 | "Industry": "Manufacturing", 32 | "Name": "Acme", 33 | "NumberOfEmployees": 680, 34 | "Phone": "(212) 555-5555", 35 | "PhotoUrl": "/services/images/photo/001B000001KJzZwIAL", 36 | "ShippingCity": "New York", 37 | "ShippingCountry": "USA", 38 | "ShippingPostalCode": "31349", 39 | "ShippingState": "NY", 40 | "ShippingStreet": "10 Main Rd.", 41 | "SicDesc": null, 42 | "Type": "Prospect", 43 | "Website": null 44 | }, 45 | { 46 | "attributes": { 47 | "type": "Account", 48 | "url": "/services/data/v51.0/sobjects/Account/001B000001KJzZxIAL" 49 | }, 50 | "Description": "Founded in March 1999, salesforce.com (http://www.salesforce.com) builds and delivers customer relationship management (CRM) applications as scalable online services. The salesforce.com product suite - Team Edition, Professional Edition, Enterprise Edition, Wireless Edition and Offline Edition - gives companies of all sizes a complete 360-degree view of the customer. The company's award-winning CRM solutions provide integrated online sales force automation, customer service and support management, and marketing automation applications to help companies meet the complex challenges of global customer communication. Salesforce.com has received considerable recognition in the industry, including Editors' Choice and two Five-Star ratings from PC Magazine, two Deploy Awards from InfoWorld, Red Herring 100, Upside Hot 100, Investor's Choice Award from Enterprise Outlook, Editors' Choice from TMCLabs, Top 10 CRM Implementation from Aberdeen Group, and InfoWorld's 2001 CRM Technology of the Year. Founded in 1999, salesforce.com is headquartered in San Francisco, with offices in Europe and Asia.", 51 | "Fax": "(415) 901-7040", 52 | "Id": "001B000001KJzZxIAL", 53 | "Industry": "Technology", 54 | "Name": "salesforce.com", 55 | "NumberOfEmployees": null, 56 | "Phone": "(415) 901-7000", 57 | "PhotoUrl": "/services/images/photo/001B000001KJzZxIAL", 58 | "ShippingCity": "San Francisco", 59 | "ShippingCountry": "USA", 60 | "ShippingPostalCode": "94105", 61 | "ShippingState": "CA", 62 | "ShippingStreet": "The Landmark @ One Market, Suite 300", 63 | "SicDesc": null, 64 | "Type": "Customer", 65 | "Website": "http://www.salesforce.com" 66 | }, 67 | { 68 | "attributes": { 69 | "type": "Account", 70 | "url": "/services/data/v51.0/sobjects/Account/001B000001KPVxBIAX" 71 | }, 72 | "Description": "Computer Software: Prepackaged Software", 73 | "Fax": null, 74 | "Id": "001B000001KPVxBIAX", 75 | "Industry": "Technology", 76 | "Name": "djZvCActuate Corporation", 77 | "NumberOfEmployees": null, 78 | "Phone": null, 79 | "PhotoUrl": "/services/images/photo/001B000001KPVxBIAX", 80 | "ShippingCity": null, 81 | "ShippingCountry": null, 82 | "ShippingPostalCode": null, 83 | "ShippingState": null, 84 | "ShippingStreet": null, 85 | "SicDesc": null, 86 | "Type": null, 87 | "Website": null 88 | }, 89 | { 90 | "attributes": { 91 | "type": "Account", 92 | "url": "/services/data/v51.0/sobjects/Account/001B000001KPVxCIAX" 93 | }, 94 | "Description": "Industrial Specialties", 95 | "Fax": null, 96 | "Id": "001B000001KPVxCIAX", 97 | "Industry": "Health Care", 98 | "Name": "jIrClAlign Technology, Inc.", 99 | "NumberOfEmployees": null, 100 | "Phone": null, 101 | "PhotoUrl": "/services/images/photo/001B000001KPVxCIAX", 102 | "ShippingCity": null, 103 | "ShippingCountry": null, 104 | "ShippingPostalCode": null, 105 | "ShippingState": null, 106 | "ShippingStreet": null, 107 | "SicDesc": null, 108 | "Type": null, 109 | "Website": null 110 | }, 111 | { 112 | "attributes": { 113 | "type": "Account", 114 | "url": "/services/data/v51.0/sobjects/Account/001B000001KPVxDIAX" 115 | }, 116 | "Description": "Computer Software: Prepackaged Software", 117 | "Fax": null, 118 | "Id": "001B000001KPVxDIAX", 119 | "Industry": "Technology", 120 | "Name": "xJdOjVocus, Inc.", 121 | "NumberOfEmployees": null, 122 | "Phone": null, 123 | "PhotoUrl": "/services/images/photo/001B000001KPVxDIAX", 124 | "ShippingCity": null, 125 | "ShippingCountry": null, 126 | "ShippingPostalCode": null, 127 | "ShippingState": null, 128 | "ShippingStreet": null, 129 | "SicDesc": null, 130 | "Type": null, 131 | "Website": null 132 | }, 133 | { 134 | "attributes": { 135 | "type": "Account", 136 | "url": "/services/data/v51.0/sobjects/Account/001B000001KPVxEIAX" 137 | }, 138 | "Description": "Savings Institutions", 139 | "Fax": null, 140 | "Id": "001B000001KPVxEIAX", 141 | "Industry": "Finance", 142 | "Name": "ukYsZUmpqua Holdings Corporation", 143 | "NumberOfEmployees": null, 144 | "Phone": null, 145 | "PhotoUrl": "/services/images/photo/001B000001KPVxEIAX", 146 | "ShippingCity": null, 147 | "ShippingCountry": null, 148 | "ShippingPostalCode": null, 149 | "ShippingState": null, 150 | "ShippingStreet": null, 151 | "SicDesc": null, 152 | "Type": null, 153 | "Website": null 154 | } 155 | ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sfdx-djc-plugin [![Build Status](https://travis-ci.org/dcarroll/sfdx-djc-plugin.svg?branch=master)](https://travis-ci.org/dcarroll/sfdx-djc-plugin) 2 | 3 | 4 | 5 | [![Version](https://img.shields.io/npm/v/datatree.svg)](https://npmjs.org/package/sfdx-djc-plugin) 6 | [![License](https://img.shields.io/npm/l/datatree.svg)](https://github.com/dcarroll/sfdx-djc-plugin/blob/master/package.json) 7 | 8 | 9 | 10 | * [sfdx-djc-plugin [![Build Status](https://travis-ci.org/dcarroll/sfdx-djc-plugin.svg?branch=master)](https://travis-ci.org/dcarroll/sfdx-djc-plugin)](#sfdx-djc-plugin--build-statushttpstravis-ciorgdcarrollsfdx-djc-pluginsvgbranchmasterhttpstravis-ciorgdcarrollsfdx-djc-plugin) 11 | 12 | 13 | 14 | A plugin for the Salesforce CLI built by Dave Carroll and containing a few of helpful commands. 15 | 16 | ## Setup 17 | 18 | ### Install from source 19 | 20 | 1. Install the SDFX CLI. 21 | 22 | 2. Clone the repository: `git clone git@github.com:wadewegner/sfdx-djc-plugin.git` 23 | 24 | 3. Install npm modules: `yarn` 25 | 26 | 4. Link the plugin: `sfdx plugins:link .` 27 | 28 | ### Install as plugin 29 | 30 | 1. Install plugin: `sfdx plugins:install sfdx-tohoom-plugin` 31 | 32 | 33 | ```sh-session 34 | $ npm install -g sfdx-djc-plugin 35 | $ sfdx-djc-plugin COMMAND 36 | running command... 37 | $ sfdx-djc-plugin (-v|--version|version) 38 | sfdx-djc-plugin/0.0.32 darwin-x64 node-v14.15.0 39 | $ sfdx-djc-plugin --help [COMMAND] 40 | USAGE 41 | $ sfdx-djc-plugin COMMAND 42 | ... 43 | ``` 44 | 45 | 46 | * [`sfdx-djc-plugin djc:cleardata -o [-v ] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-djc-plugin-djccleardata--o-string--v-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal) 47 | * [`sfdx-djc-plugin djc:export [-v ] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-djc-plugin-djcexport--v-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal) 48 | * [`sfdx-djc-plugin djc:import [-x] [-v ] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-djc-plugin-djcimport--x--v-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal) 49 | * [`sfdx-djc-plugin tohoom:data:export -o -t [-n ] [-m ] [-s] [-p] [-e] [-b] [-k] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-djc-plugin-tohoomdataexport--o-string--t-string--n-string--m-integer--s--p--e--b--k--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal) 50 | * [`sfdx-djc-plugin tohoom:data:split [-n ] [-v ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-djc-plugin-tohoomdatasplit--n-string--v-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal) 51 | 52 | ## `sfdx-djc-plugin djc:cleardata -o [-v ] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]` 53 | 54 | Delete data from a scratch org. 55 | 56 | ``` 57 | USAGE 58 | $ sfdx-djc-plugin djc:cleardata -o [-v ] [-u ] [--apiversion ] [--json] [--loglevel 59 | trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL] 60 | 61 | OPTIONS 62 | -o, --sobject=sobject (required) Object to delete all 63 | records for 64 | 65 | -u, --targetusername=targetusername username or alias for the target 66 | org; overrides default target org 67 | 68 | -v, --targetdevhubusername=targetdevhubusername username or alias for the dev hub 69 | org; overrides default dev hub org 70 | 71 | --apiversion=apiversion override the api version used for 72 | api requests made by this command 73 | 74 | --json format output as json 75 | 76 | --loglevel=(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL) [default: warn] logging level for 77 | this command invocation 78 | 79 | EXAMPLE 80 | $ sfdx djc:cleardata -o Account 81 | ``` 82 | 83 | _See code: [src/commands/djc/cleardata.ts](https://github.com/dcarroll/datatree/blob/v0.0.32/src/commands/djc/cleardata.ts)_ 84 | 85 | ## `sfdx-djc-plugin djc:export [-v ] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]` 86 | 87 | Import data to an org to use in a scratch org. 88 | 89 | ``` 90 | USAGE 91 | $ sfdx-djc-plugin djc:export [-v ] [-u ] [--apiversion ] [--json] [--loglevel 92 | trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL] 93 | 94 | OPTIONS 95 | -u, --targetusername=targetusername username or alias for the target 96 | org; overrides default target org 97 | 98 | -v, --targetdevhubusername=targetdevhubusername username or alias for the dev hub 99 | org; overrides default dev hub org 100 | 101 | --apiversion=apiversion override the api version used for 102 | api requests made by this command 103 | 104 | --json format output as json 105 | 106 | --loglevel=(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL) [default: warn] logging level for 107 | this command invocation 108 | 109 | EXAMPLE 110 | $ sfdx djc:import -p directory 111 | ``` 112 | 113 | _See code: [src/commands/djc/export.ts](https://github.com/dcarroll/datatree/blob/v0.0.32/src/commands/djc/export.ts)_ 114 | 115 | ## `sfdx-djc-plugin djc:import [-x] [-v ] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]` 116 | 117 | Import data to an org to use in a scratch org. 118 | 119 | ``` 120 | USAGE 121 | $ sfdx-djc-plugin djc:import [-x] [-v ] [-u ] [--apiversion ] [--json] [--loglevel 122 | trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL] 123 | 124 | OPTIONS 125 | -u, --targetusername=targetusername username or alias for the target 126 | org; overrides default target org 127 | 128 | -v, --targetdevhubusername=targetdevhubusername username or alias for the dev hub 129 | org; overrides default dev hub org 130 | 131 | -x, --xfiles Use the limited size files instead 132 | of full size files 133 | 134 | --apiversion=apiversion override the api version used for 135 | api requests made by this command 136 | 137 | --json format output as json 138 | 139 | --loglevel=(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL) [default: warn] logging level for 140 | this command invocation 141 | 142 | EXAMPLE 143 | $ sfdx djc:import -p directory 144 | ``` 145 | 146 | _See code: [src/commands/djc/import.ts](https://github.com/dcarroll/datatree/blob/v0.0.32/src/commands/djc/import.ts)_ 147 | 148 | ## `sfdx-djc-plugin tohoom:data:export -o -t [-n ] [-m ] [-s] [-p] [-e] [-b] [-k] [-u ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]` 149 | 150 | Extract data from an org to use in a scratch org. Just supply a list of SObjects and you *should* end up with a dataset and data plan that can be used with the official force:data:tree:import command 151 | 152 | ``` 153 | USAGE 154 | $ sfdx-djc-plugin tohoom:data:export -o -t [-n ] [-m ] [-s] [-p] [-e] [-b] [-k] [-u 155 | ] [--apiversion ] [--json] [--loglevel 156 | trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL] 157 | 158 | OPTIONS 159 | -b, --preserveobjectorder If present, uses the order of the 160 | objects from the command to 161 | determine plan order 162 | 163 | -e, --enforcereferences If present, missing child reference 164 | cause the record to be deleted, 165 | otherwise, just the reference field 166 | is removed 167 | 168 | -k, --tohoom Special Tohoom processing to handle 169 | self referential relationship 170 | 171 | -m, --maxrecords=maxrecords [default: 10] Max number of records 172 | to return in any query 173 | 174 | -n, --planname=planname [default: new-data-plan] name of the 175 | data plan to produce, deflaults to 176 | "new-plan" 177 | 178 | -o, --objects=objects (required) Comma separated list of 179 | objects to fetch 180 | 181 | -p, --spiderreferences Include refereced SObjects 182 | determined by schema examination and 183 | existing data 184 | 185 | -s, --savedescribes Save describe results (for 186 | diagnostics) 187 | 188 | -t, --targetdir=targetdir (required) target directoy to place 189 | results in 190 | 191 | -u, --targetusername=targetusername username or alias for the target 192 | org; overrides default target org 193 | 194 | --apiversion=apiversion override the api version used for 195 | api requests made by this command 196 | 197 | --json format output as json 198 | 199 | --loglevel=(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL) [default: warn] logging level for 200 | this command invocation 201 | 202 | EXAMPLE 203 | $ sfdx tohoom:data:export -o Account,Contact,Case,Opportunity -t data/exported -n my-testplan 204 | ``` 205 | 206 | _See code: [src/commands/tohoom/data/export.ts](https://github.com/dcarroll/datatree/blob/v0.0.32/src/commands/tohoom/data/export.ts)_ 207 | 208 | ## `sfdx-djc-plugin tohoom:data:split [-n ] [-v ] [--apiversion ] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]` 209 | 210 | Extract data from an org to use in a scratch org. Just supply a list of SObjects and you *should* end up with a dataset and data plan that can be used with the official force:data:tree:import command 211 | 212 | ``` 213 | USAGE 214 | $ sfdx-djc-plugin tohoom:data:split [-n ] [-v ] [--apiversion ] [--json] [--loglevel 215 | trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL] 216 | 217 | OPTIONS 218 | -n, --planname=planname [default: data-plan] name of the 219 | data plan to use with split 220 | 221 | -v, --targetdevhubusername=targetdevhubusername username or alias for the dev hub 222 | org; overrides default dev hub org 223 | 224 | --apiversion=apiversion override the api version used for 225 | api requests made by this command 226 | 227 | --json format output as json 228 | 229 | --loglevel=(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL) [default: warn] logging level for 230 | this command invocation 231 | 232 | EXAMPLE 233 | $ sfdx tohoom:data:export -o Account,Contact,Case,Opportunity -t data/exported -n my-testplan 234 | ``` 235 | 236 | _See code: [src/commands/tohoom/data/split.ts](https://github.com/dcarroll/datatree/blob/v0.0.32/src/commands/tohoom/data/split.ts)_ 237 | 238 | -------------------------------------------------------------------------------- /Sample-dataMap.json: -------------------------------------------------------------------------------- 1 | { 2 | "Contact": { 3 | "totalSize": 24, 4 | "done": true, 5 | "records": [ 6 | { 7 | "attributes": { 8 | "type": "Contact", 9 | "url": "/services/data/v44.0/sobjects/Contact/003B0000001VXqOIAW" 10 | }, 11 | "AccountId": "001B0000002OUfiIAG", 12 | "LastName": "Gonzalez", 13 | "FirstName": "Rose", 14 | "Salutation": "Ms.", 15 | "MailingStreet": "313 Constitution Place\nAustin, TX 78767\nUSA", 16 | "Phone": "(512) 757-6000", 17 | "Fax": "(512) 757-9000", 18 | "MobilePhone": "(512) 757-9340", 19 | "Email": "rose@edge.com", 20 | "Title": "SVP, Procurement", 21 | "Department": "Procurement", 22 | "LeadSource": "Trade Show", 23 | "Birthdate": "1962-08-16", 24 | "OwnerId": "005B0000001BJw8IAG", 25 | "CleanStatus": "Pending", 26 | "dip__Level__c": "Primary", 27 | "dip__Languages__c": "English", 28 | "dip__Active__c": false, 29 | "dip__Profile_Image__c": "https://s3-us-west-2.amazonaws.com/df14-dave/woman1.png", 30 | "Id": "003B0000001VXqOIAW" 31 | }, 32 | { 33 | "attributes": { 34 | "type": "Contact", 35 | "url": "/services/data/v44.0/sobjects/Contact/003B0000001VXqPIAW" 36 | } 37 | } 38 | ] 39 | } 40 | } -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | nodejs_version: "9" 3 | cache: 4 | - '%LOCALAPPDATA%\Yarn -> appveyor.yml' 5 | - node_modules -> yarn.lock 6 | 7 | install: 8 | - ps: Install-Product node $env:nodejs_version x64 9 | - yarn 10 | test_script: 11 | - yarn test 12 | after_test: 13 | - .\node_modules\.bin\nyc report --reporter text-lcov > coverage.lcov 14 | - ps: | 15 | $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH 16 | Invoke-WebRequest -Uri 'https://codecov.io/bash' -OutFile codecov.sh 17 | bash codecov.sh 18 | 19 | build: off 20 | 21 | -------------------------------------------------------------------------------- /bin/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // tslint:disable-next-line:no-var-requires 4 | require('@oclif/command').run() 5 | // tslint:disable-next-line:no-var-requires 6 | .catch(require('@oclif/errors/handle')); 7 | -------------------------------------------------------------------------------- /bin/run.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | node "%~dp0\run" %* 4 | -------------------------------------------------------------------------------- /insertResult.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "id": "0012200000XlsD5AAJ", 3 | "success": true, 4 | "errors": [] 5 | }, 6 | { 7 | "id": "0012200000XlsO2AAJ", 8 | "success": true, 9 | "errors": [] 10 | }, 11 | { 12 | "id": "0012200000XlsOHAAZ", 13 | "success": true, 14 | "errors": [] 15 | }, 16 | { 17 | "id": "0012200000XlsNxAAJ", 18 | "success": true, 19 | "errors": [] 20 | }, 21 | { 22 | "id": "0012200000XlsJ9AAJ", 23 | "success": true, 24 | "errors": [] 25 | }, 26 | { 27 | "id": "0012200000XlsOCAAZ", 28 | "success": true, 29 | "errors": [] 30 | }, 31 | { 32 | "id": "0012200000XlsO7AAJ", 33 | "success": true, 34 | "errors": [] 35 | } 36 | ] -------------------------------------------------------------------------------- /messages/examine.json: -------------------------------------------------------------------------------- 1 | { 2 | "commandDescription": "export data based on objects!", 3 | "nameFlagDescription": "objects to export" 4 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sfdx-djc-plugin", 3 | "description": "Testing", 4 | "version": "0.0.32", 5 | "author": "Dave Carroll @dcarroll", 6 | "bugs": "https://github.com/dcarroll/datatree/issues", 7 | "dependencies": { 8 | "@types/fs-extra": "^5.0.4", 9 | "@oclif/command": "^1.8.0", 10 | "@oclif/config": "^1.17.0", 11 | "@oclif/errors": "^1.3.5", 12 | "@salesforce/command": "^5.3.9", 13 | "@salesforce/core": "^4.3.1", 14 | "tslib": "^2.3.0" 15 | }, 16 | "devDependencies": { 17 | "@oclif/dev-cli": "^1.26.0", 18 | "@oclif/plugin-help": "^3.2.2", 19 | "@salesforce/dev-config": "2.1.2", 20 | "@types/jest": "^27.0.2", 21 | "@types/jsforce": "^1.9.35", 22 | "@types/node": "^16.4.6", 23 | "@typescript-eslint/eslint-plugin": "^4.33.0", 24 | "@typescript-eslint/parser": "^4.33.0", 25 | "del-cli": "^4.0.1", 26 | "eslint": "^7.32.0", 27 | "jest": "^27.2.4", 28 | "ts-jest": "^27.0.5", 29 | "typescript": "^4.4.3" 30 | }, 31 | "engines": { 32 | "node": ">=8.0.0" 33 | }, 34 | "files": [ 35 | "/lib", 36 | "/messages", 37 | "/npm-shrinkwrap.json", 38 | "/oclif.manifest.json" 39 | ], 40 | "homepage": "https://github.com/dcarroll/datatree", 41 | "keywords": [ 42 | "sfdx-plugin" 43 | ], 44 | "license": "MIT", 45 | "oclif": { 46 | "commands": "./lib/commands", 47 | "topics": { 48 | "tohoom:data:export": { 49 | "description": "export data based on list of SObjects" 50 | } 51 | }, 52 | "devPlugins": [ 53 | "@oclif/plugin-help" 54 | ] 55 | }, 56 | "repository": "dcarroll/datatree", 57 | "scripts": { 58 | "build": "tsc", 59 | "lint": "eslint . --ext .ts,.tsx --format stylish", 60 | "lint:fix": "eslint . --ext .ts,.tsx --format stylish --fix", 61 | "postpack": "del-cli -f oclif.manifest.json", 62 | "prepack": "del-cli -f lib && tsc -b && oclif-dev manifest && oclif-dev readme", 63 | "test": "jest", 64 | "version": "oclif-dev readme && git add README.md" 65 | 66 | } 67 | } -------------------------------------------------------------------------------- /planMap.json: -------------------------------------------------------------------------------- 1 | { 2 | "TicketableEvent__c": { 3 | "weight": 10, 4 | "refs": { 5 | "EventInstance__c": { 6 | "resolveRefs": true 7 | } 8 | }, 9 | "saveRefs": true 10 | }, 11 | "EventInstance__c": { 12 | "weight": 20, 13 | "refs": { 14 | "TicketAllocation__c": { 15 | "resolveRefs": true 16 | } 17 | }, 18 | "saveRefs": true 19 | }, 20 | "Account": { 21 | "weight": 30, 22 | "refs": { 23 | "Task": { 24 | "resolveRefs": true 25 | }, 26 | "Contact": { 27 | "resolveRefs": true 28 | }, 29 | "Lead": { 30 | "resolveRefs": true 31 | }, 32 | "Case": { 33 | "resolveRefs": true 34 | }, 35 | "Opportunity": { 36 | "resolveRefs": true 37 | } 38 | }, 39 | "saveRefs": true 40 | }, 41 | "Contact": { 42 | "weight": 40, 43 | "refs": { 44 | "Lead": { 45 | "resolveRefs": true 46 | }, 47 | "Case": { 48 | "resolveRefs": true 49 | } 50 | }, 51 | "saveRefs": true 52 | }, 53 | "Opportunity": { 54 | "weight": 50, 55 | "refs": { 56 | "Lead": { 57 | "resolveRefs": true 58 | } 59 | }, 60 | "saveRefs": true 61 | }, 62 | "Campaign": { 63 | "weight": 60, 64 | "refs": { 65 | "Opportunity": { 66 | "resolveRefs": true 67 | } 68 | }, 69 | "saveRefs": true 70 | } 71 | } -------------------------------------------------------------------------------- /planMapOriginal.json: -------------------------------------------------------------------------------- 1 | { 2 | "TicketableEvent__c": [ 3 | "EventInstance__c" 4 | ], 5 | "EventInstance__c": [ 6 | "TicketAllocation__c" 7 | ], 8 | "Contact": [ 9 | "Lead", 10 | "Case" 11 | ], 12 | "Account": [ 13 | "Task", 14 | "Contact", 15 | "Lead", 16 | "Case", 17 | "Opportunity" 18 | ], 19 | "Opportunity": [ 20 | "Lead" 21 | ], 22 | "Campaign": [ 23 | "Opportunity" 24 | ], 25 | "thoughts": "Iterate over this list of references. If the root exists in another reference.", 26 | "thoughts1": "then we need to ask if the root is before the other reference" 27 | } 28 | 29 | -------------------------------------------------------------------------------- /relationship_map.json: -------------------------------------------------------------------------------- 1 | { 2 | "Account": { 3 | "childRefs": [ 4 | { 5 | "cascadeDelete": false, 6 | "childSObject": "Account", 7 | "deprecatedAndHidden": false, 8 | "field": "ParentId", 9 | "junctionIdListNames": [], 10 | "junctionReferenceTo": [], 11 | "relationshipName": "ChildAccounts", 12 | "restrictedDelete": false 13 | }, 14 | { 15 | "cascadeDelete": true, 16 | "childSObject": "Contact", 17 | "deprecatedAndHidden": false, 18 | "field": "AccountId", 19 | "junctionIdListNames": [], 20 | "junctionReferenceTo": [], 21 | "relationshipName": "Contacts", 22 | "restrictedDelete": false 23 | } 24 | ], 25 | "name": "Account", 26 | "parentRefs": [ 27 | { 28 | "aggregatable": true, 29 | "aiPredictionField": false, 30 | "autoNumber": false, 31 | "byteLength": 18, 32 | "calculated": false, 33 | "calculatedFormula": null, 34 | "cascadeDelete": false, 35 | "caseSensitive": false, 36 | "compoundFieldName": null, 37 | "controllerName": null, 38 | "createable": true, 39 | "custom": false, 40 | "defaultValue": null, 41 | "defaultValueFormula": null, 42 | "defaultedOnCreate": false, 43 | "dependentPicklist": false, 44 | "deprecatedAndHidden": false, 45 | "digits": 0, 46 | "displayLocationInDecimal": false, 47 | "encrypted": false, 48 | "externalId": false, 49 | "extraTypeInfo": null, 50 | "filterable": true, 51 | "filteredLookupInfo": null, 52 | "formulaTreatNullNumberAsZero": false, 53 | "groupable": true, 54 | "highScaleNumber": false, 55 | "htmlFormatted": false, 56 | "idLookup": false, 57 | "inlineHelpText": null, 58 | "label": "Parent Account ID", 59 | "length": 18, 60 | "mask": null, 61 | "maskType": null, 62 | "name": "ParentId", 63 | "nameField": false, 64 | "namePointing": false, 65 | "nillable": true, 66 | "permissionable": true, 67 | "picklistValues": [], 68 | "polymorphicForeignKey": false, 69 | "precision": 0, 70 | "queryByDistance": false, 71 | "referenceTargetField": null, 72 | "referenceTo": [ 73 | "Account" 74 | ], 75 | "relationshipName": "Parent", 76 | "relationshipOrder": null, 77 | "restrictedDelete": false, 78 | "restrictedPicklist": false, 79 | "scale": 0, 80 | "searchPrefilterable": true, 81 | "soapType": "tns:ID", 82 | "sortable": true, 83 | "type": "reference", 84 | "unique": false, 85 | "updateable": true, 86 | "writeRequiresMasterRead": false 87 | } 88 | ] 89 | }, 90 | "Contact": { 91 | "childRefs": [ 92 | { 93 | "cascadeDelete": false, 94 | "childSObject": "Contact", 95 | "deprecatedAndHidden": false, 96 | "field": "ReportsToId", 97 | "junctionIdListNames": [], 98 | "junctionReferenceTo": [], 99 | "relationshipName": null, 100 | "restrictedDelete": false 101 | } 102 | ], 103 | "name": "Contact", 104 | "parentRefs": [ 105 | { 106 | "aggregatable": true, 107 | "aiPredictionField": false, 108 | "autoNumber": false, 109 | "byteLength": 18, 110 | "calculated": false, 111 | "calculatedFormula": null, 112 | "cascadeDelete": false, 113 | "caseSensitive": false, 114 | "compoundFieldName": null, 115 | "controllerName": null, 116 | "createable": true, 117 | "custom": false, 118 | "defaultValue": null, 119 | "defaultValueFormula": null, 120 | "defaultedOnCreate": false, 121 | "dependentPicklist": false, 122 | "deprecatedAndHidden": false, 123 | "digits": 0, 124 | "displayLocationInDecimal": false, 125 | "encrypted": false, 126 | "externalId": false, 127 | "extraTypeInfo": null, 128 | "filterable": true, 129 | "filteredLookupInfo": null, 130 | "formulaTreatNullNumberAsZero": false, 131 | "groupable": true, 132 | "highScaleNumber": false, 133 | "htmlFormatted": false, 134 | "idLookup": false, 135 | "inlineHelpText": null, 136 | "label": "Reports To ID", 137 | "length": 18, 138 | "mask": null, 139 | "maskType": null, 140 | "name": "ReportsToId", 141 | "nameField": false, 142 | "namePointing": false, 143 | "nillable": true, 144 | "permissionable": true, 145 | "picklistValues": [], 146 | "polymorphicForeignKey": false, 147 | "precision": 0, 148 | "queryByDistance": false, 149 | "referenceTargetField": null, 150 | "referenceTo": [ 151 | "Contact" 152 | ], 153 | "relationshipName": "ReportsTo", 154 | "relationshipOrder": null, 155 | "restrictedDelete": false, 156 | "restrictedPicklist": false, 157 | "scale": 0, 158 | "searchPrefilterable": true, 159 | "soapType": "tns:ID", 160 | "sortable": true, 161 | "type": "reference", 162 | "unique": false, 163 | "updateable": true, 164 | "writeRequiresMasterRead": false 165 | } 166 | ] 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/commands/djc/cleardata.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { join } from 'path'; 3 | import * as fs from 'fs'; 4 | import { Connection, Messages, SfError, AuthInfo } from '@salesforce/core'; 5 | import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; 6 | import { JsonMap } from '@salesforce/ts-types'; 7 | import { ux } from '@oclif/core'; 8 | 9 | Messages.importMessagesDirectory(join(__dirname, '..', '..', '..')); 10 | 11 | export type ClearDataResult = { 12 | message: string; 13 | data: JsonMap; 14 | }; 15 | 16 | export default class ClearData extends SfCommand { 17 | public static description = `Delete data from a scratch org. `; 18 | 19 | public static examples = [ 20 | `$ sfdx djc:cleardata -o Account 21 | ` 22 | ]; 23 | 24 | protected static flagsConfig = { 25 | // flag with a value (-n, --name=VALUE) 26 | sobject: Flags.string({char: 'o', required: true, description: 'Object to delete all records for'}) 27 | }; 28 | 29 | // Comment this out if your command does not require an org username 30 | protected static requiresUsername = true; 31 | 32 | // Comment this out if your command does not support a hub org username 33 | protected static supportsDevhubUsername = true; 34 | 35 | protected conn:Connection; 36 | 37 | // Set this to true if your command requires a project workspace; 'requiresProject' is false by default 38 | public static requiresProject = true; 39 | 40 | 41 | // tslint:disable-next-line:no-any 42 | public async run(): Promise { 43 | const { flags } = await this.parse(ClearData); 44 | //await this.clearData(conn, 'Entitlement'); 45 | // await this.clearDataViaBulk(conn, this.flags.sobject); 46 | const authInfo = await AuthInfo.create({ username: flags.username }); 47 | this.conn = await Connection.create({ authInfo }); 48 | await this.handleBigData(flags.sobject, await this.getDataToDelete(flags.sobject)); 49 | //await this.clearData(conn, 'Contact'); 50 | } 51 | 52 | protected async getDataToDelete(sobject: string): Promise> { 53 | const results = await this.conn.autoFetchQuery(`Select Id From ${sobject}`, { maxFetch: 20000}); 54 | ux.log(`Discovered a total of ${results.totalSize} ${sobject} records for deletion.`); 55 | return results.records; 56 | 57 | } 58 | 59 | protected async handleBigData(sobject: string, dataToDelete: Array): Promise { 60 | const chunkSize = 10000; 61 | let numberImported: number = 0; 62 | let batchNumber: number = 0; 63 | for (let i=0;i, batchNumber: number): Promise { 73 | const conn = this.org.getConnection(); 74 | const data = await conn.autoFetchQuery(`Select Id From ${sobject}`, { maxFetch: 20000}); 75 | const job = conn.bulk.createJob(sobject, "delete"); 76 | const batch = job.createBatch(); 77 | //this.ux.log(`Found ${data.totalSize} ${sobject} records to delete`); 78 | const cmd:ClearData = this; 79 | // start job 80 | if (data.totalSize > 0) { 81 | return new Promise(function(resolve, reject) { 82 | batch.execute(data.records); 83 | // listen for events 84 | batch.on("error", function(batchInfo) { // fired when batch request is queued in server. 85 | console.log('Error, batchInfo:', batchInfo); 86 | reject('Error, batchInfo:'+ JSON.stringify(batchInfo, null, 4)); 87 | }); 88 | batch.on("queue", function() { // fired when batch request is queued in server. 89 | ux.log(`Queueing the deletion of ${data.records.length} ${sobject} records in batches of 10,000.`) 90 | batch.poll(2000 /* interval(ms) */, 200000 /* timeout(ms) */); // start polling - Do not poll until the batch has started 91 | }); 92 | batch.on("response", function(rets) { // fired when batch finished and result retrieved 93 | let successCount: number = 0; 94 | let errorCount: number =0; 95 | let errorOutput:string = ''; 96 | for (var i=0; i < rets.length; i++) { 97 | if (rets[i].success) { 98 | successCount++; 99 | } else { 100 | errorCount++; 101 | for (let x = 0;x < rets[i].errors.length; x++) { 102 | errorOutput = errorOutput + `Error on create: ${rets[i].errors[x]}\n`; 103 | } 104 | } 105 | } 106 | if (errorCount > 0) { 107 | ux.log('Errors'); 108 | fs.writeFileSync(`${sobject}_delete_errors.txt`, errorOutput); 109 | } 110 | ux.log(`Batch delete finished 111 | ${successCount} ${sobject} records successfully deleted 112 | ${errorCount} erros occured, check ${sobject}_delete_errors.txt`); 113 | resolve(1); 114 | }); 115 | }); 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/commands/djc/export.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { join } from 'path'; 3 | import * as fs from 'fs'; 4 | import { SfCommand } from '@salesforce/sf-plugins-core'; 5 | import { JsonMap } from '@salesforce/ts-types'; 6 | import { Connection, Messages, SfError, AuthInfo } from '@salesforce/core'; 7 | import { ux } from '@oclif/core'; 8 | 9 | export type ExportResult = { 10 | message: string; 11 | data: JsonMap; 12 | }; 13 | 14 | Messages.importMessagesDirectory(join(__dirname, '..', '..', '..')); 15 | export default class Export extends SfCommand { 16 | public static description = `Import data to an org to use in a scratch org. `; 17 | 18 | public static examples = [ 19 | `$ sfdx djc:import -p directory 20 | ` 21 | ]; 22 | 23 | protected static flagsConfig = { 24 | // flag with a value (-n, --name=VALUE) 25 | // name: flags.string({char: 'n', description: messages.getMessage('nameFlagDescription')}) 26 | }; 27 | 28 | // Comment this out if your command does not require an org username 29 | protected static requiresUsername = true; 30 | 31 | // Comment this out if your command does not support a hub org username 32 | protected static supportsDevhubUsername = true; 33 | 34 | protected conn:Connection; 35 | // Set this to true if your command requires a project workspace; 'requiresProject' is false by default 36 | public static requiresProject = true; 37 | 38 | 39 | 40 | // tslint:disable-next-line:no-any 41 | public async run(): Promise { 42 | const { flags } = await this.parse(Export); 43 | const accountQuery:string = 'Select Description, Fax, Id, Industry, Name, NumberOfEmployees, Phone, ShippingCity, ShippingCountry, ShippingPostalCode, ShippingState, ShippingStreet, SicDesc, Type, Website From Account'; 44 | const contactQuery:string = 'Select Id, AccountId, Activation_Date__c, Active_Customer__c, Age__c, Customer_Category__c, Customer_Code__c, Customer_Number__c, Customer_Region__c, Customer_Relationship_Type__c, Customer_Tenure__c, Customer_Type__c, Debt_Service__c, Deceased__c, Delinquent_Status__c, Department, Description, Email, Employment_Status__c, ExternalId__c, FirstName, Foreigner__c, Gender__c, Home_Branch_Location__c, HomePhone, Household_Income__c, Industry__c, Joined_By_Channel__c, Last_Date_As_Primary_Customer__c, LastName, MailingCity, MailingCountry, MailingPostalCode, MailingState, MailingStreet, MobilePhone, New_Customer__c, Parent_Legal_Entity__c, Phone, Premier_Customer__c, Primary_Address__c, Primary_Customer__c, Province_Code__c, Race__c From Contact'; 45 | const bankProductQuery:string = 'Select Bank_Code__c, Category__c, ExternalId__c, Description__c, Id, Minimum_Deposit__c, Name, Online_Application__c, Product_Category__c, Promotion_Type__c From Bank_Product__c'; 46 | const bankAccountQuery:string = 'Select Account_Age__c, Bank_Product__c, Contact__c, Id, Name From Bank_Account__c'; 47 | const authInfo:AuthInfo = await AuthInfo.create({ username: flags.username }); 48 | this.conn = await Connection.create({ authInfo }); 49 | // First, query for accounts from source org 50 | await this.getData(this.conn, accountQuery, 'Account'); 51 | await this.getData(this.conn, contactQuery, 'Contact'); 52 | await this.getData(this.conn, bankProductQuery, 'BankProduct'); 53 | await this.getData(this.conn, bankAccountQuery, 'BankAccount'); 54 | 55 | } 56 | 57 | private async getData (conn: Connection, query:string, objectType: string) { 58 | // First, query for accounts from source org 59 | // let execOptions:QueryOptions = { autoFetch: true, maxFetch: 50000 }; 60 | // execOptions.autoFetch = true; 61 | let result = await conn.autoFetchQuery(query); 62 | 63 | if (!result.records || result.records.length <= 0) { 64 | ux.log('Ooops'); 65 | throw new SfError('No records returned for Query!'); 66 | } 67 | 68 | let allResults:Array = []; 69 | allResults = allResults.concat(result.records); 70 | ux.log(`Gonna write ${objectType} file: ${allResults.length} records out of ${result.totalSize}`); 71 | fs.writeFileSync(`${objectType}Data.json`, JSON.stringify(allResults, null, 4)); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/commands/djc/import.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; 3 | import { join } from 'path'; 4 | import * as fs from 'fs'; 5 | import { Connection, Messages, AuthInfo } from '@salesforce/core'; 6 | import { ux } from '@oclif/core'; 7 | import { JsonMap } from '@salesforce/ts-types'; 8 | 9 | Messages.importMessagesDirectory(join(__dirname, '..', '..', '..')); 10 | interface Attributes { 11 | type: string; 12 | url: string; 13 | } 14 | 15 | interface Contact { 16 | attributes: Attributes; 17 | Id: string; 18 | AccountId: string; 19 | Activation_Date__c: string; 20 | Active_Customer__c: boolean; 21 | Age__c: number; 22 | Customer_Category__c: string; 23 | Customer_Code__c: string; 24 | Customer_Number__c: number; 25 | Customer_Region__c: string; 26 | Customer_Relationship_Type__c: string; 27 | Customer_Tenure__c: string; 28 | Customer_Type__c: string; 29 | Debt_Service__c: string; 30 | Deceased__c: boolean; 31 | Delinquent_Status__c: string; 32 | Department: string; 33 | Description: string; 34 | Email: string; 35 | Employment_Status__c: string; 36 | ExternalId__c: string; 37 | FirstName: string; 38 | Foreigner__c: false; 39 | Gender__c: string; 40 | Home_Branch_Location__c: string; 41 | HomePhone: string; 42 | Household_Income__c: number; 43 | Industry__c: string; 44 | Joined_By_Channel__c: string; 45 | Last_Date_As_Primary_Customer__c: string; 46 | LastName: string; 47 | MailingCity: string; 48 | MailingCountry: string; 49 | MailingPostalCode: string; 50 | MailingState: string; 51 | MailingStreet: string; 52 | MobilePhone: string; 53 | Name: string; 54 | New_Customer__c: string; 55 | Parent_Legal_Entity__c: string; 56 | Phone: string; 57 | Premier_Customer__c: boolean; 58 | Primary_Address__c: boolean; 59 | Primary_Customer__c: boolean; 60 | Province_Code__c: string; 61 | Race__c: string; 62 | UpdatedId: string; 63 | } 64 | 65 | interface Account { 66 | attributues: Attributes; 67 | Description: string; 68 | Fax: string; 69 | Id: string; 70 | Industry: string; 71 | IsBuyer: boolean; 72 | Name: string; 73 | NumberOfEmployees: number; 74 | Phone: string; 75 | PhotoUrl: string; 76 | ShippingCity: string; 77 | ShippingCountry: string; 78 | ShippingPostalCode: string; 79 | ShippingState: string; 80 | ShippingStreet: string; 81 | SicDesc: string; 82 | Type: string; 83 | Website: string; 84 | UpdatedId: string; 85 | } 86 | 87 | interface Bank_Account { 88 | attributes: Attributes; 89 | Account_Age__c: number; 90 | Bank_Product__c: string; 91 | Contact__c: string; 92 | Id: string; 93 | Name: string; 94 | UpdatedId: string; 95 | } 96 | 97 | interface Bank_Product { 98 | attributes: Attributes; 99 | Bank_Code__c: string; 100 | Category__c: string; 101 | ExternalId__c: string; 102 | Id: string; 103 | Minimum_Deposit__c: number; 104 | Name: string; 105 | Online_Application__c: boolean; 106 | Product_Category__c: string; 107 | Promotion_Type__c: string; 108 | UpdatedId: string; 109 | } 110 | 111 | export type ImportResult = { 112 | message: string; 113 | data: JsonMap; 114 | }; 115 | 116 | export default class Import extends SfCommand { 117 | public static description = `Import data to an org to use in a scratch org. `; 118 | 119 | public static examples = [ 120 | `$ sfdx djc:import -p directory 121 | ` 122 | ]; 123 | 124 | protected static flagsConfig = { 125 | // flag with a value (-n, --name=VALUE) 126 | xfiles: Flags.boolean({ char: 'x', description: 'Use the limited size files instead of full size files'}) 127 | }; 128 | 129 | // Comment this out if your command does not require an org username 130 | protected static requiresUsername = true; 131 | 132 | // Comment this out if your command does not support a hub org username 133 | protected static supportsDevhubUsername = true; 134 | 135 | protected connection: Connection; 136 | 137 | // Set this to true if your command requires a project workspace; 'requiresProject' is false by default 138 | public static requiresProject = true; 139 | 140 | private accounts: Array; 141 | private contacts: Array; 142 | private bankproducts: Array; 143 | private bankaccounts: Array; 144 | 145 | // Import Accounts 146 | // Update the accounts with new Ids 147 | // Read in Contacts 148 | // Update the AccountId with the new Account Id 149 | // Import the contacts 150 | // Update the contact with the new Ids 151 | // Import the Bank Products 152 | // Update the bank products with the new Ids 153 | // Read in the Bank Accounts 154 | // Update the Contact Id and the Bank Product Id with the new Ids 155 | // Import the Bank Accounts 156 | // tslint:disable-next-line:no-any 157 | public async run(): Promise { 158 | const { flags } = await this.parse(Import); 159 | this.loadDataFiles(flags.xfiles); 160 | const authInfo = await AuthInfo.create({username: flags.username }); 161 | this.connection = await Connection.create({ authInfo }); 162 | //this.excludeAccountsWithNoContacts(); 163 | //this.excludeContactsWithoutAnAccountInGoldFileX(); 164 | //this.excludeBankAccountsWithNoContactOrProduct(); 165 | 166 | this.summarizeImportOperation(); 167 | await this.handleBigData('Account', this.accounts) 168 | .then(() => { 169 | return this.updateContactAccountIds(); 170 | }).then(() => { 171 | return this.handleBigData('Contact', this.contacts); 172 | }).then(() => { 173 | return this.handleBigData('Bank_Product__c', this.bankproducts); 174 | }).then(() => { 175 | return this.updateBankAccountIds() 176 | }).then(() => { 177 | return this.handleBigData('Bank_Account__c', this.bankaccounts); 178 | }).catch((reason: any) => { 179 | return reason 180 | }); 181 | } 182 | 183 | protected async handleBigData(sobject: string, dataToLoad: Array): Promise { 184 | const chunkSize = 10000; 185 | let numberImported: number = 0; 186 | let batchNumber: number = 0; 187 | for (let i=0;i, batchNumber: number): Promise { 197 | // Create job and batch 198 | const job = this.connection.bulk.createJob(sobject, "insert"); 199 | const batch = job.createBatch(); 200 | // start job 201 | const cmd:Import = this; 202 | 203 | return new Promise(function(resolve, reject) { 204 | batch.execute(dataToLoad); 205 | // listen for events 206 | batch.on("error", function(batchInfo) { // fired when batch request is queued in server. 207 | ux.error('Error, batchInfo:', batchInfo); 208 | reject('Error, batchInfo:'+ JSON.stringify(batchInfo, null, 4)); 209 | }); 210 | batch.on("queue", function(batchInfo) { // fired when batch request is queued in server. 211 | ux.log(`Queued batch for ${dataToLoad.length} ${sobject} records.`); 212 | // poll(interval(ms), timeout(ms)) 213 | batch.poll(2000, 200000); // start polling - Do not poll until the batch has started 214 | }); 215 | batch.on("response", function(rets) { // fired when batch finished and result retrieved 216 | let successCount:number = 0; 217 | let errorCount:number = 0; 218 | let errorOutput:string = ''; 219 | for (var i=0; i < rets.length; i++) { 220 | if (rets[i].success) { 221 | dataToLoad[i].UpdatedId = rets[i].id; 222 | successCount++; 223 | } else { 224 | errorCount++; 225 | for (let x = 0;x < rets[i].errors.length; x++) { 226 | errorOutput = errorOutput + `Error on create: ${rets[i].errors[x]}\n`; 227 | } 228 | } 229 | } 230 | if (errorCount > 0) { 231 | ux.log('Errors'); 232 | fs.writeFileSync(`${sobject}_insert_errors-${batchNumber}.txt`, errorOutput); 233 | } 234 | ux.log(`Batch insert finished 235 | ${successCount} ${sobject} records successfully inserted 236 | ${errorCount} erros occured, check ${sobject}_insert_errors.txt`); 237 | resolve(1); 238 | }); 239 | }); 240 | } 241 | 242 | protected summarizeImportOperation() { 243 | ux.log(` 244 | Will import the following data: 245 | ${this.accounts.length} Accounts 246 | ${this.contacts.length} Contacts 247 | ${this.bankproducts.length} Bank Products 248 | ${this.bankaccounts.length} Bank Accounts`); 249 | } 250 | 251 | protected loadDataFiles(xfiles: boolean) { 252 | //const filepostfix: string = (xfiles ? 'x' : ''); 253 | //this.accounts = JSON.parse(fs.readFileSync(`AccountData_GF${filepostfix}.json`, 'utf8').toString()); 254 | //this.contacts = JSON.parse(fs.readFileSync('ContactData_GF.json', 'utf8').toString()); 255 | //this.bankproducts = JSON.parse(fs.readFileSync('BankProductData_GF.json', 'utf8').toString()); 256 | //this.bankaccounts = JSON.parse(fs.readFileSync('BankAccountData_GF.json', 'utf8').toString()); 257 | this.accounts = JSON.parse(fs.readFileSync(`AccountData.json`, 'utf8').toString()); 258 | this.contacts = JSON.parse(fs.readFileSync('ContactData.json', 'utf8').toString()); 259 | this.bankproducts = JSON.parse(fs.readFileSync('BankProductData.json', 'utf8').toString()); 260 | this.bankaccounts = JSON.parse(fs.readFileSync('BankAccountData.json', 'utf8').toString()); 261 | } 262 | 263 | protected findRecord(dataToSearch:Array, idTofind:string, fieldToLookIn:string) { 264 | return dataToSearch.find(record => { 265 | return record[fieldToLookIn] === idTofind; 266 | }) 267 | } 268 | 269 | protected async updateContactAccountIds(): Promise { 270 | for (let i:number = 0; i < this.contacts.length; i++ ) { 271 | const account = this.accounts.find(d => d.Id === this.contacts[i].AccountId); 272 | if (account !== undefined) { 273 | this.contacts[i].AccountId = account.UpdatedId; 274 | } 275 | } 276 | } 277 | 278 | protected async updateBankAccountIds(): Promise { 279 | for (let i:number = 0; i < this.bankaccounts.length; i++ ) { 280 | const contact = this.contacts.find(d => d.Id === this.bankaccounts[i].Contact__c); 281 | if (contact !== undefined) { 282 | this.bankaccounts[i].Contact__c = contact.UpdatedId; 283 | const bankProduct = this.bankproducts.find(d => d.Id === this.bankaccounts[i].Bank_Product__c); 284 | if (bankProduct) { 285 | this.bankaccounts[i].Bank_Product__c = bankProduct.UpdatedId; 286 | } 287 | } 288 | } 289 | return; 290 | } 291 | 292 | /*protected async importAndUpdateAccounts(conn: Connection, cmd: Import): Promise { 293 | //this.accounts = JSON.parse(fs.readFileSync('AccountData_GFx.json', 'utf8').toString()); 294 | const results:any = await conn.insert('Account', this.accounts) 295 | cmd.ux.log('Got results?'); 296 | let successes:number = 0; 297 | let failures:number = 0; 298 | for (let i = 0; i < results.length; i++) { 299 | if (results[i].success === true) { 300 | successes++; 301 | this.accounts[i].UpdatedId = results[i].id; 302 | } else { 303 | failures++; 304 | } 305 | } 306 | this.ux.log(`Imported ${successes} records with ${failures} failures`); 307 | fs.writeFileSync('importedAccounts.json', JSON.stringify(this.accounts, null, 4)); 308 | }*/ 309 | 310 | /*protected async importAndUpdateContacts(conn: Connection): Promise { 311 | //this.contacts = JSON.parse(fs.readFileSync('ContactData_GFx.json', 'utf8').toString()); 312 | const results:any = await conn.insert('Contact', this.contacts) 313 | this.ux.log('Got results?'); 314 | for (let i = 0; i < results.length; i++) { 315 | this.contacts[i].UpdatedId = results[i].id; 316 | } 317 | fs.writeFileSync('importedContacts.json', JSON.stringify(this.contacts, null, 4)); 318 | }*/ 319 | 320 | /*protected async importAndUpdateBankProducts(): Promise { 321 | //this.bankproducts = JSON.parse(fs.readFileSync('BankProductData_GF.json', 'utf8').toString()); 322 | const results:any = await this.org.getConnection().insert('Bank_Product__c', this.bankproducts) 323 | this.ux.log('Got results?'); 324 | for (let i = 0; i < results.length; i++) { 325 | this.bankproducts[i].UpdatedId = results[i].id; 326 | } 327 | fs.writeFileSync('importedBankProducts.json', JSON.stringify(this.accounts, null, 4)); 328 | return; 329 | }*/ 330 | 331 | protected async importAndUpdateBankAccounts(conn: Connection, cmd: Import) { 332 | //this.bankaccounts = JSON.parse(fs.readFileSync('BankAccountData_GFx.json', 'utf8').toString()); 333 | for (let i:number = 0; i < this.bankaccounts.length; i++ ) { 334 | const contact = this.contacts.find(d => d.Id === this.bankaccounts[i].Contact__c); 335 | if (contact) { 336 | this.bankaccounts[i].Contact__c = contact.UpdatedId; 337 | } 338 | const bankProduct = this.bankproducts.find(d => d.Id === this.bankaccounts[i].Bank_Product__c); 339 | if (bankProduct) { 340 | this.bankaccounts[i].Bank_Product__c = bankProduct.UpdatedId; 341 | } 342 | } 343 | 344 | const results:any = await conn.insert('Bank_Account__c', this.bankaccounts) 345 | ux.log('Got results?'); 346 | for (let i = 0; i < results.length; i++) { 347 | this.bankaccounts[i].UpdatedId = results[i].id; 348 | } 349 | fs.writeFileSync('importedBankAccounts.json', JSON.stringify(this.bankaccounts, null, 4)); 350 | 351 | } 352 | 353 | protected excludeAccountsWithNoContacts() { 354 | //const accounts:Array = JSON.parse(fs.readFileSync('AccountData_GF.json', 'utf8').toString()); 355 | //const contacts:Array = JSON.parse(fs.readFileSync('ContactData_GF.json', 'utf8').toString()); 356 | let accountsDeleted:number = 0; 357 | let accountsChecked:number = 0; 358 | const filteredAccounts:Array = []; 359 | for (let i=0; i { 363 | return contact.AccountId === account.Id; 364 | }) 365 | if (account != null && foundContact === undefined) { 366 | accountsDeleted++; 367 | delete this.accounts[i]; 368 | } else { 369 | filteredAccounts.push(this.accounts[i]); 370 | } 371 | } 372 | ux.log(`Removed ${accountsDeleted} accounts since they have not contacts.`); 373 | ux.log(`Checked ${accountsChecked} accounts.`) 374 | this.accounts = filteredAccounts; 375 | //this.writeUpdatedDataFile('Account', this.accounts); 376 | } 377 | 378 | protected excludeContactsWithoutAnAccountInGoldFileX() { 379 | //this.contacts = JSON.parse(fs.readFileSync('ContactData_GF.json', 'utf8').toString()); 380 | //this.loadAccountsFromFile(); 381 | let contactsDeleted:number = 0; 382 | let contactsChecked:number = this.contacts.length; 383 | const filterdContacts:Array = []; 384 | for (let i:number = 0; i < this.contacts.length; i++ ) { 385 | const contact:Contact = this.contacts[i]; 386 | const account = this.accounts.find(acct => { 387 | if (acct === null) { 388 | return undefined; 389 | } else { 390 | return acct.Id === contact.AccountId; 391 | } 392 | }); 393 | if (account === undefined) { 394 | contactsDeleted++; 395 | this.contacts.splice(i, 1); 396 | i--; 397 | } else { 398 | filterdContacts.push(this.contacts[i]); 399 | } 400 | } 401 | ux.log(`Removed ${contactsDeleted} contacts since they have no accounts in the extracted data.`); 402 | ux.log(`Checked ${contactsChecked} contacts.`) 403 | this.contacts = filterdContacts; 404 | //this.writeUpdatedDataFile('Contact', this.contacts); 405 | } 406 | 407 | protected excludeBankAccountsWithNoContactOrProduct() { 408 | //const bankaccounts:Array = JSON.parse(fs.readFileSync('BankAccountData_GF.json', 'utf8').toString()); 409 | //const contacts:Array = JSON.parse(fs.readFileSync('ContactData_GF.json', 'utf8').toString()); 410 | //const bankproducts:Array = JSON.parse(fs.readFileSync('BankProductData_GF.json', 'utf8').toString()); 411 | let bankAccountsDeleted:number = 0; 412 | let bankAccountsChecked:number = this.bankaccounts.length; 413 | const filteredBankAccounts: Array = []; 414 | for (let i=0; i d.Id === bankaccount.Contact__c); 417 | const foundProduct = this.bankproducts.find(d => d.Id === bankaccount.Bank_Product__c); 418 | if ( (bankaccount != null && foundContact === undefined) || (bankaccount != null && foundProduct === undefined) ) { 419 | bankAccountsDeleted++; 420 | delete this.bankaccounts[i]; 421 | } else { 422 | filteredBankAccounts.push(this.bankaccounts[i]); 423 | } 424 | } 425 | if (bankAccountsDeleted > 0) { 426 | this.bankaccounts = filteredBankAccounts; 427 | ux.log(`Removed ${bankAccountsDeleted} bank accounts since they have no contacts.`); 428 | ux.log(`Checked ${bankAccountsChecked} bank accounts.`) 429 | //this.writeUpdatedDataFile('BankAccount', this.bankaccounts); 430 | } 431 | } 432 | } 433 | -------------------------------------------------------------------------------- /src/commands/tohoom/data/export.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { join } from 'path'; 3 | import * as fs from 'fs'; 4 | import * as fsExtra from 'fs-extra'; 5 | import * as path from 'path'; 6 | import { isUndefined } from 'util'; 7 | import { DescribeSObjectResult, QueryResult } from 'jsforce'; 8 | import TohoomExtension from '../../../tohoom'; 9 | import { Connection, Messages, AuthInfo } from '@salesforce/core'; 10 | import { JsonMap } from '@salesforce/ts-types'; 11 | import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; 12 | import { ux } from '@oclif/core'; 13 | 14 | Messages.importMessagesDirectory(join(__dirname, '..', '..', '..')); 15 | // const messages = core.Messages.loadMessages('data', 'export'); 16 | interface ChildRelationship { 17 | cascadeDelete: boolean; 18 | childSObject: string; 19 | deprecatedAndHidden: boolean; 20 | field: string; 21 | junctionIdListNames: string[]; 22 | junctionReferenceTo: string[]; 23 | relationshipName: string; 24 | restrictedDelete: boolean; 25 | fieldReferenceTo: string; 26 | } 27 | 28 | interface Field { 29 | custom: boolean; 30 | defaultValue?: string | boolean; 31 | // encrypted: boolean; 32 | externalId: boolean; 33 | // extraTypeInfo: string; 34 | filterable: boolean; 35 | idLookup: boolean; 36 | label: string; 37 | mask?: string; 38 | maskType?: string; 39 | name: string; 40 | nameField: boolean; 41 | namePointing: boolean; 42 | polymorphicForeignKey: boolean; 43 | referenceTargetField?: string; 44 | referenceTo?: string[]; 45 | relationshipName?: string; 46 | relationshipOrder?: number; 47 | // tslint:disable-next-line:no-reserved-keywords 48 | type: string; 49 | } 50 | 51 | interface IDescribeSObjectResult { 52 | fields: Field[]; 53 | childRelationships: ChildRelationship[]; 54 | layoutable: boolean; 55 | } 56 | 57 | interface RelationshipMap { 58 | parentRefs: Field[]; 59 | childRefs: ChildRelationship[]; 60 | } 61 | 62 | interface PlanEntry { 63 | sobject: string; 64 | saveRefs: boolean; 65 | resolveRefs: boolean; 66 | files: string[]; 67 | } 68 | 69 | export type ExportResult = { 70 | message: string; 71 | data: JsonMap; 72 | }; 73 | 74 | export default class Export extends SfCommand { 75 | public static description = `Extract data from an org to use in a scratch org. Just supply a list of SObjects and you *should* end up with a dataset and data plan that can be used with the official force:data:tree:import command`; // messages.getMessage('commandDescription'); 76 | 77 | public static examples = [ 78 | `$ sfdx tohoom:data:export -o Account,Contact,Case,Opportunity -t data/exported -n my-testplan 79 | ` 80 | ]; 81 | 82 | protected static flagsConfig = { 83 | // flag with a value (-n, --name=VALUE) 84 | // name: flags.string({char: 'n', description: messages.getMessage('nameFlagDescription')}) 85 | objects: Flags.string({ required: true, char: 'o', description: 'Comma separated list of objects to fetch' }), 86 | planname: Flags.string({ default: 'new-data-plan', description: 'name of the data plan to produce, deflaults to "new-plan"', char: 'n'}), 87 | targetdir: Flags.string({ required: true, char: 't', description: 'target directoy to place results in'}), 88 | maxrecords: Flags.integer({ default: 10, char: 'm', description: 'Max number of records to return in any query'}), 89 | savedescribes: Flags.boolean({ char: 's', description: 'Save describe results (for diagnostics)'}), 90 | spiderreferences: Flags.boolean({ char: 'p', description: 'Include refereced SObjects determined by schema examination and existing data'}), 91 | enforcereferences: Flags.boolean({ char: 'e', description: 'If present, missing child reference cause the record to be deleted, otherwise, just the reference field is removed'}), 92 | preserveobjectorder: Flags.boolean({ char: 'b', description: 'If present, uses the order of the objects from the command to determine plan order'}), 93 | tohoom: Flags.boolean({ char: 'k', description: 'Special Tohoom processing to handle self referential relationship'}) 94 | }; 95 | 96 | // Comment this out if your command does not require an org username 97 | protected static requiresUsername = true; 98 | 99 | // Comment this out if your command does not support a hub org username 100 | protected static supportsDevhubUsername = false; 101 | 102 | protected conn:Connection; 103 | // Set this to true if your command requires a project workspace; 'requiresProject' is false by default 104 | public static requiresProject = true; 105 | 106 | private describeMap = {}; // Objectname describe result map 107 | private relMap: RelationshipMap; // map of object name and childRelationships and/or parents 108 | private objects: Array; 109 | private dataMap = {}; 110 | private planEntries: PlanEntry[]; 111 | private globalIds: string[] = [] as string[]; 112 | 113 | // tslint:disable-next-line:no-any 114 | public async run(): Promise { 115 | const { flags } = await this.parse(Export); 116 | // We take in a set of object that we want to generate data for. We will 117 | // examine the relationships of the included objects to one another to datermine 118 | // what to export and in what order. 119 | this.objects = flags.objects.split(','); // [ 'Account', 'Contact', 'Lead', 'Property__c', 'Broker__c']; 120 | // await this.newTest(); 121 | // return; 122 | const authInfo = await AuthInfo.create({username: flags.username}); 123 | this.conn = await Connection.create({ authInfo }); 124 | // Create a map of object describes keyed on object name, based on 125 | // describe calls. This should be cacheing the describe, at least 126 | // for development purposes. 127 | ux.log('Determining relationships for ' + this.objects.length + ' objects...'); 128 | this.describeMap = await this.makeDescribeMap(this.objects, this.conn, flags); 129 | // Create a relationship map. A relationship map object is keyed on the 130 | // object name and has the following structure. 131 | // { 132 | // parentRefs: Field[]; 133 | // childRefs: ChildRelationship[]; 134 | // } 135 | this.relMap = this.makeRelationshipMap(); 136 | 137 | // Run the queries and put the data into individual json files. 138 | await this.runCountQueries(this.conn); 139 | 140 | ux.log('Running queries for ' + _.keys(this.relMap).length + ' objects...'); 141 | this.planEntries = await this.createDataPlan(); 142 | await this.runQueries(this.conn, flags); 143 | ux.log('Saving data...'); 144 | 145 | await this.saveData(flags); 146 | 147 | if (process.env.NODE_OPTIONS === '--inspect-brk' || flags.savedescribes ) { 148 | this.saveDescribeMap(); 149 | } 150 | // return this.planEntries; 151 | if (flags.tohoom) { 152 | let ext = new TohoomExtension(); 153 | ext.run(flags.planname, flags.targetdir, this); 154 | } 155 | ux.log('Finished exporting data and plan.'); 156 | 157 | } 158 | 159 | private reorderPlan() { 160 | const newOrder: Array = []; 161 | //var pe: PlanEntry[]; 162 | _.forEach(this.objects, (data, ind) => { 163 | const e = this.planEntries.find(element => element.sobject === data) 164 | if (e) { 165 | newOrder.push(e) 166 | } 167 | }); 168 | this.planEntries = newOrder; 169 | } 170 | 171 | // Save data iterates over the in-memory data sets. For each data set, 172 | // each record is examined and the referenceId returned from the query 173 | // is set to just the id, rather than a url. After the data sets have been 174 | // adjusted, the data is written to the file system at the location passed 175 | // on the --targetdir flag. 176 | private async saveData(flags) { 177 | // tslint:disable-next-line:forin 178 | for (let objName in this.dataMap) { 179 | objName = objName.split('.')[0]; 180 | // tslint:disable-next-line:forin 181 | for (const ind in this.dataMap[objName].records) { 182 | const record = this.dataMap[objName].records[ind]; 183 | if (!isUndefined(record.attributes)) { 184 | record.attributes['referenceId'] = record.Id; 185 | } else { 186 | this.dataMap[objName].records.splice(ind, 1); 187 | } 188 | } 189 | this.createRefs(this.dataMap[objName]); 190 | } 191 | 192 | this.pruneBadReferences(flags); 193 | if (!fs.existsSync(flags.targetdir)) { 194 | fsExtra.ensureDirSync(flags.targetdir); 195 | } 196 | // tslint:disable-next-line:forin 197 | for (let objName in this.dataMap) { 198 | objName = objName.split('.')[0]; 199 | fs.writeFileSync(path.join(flags.targetdir, objName + '.json'), JSON.stringify(this.dataMap[objName], null, 4)); 200 | } 201 | if (flags.preserveobjectorder) { 202 | this.reorderPlan(); 203 | } 204 | fs.writeFileSync(path.join(flags.targetdir, flags.planname + '.json'), JSON.stringify(this.planEntries, null, 4)); 205 | } 206 | 207 | private pruneBadReferences(flags) { 208 | _.forOwn(this.dataMap, (dataMapItem, key) => { 209 | // tslint:disable-next-line:no-any 210 | const records: Array<{}> = (dataMapItem as any).records; 211 | _.forEach(records, (record, index) => { 212 | _.forOwn(record, (field: string, fieldName: string) => { 213 | if (fieldName !== 'attributes' && typeof field === 'string') { 214 | if (field.startsWith('@ref')) { 215 | if (!this.globalIds.includes(field.split('@ref')[1])) { 216 | if (flags.enforcereferences) { 217 | this.dataMap[key].records.splice(index, 1); 218 | } else { 219 | delete record[fieldName]; 220 | } 221 | } 222 | } 223 | } 224 | }); 225 | }); 226 | }); 227 | } 228 | 229 | private async listGen(): Promise> { 230 | const listMap = {}; 231 | for (const key in this.relMap) { 232 | const obj = this.relMap[key]; 233 | // tslint:disable-next-line:forin 234 | for (const ind in obj.parentRefs) { 235 | const refTo = obj.parentRefs[ind].referenceTo; 236 | // tslint:disable-next-line:prefer-for-of 237 | for (let i = 0; i < refTo.length; i++) { 238 | const refToValue = refTo[i]; 239 | //if (refToValue !== key && !isUndefined(this.relMap[refToValue])) { 240 | if (isUndefined(listMap[refTo[i]])) { 241 | listMap[refToValue] = []; 242 | } 243 | if (!listMap[refToValue].includes(key)) { 244 | listMap[refToValue].push(key); 245 | } 246 | } 247 | //} 248 | } 249 | } 250 | return await this.getDataPlanOrder(listMap); 251 | } 252 | 253 | private async getDataPlanOrder(listMap): Promise> { 254 | const tempList: Array = []; 255 | // tslint:disable-next-line:no-any 256 | // const listMap: any = await core.json.readJson('./planMapOriginal.json'); 257 | // tslint:disable-next-line:forin 258 | for (const topLevelObject in listMap) { 259 | listMap[topLevelObject].forEach(child => { 260 | if (!tempList.includes(child)) { 261 | tempList.push(child); 262 | } 263 | }); 264 | // Determine if this object is referenced by any other objext in our list 265 | const isObjRefResult = this.isObjectReferenced(topLevelObject, listMap); 266 | if (isObjRefResult.result) { 267 | // This object is referenced by another object so we need to insert it in the right place 268 | // What is the right place??? Not sure 269 | isObjRefResult.refrerencingObject.forEach(refObj => { 270 | if (!tempList.includes(refObj)) { 271 | // See if topLevelObject is in the list, if it is, we should create the refObj 272 | // just above the topLevel object 273 | if (tempList.includes(topLevelObject)) { 274 | tempList.splice(tempList.indexOf(topLevelObject), 0, refObj); 275 | } else { 276 | // Neither the topLevelObject nor the refObj are in the list 277 | tempList.unshift(refObj); 278 | } 279 | } 280 | const ind = tempList.indexOf(topLevelObject); 281 | // Add the top level just after the refObj if it's not already in the list 282 | if (ind === -1) { 283 | tempList.splice(tempList.indexOf(refObj) + 1, 0, topLevelObject); 284 | } 285 | // If the toplevel was already in the array, remove the orginal one indexed by 'ind' 286 | //if (ind !== -1) { 287 | // tempList.splice(ind + 1, 1); 288 | //} 289 | }); 290 | } else { 291 | if (!tempList.includes(topLevelObject)) { 292 | tempList.unshift(topLevelObject); 293 | } 294 | } 295 | } 296 | return tempList; 297 | } 298 | 299 | private isObjectReferenced(objectName, listMap) { 300 | // deepcode ignore ArrayConstructor: 301 | const result = { result: false, refrerencingObject: new Array() }; 302 | for (const topLevelObject in listMap) { 303 | if (topLevelObject === objectName) { 304 | // Skip, this is the thing itself 305 | } else { 306 | // Loop over the the children of the object, if the child is the object we are checking 307 | // to see if it referenced, then we create a result object indicating that is in fact 308 | // reference by another object also make note of the object that references it 309 | listMap[topLevelObject].forEach(childElement => { 310 | if (childElement === objectName) { 311 | result.result = true; 312 | result.refrerencingObject.push(topLevelObject); 313 | } 314 | }); 315 | } 316 | } 317 | return result; 318 | } 319 | 320 | private async createDataPlan(): Promise { 321 | const listPlan: Array = await this.listGen(); 322 | const planEntries: PlanEntry[] = [] as PlanEntry[]; 323 | 324 | listPlan.forEach(key => { 325 | const obj: RelationshipMap = this.relMap[key]; 326 | if (!isUndefined(obj)) { 327 | if (obj.childRefs !== undefined && obj.childRefs.length > 0) { 328 | // This is an object that has childRelationships, so should bubble up to the top of the plan 329 | planEntries.push(this.makeParentPlanEntry(key, obj)); 330 | } else if (obj.parentRefs.length > 0) { 331 | planEntries.push(this.makePlanEntry(key, obj)); 332 | } 333 | } 334 | }); 335 | // tslint:disable-next-line:prefer-for-of 336 | for (let i: number = 0; i < planEntries.length; i++) { 337 | if (isUndefined(planEntries[i].resolveRefs)) { 338 | const ent = planEntries.splice(i, 1); 339 | planEntries.unshift(ent[0]); 340 | } 341 | } 342 | return planEntries; 343 | } 344 | 345 | private makePlanEntry(sObjectName: string, obj: RelationshipMap): PlanEntry { 346 | const planEntry: PlanEntry = {} as PlanEntry; 347 | planEntry.sobject = sObjectName; 348 | planEntry.resolveRefs = true; 349 | planEntry.files = []; 350 | planEntry.files.push(sObjectName + '.json'); 351 | return planEntry; 352 | } 353 | 354 | private makeParentPlanEntry(sObjectName: string, obj: RelationshipMap): PlanEntry { 355 | const planEntry: PlanEntry = {} as PlanEntry; 356 | planEntry.sobject = sObjectName; 357 | planEntry.saveRefs = true; 358 | if (obj.parentRefs !== undefined && obj.parentRefs.length > 0) { 359 | planEntry.resolveRefs = true; 360 | } 361 | planEntry.files = []; 362 | planEntry.files.push(sObjectName + '.json'); 363 | return planEntry; 364 | } 365 | 366 | // tslint:disable-next-line:no-any 367 | private createRefs(data: any) { 368 | const idMap = {}; 369 | const regex = /[a-zA-Z0-9]{15}|[a-zA-Z0-9]{18}/; 370 | data.records.forEach(element => { 371 | // tslint:disable-next-line:forin 372 | for (const key in element) { 373 | const value = element[key] + ''; 374 | if ((value.length === 18 || value.length === 15) && value.match(regex)) { 375 | if (key === 'OwnerId') { 376 | delete element[key]; 377 | } else { 378 | if (idMap.hasOwnProperty(value)) { 379 | element[key] = idMap[value]['ref']; 380 | } else { 381 | idMap[value] = { key, ref: '@ref' + value }; 382 | element[key] = '@ref' + value; 383 | } 384 | if (key === 'Id') { 385 | element['attributes']['referenceId'] = 'ref' + value; 386 | delete element['attributes']['url']; 387 | delete element[key]; 388 | } 389 | } 390 | } 391 | } 392 | }); 393 | } 394 | 395 | private removeNulls(rootData) { 396 | rootData.records.forEach(element => { 397 | // tslint:disable-next-line:forin 398 | for (const key in element) { 399 | if (element[key] === null) { 400 | delete element[key]; 401 | } 402 | } 403 | }); 404 | return rootData; 405 | } 406 | 407 | private shouldQueryThisField(childSObject): boolean { 408 | return !isUndefined(childSObject.relationshipName) && !isUndefined(this.relMap[childSObject.childSObject]); 409 | } 410 | 411 | private _validRootObj(rootObj): boolean { 412 | if (!isUndefined(rootObj)) { 413 | if (!isUndefined(rootObj.childRefs)) { 414 | return true; 415 | } 416 | } 417 | return false; 418 | } 419 | 420 | private async runQueries(connection: Connection, flags) { 421 | for (const sobjectName in this.relMap) { 422 | if (this.relMap.hasOwnProperty(sobjectName)) { 423 | if (_.findIndex(this.planEntries, [ 'sobject', sobjectName]) === -1) { 424 | // if (!_.has(this.planEntries, key)) { 425 | delete this.relMap[sobjectName]; 426 | delete this.describeMap[sobjectName]; 427 | } else { 428 | const rootObj = this.relMap[sobjectName]; 429 | 430 | if (this._validRootObj(rootObj)) { 431 | // Run query and store in qrMap 432 | await connection.query(this.generateSimpleQuery(sobjectName, flags)).then(rootData => { 433 | rootData = this.removeNulls(rootData); 434 | if (rootData.totalSize > 0) { 435 | this.dataMap[sobjectName] = rootData; 436 | const ids = this.pullIds(this.dataMap[sobjectName]); 437 | 438 | // tslint:disable-next-line:forin 439 | for (const dependent in rootObj.childRefs) { 440 | // Run query using ids from rootObj in where clause for dependent 441 | const childSObject = rootObj.childRefs[dependent]; 442 | if (rootObj.name !== childSObject.childSObject && this.shouldQueryThisField(childSObject)) { 443 | connection.query(this.generateDependentQuery(childSObject.childSObject, ids, childSObject.field, flags)).then(data => { 444 | this.removeNulls(data); 445 | if (data.totalSize > 0) { 446 | this.addToDatamap(childSObject.childSObject, data); 447 | } 448 | }).catch((reason: any) => { 449 | return reason; 450 | }); 451 | } 452 | } 453 | } else { 454 | delete this.describeMap[sobjectName]; 455 | delete this.relMap[sobjectName]; 456 | } 457 | }).catch((reason: any) => { 458 | return reason; 459 | }); 460 | } else if (isUndefined(rootObj.childRefs) && !isUndefined(rootObj.parentRefs)) { 461 | // Run query and add to map 462 | await connection.query(this.generateSimpleQuery(sobjectName, flags)).then(rootData => { 463 | rootData = this.removeNulls(rootData); 464 | if (rootData.totalSize > 0) { 465 | this.dataMap[sobjectName] = rootData; 466 | this.pullIds(this.dataMap[sobjectName]); 467 | } 468 | }).catch((reason: any) => { 469 | return reason; 470 | }); 471 | } else { 472 | delete this.describeMap[sobjectName]; 473 | delete this.relMap[sobjectName]; 474 | } 475 | } 476 | } 477 | } 478 | } 479 | 480 | private async runCountQueries(connection: Connection) { 481 | for (const key in this.relMap) { 482 | if (this.relMap.hasOwnProperty(key)) { 483 | if (this._validRootObj(this.relMap[key])) { 484 | // Run query and store in qrMap 485 | await connection.query(this.generateSimpleCountQuery(key)).then(rootData => { 486 | if (rootData.totalSize === 0) { 487 | delete this.describeMap[key]; 488 | delete this.relMap[key]; 489 | } 490 | }).catch((reason: any) => { 491 | return reason; 492 | }); 493 | } else { 494 | delete this.describeMap[key]; 495 | delete this.relMap[key]; 496 | } 497 | } 498 | } 499 | } 500 | 501 | private addToDatamap(dataMapIndex: string, dependentData: QueryResult<{}>) { 502 | if (this.dataMap.hasOwnProperty(dataMapIndex)) { 503 | // remove duplicates and add to map 504 | const newRecords = this.removeDuplicates(this.dataMap[dataMapIndex], dependentData); 505 | this.dataMap[dataMapIndex].records.concat(newRecords.records); 506 | } else { 507 | this.dataMap[dataMapIndex] = dependentData; 508 | } 509 | } 510 | 511 | private removeDuplicates(mapData: QueryResult<{}>, newData: QueryResult<{}>): QueryResult<{}> { 512 | mapData.records.forEach(element => { 513 | const foundIndex = _.findIndex(newData.records, ['Id', element['Id']]); 514 | if ( foundIndex !== -1) { 515 | newData.records.splice(foundIndex, 1); 516 | newData.totalSize = newData.totalSize - 1; 517 | } 518 | }); 519 | return newData; 520 | } 521 | 522 | private pullIds(data) { 523 | const ids: string[] = []; 524 | // tslint:disable-next-line:forin 525 | for (const ind in data.records) { 526 | ids.push(data.records[ind].Id); 527 | this.globalIds.push(data.records[ind].Id); 528 | } 529 | return ids; 530 | } 531 | 532 | private generateSimpleCountQuery(objName) { 533 | return 'Select Count() From ' + objName; 534 | } 535 | 536 | private generateSimpleQuery(objName, flags) { 537 | return this.generateQuery(objName) + ' Limit ' + flags.maxrecords; 538 | } 539 | 540 | private generateDependentQuery(objName: string, ids: string[], filterField: string, flags) { 541 | return this.generateQuery(objName) + ' Where ' + filterField + ' in (\'' + ids.join('\',\'') + '\') Limit ' + flags.maxrecords; 542 | } 543 | 544 | private generateQuery(objName) { 545 | const selectClause = new Array(); 546 | // tslint:disable-next-line:forin 547 | for (const fieldIndex in this.describeMap[objName].fields) { 548 | const field = this.describeMap[objName].fields[fieldIndex]; 549 | if (field.createable) { 550 | selectClause.push(field.name); 551 | } 552 | } 553 | selectClause.push('Id'); 554 | 555 | return 'Select ' + selectClause.join(',') + ' From ' + objName; 556 | } 557 | 558 | private getObjectChildRelationships(): RelationshipMap { 559 | const relationshipMap = {}; 560 | // Iterate over the describeMap to visit each object describe 561 | for (const value in this.describeMap) { 562 | if (!isUndefined(value)) { 563 | let index = 0; 564 | for (const child of this.describeMap[value].childRelationships) { 565 | if (!isUndefined(this.describeMap[child.childSObject]) && this.describeMap[child.childSObject].layoutable) { 566 | _.set(relationshipMap, [value, 'childRefs', index], child); 567 | index++; 568 | } 569 | } 570 | if (!isUndefined(relationshipMap[value])) { 571 | _.set(relationshipMap, [value, 'name'], value); 572 | } else { 573 | _.set(relationshipMap, [value, 'childRefs'], []); 574 | } 575 | } 576 | } 577 | return relationshipMap as RelationshipMap; 578 | } 579 | 580 | private getObjectParentRelationships(): RelationshipMap { 581 | const relationshipMap = {}; 582 | // tslint:disable-next-line:no-any 583 | _.map(this.describeMap, (value: any, key) => { 584 | let relIndex = 0; 585 | _.forEach(value.fields, field => { 586 | if (field.type === 'reference') { 587 | // tslint:disable-next-line:prefer-for-of 588 | for (let i = 0; i < field.referenceTo.length; i++ ) { 589 | if (!isUndefined(this.describeMap[field.referenceTo[i]])) { 590 | _.set(relationshipMap, [key, 'parentRefs', relIndex++], field); 591 | } 592 | } 593 | } 594 | }); 595 | if (relationshipMap[key]) { 596 | _.remove(relationshipMap[key]['parentRefs'], n => { 597 | return _.isUndefined(n); 598 | }); 599 | } 600 | if (!isUndefined(relationshipMap[key])) { 601 | _.set(relationshipMap, [key, 'name'], key); 602 | } 603 | }); 604 | return relationshipMap as RelationshipMap; 605 | } 606 | 607 | private async getSobjectDescribe(objName: string, conn): Promise { 608 | let describeResult: IDescribeSObjectResult; 609 | if (fs.existsSync('./describes/' + objName + '.json')) { 610 | describeResult = JSON.parse(fs.readFileSync('./describes/' + objName + '.json').toString()); 611 | } else { 612 | describeResult = await conn.describe(objName); 613 | } 614 | return describeResult; 615 | } 616 | 617 | private saveDescribeMap() { 618 | if (!fs.existsSync('./describes')) { 619 | fs.mkdirSync('./describes'); 620 | } 621 | for (const key in this.describeMap) { 622 | if (this.describeMap.hasOwnProperty(key)) { 623 | if (this.describeMap[key].layoutable) { 624 | fs.writeFileSync('./describes/' + key + '.json', JSON.stringify(this.describeMap[key], null, 4)); 625 | } 626 | } 627 | } 628 | } 629 | 630 | private async makeDescribeMap(objects, conn, flags) { 631 | const describeMap = {}; // Objectname describe result map 632 | for (const object of this.objects) { 633 | 634 | await this.getSobjectDescribe(object, conn).then(async describeResult => { 635 | if (describeResult.layoutable) { 636 | describeMap[object] = { 637 | fields: describeResult.fields, 638 | childRelationships: describeResult['childRelationships'], 639 | layoutable: describeResult.layoutable 640 | }; 641 | 642 | if (flags.spiderreferences) { 643 | await this.spiderReferences(describeMap[object], describeMap, conn, object); 644 | } 645 | } 646 | }).catch((reason: any) => { 647 | return reason; 648 | }); 649 | } 650 | return describeMap; 651 | } 652 | 653 | private async spiderReferences(describeResult: DescribeSObjectResult, describeMap, conn, object) { 654 | // tslint:disable-next-line:prefer-for-of 655 | for (let i = 0; i < describeResult.fields.length; i++) { 656 | const field: Field = describeResult.fields[i] as unknown as Field; 657 | if (field.referenceTo) { 658 | if (field.type === 'reference' && !field.referenceTo.includes('User')) { 659 | // tslint:disable-next-line:prefer-for-of 660 | for (let index = 0; index < field.referenceTo.length; index++) { 661 | const objectReference = field.referenceTo[index]; 662 | if (isUndefined(describeMap[objectReference])) { 663 | await this.getSobjectDescribe(objectReference, conn).then(describeSObjectResult => { 664 | if (describeSObjectResult.layoutable) { 665 | // this.objects.push(objectReference); 666 | describeMap[objectReference] = { 667 | fields: describeSObjectResult.fields, 668 | childRelationships: describeSObjectResult['childRelationships'], 669 | layoutable: describeSObjectResult.layoutable 670 | }; 671 | } 672 | }).catch((reason: any) => { 673 | return reason; 674 | }); 675 | } 676 | } 677 | } 678 | } 679 | } 680 | } 681 | 682 | private makeRelationshipMap() { 683 | const relationshipMap: RelationshipMap = {} as RelationshipMap; 684 | _.merge(relationshipMap, 685 | this.getObjectChildRelationships(), 686 | this.getObjectParentRelationships()); 687 | 688 | return relationshipMap; 689 | } 690 | } 691 | -------------------------------------------------------------------------------- /src/commands/tohoom/data/split.ts: -------------------------------------------------------------------------------- 1 | import { flags, SfdxCommand } from "@salesforce/command"; 2 | import { Messages } from "@salesforce/core"; 3 | import { join } from "path"; 4 | import DataApi from "../../../dataApi"; 5 | 6 | Messages.importMessagesDirectory(join(__dirname, '..', '..', '..')); 7 | // const messages = core.Messages.loadMessages('data', 'export'); 8 | 9 | export default class Split extends SfdxCommand { 10 | public static description = `Extract data from an org to use in a scratch org. Just supply a list of SObjects and you *should* end up with a dataset and data plan that can be used with the official force:data:tree:import command`; // messages.getMessage('commandDescription'); 11 | 12 | public static examples = [ 13 | `$ sfdx tohoom:data:export -o Account,Contact,Case,Opportunity -t data/exported -n my-testplan 14 | ` 15 | ]; 16 | 17 | protected static flagsConfig = { 18 | // flag with a value (-n, --name=VALUE) 19 | // name: flags.string({char: 'n', description: messages.getMessage('nameFlagDescription')}) 20 | planname: flags.string({ default: 'data-plan', description: 'name of the data plan to use with split', char: 'n'}) 21 | }; 22 | 23 | // Comment this out if your command does not require an org username 24 | protected static requiresUsername = false; 25 | 26 | // Comment this out if your command does not support a hub org username 27 | protected static supportsDevhubUsername = true; 28 | 29 | // Set this to true if your command requires a project workspace; 'requiresProject' is false by default 30 | protected static requiresProject = true; 31 | 32 | // tslint:disable-next-line:no-any 33 | public async run(): Promise { 34 | this.ux.startSpinner('Determining relationships for '); 35 | this.ux.stopSpinner(''); 36 | 37 | const dapi = new DataApi(); 38 | dapi.run(this.ux, this.flags.planname, this); 39 | this.ux.startSpinner('Running queries for objects...'); 40 | this.ux.stopSpinner('Saving data...'); 41 | 42 | this.ux.log('Finished exporting data and plan.'); 43 | 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/dataApi.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path'; 2 | import * as fs from 'fs'; 3 | import _ from 'lodash'; 4 | import { SfdxCommand, UX } from '@salesforce/command'; 5 | 6 | 7 | export default class DataApi { 8 | 9 | protected temparray: string[]; 10 | 11 | public async run(ux: UX, dataPlan: string, cmd: SfdxCommand) { 12 | this.splitFiles(ux, dataPlan, cmd); 13 | } 14 | 15 | private async validateFile(path: string) { 16 | return fs.existsSync(path); 17 | }; 18 | 19 | private async writeFile(name, contents) { 20 | var fs = require('fs'); 21 | fs.writeFile(name, contents, function(err) { 22 | if(err) { 23 | return console.log(err); 24 | } 25 | console.log("The file was saved!\n" + name); 26 | }); 27 | }; 28 | 29 | private async writeDataFile(datafolder, recordData) { 30 | this.writeFile(path.join(datafolder, recordData.fileName), JSON.stringify(recordData.data, null, 4)); 31 | this.temparray.push(recordData.fileName); 32 | }; 33 | 34 | private splitFiles(ux: UX, dataPlan: string, cmd: SfdxCommand) { 35 | const filepath = path.resolve(process.cwd(), dataPlan); 36 | const datafolder = path.dirname(filepath); 37 | if (!this.validateFile(filepath)) { 38 | cmd.error('Error splitting files.'); 39 | } 40 | let plan = require(filepath); 41 | const that = this; 42 | _.forEach(plan, function(p) { 43 | _.forEach(p.files, function(f) { 44 | that.breakupDataFile(datafolder, f); 45 | console.log(f); 46 | }); 47 | p.files = that.temparray; 48 | that.writeFile(filepath, JSON.stringify(plan, null, 4)); 49 | }); 50 | } 51 | 52 | protected breakupDataFile(datafolder: string, f: string) { 53 | let records = require(path.join(datafolder, f)).records; 54 | if (records.length <= 200) { 55 | this.temparray.push(f); 56 | return 57 | } 58 | let i: number, j: number; 59 | const chunk = 200; 60 | this.temparray = []; 61 | for (i=0,j=records.length; i; 29 | } 30 | 31 | export default class TohoomExtension { 32 | public static description = 'This command is specific to post processing the Tohoom dataset for handling the self referential Hoom_Team_Member object'; 33 | 34 | private planname: string; 35 | private targetdir: string; 36 | private cmd: SfCommand; 37 | 38 | public async run(planName: string, targetDir: string, cmd: SfCommand): Promise { 39 | 40 | this.planname = planName; 41 | this.targetdir = targetDir; 42 | this.cmd = cmd; 43 | 44 | ux.log(`Re-jiggering the ${this.planname} data plan found in the ${this.targetdir} direcotry...`); 45 | await this.addSeedFileToPlan(); 46 | await this.copyHoomTMtoSeedFile(); 47 | ux.log('Finished modifying plan for Tohoom data.'); 48 | } 49 | 50 | private async addSeedFileToPlan() { 51 | const pathToPlan = path.join(this.targetdir, this.planname + '.json'); 52 | // Read the file into a json structure 53 | if (!fs.existsSync(pathToPlan)) { 54 | this.cmd.error(`Could not find the data plan file ${pathToPlan}`); 55 | } else { 56 | const plandata: Buffer = fs.readFileSync(pathToPlan); 57 | const planarray: Array = JSON.parse(plandata.toString()); 58 | const ind = planarray.findIndex(element => element.sobject === 'Hoom_Team_Member__c'); 59 | // Found the right item to copy, no we need a copy of it before modifying it and sticking 60 | // back into the plan entries. 61 | const planEntry: PlanEntry = JSON.parse(JSON.stringify(planarray[ind])); 62 | const fileName = planEntry.files[0]; 63 | // Replace the file with the new name 64 | planEntry.files[0] = fileName.split('.')[0] + '_seed' + '.json'; 65 | // Insert the new plan entry ahead of the original, so this would be ind-1 66 | planarray.splice(ind, 0, planEntry); 67 | // Now, write the file back to where it came from 68 | fs.writeFileSync(pathToPlan, JSON.stringify(planarray, null, 4)); 69 | console.log(JSON.stringify(planarray[0], null, 4)); 70 | } 71 | } 72 | 73 | private async copyHoomTMtoSeedFile() { 74 | const pathToData = path.join(this.targetdir, 'Hoom_Team_Member__c.json'); 75 | const pathToSeedData = path.join(this.targetdir, 'Hoom_Team_Member__c_seed.json'); 76 | const HTMData = fs.readFileSync(pathToData); 77 | const HTMJson = JSON.parse(HTMData.toString()); 78 | const records: Array = HTMJson.records; 79 | // Find the first one, this needs to be add to the org before the circular references take place 80 | const CRO: TeamMember = JSON.parse(JSON.stringify(records.find(record => record.Sales_Heirarchy__c === 'CRO'))); 81 | const FLSM: TeamMember = JSON.parse(JSON.stringify(records.find(record => record.Sales_Heirarchy__c === 'FLSM'))); 82 | delete FLSM.Reports_To__c; 83 | 84 | // Ok, now write theese to reecords to the seed file 85 | const dataFile: TeamMemberDataFile = {} as TeamMemberDataFile; 86 | dataFile.done = true; 87 | dataFile.totalSize = 2; 88 | dataFile.records = [ CRO, FLSM ]; 89 | fs.writeFileSync(pathToSeedData, JSON.stringify(dataFile, null, 4)); 90 | } 91 | } 92 | 93 | -------------------------------------------------------------------------------- /test/commands/djc/data/data.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from '@salesforce/command/dist/test'; 2 | 3 | describe('djc:data:export', () => { 4 | test 5 | .withOrg({ username: 'test@org.com' }, true) 6 | .withConnectionRequest(request => { 7 | if (request.url.match(/Organization/)) { 8 | return Promise.resolve({ records: [ { Name: 'Super Awesome Org', TrialExpirationDate: '2018-03-20T23:24:11.000+0000'}] }); 9 | } 10 | return Promise.resolve({ records: [] }); 11 | }) 12 | .stdout() 13 | .command(['djc:data:export', '--targetusername', 'test@org.com']) 14 | .it('runs djc:data:export --targetusername test@org.com', ctx => { 15 | expect(ctx.stdout).to.contain('Hello world! This is org: Super Awesome Org and I will be around until Tue Mar 20 2018!'); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /test/helpers/init.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | process.env.TS_NODE_PROJECT = path.resolve('test/tsconfig.json') 3 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --require test/helpers/init.js 2 | --require ts-node/register 3 | --require source-map-support/register 4 | --watch-extensions ts 5 | --recursive 6 | --reporter spec 7 | --timeout 5000 8 | -------------------------------------------------------------------------------- /test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig" 3 | } 4 | 5 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/@salesforce/dev-config/tsconfig", 3 | "compilerOptions": { 4 | "declaration": true, 5 | "outDir": "./lib", 6 | "importHelpers": true, 7 | "types": ["jest"] 8 | }, 9 | "include": [ 10 | "./src/**/*" 11 | ] 12 | } 13 | --------------------------------------------------------------------------------