├── _config.yml ├── src ├── images │ ├── logo.png │ ├── splash.png │ ├── diagram.jpg │ ├── favicon.png │ └── logo-black.png ├── index.ts ├── addons │ ├── components │ │ ├── sfdmu-run │ │ │ ├── custom │ │ │ │ ├── index.ts │ │ │ │ └── SfdmuRunCustomAddonApiService.ts │ │ │ ├── ISfdmuRunAddonResult.ts │ │ │ ├── ISfdmuRunScriptObject.ts │ │ │ ├── ISfdmuRunScript.ts │ │ │ ├── sfdmuRunAddonModule.ts │ │ │ ├── SfdmuRunAddonTaskData.ts │ │ │ ├── sfdmuRunAddonJob.ts │ │ │ └── sfdmuRunAddonTask.ts │ │ └── common │ │ │ ├── IAddonContext.ts │ │ │ ├── addonResult.ts │ │ │ ├── addonModule.ts │ │ │ └── addonRuntime.ts │ ├── addonsCore.json │ ├── modules │ │ └── sfdmu-run │ │ │ ├── RecordsFilter │ │ │ ├── IRecordsFilter.ts │ │ │ ├── index.ts │ │ │ └── BadWordsFilter.ts │ │ │ ├── OnAfter │ │ │ └── index.ts │ │ │ ├── OnBefore │ │ │ └── index.ts │ │ │ └── custom-addons │ │ │ ├── package │ │ │ ├── ISfdmuRunCustomAddonScriptMappingItem.ts │ │ │ ├── ISfdmuRunCustomAddonScriptOrg.ts │ │ │ ├── ISfdmuRunCustomAddonScriptMockField.ts │ │ │ ├── ISfdmuRunCustomAddonScriptAddonManifestDefinition.ts │ │ │ ├── ISfdmuRunCustomAddonSFieldDescribe.ts │ │ │ ├── ISfdmuRunCustomAddonCommandRunInfo.ts │ │ │ ├── ISfdmuRunCustomAddonResult.ts │ │ │ ├── ISfdmuRunCustomAddonRuntime.ts │ │ │ ├── ISfdmuRunCustomAddonPluginInfo.ts │ │ │ ├── index.ts │ │ │ ├── ISfdmuRunCustomAddonProcessedData.ts │ │ │ ├── ISFdmuRunCustomAddonJob.ts │ │ │ ├── ISfdmuRunCustomAddonTaskData.ts │ │ │ ├── ISfdmuRunCustomAddonScript.ts │ │ │ ├── common.ts │ │ │ ├── ISfdmuRunCustomAddonContext.ts │ │ │ ├── ISfdmuRunCustomAddonScriptObject.ts │ │ │ ├── ISfdmuRunCustomAddonModule.ts │ │ │ ├── ISfdmuRunCustomAddonApiService.ts │ │ │ └── ISFdmuRunCustomAddonTask.ts │ │ │ └── CustomSfdmuRunAddonTemplate │ │ │ └── index.ts │ └── messages │ │ └── sfdmuRunAddonMessages.ts ├── modules │ ├── components │ │ ├── common_components │ │ │ ├── bulitinMessages.ts │ │ │ ├── enumerations.ts │ │ │ └── mockGenerator.ts │ │ └── api_engines │ │ │ ├── bulkApiV1_0Engine.ts │ │ │ └── restApiEngine.ts │ ├── models │ │ ├── common_models │ │ │ ├── ISfdmuStatics.ts │ │ │ ├── ISfdxCommand.ts │ │ │ ├── IPluginInfo.ts │ │ │ ├── orgInfo.ts │ │ │ ├── ICommandRunInfo.ts │ │ │ ├── helper_interfaces.ts │ │ │ └── errors.ts │ │ ├── api_models │ │ │ ├── index.ts │ │ │ ├── ApiResultRecord.ts │ │ │ ├── ApiInfo.ts │ │ │ └── helper_interfaces.ts │ │ ├── script_models │ │ │ ├── scriptObjectSet.ts │ │ │ ├── scriptMappingItem.ts │ │ │ ├── scriptMockField.ts │ │ │ ├── scriptAddonManifestDefinition.ts │ │ │ └── scriptOrg.ts │ │ ├── sf_models │ │ │ ├── sobjectDescribe.ts │ │ │ └── contentVersion.ts │ │ └── index.ts │ ├── app │ │ ├── appSfdmuRunModule.ts │ │ ├── IAppSfdmuRunModuleArgs.ts │ │ ├── appMessagesBase.ts │ │ ├── appEmbeddedMessages.ts │ │ ├── appJsonMessages.ts │ │ ├── appConsoleUxLogger.ts │ │ ├── appModels.ts │ │ └── appSfdmuRunApp.ts │ └── commands_processors │ │ ├── IRunProcess.ts │ │ └── runCommandExecutor.ts ├── sfdmu_run.ts ├── sfdmu_run_module_demo.ts └── commands │ └── sfdmu │ └── run.ts ├── bin └── run ├── CODEOWNERS ├── .editorconfig ├── sfdmu-run.cmd ├── debug.cmd ├── sfdmu-run-debug.cmd ├── .npmignore ├── .gitignore ├── SECURITY.md ├── tsconfig.json ├── .github ├── workflows │ ├── onPushToMain.yml │ ├── onRelease.yml │ ├── codeql-analysis.yml │ ├── manualRelease.yml │ ├── failureNotifications.yml │ ├── stale.yml │ └── issue-response-handler.yml ├── dependabot.yml └── ISSUE_TEMPLATE │ └── critical-runtime--errors-report.md ├── appveyor.yml ├── tslint.json ├── package.json ├── CODE_OF_CONDUCT.md ├── messages └── run.json ├── CONTRIBUTING.md └── README.md /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-dinky -------------------------------------------------------------------------------- /src/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/SFDX-Data-Move-Utility/HEAD/src/images/logo.png -------------------------------------------------------------------------------- /src/images/splash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/SFDX-Data-Move-Utility/HEAD/src/images/splash.png -------------------------------------------------------------------------------- /bin/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | require('@oclif/command').run() 4 | .catch(require('@oclif/errors/handle')) 5 | -------------------------------------------------------------------------------- /src/images/diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/SFDX-Data-Move-Utility/HEAD/src/images/diagram.jpg -------------------------------------------------------------------------------- /src/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/SFDX-Data-Move-Utility/HEAD/src/images/favicon.png -------------------------------------------------------------------------------- /src/images/logo-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/SFDX-Data-Move-Utility/HEAD/src/images/logo-black.png -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Comment line immediately above ownership line is reserved for related other information. Please be careful while editing. 2 | #ECCN:Open Source 3 | #GUSINFO:Open Source,Open Source Workflow 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.md] 11 | trim_trailing_whitespace = false 12 | -------------------------------------------------------------------------------- /sfdmu-run.cmd: -------------------------------------------------------------------------------- 1 | echo off 2 | 3 | goto(){ 4 | # Linux code here 5 | node ./lib/sfdmu_run.js $* 6 | } 7 | 8 | goto $@ 9 | exit 10 | 11 | :(){ 12 | rem Windows script here 13 | node ./lib/sfdmu_run.js %* 14 | exit 15 | 16 | @echo off 17 | 18 | -------------------------------------------------------------------------------- /debug.cmd: -------------------------------------------------------------------------------- 1 | echo off 2 | 3 | goto(){ 4 | # Linux code here 5 | node --inspect "bin/run" sfdmu:run $* 6 | } 7 | 8 | goto $@ 9 | exit 10 | 11 | :(){ 12 | rem Windows script here 13 | node --inspect "%~dp0\bin\run" sfdmu:run %* 14 | exit 15 | 16 | @echo off 17 | 18 | -------------------------------------------------------------------------------- /sfdmu-run-debug.cmd: -------------------------------------------------------------------------------- 1 | echo off 2 | 3 | goto(){ 4 | # Linux code here 5 | node --inspect ./lib/sfdmu_run.js $* 6 | } 7 | 8 | goto $@ 9 | exit 10 | 11 | :(){ 12 | rem Windows script here 13 | node --inspect ./lib/sfdmu_run.js %* 14 | exit 15 | 16 | @echo off 17 | 18 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | export default {}; 9 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | *-debug.log 2 | *-error.log 3 | /.nyc_output 4 | /dist 5 | /dist2 6 | /lib 7 | /package-lock.json 8 | /tmp 9 | node_modules 10 | /export*.json 11 | ._* 12 | .DS_Store 13 | /logs 14 | /source 15 | /target 16 | /development 17 | /test-addons 18 | /documentation 19 | /src/addons/modules/sfdmu-run/custom-addons/TestModule* 20 | /temp 21 | *-private* 22 | -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/custom/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | export { default as SfdmuRunCustomAddonService } from "./SfdmuRunCustomAddonApiService"; 9 | -------------------------------------------------------------------------------- /src/modules/components/common_components/bulitinMessages.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | export enum BUILTIN_MESSAGES { 11 | 12 | Break = '\n' 13 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *-debug.log 2 | *-error.log 3 | /.nyc_output 4 | /dist 5 | /dist2 6 | /lib 7 | /package-lock.json 8 | /tmp 9 | node_modules 10 | /export*.json 11 | ._* 12 | .DS_Store 13 | /logs 14 | /source 15 | /target 16 | /development 17 | /documentation 18 | /src/addons/modules/sfdmu-run/custom-addons/TestModule* 19 | typedoc-sfdmu-run-addons.json 20 | /temp 21 | *-private* 22 | /.vscode 23 | /.images 24 | 25 | 26 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Security 2 | 3 | Please report any security issue to [security@salesforce.com](mailto:security@salesforce.com) 4 | as soon as it is discovered. This library limits its runtime dependencies in 5 | order to reduce the total cost of ownership as much as can be, but all consumers 6 | should remain vigilant and have their security stakeholders review all third-party 7 | products (3PP) like this one and their dependencies. -------------------------------------------------------------------------------- /src/modules/models/common_models/ISfdmuStatics.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | export default interface ISfdmuStatics { 10 | plugin: { 11 | root: string, 12 | name: string 13 | }, 14 | name: string 15 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/@salesforce/dev-config/tsconfig", 3 | "compilerOptions": { 4 | "outDir": "./lib", 5 | "rootDir": "./src", 6 | "experimentalDecorators": true, 7 | "emitDecoratorMetadata": true, 8 | "removeComments": true, 9 | "resolveJsonModule": true, 10 | "skipLibCheck": true, 11 | "types": ["node"] 12 | }, 13 | "include": [ 14 | "./src/**/*", 15 | "src/addons/addonsCore.json" 16 | ] 17 | } -------------------------------------------------------------------------------- /src/modules/models/common_models/ISfdxCommand.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import ISfdmuStatics from './ISfdmuStatics'; 9 | 10 | export default interface ISfdmuCommand { 11 | 12 | statics: ISfdmuStatics; 13 | argv: Array; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/ISfdmuRunAddonResult.ts: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (c) 2020, salesforce.com, inc. 4 | * All rights reserved. 5 | * SPDX-License-Identifier: BSD-3-Clause 6 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 7 | */ 8 | 9 | import { ISfdmuRunCustomAddonResult } from "../../modules/sfdmu-run/custom-addons/package"; 10 | 11 | 12 | export default interface ISfdmuRunAddonResult extends ISfdmuRunCustomAddonResult { 13 | 14 | } -------------------------------------------------------------------------------- /src/modules/models/api_models/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | export * from "./helper_interfaces"; 10 | export { default as ApiEngineBase } from './ApiEngineBase'; 11 | export { default as ApiInfo } from './ApiInfo'; 12 | export { default as ApiResultRecord } from './ApiResultRecord'; 13 | -------------------------------------------------------------------------------- /src/addons/components/common/IAddonContext.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import ISfdmuRunCustomAddonContext from "../../modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonContext"; 9 | 10 | 11 | export default interface IAddonContext extends ISfdmuRunCustomAddonContext { 12 | isCore: boolean; 13 | } 14 | -------------------------------------------------------------------------------- /src/addons/addonsCore.json: -------------------------------------------------------------------------------- 1 | { 2 | "addons": [ 3 | { 4 | "command": "sfdmu:run", 5 | "path": null, 6 | "module": "core:OnBefore", 7 | "event": "onBefore", 8 | "excluded": true, 9 | "args": null 10 | }, 11 | { 12 | "command": "sfdmu:run", 13 | "path": null, 14 | "module": "core:OnAfter", 15 | "event": "onAfter", 16 | "excluded": true, 17 | "args": null 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /.github/workflows/onPushToMain.yml: -------------------------------------------------------------------------------- 1 | # test 2 | name: version, tag and github release 3 | 4 | on: 5 | push: 6 | branches: [master] 7 | 8 | jobs: 9 | release: 10 | uses: salesforcecli/github-workflows/.github/workflows/githubRelease.yml@main 11 | secrets: inherit 12 | 13 | # most repos won't use this 14 | # depends on previous job to avoid git collisions, not for any functionality reason 15 | # docs: 16 | # uses: salesforcecli/github-workflows/.github/workflows/publishTypedoc.yml@main 17 | # secrets: inherit 18 | # needs: release 19 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | nodejs_version: "10" 3 | cache: 4 | - '%LOCALAPPDATA%\Yarn -> appveyor.yml' 5 | - node_modules -> yarn.lock 6 | 7 | install: 8 | - ps: Install-Product node $env:nodejs_version x64 9 | - yarn 10 | test_script: 11 | - yarn test 12 | 13 | after_test: 14 | - .\node_modules\.bin\nyc report --reporter text-lcov > coverage.lcov 15 | - ps: | 16 | $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH 17 | Invoke-WebRequest -Uri 'https://codecov.io/bash' -OutFile codecov.sh 18 | bash codecov.sh 19 | 20 | 21 | build: off 22 | 23 | -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/ISfdmuRunScriptObject.ts: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (c) 2020, salesforce.com, inc. 4 | * All rights reserved. 5 | * SPDX-License-Identifier: BSD-3-Clause 6 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 7 | */ 8 | 9 | 10 | import { ISfdmuRunCustomAddonScriptObject } from "../../modules/sfdmu-run/custom-addons/package"; 11 | 12 | 13 | export default interface ISfdmuRunScriptObject extends ISfdmuRunCustomAddonScriptObject { 14 | 15 | extraFieldsToUpdate?: Array; 16 | 17 | 18 | } -------------------------------------------------------------------------------- /.github/workflows/onRelease.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | on: 4 | release: 5 | types: [released] 6 | # support manual release in case something goes wrong and needs to be repeated or tested 7 | workflow_dispatch: 8 | inputs: 9 | tag: 10 | description: tag that needs to publish 11 | type: string 12 | required: true 13 | jobs: 14 | npm: 15 | uses: salesforcecli/github-workflows/.github/workflows/npmPublish.yml@main 16 | with: 17 | sign: true 18 | tag: latest 19 | githubTag: ${{ github.event.release.tag_name || inputs.tag }} 20 | secrets: inherit 21 | -------------------------------------------------------------------------------- /src/sfdmu_run.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import IAppSfdmuRunModuleArgs from "./modules/app/IAppSfdmuRunModuleArgs"; 10 | import AppSfdmuRunApp from "./modules/app/appSfdmuRunApp"; 11 | 12 | const args: IAppSfdmuRunModuleArgs = { 13 | argv: process.argv 14 | }; 15 | 16 | const application = new AppSfdmuRunApp(args); 17 | (async () => await application.runCommand())(); 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/modules/models/common_models/IPluginInfo.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { 9 | ISfdmuRunCustomAddonPluginInfo, 10 | } from '../../../addons/modules/sfdmu-run/custom-addons/package'; 11 | 12 | type IPlugInfoBase = ISfdmuRunCustomAddonPluginInfo; 13 | 14 | /** 15 | * The common information about the SFDMU Plugin 16 | */ 17 | export default interface IPluginInfo extends IPlugInfoBase { 18 | 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/modules/models/script_models/scriptObjectSet.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { Type } from 'class-transformer'; 9 | 10 | import ScriptObject from './scriptObject'; 11 | 12 | export default class ScriptObjectSet { 13 | 14 | constructor(objects?: ScriptObject[]) { 15 | this.objects = objects; 16 | } 17 | 18 | @Type(() => ScriptObject) 19 | objects: ScriptObject[] = new Array(); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/addons/components/common/addonResult.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISfdmuRunCustomAddonResult } from "../../modules/sfdmu-run/custom-addons/package"; 9 | 10 | 11 | export default class AddonResult implements ISfdmuRunCustomAddonResult { 12 | 13 | constructor(init: Partial) { 14 | if (init) { 15 | Object.assign(this, init); 16 | } 17 | } 18 | 19 | cancel: boolean; 20 | } -------------------------------------------------------------------------------- /src/modules/app/appSfdmuRunModule.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import IAppSfdmuRunModuleArgs from "./IAppSfdmuRunModuleArgs"; 9 | import AppSfdmuRunApp from "./appSfdmuRunApp"; 10 | 11 | 12 | export class AppSfdmuRunModule { 13 | 14 | public static async runCommand(args: IAppSfdmuRunModuleArgs) { 15 | args.runProcess = new AppSfdmuRunApp(args); 16 | await args.runProcess.runCommand(); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "07:00" 8 | allow: 9 | - dependency-type: "direct" # Optional: Focus on direct dependencies 10 | open-pull-requests-limit: 10 11 | versioning-strategy: increase 12 | ignore: 13 | - dependency-name: "csv-parse" 14 | - dependency-name: "globby" 15 | - dependency-name: "madge" 16 | - dependency-name: "mocha" 17 | - dependency-name: "nyc" 18 | - dependency-name: "@oclif/*" 19 | - dependency-name: "soql-parser-js" 20 | - dependency-name: "@babel/*" 21 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/RecordsFilter/IRecordsFilter.ts: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (c) 2020, salesforce.com, inc. 4 | * All rights reserved. 5 | * SPDX-License-Identifier: BSD-3-Clause 6 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 7 | */ 8 | 9 | 10 | export interface IRecordsFilterArgs { 11 | filterType: string; 12 | settings: IRecordsFilterSetting; 13 | } 14 | 15 | export interface IRecordsFilterSetting { 16 | // TODO: Can add extra props for all filters 17 | } 18 | 19 | 20 | export interface IRecordsFilter { 21 | isInitialized: boolean; 22 | filterRecords(records: any[]): Promise; 23 | } 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/modules/app/IAppSfdmuRunModuleArgs.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { IRunProcess } from "../commands_processors/IRunProcess"; 9 | import { IResourceBundle, IUxLogger } from "../components/common_components/logger"; 10 | 11 | export default interface IAppSfdmuRunModuleArgs { 12 | argv: Array; 13 | runProcess?: IRunProcess; 14 | logger?: IUxLogger; 15 | exportJson?: string; 16 | commandMessages?: IResourceBundle; 17 | resources?: IResourceBundle; 18 | exitProcess?: boolean; 19 | } 20 | -------------------------------------------------------------------------------- /src/modules/models/script_models/scriptMappingItem.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { 9 | ISfdmuRunCustomAddonScriptMappingItem, 10 | } from '../../../addons/modules/sfdmu-run/custom-addons/package'; 11 | 12 | /** 13 | * Parsed FieldMapping object of the script. 14 | * Represents field mapping 15 | * 16 | * @export 17 | * @class ScriptMapping 18 | */ 19 | export default class ScriptMappingItem implements ISfdmuRunCustomAddonScriptMappingItem { 20 | targetObject: string; 21 | sourceField: string; 22 | targetField: string; 23 | } 24 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/OnAfter/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import AddonResult from "../../../components/common/addonResult"; 10 | import IAddonContext from "../../../components/common/IAddonContext"; 11 | import SfdmuRunAddonModule from "../../../components/sfdmu-run/sfdmuRunAddonModule"; 12 | 13 | export default class CoreOnAfter extends SfdmuRunAddonModule { 14 | 15 | async onExecute(context: IAddonContext, args : any) : Promise { 16 | // TODO: Implement the core OnAfter functionality here 17 | return null; 18 | } 19 | 20 | } -------------------------------------------------------------------------------- /src/modules/models/common_models/orgInfo.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | /** 11 | * Force:org:display command response 12 | * 13 | * @export 14 | * @class OrgInfo 15 | */ 16 | export default class OrgInfo { 17 | AccessToken: string; 18 | ClientId: string; 19 | ConnectedStatus: string; 20 | Status: string; 21 | OrgId: string; 22 | UserId: string; 23 | InstanceUrl: string; 24 | Username: string; 25 | 26 | get isConnected() { 27 | return this.ConnectedStatus == "Connected" || this.Status == "Active"; 28 | } 29 | } 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/OnBefore/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | import AddonResult from "../../../components/common/addonResult"; 11 | import IAddonContext from "../../../components/common/IAddonContext"; 12 | import SfdmuRunAddonModule from "../../../components/sfdmu-run/sfdmuRunAddonModule"; 13 | 14 | 15 | export default class CoreOnBefore extends SfdmuRunAddonModule { 16 | 17 | async onExecute(context: IAddonContext, args : any) : Promise { 18 | // TODO: Implement the core OnBefore functionality here 19 | 20 | return null; 21 | } 22 | 23 | } -------------------------------------------------------------------------------- /src/modules/models/common_models/ICommandRunInfo.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { 9 | ISfdmuRunCustomAddonCommandRunInfo, 10 | } from '../../../addons/modules/sfdmu-run/custom-addons/package'; 11 | import IPluginInfo from './IPluginInfo'; 12 | 13 | type ICommandRunInfoBase = ISfdmuRunCustomAddonCommandRunInfo; 14 | 15 | 16 | /** 17 | * The information about currently running SFDMU command 18 | */ 19 | export default interface ICommandRunInfo extends ICommandRunInfoBase { 20 | 21 | /** 22 | * the information about the Plugin and the framework 23 | */ 24 | readonly pinfo: IPluginInfo 25 | } 26 | -------------------------------------------------------------------------------- /src/addons/components/common/addonModule.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | import AddonResult from "./addonResult"; 11 | import AddonRuntime from "./addonRuntime"; 12 | import IAddonContext from "./IAddonContext"; 13 | 14 | 15 | export default abstract class AddonModule { 16 | 17 | constructor(runtime: AddonRuntime) { 18 | this.runtime = runtime; 19 | } 20 | 21 | context: IAddonContext; 22 | runtime: AddonRuntime; 23 | 24 | abstract onExecute(context: IAddonContext, args: any): Promise; 25 | abstract onInit(context: IAddonContext, args: any): Promise; 26 | 27 | } -------------------------------------------------------------------------------- /src/modules/app/appMessagesBase.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { IResourceBundle } from "../components/common_components/logger"; 9 | 10 | export default abstract class AppMessagesBase implements IResourceBundle { 11 | 12 | protected messages: Map = new Map(); 13 | 14 | getMessage(key: string, tokens?: any): string { 15 | let message = this.messages.get(key) || ''; 16 | let counter = 0; 17 | tokens = tokens || []; 18 | return message.replace(/(%s)/gi, () => { 19 | return tokens[counter++] || ''; 20 | }); 21 | } 22 | 23 | } -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/ISfdmuRunScript.ts: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (c) 2020, salesforce.com, inc. 4 | * All rights reserved. 5 | * SPDX-License-Identifier: BSD-3-Clause 6 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 7 | */ 8 | 9 | import ScriptAddonManifestDefinition from "../../../modules/models/script_models/scriptAddonManifestDefinition"; 10 | import { ISfdmuRunCustomAddonScript } from "../../modules/sfdmu-run/custom-addons/package"; 11 | import ISfdmuRunScriptObject from "./ISfdmuRunScriptObject"; 12 | 13 | 14 | 15 | export default interface ISfdmuRunScript extends ISfdmuRunCustomAddonScript { 16 | 17 | objects: ISfdmuRunScriptObject[]; 18 | 19 | getAllAddOns(): ScriptAddonManifestDefinition[]; 20 | addScriptObject(object: ISfdmuRunScriptObject): ISfdmuRunScriptObject; 21 | 22 | } -------------------------------------------------------------------------------- /src/modules/models/script_models/scriptMockField.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { 9 | ISfdmuRunCustomAddonScriptMockField, 10 | } from '../../../addons/modules/sfdmu-run/custom-addons/package'; 11 | 12 | /** 13 | * Parsed MockField object 14 | * from the script file 15 | * 16 | * @export 17 | * @class ScriptMockField 18 | */ 19 | export default class ScriptMockField implements ISfdmuRunCustomAddonScriptMockField { 20 | // ------------- JSON -------------- 21 | name: string = ""; 22 | pattern: string = ""; 23 | excludedRegex: string = ""; 24 | includedRegex: string = ""; 25 | excludeNames: string[] = []; 26 | } 27 | -------------------------------------------------------------------------------- /src/modules/commands_processors/IRunProcess.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { 9 | IResourceBundle, 10 | IUxLogger, 11 | } from '../components/common_components/logger'; 12 | import ISfdmuCommand from '../models/common_models/ISfdxCommand'; 13 | import { RunCommand } from './runCommand'; 14 | 15 | export interface IRunProcess { 16 | argv: Array; 17 | command: RunCommand; 18 | cmd: ISfdmuCommand; 19 | m_ux: IUxLogger; 20 | m_flags: any; 21 | exportJson: string; 22 | commandMessages: IResourceBundle; 23 | resources: IResourceBundle; 24 | exitProcess: boolean; 25 | runCommand(): Promise; 26 | } 27 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScriptMappingItem.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | /** 11 | * The field mapping item provided with the parent {@link ISfdmuRunCustomAddonScriptObject}. 12 | * @see {@link https://help.sfdmu.com/full-documentation/configuration-and-running/full-exportjson-format | Full export.json format} for the detailed information about the fields. 13 | * 14 | * @export 15 | * @interface ISfdmuRunCustomAddonScriptMappingItem 16 | */ 17 | export default interface ISfdmuRunCustomAddonScriptMappingItem { 18 | 19 | targetObject: string; 20 | sourceField: string; 21 | targetField: string; 22 | 23 | } -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScriptOrg.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | /** 11 | * The authentication data provided with the currently running {@link ISfdmuRunCustomAddonScript}. 12 | * @see {@link https://help.sfdmu.com/full-documentation/configuration-and-running/full-exportjson-format | Full export.json format} for the detailed information about the fields. 13 | * 14 | * @export 15 | * @interface ISfdmuRunCustomAddonScriptOrg 16 | */ 17 | export default interface ISfdmuRunCustomAddonScriptOrg { 18 | 19 | name: string; 20 | orgUserName: string; 21 | instanceUrl: string; 22 | accessToken: string; 23 | 24 | } -------------------------------------------------------------------------------- /src/modules/models/api_models/ApiResultRecord.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | export default class ApiResultRecord { 10 | 11 | constructor(init: Partial) { 12 | Object.assign(this, init); 13 | } 14 | 15 | id: string; 16 | sourceRecord: object; 17 | targetRecord: object; 18 | 19 | isFailed: boolean; 20 | isUnprocessed: boolean; 21 | isMissingSourceTargetMapping: boolean; 22 | 23 | get isSuccess(): boolean { 24 | return !this.isFailed 25 | && !this.isUnprocessed 26 | && !this.isMissingSourceTargetMapping; 27 | } 28 | 29 | isCreated: boolean; 30 | errorMessage: string; 31 | } -------------------------------------------------------------------------------- /src/modules/models/sf_models/sobjectDescribe.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { SFieldDescribe } from '../'; 9 | 10 | /** 11 | * Description of the sobject 12 | * 13 | * @export 14 | * @class SObjectDescribe 15 | */ 16 | export default class SObjectDescribe { 17 | 18 | constructor(init?: Partial) { 19 | if (init) { 20 | Object.assign(this, init); 21 | } 22 | } 23 | 24 | name: string = ""; 25 | label: string = ""; 26 | updateable: boolean = false; 27 | createable: boolean = false; 28 | custom: boolean = false; 29 | fieldsMap: Map = new Map(); 30 | } 31 | -------------------------------------------------------------------------------- /src/sfdmu_run_module_demo.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import IAppSfdmuRunModuleArgs from "./modules/app/IAppSfdmuRunModuleArgs"; 10 | import AppSfdmuRunApp from "./modules/app/appSfdmuRunApp"; 11 | 12 | const args: IAppSfdmuRunModuleArgs = { 13 | argv: [ 14 | // The 2 first members of the array should always be empty 15 | "", 16 | "", 17 | 18 | // list of the CLI flags 19 | "--path", 20 | "C:\\PathToExportJson", 21 | "--sourceusername", 22 | "source@mymail.com", 23 | "--targetusername", 24 | "target@mymail.com", 25 | "--verbose" 26 | ] 27 | }; 28 | 29 | const app = new AppSfdmuRunApp(args); 30 | (async () => await app.runCommand())(); 31 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScriptMockField.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | /** 11 | * The mock item provided with the parent {@link ISfdmuRunCustomAddonScriptObject}. 12 | * @see {@link https://help.sfdmu.com/full-documentation/configuration-and-running/full-exportjson-format | Full export.json format} for the detailed information about the fields. 13 | * 14 | * @export 15 | * @interface ISfdmuRunCustomAddonScriptMockField 16 | */ 17 | export default interface ISfdmuRunCustomAddonScriptMockField { 18 | 19 | name: string; 20 | excludeNames: string[]; 21 | pattern: string; 22 | excludedRegex: string; 23 | includedRegex: string; 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/modules/app/appEmbeddedMessages.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { IResourceBundle } from "../components/common_components/logger"; 9 | import AppMessagesBase from "./appMessagesBase"; 10 | 11 | export default class AppEmbeddedMessages extends AppMessagesBase implements IResourceBundle { 12 | 13 | constructor() { 14 | super(); 15 | } 16 | 17 | setup(resourceInstance: RType): AppEmbeddedMessages { 18 | resourceInstance = (resourceInstance as any) || {}; 19 | this.messages.clear(); 20 | Object.keys(resourceInstance).forEach(key => { 21 | this.messages.set(key, resourceInstance[key] || ''); 22 | }); 23 | return this; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScriptAddonManifestDefinition.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | /** 11 | * The manifest of the Add-On module defined for the whole {@link ISfdmuRunCustomAddonScript} 12 | * or for the particular {@link ISfdmuRunCustomAddonScriptObject}. 13 | * @see {@link https://help.sfdmu.com/full-documentation/configuration-and-running/full-exportjson-format | Full export.json format} for the detailed information. 14 | * 15 | * @export 16 | * @interface ISfdmuRunCustomAddonScriptAddonManifestDefinition 17 | */ 18 | export default interface ISfdmuRunCustomAddonScriptAddonManifestDefinition { 19 | path: string; 20 | module: string; 21 | description: string; 22 | excluded: boolean; 23 | args: any; 24 | } -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | permissions: 4 | contents: read 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | paths: 10 | - 'src/**/*' # Trigger only when files in src folder are changed 11 | pull_request: 12 | branches: [ master ] 13 | paths: 14 | - 'src/**/*' # Trigger only when files in src folder are changed 15 | schedule: 16 | - cron: '0 14 * * 0' # Schedule a weekly run 17 | workflow_dispatch: # Allows the workflow to be run manually 18 | 19 | 20 | jobs: 21 | analyze: 22 | name: Analyze 23 | runs-on: ubuntu-latest 24 | permissions: 25 | actions: read 26 | contents: read 27 | security-events: write 28 | 29 | steps: 30 | - name: Checkout repository 31 | uses: actions/checkout@v2 32 | 33 | - name: Initialize CodeQL 34 | uses: github/codeql-action/init@v3 35 | with: 36 | languages: typescript 37 | 38 | - name: Perform CodeQL Analysis 39 | uses: github/codeql-action/analyze@v3 40 | -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/sfdmuRunAddonModule.ts: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (c) 2020, salesforce.com, inc. 4 | * All rights reserved. 5 | * SPDX-License-Identifier: BSD-3-Clause 6 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 7 | */ 8 | 9 | 10 | import ISfdmuRunCustomAddonModule from "../../modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonModule"; 11 | import AddonModule from "../common/addonModule"; 12 | import AddonResult from "../common/addonResult"; 13 | import IAddonContext from "../common/IAddonContext"; 14 | import SfdmuRunAddonRuntime from "./sfdmuRunAddonRuntime"; 15 | 16 | 17 | export default abstract class SfdmuRunAddonModule extends AddonModule implements ISfdmuRunCustomAddonModule { 18 | 19 | runtime: SfdmuRunAddonRuntime; 20 | 21 | abstract onExecute(context: IAddonContext, args: any): Promise; 22 | 23 | onInit(context: IAddonContext, args: any): Promise { 24 | return null; 25 | } 26 | 27 | } -------------------------------------------------------------------------------- /src/modules/app/appJsonMessages.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { IResourceBundle } from "../components/common_components/logger"; 9 | import * as path from 'path'; 10 | import AppMessagesBase from "./appMessagesBase"; 11 | 12 | export default class AppJsonMessages extends AppMessagesBase implements IResourceBundle { 13 | 14 | jsonPath: string; 15 | 16 | constructor(rootPath: string, bundleName: string) { 17 | super(); 18 | this.jsonPath = path.join(rootPath, "messages", bundleName + ".json"); 19 | try { 20 | let json = require(this.jsonPath); 21 | this.messages = Object.keys(json).reduce((acc: Map, key: string) => { 22 | acc.set(key, String(json[key])); 23 | return acc; 24 | }, new Map()); 25 | } catch (e: any) { } 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonSFieldDescribe.ts: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (c) 2020, salesforce.com, inc. 4 | * All rights reserved. 5 | * SPDX-License-Identifier: BSD-3-Clause 6 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 7 | */ 8 | 9 | import { FieldType } from "."; 10 | 11 | 12 | /** 13 | * The description of the SF object field. 14 | */ 15 | export default interface ISfdmuRunCustomAddonSFieldDescribe { 16 | 17 | 18 | objectName: string; 19 | name: string; 20 | type: FieldType | "dynamic"; 21 | label: string; 22 | updateable: boolean; 23 | creatable: boolean; 24 | cascadeDelete: boolean; 25 | autoNumber: boolean; 26 | unique: boolean; 27 | nameField: boolean; 28 | custom: boolean; 29 | calculated: boolean; 30 | 31 | lookup: boolean; 32 | referencedObjectType: string; 33 | polymorphicReferenceObjectType: string; 34 | 35 | length: number; 36 | 37 | 38 | isPolymorphicField: boolean; 39 | readonly: boolean; 40 | isMasterDetail: boolean; 41 | 42 | 43 | } -------------------------------------------------------------------------------- /src/modules/models/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | export * from "./common_models/errors"; 11 | export * from "./common_models/helpers_classes"; 12 | export { default as OrgInfo } from './common_models/orgInfo'; 13 | export { default as Script } from './script_models/script'; 14 | export { default as ScriptMockField } from './script_models/scriptMockField'; 15 | export { 16 | default as ScriptMappingItem, 17 | } from './script_models/scriptMappingItem'; 18 | export { default as ScriptObject } from './script_models/scriptObject'; 19 | export { default as ScriptObjectSet } from './script_models/scriptObjectSet'; 20 | export { default as ScriptOrg } from './script_models/scriptOrg'; 21 | export { default as SFieldDescribe } from './sf_models/sfieldDescribe'; 22 | export { default as SObjectDescribe } from './sf_models/sobjectDescribe'; 23 | export { default as MigrationJobTask } from './job_models/migrationJobTask'; 24 | export { default as MigrationJob } from './job_models/migrationJob'; 25 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonCommandRunInfo.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISfdmuRunCustomAddonPluginInfo } from "."; 9 | 10 | 11 | /** 12 | * The information about currently running SFDMU command. 13 | * Contains the CLI command flags which were used 14 | * to run the Sfdmu job and the common information about the Plugin. 15 | */ 16 | export default interface ISfdmuRunCustomAddonCommandRunInfo { 17 | /** 18 | * The --sourceusername command flag. 19 | */ 20 | sourceUsername: string, 21 | 22 | /** 23 | * The --targetusername command flag. 24 | */ 25 | targetUsername: string, 26 | 27 | /** 28 | * The --apiversion command flag. 29 | */ 30 | apiVersion: string, 31 | 32 | /** 33 | * The directory location where the Plugin was started. 34 | */ 35 | readonly basePath: string, 36 | 37 | /** 38 | * The information about the Plugin and the framework. 39 | */ 40 | readonly pinfo: ISfdmuRunCustomAddonPluginInfo 41 | } -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/critical-runtime--errors-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Critical Runtime Errors Report 3 | about: NOTE! Report only if you program stops with “Uncaught TypeError” or “ReferenceError” 4 | messages! 5 | title: '[RUNTIME ERROR] - Replace this placeholder with a specific title for the issue. 6 | Do not delete "BUG".' 7 | labels: runtime error 8 | assignees: hknokh 9 | 10 | --- 11 | 12 | **Describe the runtime error** 13 | A clear and concise description of the runtime error you encountered, the purpose of your migration job, and the exact error message displayed. 14 | 15 | **To Reproduce** 16 | Describe the steps to reproduce the behavior. 17 | 18 | **export.json** 19 | Attach your **export.json** file. 20 | Reference: https://help.sfdmu.com/full-documentation/export-json-file-objects-specification/export-json-file-overview 21 | 22 | **Log file** 23 | Attach your full **.log** file. 24 | Reference: https://help.sfdmu.com/full-documentation/reports/the-execution-log 25 | 26 | **_target.csv** file 27 | If you have an issue with failed rows, attach a dump of the **_target.csv** file containing the error messages (include at least 1–2 full relevant rows). 28 | Reference: https://help.sfdmu.com/full-documentation/reports/the-target-csv-files 29 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonResult.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | /* 4 | * Copyright (c) 2020, salesforce.com, inc. 5 | * All rights reserved. 6 | * SPDX-License-Identifier: BSD-3-Clause 7 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 8 | */ 9 | 10 | /** 11 | * The class returned from the {@link ISfdmuRunCustomAddonModule.onExecute } method. 12 | * The Add-On can interact with the parent process by returning this object back to the caller. 13 | * 14 | * @export 15 | * @interface ISfdmuRunCustomAddonResult 16 | */ 17 | export default interface ISfdmuRunCustomAddonResult { 18 | 19 | /** 20 | * Set this property to true will abort the current Plugin job. 21 | * Use this property if you want to interrupt the migration process after finishing the Add-On execution. 22 | * @example 23 | * ```ts 24 | * async onExecute(context: ISfdmuRunCustomAddonContext, args: any): Promise { 25 | * // Return cancel = true if you want to abort the current miration job. 26 | * return { cancel: true }; 27 | * } 28 | * ``` 29 | * 30 | * @type {boolean} 31 | * @memberof ISfdmuRunCustomAddonResult 32 | */ 33 | cancel: boolean; 34 | 35 | } -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@salesforce/dev-config/tslint", 3 | "rules": { 4 | "member-access": false, 5 | "ordered-imports": false, 6 | "trailing-comma": false, 7 | "quotemark": false, 8 | "member-ordering": false, 9 | "no-any": false, 10 | "comment-format": false, 11 | "no-consecutive-blank-lines": false, 12 | "prefer-const": false, 13 | "one-line": false, 14 | "no-trailing-whitespace": false, 15 | "no-var-keyword": false, 16 | "triple-equals": false, 17 | "curly": false, 18 | "no-angle-bracket-type-assertion": false, 19 | "whitespace": false, 20 | "prefer-for-of": false, 21 | "no-shadowed-variable": false, 22 | "only-arrow-functions": false, 23 | "space-before-function-paren": false, 24 | "semicolon": false, 25 | "radix": false, 26 | "array-type": false, 27 | "ban-types": false, 28 | "jsdoc-format": false, 29 | "align": false, 30 | "arrow-parens": false, 31 | "object-literal-shorthand": false, 32 | "no-eval": false, 33 | "no-function-constructor-with-string-args": false, 34 | "no-var-requires": false, 35 | "one-variable-per-declaration": false, 36 | "typedef-whitespace":false, 37 | "eofline": false, 38 | "variable-name": false, 39 | "object-literal-key-quotes": false 40 | } 41 | } -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonRuntime.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISfdmuRunCustomAddonApiService, ISfdmuRunCustomAddonScript } from "."; 9 | 10 | 11 | /** 12 | * The Custom Add-On runtime. 13 | *
14 | * Besides other runtime information, This class exposes the instance of the Custom Add-On Api service 15 | * using the {@link ISfdmuRunCustomAddonRuntime.service} property. 16 | * 17 | * @export 18 | * @interface ISfdmuRunCustomAddonRuntime 19 | */ 20 | export default interface ISfdmuRunCustomAddonRuntime { 21 | 22 | /** 23 | * The instance of the Add-On Api service. 24 | * 25 | * @type {ISfdmuRunCustomAddonApiService} 26 | * 27 | * @memberof ISfdmuRunCustomAddonRuntime 28 | */ 29 | service: ISfdmuRunCustomAddonApiService; 30 | 31 | /** 32 | * Returns the data of the currently running {@link ISfdmuRunCustomAddonScript | export.json script}. 33 | * 34 | * @return {*} {ISfdmuRunCustomAddonScript} 35 | * @memberof ISfdmuRunCustomAddonRuntime 36 | */ 37 | getScript(): ISfdmuRunCustomAddonScript; 38 | 39 | } -------------------------------------------------------------------------------- /.github/workflows/manualRelease.yml: -------------------------------------------------------------------------------- 1 | name: manual release 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | release: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v3 11 | with: 12 | token: ${{ secrets.SVC_CLI_BOT_GITHUB_TOKEN }} 13 | - name: Conventional Changelog Action 14 | id: changelog 15 | uses: TriPSs/conventional-changelog-action@d360fad3a42feca6462f72c97c165d60a02d4bf2 16 | # overriding some of the basic behaviors to just get the changelog 17 | with: 18 | git-user-name: SF-CLI-BOT 19 | git-user-email: alm-cli@salesforce.com 20 | github-token: ${{ secrets.SVC_CLI_BOT_GITHUB_TOKEN }} 21 | output-file: false 22 | # always do the release, even if there are no semantic commits 23 | skip-on-empty: false 24 | tag-prefix: '' 25 | - uses: notiz-dev/github-action-json-property@2192e246737701f108a4571462b76c75e7376216 26 | id: packageVersion 27 | with: 28 | path: 'package.json' 29 | prop_path: 'version' 30 | - name: Create Github Release 31 | uses: actions/create-release@v1 32 | env: 33 | GITHUB_TOKEN: ${{ secrets.SF_CLI_BOT_GITHUB_TOKEN }} 34 | with: 35 | tag_name: ${{ steps.packageVersion.outputs.prop }} 36 | release_name: ${{ steps.packageVersion.outputs.prop }} 37 | -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/SfdmuRunAddonTaskData.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import { MigrationJobTask, TaskOrgData } from "../../../modules/models"; 10 | import { DATA_MEDIA_TYPE } from "../../../modules/components/common_components/enumerations"; 11 | import { ISfdmuRunCustomAddonTaskData } from "../../modules/sfdmu-run/custom-addons/package"; 12 | 13 | export default class SfdmuRunAddonTaskData implements ISfdmuRunCustomAddonTaskData { 14 | 15 | #taskOrgData: TaskOrgData; 16 | 17 | constructor(taskOrgData: TaskOrgData) { 18 | this.#taskOrgData = taskOrgData; 19 | } 20 | 21 | get task(): MigrationJobTask { 22 | return this.#taskOrgData.task; 23 | } 24 | 25 | get mediaType(): DATA_MEDIA_TYPE { 26 | return this.#taskOrgData.org.media; 27 | } 28 | 29 | get sObjectName(): string { 30 | return this.#taskOrgData.task.sObjectName; 31 | } 32 | 33 | get isSource(): boolean { 34 | return this.#taskOrgData.isSource; 35 | } 36 | 37 | get idRecordsMap(): Map { 38 | return this.#taskOrgData.idRecordsMap; 39 | } 40 | 41 | get extIdRecordsMap(): Map { 42 | return this.#taskOrgData.extIdRecordsMap; 43 | } 44 | 45 | get records(): any[] { 46 | return this.#taskOrgData.records; 47 | } 48 | } -------------------------------------------------------------------------------- /src/modules/models/sf_models/contentVersion.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | export default class ContentVersion { 10 | constructor(init: Partial) { 11 | if (init) { 12 | Object.assign(this, init); 13 | if (typeof this.ContentModifiedDate == 'string') { 14 | this.ContentModifiedDate = new Date(this.ContentModifiedDate); 15 | } 16 | } 17 | } 18 | Id: string; 19 | ContentDocumentId: string; 20 | Title: string; 21 | Description: string; 22 | PathOnClient: string; 23 | VersionData: string; 24 | ContentModifiedDate: Date; 25 | ContentSize: number; 26 | Checksum: string; 27 | ContentUrl: string; 28 | ContentBodyId: string; 29 | targetId: string; 30 | targetContentDocumentId: string; 31 | isError: boolean; 32 | get isUrlContent() { 33 | return !!this.ContentUrl; 34 | } 35 | isNewer(old: ContentVersion) { 36 | return this.isUrlContent != old.isUrlContent 37 | || this.isUrlContent && (this.ContentModifiedDate > old.ContentModifiedDate || this.ContentUrl != old.ContentUrl) 38 | || !this.isUrlContent && this.Checksum != old.Checksum; 39 | } 40 | get reasonForChange(): string { 41 | return this.ContentDocumentId ? 'Updated' : undefined; 42 | } 43 | 44 | } -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/sfdmuRunAddonJob.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | import { MigrationJob } from "../../../modules/models"; 11 | import { ISFdmuRunCustomAddonJob, ISFdmuRunCustomAddonTask } from "../../modules/sfdmu-run/custom-addons/package"; 12 | import SfdmuRunAddonTask from "./sfdmuRunAddonTask"; 13 | 14 | 15 | export default class SfdmuRunAddonJob implements ISFdmuRunCustomAddonJob { 16 | 17 | #migrationJob: MigrationJob; 18 | #pluginTasks: SfdmuRunAddonTask[]; 19 | 20 | constructor(migrationJob: MigrationJob) { 21 | this.#migrationJob = migrationJob; 22 | this.#pluginTasks = this.#migrationJob.tasks.map(jobTask => new SfdmuRunAddonTask(jobTask)); 23 | } 24 | 25 | 26 | get tasks(): SfdmuRunAddonTask[] { 27 | return this.#pluginTasks; 28 | } 29 | 30 | getTaskByFieldPath(fieldPath: string): { task: ISFdmuRunCustomAddonTask; field: string; } { 31 | let out = this.#migrationJob.getTaskByFieldPath(fieldPath); 32 | if (!out) { 33 | return { 34 | field: fieldPath.split('.').pop(), 35 | task: null 36 | } 37 | } 38 | let task = this.tasks.find(task => task.sObjectName == out.task.sObjectName); 39 | return { 40 | field: out.field, 41 | task 42 | } 43 | } 44 | } -------------------------------------------------------------------------------- /.github/workflows/failureNotifications.yml: -------------------------------------------------------------------------------- 1 | name: failureNotifications 2 | 3 | on: 4 | workflow_run: 5 | workflows: 6 | - version, tag and github release 7 | - publish 8 | types: 9 | - completed 10 | 11 | jobs: 12 | failure-notify: 13 | runs-on: ubuntu-latest 14 | if: ${{ github.event.workflow_run.conclusion == 'failure' }} 15 | steps: 16 | - name: Announce Failure 17 | id: slack 18 | uses: slackapi/slack-github-action@v1.21.0 19 | env: 20 | # for non-CLI-team-owned plugins, you can send this anywhere you like 21 | SLACK_WEBHOOK_URL: ${{ secrets.CLI_ALERTS_SLACK_WEBHOOK }} 22 | SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK 23 | with: 24 | payload: | 25 | { 26 | "text": "${{ github.event.workflow_run.name }} failed: ${{ github.event.workflow_run.repository.name }}", 27 | "blocks": [ 28 | { 29 | "type": "header", 30 | "text": { 31 | "type": "plain_text", 32 | "text": ":bh-alert: ${{ github.event.workflow_run.name }} failed: ${{ github.event.workflow_run.repository.name }} :bh-alert:" 33 | } 34 | }, 35 | { 36 | "type": "section", 37 | "text": { 38 | "type": "mrkdwn", 39 | "text": "Repo: ${{ github.event.workflow_run.repository.html_url }}\nWorkflow name: `${{ github.event.workflow_run.name }}`\nJob url: ${{ github.event.workflow_run.html_url }}" 40 | } 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonPluginInfo.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISfdmuAddonInfo } from "."; 9 | 10 | 11 | 12 | 13 | /** 14 | * Contains the information about the Sfdmu Plugin. 15 | */ 16 | export default interface ISfdmuRunCustomAddonPluginInfo { 17 | 18 | /** 19 | * The Plugin name (f.ex. ```sfdmu```). 20 | */ 21 | pluginName: string, 22 | 23 | /** 24 | * The executed command (f.ex. ```run```). 25 | */ 26 | commandName: string, 27 | 28 | /** 29 | * The current version of the running Plugin (f.ex. ```5.0.0```). 30 | */ 31 | version: string, 32 | 33 | /** 34 | * Path to the directory where the Sfdmu Plugin is installed. 35 | */ 36 | path: string, 37 | 38 | /** 39 | * Full CLI command string used to run the command (f.ex ```sfdx sfdmu:run --sourceusername my-source@mail.com --targetusername my-target@mail.com```) 40 | */ 41 | commandString: string, 42 | 43 | /** 44 | * The array of CLI arguments 45 | * @example 46 | * ```ts 47 | * ['--sourceusername', 'my-source@mail.com', '--targetusername', 'my-target@mail.com']; 48 | * ``` 49 | */ 50 | argv: string[], 51 | 52 | /** 53 | * Contains the information about the current version of the Add-On Api 54 | * related to the sfdmu:run command. 55 | * 56 | * @type {ISfdmuAddonApiInfo} 57 | * @memberof ISfdmuRunCustomAddonPluginInfo 58 | */ 59 | runAddOnApiInfo: ISfdmuAddonInfo 60 | } -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/index.ts: -------------------------------------------------------------------------------- 1 | export { default as ISfdmuRunCustomAddonApiService } from "./ISfdmuRunCustomAddonApiService"; 2 | export { default as ISfdmuRunCustomAddonCommandRunInfo } from "./ISfdmuRunCustomAddonCommandRunInfo"; 3 | export { default as ISfdmuRunCustomAddonContext } from "./ISfdmuRunCustomAddonContext"; 4 | export { default as ISFdmuRunCustomAddonJob } from "./ISFdmuRunCustomAddonJob"; 5 | export { default as ISfdmuRunCustomAddonModule } from "./ISfdmuRunCustomAddonModule"; 6 | export { default as ISfdmuRunCustomAddonPluginInfo } from "./ISfdmuRunCustomAddonPluginInfo"; 7 | export { default as ISfdmuRunCustomAddonProcessedData } from "./ISfdmuRunCustomAddonProcessedData"; 8 | export { default as ISfdmuRunCustomAddonResult } from "./ISfdmuRunCustomAddonResult"; 9 | export { default as ISfdmuRunCustomAddonRuntime } from "./ISfdmuRunCustomAddonRuntime"; 10 | export { default as ISfdmuRunCustomAddonScript } from "./ISfdmuRunCustomAddonScript"; 11 | export { default as ISfdmuRunCustomAddonScriptAddonManifestDefinition } from "./ISfdmuRunCustomAddonScriptAddonManifestDefinition"; 12 | export { default as ISfdmuRunCustomAddonScriptMappingItem } from "./ISfdmuRunCustomAddonScriptMappingItem"; 13 | export { default as ISfdmuRunCustomAddonScriptMockField } from "./ISfdmuRunCustomAddonScriptMockField"; 14 | export { default as ISfdmuRunCustomAddonScriptObject } from "./ISfdmuRunCustomAddonScriptObject"; 15 | export { default as ISfdmuRunCustomAddonScriptOrg } from "./ISfdmuRunCustomAddonScriptOrg"; 16 | export { default as ISfdmuRunCustomAddonSFieldDescribe } from "./ISfdmuRunCustomAddonSFieldDescribe"; 17 | export { default as ISFdmuRunCustomAddonTask } from "./ISFdmuRunCustomAddonTask"; 18 | export { default as ISfdmuRunCustomAddonTaskData } from "./ISfdmuRunCustomAddonTaskData"; 19 | export * from "./common"; 20 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonProcessedData.ts: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (c) 2020, salesforce.com, inc. 4 | * All rights reserved. 5 | * SPDX-License-Identifier: BSD-3-Clause 6 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 7 | */ 8 | 9 | import { ISfdmuRunCustomAddonSFieldDescribe } from "."; 10 | 11 | 12 | /** 13 | * Contains necessary information about the records are being processed within the current update step. 14 | * 15 | * @export 16 | * @interface ISfdmuRunCustomAddonProcessedData 17 | */ 18 | export default interface ISfdmuRunCustomAddonProcessedData { 19 | 20 | /** 21 | * The list of api names of the SF object fields are about to be updated in the Target. 22 | * 23 | * @type {Array} 24 | * @memberof ISfdmuRunCustomAddonProcessedData 25 | */ 26 | readonly fieldNames: Array; 27 | 28 | 29 | /** 30 | * The records to be updated in the Target. 31 | * 32 | * @type {Array} 33 | * @memberof ISfdmuRunCustomAddonProcessedData 34 | */ 35 | recordsToUpdate: Array; 36 | 37 | 38 | 39 | /** 40 | * The records to be inserted in the Target. 41 | * 42 | * @type {Array} 43 | * @memberof ISfdmuRunCustomAddonProcessedData 44 | */ 45 | recordsToInsert: Array; 46 | 47 | 48 | 49 | /** 50 | * The list of descriptions of SF object fields are about to be updated in the Target. 51 | * The {@link ISfdmuRunCustomAddonProcessedData.fieldNames} property contains only the Api names of the same fields. 52 | * 53 | * @type {Array} 54 | * @memberof ISfdmuRunCustomAddonProcessedData 55 | */ 56 | fields: Array; 57 | 58 | } -------------------------------------------------------------------------------- /src/modules/components/common_components/enumerations.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | export enum DATA_MEDIA_TYPE { 9 | Org, 10 | File 11 | } 12 | 13 | export enum OPERATION { 14 | Insert, 15 | Update, 16 | Upsert, 17 | Readonly, 18 | Delete, 19 | DeleteSource, 20 | DeleteHierarchy, 21 | HardDelete, 22 | Unknown 23 | } 24 | 25 | export enum API_ENGINE { 26 | DEFAULT_ENGINE, 27 | REST_API, 28 | BULK_API_V1, 29 | BULK_API_V2 30 | } 31 | 32 | export enum RESULT_STATUSES { 33 | Undefined = "Undefined", 34 | ApiOperationStarted = "ApiOperationStarted", 35 | ApiOperationFinished = "ApiOperationFinished", 36 | Information = "Information", 37 | JobCreated = "JobCreated", 38 | BatchCreated = "BatchCreated", 39 | DataUploaded = "DataUploaded", 40 | InProgress = "InProgress", 41 | Completed = "Completed", 42 | FailedOrAborted = "FailedOrAborted", 43 | ProcessError = "ProcessError" 44 | } 45 | 46 | export enum MESSAGE_IMPORTANCE { 47 | Silent, 48 | Low, 49 | Normal, 50 | High, 51 | Warn, 52 | Error 53 | } 54 | 55 | export enum ADDON_EVENTS { 56 | none = 'none', 57 | onBefore = "onBefore", 58 | onAfter = "onAfter", 59 | onBeforeUpdate = "onBeforeUpdate", 60 | onAfterUpdate = "onAfterUpdate", 61 | onDataRetrieved = "onDataRetrieved", 62 | 63 | filterRecordsAddons = "filterRecordsAddons" 64 | } 65 | 66 | export enum SPECIAL_MOCK_PATTERN_TYPES { 67 | haveAnyValue, 68 | missingValue 69 | } 70 | 71 | export enum DATA_CACHE_TYPES { 72 | InMemory = "InMemory", 73 | CleanFileCache = "CleanFileCache", 74 | FileCache = "FileCache" 75 | } 76 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Stale Issues 2 | on: 3 | schedule: 4 | - cron: "30 1 * * *" 5 | 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | close-stale-issues: 13 | runs-on: ubuntu-latest 14 | permissions: 15 | issues: write 16 | pull-requests: write 17 | steps: 18 | - uses: actions/stale@6f05e4244c9a0b2ed3401882b05d701dd0a7289b 19 | with: 20 | days-before-issue-stale: 3 21 | days-before-issue-close: 3 22 | stale-issue-label: "to-be-closed" 23 | exempt-issue-labels: "in-progress" 24 | close-issue-reason: "completed" 25 | stale-issue-message: "This case has been marked as 'to-be-closed', since it has no activity for the 3 days.
It will be automatically closed in another 3 days of inactivity." 26 | close-issue-message: "This case has been closed, since it has no activity for the last 6 days. Feel free to reopen it, if you need more help." 27 | repo-token: ${{ secrets.GITHUB_TOKEN }} 28 | 29 | - name: Remove 'to-be-closed' label from closed issues 30 | uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea 31 | with: 32 | github-token: ${{ secrets.GITHUB_TOKEN }} 33 | script: | 34 | const { repo, owner } = context.repo; 35 | const labelToRemove = "to-be-closed"; 36 | const query = `is:issue is:closed label:"${labelToRemove}" repo:${owner}/${repo}`; 37 | const issues = await github.paginate(github.rest.search.issuesAndPullRequests, { q: query }); 38 | for (const issue of issues) { 39 | await github.rest.issues.removeLabel({ 40 | owner, 41 | repo, 42 | issue_number: issue.number, 43 | name: labelToRemove, 44 | }); 45 | } 46 | 47 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISFdmuRunCustomAddonJob.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISFdmuRunCustomAddonTask } from "."; 9 | 10 | /** 11 | * The currently running migration job. 12 | * This object holds array of {@link ISFdmuRunCustomAddonJob.tasks | Tasks}. 13 | * 14 | * @export 15 | * @interface ISFdmuRunCustomAddonJob 16 | */ 17 | export default interface ISFdmuRunCustomAddonJob { 18 | 19 | /** 20 | * The migration Tasks related to this Job. 21 | * 22 | * @type {ISFdmuRunCustomAddonTask[]} 23 | * @memberof ISFdmuRunCustomAddonJob 24 | */ 25 | tasks: ISFdmuRunCustomAddonTask[]; 26 | 27 | 28 | /** 29 | * Finds the sobject by the provided field path, then returns the {@link ISFdmuRunCustomAddonTask | Task}, 30 | * associated with this sobject. 31 | * This method can help you to locate and access the source/target records which contain the desired field. 32 | * 33 | * @param {string} fieldPath The full field path to the field, e.g. ```Account.Test1__r.Text2__r.Name```. 34 | * In this case the method will find the sobject referenced by the lookup field ```Text2__c```. 35 | * So you will be able to access the records of this sobject including the desired Name field. 36 | * @return {{ 37 | * task: ISFdmuRunCustomAddonTask, 38 | * field: string 39 | * }} Returns the Task and the field name, for example { task: [Task of Text2__c], field: 'Name' } 40 | * @memberof ISFdmuRunCustomAddonJob 41 | */ 42 | getTaskByFieldPath(fieldPath: string): { 43 | task: ISFdmuRunCustomAddonTask, 44 | field: string 45 | }; 46 | 47 | } -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonTaskData.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { DATA_MEDIA_TYPE } from "."; 9 | 10 | 11 | /** 12 | * The data associated with the given migration task (the Source or the Target data) 13 | * 14 | * @export 15 | * @interface ISfdmuRunCustomAddonTaskData 16 | */ 17 | export default interface ISfdmuRunCustomAddonTaskData { 18 | 19 | 20 | /** 21 | * The type of media. 22 | *
23 | * For example the data source can be of the {@link DATA_MEDIA_TYPE.Org} media type 24 | * and the Target is of the {@link DATA_MEDIA_TYPE.File} media type. 25 | * 26 | * @type {DATA_MEDIA_TYPE} 27 | * @memberof ISfdmuRunCustomAddonTaskData 28 | */ 29 | readonly mediaType: DATA_MEDIA_TYPE; 30 | 31 | 32 | /** 33 | * Returns true if this object contains the data retireved from the Source 34 | * and false if it's the data retireved from the Target. 35 | * 36 | * @type {boolean} 37 | * @memberof ISfdmuRunCustomAddonTaskData 38 | */ 39 | readonly isSource: boolean; 40 | 41 | 42 | /** 43 | * The mapping between the record Id to the record object. 44 | * 45 | * @type {Map} 46 | * @memberof ISfdmuRunCustomAddonTaskData 47 | */ 48 | readonly idRecordsMap: Map; 49 | 50 | 51 | /** 52 | * The mapping between the externalId value to the record Id value. 53 | * 54 | * @type {Map} 55 | * @memberof ISfdmuRunCustomAddonTaskData 56 | */ 57 | readonly extIdRecordsMap: Map; 58 | 59 | 60 | /** 61 | * The array of the records (the source or the target). 62 | * 63 | * @type {any[]} 64 | * @memberof ISfdmuRunCustomAddonTaskData 65 | */ 66 | readonly records: any[]; 67 | 68 | } -------------------------------------------------------------------------------- /src/modules/app/appConsoleUxLogger.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import { IUxLogger } from "../components/common_components/logger"; 10 | 11 | const readline = require('readline'); 12 | const { 13 | stdin: input, 14 | stdout: output 15 | } = require('process'); 16 | const rl = readline.createInterface({ input, output }); 17 | 18 | export default class AppConsoleLogger implements IUxLogger { 19 | 20 | log = (message: any): void => { 21 | console.log(`${message}`); 22 | } 23 | 24 | styledJSON = (message: string): void => console.log('\x1b[34m%s\x1b[0m', JSON.stringify(JSON.parse(message || '{}'), null, 4)); 25 | 26 | warn = (message: any): void => console.warn('\x1b[33m%s\x1b[0m', `${message}`); 27 | error = (message: any): void => console.error('\x1b[31m%s\x1b[0m', `${message}`); 28 | styledObject = (message: any): void => console.log('\x1b[34m%s\x1b[0m', JSON.stringify(message || {}, null, 4)); 29 | table = (message: any): void => console.log(message); 30 | 31 | prompt = (message: string, params: { 32 | default: string, 33 | timeout: number 34 | }): Promise => { 35 | return new Promise(resolve => { 36 | let timeout: any; 37 | if (params && params.timeout) { 38 | timeout = setTimeout(() => { 39 | rl.close(); 40 | resolve(params.default); 41 | }, params.timeout); 42 | } 43 | rl.question(message + ' ', function (answer: string) { 44 | if (timeout) { 45 | clearTimeout(timeout); 46 | } 47 | rl.close(); 48 | resolve(answer); 49 | }); 50 | }); 51 | }; 52 | styledHeader = (message: any): void => console.log('\x1b[32m%s\x1b[0m', '\n' 53 | + String(message || '').toUpperCase() 54 | + '\n=================\n'); 55 | 56 | startSpinner = (): void => void (0); 57 | stopSpinner = (): void => void (0); 58 | setSpinnerStatus = (): void => void (0); 59 | 60 | } 61 | 62 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScript.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { DATA_CACHE_TYPES, ISfdmuRunCustomAddonScriptAddonManifestDefinition, ISfdmuRunCustomAddonScriptObject, ISfdmuRunCustomAddonScriptOrg } from "."; 9 | 10 | 11 | 12 | /** 13 | * Provides an access to the currently running export.json script. 14 | * @see {@link https://help.sfdmu.com/full-documentation/configuration-and-running/full-exportjson-format | Full export.json format} for the detailed information about the fields. 15 | * 16 | * @export 17 | * @interface ISfdmuRunCustomAddonScript 18 | */ 19 | export default interface ISfdmuRunCustomAddonScript { 20 | 21 | orgs?: ISfdmuRunCustomAddonScriptOrg[]; 22 | objects?: ISfdmuRunCustomAddonScriptObject[]; 23 | 24 | pollingIntervalMs?: number; 25 | pollingQueryTimeoutMs?: number; 26 | concurrencyMode?: "Serial" | "Parallel"; 27 | bulkThreshold?: number; 28 | queryBulkApiThreshold?: number; 29 | bulkApiVersion?: string; 30 | bulkApiV1BatchSize?: number; 31 | restApiBatchSize?: number; 32 | allOrNone?: boolean; 33 | //promptOnUpdateError: boolean; 34 | promptOnMissingParentObjects?: boolean; 35 | promptOnIssuesInCSVFiles?: boolean; 36 | validateCSVFilesOnly?: boolean; 37 | apiVersion?: string; 38 | createTargetCSVFiles?: boolean; 39 | importCSVFilesAsIs?: boolean; 40 | alwaysUseRestApiToUpdateRecords?: boolean; 41 | excludeIdsFromCSVFiles?: boolean; 42 | //fileLog: boolean; 43 | keepObjectOrderWhileExecute?: boolean; 44 | allowFieldTruncation?: boolean; 45 | simulationMode?: boolean; 46 | proxyUrl?: string; 47 | binaryDataCache?: DATA_CACHE_TYPES; 48 | sourceRecordsCache?: DATA_CACHE_TYPES; 49 | parallelBinaryDownloads?: number; 50 | parallelBulkJobs?: number; 51 | parallelRestJobs?: number; 52 | 53 | beforeAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[]; 54 | afterAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[]; 55 | dataRetrievedAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[]; 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/modules/models/common_models/helper_interfaces.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | export interface IOrgConnectionData { 10 | instanceUrl: string; 11 | accessToken: string; 12 | apiVersion: string; 13 | proxyUrl: string; 14 | } 15 | 16 | export interface IMockField { 17 | fn: string; 18 | regIncl: string; 19 | regExcl: string; 20 | disallowMockAllRecord: boolean; 21 | allowMockAllRecord: boolean; 22 | } 23 | 24 | export interface ICSVIssueCsvRow { 25 | "Date update": string, 26 | "Field value": string, 27 | "sObject name": string, 28 | "Field name": string, 29 | "Parent field value": string, 30 | "Parent SObject name": string, 31 | "Parent field name": string, 32 | "Error": string 33 | } 34 | 35 | export interface IMissingParentLookupRecordCsvRow { 36 | "Date update": string, 37 | "Record Id": string, 38 | "sObject name": string; 39 | "Lookup field name": string; 40 | "Lookup reference field name": string; 41 | "Parent SObject name": string; 42 | "Parent ExternalId field name": string; 43 | "Missing parent External Id value": string; 44 | } 45 | 46 | 47 | export interface IFieldMapping { 48 | sourceQueryToTarget: (query: string, sourceObjectName: string) => IFieldMappingResult; 49 | sourceRecordsToTarget: (records: Array, sourceObjectName: string) => IFieldMappingResult; 50 | targetRecordsToSource: (records: Array, sourceObjectName: string) => IFieldMappingResult; 51 | transformQuery: (query: string, sourceObjectName: string) => IFieldMappingResult; 52 | } 53 | 54 | export interface IFieldMappingResult { 55 | query?: string; 56 | records?: Array; 57 | targetSObjectName?: string; 58 | } 59 | 60 | export interface IIdentityInfo { 61 | user_id: string, 62 | organization_id: string, 63 | username: string, 64 | display_name: string 65 | } 66 | 67 | 68 | /** 69 | * Metadata to write table into log 70 | */ 71 | export interface ITableMessage { 72 | tableBody: Array, 73 | tableColumns: Array<{ 74 | key: string, 75 | label: string, 76 | width?: number 77 | }> 78 | } 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /src/modules/app/appModels.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { DATA_CACHE_TYPES } from "../components/common_components/enumerations"; 9 | import { LOG_MESSAGE_TYPE, LOG_MESSAGE_VERBOSITY } from "../components/common_components/logger"; 10 | import { ObjectFieldMapping, SFieldDescribe, SObjectDescribe } from "../models"; 11 | import { IBlobField } from "../models/api_models/helper_interfaces"; 12 | import { IIdentityInfo, ITableMessage } from "../models/common_models/helper_interfaces"; 13 | 14 | export interface IAppLogger { 15 | log(message: string | object | ITableMessage, 16 | type?: LOG_MESSAGE_TYPE, 17 | verbosity?: LOG_MESSAGE_VERBOSITY, 18 | ...tokens: string[] 19 | ): void; 20 | infoMinimal(message: string, ...tokens: string[]): void; 21 | infoNormal(message: string, ...tokens: string[]): void; 22 | infoVerbose(message: string, ...tokens: string[]): void 23 | } 24 | 25 | export interface IAppScript { 26 | logger: IAppLogger; 27 | sourceRecordsCache: DATA_CACHE_TYPES; 28 | binaryDataCache: DATA_CACHE_TYPES; 29 | parallelBinaryDownloads: number; 30 | binaryCacheDirectory: string; 31 | sourceRecordsCacheDirectory: string; 32 | } 33 | 34 | export interface IAppScriptOrg { 35 | script: IAppScript; 36 | getConnection(): any; 37 | isSource: boolean; 38 | } 39 | 40 | export interface IAppSfdxService { 41 | org: IAppScriptOrg; 42 | readonly logger: IAppLogger; 43 | queryOrgOrCsvAsync(soql: string, useBulkQueryApi: boolean, csvFullFilename?: string, sFieldsDescribeMap?: Map, useQueryAll?: boolean, bulkQueryPollTimeout?: number): Promise>; 44 | queryOrgAsync(soql: string, useBulkQueryApi: boolean, useQueryAll?: boolean, bulkQueryPollTimeout?: number): Promise>; 45 | describeOrgAsync(): Promise>; 46 | getPolymorphicObjectFields(sObjectName: string): Promise; 47 | identityAsync(): Promise; 48 | describeSObjectAsync(objectName: string, objectFieldMapping?: ObjectFieldMapping): Promise; 49 | downloadBlobFieldDataAsync(recordIds: Array, blobField: IBlobField): Promise>; 50 | } 51 | -------------------------------------------------------------------------------- /src/modules/models/common_models/errors.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | /** 11 | * Errors in the Org metadata validation 12 | * 13 | * @export 14 | * @class OrgMetadataError 15 | * @extends {Error} 16 | */ 17 | export class OrgMetadataError extends Error { 18 | constructor(m: string) { 19 | super(m); 20 | } 21 | } 22 | 23 | /** 24 | * Errors occur while the command is being initializating 25 | * 26 | * @export 27 | * @class CommandInitializationError 28 | * @extends {Error} 29 | */ 30 | export class CommandInitializationError extends Error { 31 | constructor(m: string) { 32 | super(m); 33 | } 34 | } 35 | 36 | /** 37 | * Errors during command execution 38 | * 39 | * @export 40 | * @class CommandExecutionError 41 | * @extends {Error} 42 | */ 43 | export class CommandExecutionError extends Error { 44 | constructor(m: string) { 45 | super(m); 46 | } 47 | } 48 | 49 | /** 50 | * Unresolvable warning 51 | * 52 | * @export 53 | * @class UnresolvableWarning 54 | * @extends {Error} 55 | */ 56 | export class UnresolvableWarning extends Error { 57 | constructor(m: string) { 58 | super(m); 59 | } 60 | } 61 | 62 | /** 63 | * User has stopped execution of the command 64 | * 65 | * @export 66 | * @class CommandAbortedByUserError 67 | * @extends {Error} 68 | */ 69 | export class CommandAbortedByUserError extends Error { 70 | constructor(m: string) { 71 | super(m); 72 | } 73 | } 74 | 75 | /** 76 | * Add-On module has stopped execution of the command 77 | * 78 | * @export 79 | * @class CommandAbortedByAddOnError 80 | * @extends {Error} 81 | */ 82 | export class CommandAbortedByAddOnError extends Error { 83 | constructor(m: string) { 84 | super(m); 85 | } 86 | } 87 | 88 | /** 89 | * When thrown the command need to be aborted with success result 90 | * 91 | * @export 92 | * @class SuccessExit 93 | * @extends {Error} 94 | */ 95 | export class SuccessExit extends Error { 96 | constructor(m?: string) { 97 | super(m); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/common.ts: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (c) 2020, salesforce.com, inc. 4 | * All rights reserved. 5 | * SPDX-License-Identifier: BSD-3-Clause 6 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 7 | */ 8 | 9 | /** 10 | * The available types of the SF object field. 11 | * Mostly is compatible with the official SF documentaiton. 12 | */ 13 | export type FieldType = 14 | | 'string' 15 | | 'boolean' 16 | | 'int' 17 | | 'double' 18 | | 'date' 19 | | 'datetime' 20 | | 'base64' 21 | | 'id' 22 | | 'reference' 23 | | 'currency' 24 | | 'textarea' 25 | | 'percent' 26 | | 'phone' 27 | | 'url' 28 | | 'email' 29 | | 'combobox' 30 | | 'picklist' 31 | | 'multipicklist' 32 | | 'anyType' 33 | | 'location' 34 | // the following are not found in official documentation, but still occur when describing an sobject 35 | | 'time' 36 | | 'encryptedstring' 37 | | 'address' 38 | | 'complexvalue' 39 | 40 | 41 | /** 42 | * The available operations 43 | * 44 | * @export 45 | * @enum {number} 46 | */ 47 | export enum OPERATION { 48 | Insert, 49 | Update, 50 | Upsert, 51 | Readonly, 52 | Delete, 53 | DeleteSource, 54 | DeleteHierarchy, 55 | HardDelete, 56 | Unknown 57 | } 58 | 59 | 60 | /** 61 | * The available media sources 62 | * 63 | * @export 64 | * @enum {number} 65 | */ 66 | export enum DATA_MEDIA_TYPE { 67 | Org, 68 | File 69 | } 70 | 71 | 72 | /** 73 | * The detailed information about the current version of the Add-On Api. 74 | * 75 | * @export 76 | * @interface ISfdmuAddonInfo 77 | */ 78 | export interface ISfdmuAddonInfo { 79 | 80 | /** 81 | * The number of the Api version (f.ex. ```1.0.0```). 82 | * 83 | * @type {string} 84 | * @memberof ISfdmuAddonInfo 85 | */ 86 | version: string; 87 | } 88 | 89 | /** 90 | * The type of caching (for the bimary data or for records) 91 | * when the caching feature is enabled. 92 | * @see {@link https://help.sfdmu.com/full-documentation/configuration-and-running/full-exportjson-format | Full export.json format} for the detailed information. 93 | * 94 | * @export 95 | * @enum {number} 96 | */ 97 | export enum DATA_CACHE_TYPES { 98 | InMemory = "InMemory", 99 | CleanFileCache = "CleanFileCache", 100 | FileCache = "FileCache" 101 | } 102 | 103 | -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/custom/SfdmuRunCustomAddonApiService.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import { ISfdmuRunCustomAddonCommandRunInfo, ISfdmuRunCustomAddonContext, ISFdmuRunCustomAddonJob, ISfdmuRunCustomAddonModule, ISfdmuRunCustomAddonProcessedData, ISFdmuRunCustomAddonTask } from "../../../modules/sfdmu-run/custom-addons/package"; 10 | import ISfdmuRunCustomAddonApiService from "../../../modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonApiService"; 11 | import SfdmuRunAddonJob from "../sfdmuRunAddonJob"; 12 | import SfdmuRunAddonRuntime from "../sfdmuRunAddonRuntime"; 13 | import SfdmuRunAddonTask from "../sfdmuRunAddonTask"; 14 | 15 | 16 | export default class SfdmuRunCustomAddonApiService implements ISfdmuRunCustomAddonApiService { 17 | 18 | runtime: SfdmuRunAddonRuntime; 19 | 20 | constructor(runtime: SfdmuRunAddonRuntime) { 21 | this.runtime = runtime; 22 | } 23 | 24 | 25 | getPluginRunInfo(): ISfdmuRunCustomAddonCommandRunInfo { 26 | return this.runtime.runInfo; 27 | } 28 | 29 | getProcessedData(context: ISfdmuRunCustomAddonContext): ISfdmuRunCustomAddonProcessedData { 30 | return this.#getPluginTask(context).processedData; 31 | } 32 | 33 | log(module: ISfdmuRunCustomAddonModule, message: string | object, messageType?: "INFO" | "WARNING" | "ERROR" | "JSON", ...tokens: string[]): void { 34 | if (typeof message === 'string') { 35 | (this.runtime as any).logFormatted(module, message, messageType, ...tokens); 36 | } else { 37 | (this.runtime as any).log(message, messageType == 'JSON' ? messageType : 'OBJECT', ...tokens); 38 | } 39 | } 40 | 41 | getPluginJob(): ISFdmuRunCustomAddonJob { 42 | return this.#pluginJob; 43 | } 44 | getPluginTask(context: ISfdmuRunCustomAddonContext): ISFdmuRunCustomAddonTask { 45 | return this.#getPluginTask(context); 46 | } 47 | 48 | 49 | // ------------- Helpers ------------- // 50 | get #pluginJob(): SfdmuRunAddonJob { 51 | return this.runtime.pluginJob; 52 | } 53 | 54 | #getPluginTask(context: ISfdmuRunCustomAddonContext): SfdmuRunAddonTask { 55 | return this.#pluginJob.tasks.find(task => task.sObjectName == context.objectName); 56 | } 57 | 58 | 59 | } -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonContext.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | /** 10 | * The Custom Add-On module runtime context. 11 | * @export 12 | * @interface ISfdmuRunCustomAddonContext 13 | */ 14 | export default interface ISfdmuRunCustomAddonContext { 15 | 16 | /** 17 | * The name of the triggered Add-On event. 18 | *
19 | * @example 20 | * ```ts 21 | * async onExecute(context: ISfdmuRunCustomAddonContext, args: any): Promise { 22 | * console.log(context.eventName); // For the BeforeUpdate event, outputs 'onBeforeUpdate' 23 | * } 24 | * ``` 25 | * 26 | * @type {string} 27 | * @memberof ISfdmuRunCustomAddonContext 28 | */ 29 | eventName: string; 30 | 31 | 32 | /** 33 | * The name of the current Add-On module, including it's type (core or custom) 34 | * @example ```custom:CustomSfdmuRunAddonTemplate```, ```core:ExportFile``` 35 | * 36 | * @type {string} 37 | * @memberof ISfdmuRunCustomAddonContext 38 | */ 39 | moduleDisplayName: string; 40 | 41 | 42 | /** 43 | * The Api name of the SF object which is being currently processed by the Plugin. 44 | * @example ```AccountTeamMember``` 45 | * 46 | * @type {string} 47 | * @memberof ISfdmuRunCustomAddonContext 48 | */ 49 | objectName: string; 50 | 51 | 52 | /** 53 | * The display name of the processed SF object (typically it's the object label). 54 | * @example ```Account Team Member``` 55 | * 56 | * @type {string} 57 | * @memberof ISfdmuRunCustomAddonContext 58 | */ 59 | objectDisplayName: string; 60 | 61 | 62 | /** 63 | * The description of the current Add-On module. 64 | * Defined in the ```object/[addons]``` section of the Script, as in the example below: 65 | *
66 | * @example 67 | * ```json 68 | * { 69 | "description": "This test AddOn manipulates with the source Json string right before the target update. It extracts the Json value from the LongText__c, then stores the extracted string into the TEST1__c." 70 | } 71 | * ``` 72 | * @type {string} 73 | * @memberof ISfdmuRunCustomAddonContext 74 | */ 75 | description: string; 76 | 77 | } -------------------------------------------------------------------------------- /src/modules/app/appSfdmuRunApp.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import ISfdmuCommand from "../models/common_models/ISfdxCommand"; 10 | import { RunCommand } from "../commands_processors/runCommand"; 11 | import AppJsonMessages from './appJsonMessages'; 12 | import AppConsoleLogger from './appConsoleUxLogger'; 13 | 14 | import * as path from 'path'; 15 | import { IRunProcess } from "../commands_processors/IRunProcess"; 16 | import { IResourceBundle, IUxLogger } from "../components/common_components/logger"; 17 | import ISfdmuStatics from "../models/common_models/ISfdmuStatics"; 18 | import RunCommandExecutor from "../commands_processors/runCommandExecutor"; 19 | import { Common } from "../components/common_components/common"; 20 | import IAppSfdmuRunModuleArgs from "./IAppSfdmuRunModuleArgs"; 21 | 22 | const root = path.resolve(__dirname, '../../../'); 23 | const commandMessages = new AppJsonMessages(root, "run"); 24 | const resources = new AppJsonMessages(root, "resources"); 25 | 26 | export default class AppSfdmuRunApp implements IRunProcess { 27 | 28 | argv: Array; 29 | exportJson: string; 30 | 31 | cmd: ISfdmuCommand; 32 | command: RunCommand; 33 | statics: ISfdmuStatics; 34 | 35 | exitProcess: boolean; 36 | 37 | m_ux: IUxLogger; 38 | m_flags: any = {}; 39 | 40 | commandMessages: IResourceBundle; 41 | resources: IResourceBundle; 42 | 43 | constructor(args: IAppSfdmuRunModuleArgs) { 44 | 45 | this.m_ux = args.logger || new AppConsoleLogger(); 46 | this.argv = args.argv.splice(2); 47 | 48 | this.exportJson = args.exportJson; 49 | 50 | this.commandMessages = args.commandMessages || commandMessages; 51 | this.resources = args.resources || resources; 52 | 53 | this.exitProcess = args.exitProcess; 54 | 55 | 56 | let flags = Common.parseArgv(...this.argv); 57 | Object.assign(this.m_flags, flags); 58 | 59 | this.statics = { 60 | name: "Run", 61 | plugin: { 62 | name: "sfdmu", 63 | root 64 | } 65 | }; 66 | this.cmd = { 67 | statics: this.statics, 68 | argv: this.argv 69 | }; 70 | } 71 | 72 | async runCommand(): Promise { 73 | await RunCommandExecutor.execute(this); 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /.github/workflows/issue-response-handler.yml: -------------------------------------------------------------------------------- 1 | name: Issue Response Handler 2 | 3 | on: 4 | issues: 5 | types: [opened] 6 | 7 | permissions: 8 | contents: read 9 | 10 | jobs: 11 | respond: 12 | runs-on: ubuntu-latest 13 | steps: 14 | 15 | - name: Random Delay 16 | run: | 17 | # Set minimum and maximum delay in minutes 18 | MIN_DELAY=10 # n1 minutes 19 | MAX_DELAY=30 # n2 minutes 20 | 21 | # Generate a random delay between MIN_DELAY and MAX_DELAY 22 | DELAY=$(( (RANDOM % (MAX_DELAY - MIN_DELAY + 1) + MIN_DELAY) * 60 )) 23 | 24 | # Sleep for the randomly determined number of seconds 25 | echo "Delaying for $((DELAY / 60)) minutes" 26 | sleep $DELAY 27 | 28 | - name: Post Comment for 'bug' 29 | if: contains(github.event.issue.labels.*.name, 'bug') 30 | run: | 31 | curl -X POST \ 32 | -H "Authorization: token ${{ secrets.REPO_TOKEN }}" \ 33 | -H "Accept: application/vnd.github.v3+json" \ 34 | -d '{"body":"Hello, @${{ github.event.issue.user.login }}\n\nThank you for reporting a bug.\nI will take a look at it as soon as possible and let you know of any updates.\n\nCheers"}' \ 35 | https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments 36 | 37 | - name: Post Comment for 'feature-request' 38 | if: contains(github.event.issue.labels.*.name, 'feature-request') 39 | run: | 40 | curl -X POST \ 41 | -H "Authorization: token ${{ secrets.REPO_TOKEN }}" \ 42 | -H "Accept: application/vnd.github.v3+json" \ 43 | -d '{"body":"Hello, @${{ github.event.issue.user.login }}\n\nThank you for your feature request.\nI will review it as soon as possible and provide updates as they become available.\n\nCheers"}' \ 44 | https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments 45 | 46 | - name: Post Comment for 'help-wanted' 47 | if: contains(github.event.issue.labels.*.name, 'help-wanted') 48 | run: | 49 | curl -X POST \ 50 | -H "Authorization: token ${{ secrets.REPO_TOKEN }}" \ 51 | -H "Accept: application/vnd.github.v3+json" \ 52 | -d '{"body":"Hello, @${{ github.event.issue.user.login }}\n\nThank you for reaching out.\nI will do my best to assist you as quickly as possible and will keep you updated on my progress.\n\nCheers"}' \ 53 | https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments 54 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScriptObject.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISfdmuRunCustomAddonScriptAddonManifestDefinition, ISfdmuRunCustomAddonScriptMappingItem, ISfdmuRunCustomAddonScriptMockField, OPERATION } from "."; 9 | 10 | 11 | 12 | /** 13 | * Provides an access to the object included in the currently running {@link ISfdmuRunCustomAddonScript}. 14 | * @see {@link https://help.sfdmu.com/full-documentation/configuration-and-running/full-exportjson-format | Full export.json format} for the detailed information about the fields. 15 | * 16 | * @export 17 | * @interface ISfdmuRunCustomAddonScriptObject 18 | */ 19 | export default interface ISfdmuRunCustomAddonScriptObject { 20 | 21 | mockFields?: ISfdmuRunCustomAddonScriptMockField[]; 22 | fieldMapping?: ISfdmuRunCustomAddonScriptMappingItem[]; 23 | query?: string; 24 | deleteQuery?: string; 25 | operation?: OPERATION; 26 | externalId?: string; 27 | deleteOldData?: boolean; 28 | deleteFromSource?: boolean; 29 | deleteByHierarchy?: boolean; 30 | hardDelete?: boolean; 31 | updateWithMockData?: boolean; 32 | //mockCSVData?: boolean; 33 | sourceRecordsFilter?: string; 34 | targetRecordsFilter?: string; 35 | excluded?: boolean; 36 | useCSVValuesMapping?: boolean; 37 | useFieldMapping?: boolean; 38 | useValuesMapping?: boolean; 39 | allRecords?: boolean; 40 | master?: boolean; 41 | excludedFields?: Array; 42 | excludedFromUpdateFields?: Array; 43 | restApiBatchSize?: number; 44 | bulkApiV1BatchSize?: number; 45 | parallelBulkJobs?: number; 46 | parallelRestJobs?: number; 47 | 48 | useSourceCSVFile?: boolean; 49 | skipRecordsComparison?: boolean; 50 | 51 | 52 | beforeAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[]; 53 | afterAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[]; 54 | beforeUpdateAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[]; 55 | afterUpdateAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[]; 56 | filterRecordsAddons?: ISfdmuRunCustomAddonScriptAddonManifestDefinition[]; 57 | 58 | // ---- Runtime ----- 59 | /** 60 | * The API name of the current sObject 61 | * 62 | * @type {string} 63 | * @memberof ISfdmuRunCustomAddonScriptObject 64 | */ 65 | readonly name?: string; 66 | objectName?: string 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonModule.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISfdmuRunCustomAddonContext, ISfdmuRunCustomAddonResult, ISfdmuRunCustomAddonRuntime } from "."; 9 | 10 | 11 | /** 12 | * The base interface to be implemented in every custom Sfdmu Add-On module. 13 | * @export 14 | * @interface ISfdmuRunCustomAddonModule 15 | */ 16 | export default interface ISfdmuRunCustomAddonModule { 17 | 18 | 19 | /** 20 | * The instance of the Custom Add-On module runtime. 21 | *
22 | * Uses the public property {@link ISfdmuRunCustomAddonRuntime.service} 23 | * to share the Custom Add-On module Api with the current module instance. 24 | * 25 | * @type {ISfdmuRunCustomAddonRuntime} 26 | * @memberof ISfdmuRunCustomAddonModule 27 | * 28 | */ 29 | runtime: ISfdmuRunCustomAddonRuntime; 30 | 31 | 32 | 33 | /** 34 | * The entry point which is executed by the Plugin when the Add-On event is triggered. 35 | * 36 | * @param {ISfdmuRunCustomAddonContext} context The current Add-On runtime context. 37 | * @param {*} args The JS object passed into the function from the ```arg``` property 38 | * defined in the ```object/[addons]``` section of the Script. 39 | *
40 | *
41 | * For example, the portion of the json as below: 42 | *
43 | * ```json 44 | * "args" : { 45 | "TEST__c": "Another test, assigning this text to the field TEST__c of each record being processed" 46 | } 47 | * ``` 48 | Will pass to the method the following args: 49 | ```ts 50 | args = { 51 | TEST__c : "Another test, assigning this text to the field TEST__c of each record being processed" 52 | } 53 | ``` 54 | * @return {Promise} 55 | * @memberof ISfdmuRunCustomAddonModule 56 | */ 57 | onExecute(context: ISfdmuRunCustomAddonContext, args: any): Promise; 58 | 59 | 60 | /** 61 | * If implemented, this method wil run for each Add-On module (both of core and custom types) immediatelly AFTER the export.json file is parsed 62 | * but BEFORE the migration job is actually started. 63 | * Allows to modify the script and to make other necessary prerequisites to get the migration job ready. 64 | * 65 | * @param {ISfdmuRunCustomAddonContext} context The current Add-On runtime context. 66 | * @param {*} args The JS object passed into the function from the ```arg``` property 67 | * defined in the ```object/[addons]``` section of the Script. 68 | * @return {Promise} 69 | * @memberof ISfdmuRunCustomAddonModule 70 | */ 71 | onInit?(context: ISfdmuRunCustomAddonContext, args: any): Promise; 72 | 73 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sfdmu", 3 | "description": "The SFDX Data Move Utility (SFDMU) is the most modern and powerful salesforce data migration tool. It will help you to populate your org with data in minutes.", 4 | "version": "4.38.0", 5 | "author": "Salesforce", 6 | "license": "BSD-3-Clause", 7 | "developedBy": "Haim Knokh", 8 | "bugs": "https://github.com/forcedotcom/SFDX-Data-Move-Utility/issues", 9 | "dependencies": { 10 | "@babel/traverse": "^7.24.1", 11 | "@oclif/command": "^1.8.36", 12 | "@oclif/config": "^1.18.17", 13 | "@oclif/errors": "^1.3.6", 14 | "@salesforce/command": "^5.3.9", 15 | "@salesforce/core": "^7.3.3", 16 | "@salesforce/dev-config": "^4.1.0", 17 | "@types/bunyan": "^1.8.11", 18 | "alasql": "^4.5.2", 19 | "casual": "^1.6.2", 20 | "class-transformer": "^0.5.1", 21 | "csv-parse": "^4.12.0", 22 | "csv-writer": "^1.6.0", 23 | "deep.clone": "^2.1.2", 24 | "es6-shim": "^0.35.5", 25 | "fastest-levenshtein": "^1.0.16", 26 | "glob": "^10.3.12", 27 | "jsforce": "^1.11.1", 28 | "madge": "^8.0.0", 29 | "promise-parallel-throttle": "^3.4.1", 30 | "reflect-metadata": "^0.1.13", 31 | "soql-parser-js": "^1.2.1", 32 | "tslib": "^2.7.0" 33 | }, 34 | "devDependencies": { 35 | "@oclif/dev-cli": "^1.26.10", 36 | "@oclif/plugin-help": "^6.0.21", 37 | "@oclif/test": "^3.2.10", 38 | "@types/chai": "^4.3.14", 39 | "@types/mocha": "^10.0.6", 40 | "@types/node": "^20.12.7", 41 | "chai": "^5.1.0", 42 | "globby": "11.0.0", 43 | "mocha": "^10.4.0", 44 | "nyc": "^15.1.0", 45 | "ts-node": "^10.9.2", 46 | "tslint": "^6.1.3", 47 | "typescript": "^5.6.2" 48 | }, 49 | "engines": { 50 | "node": ">=8.0.0" 51 | }, 52 | "files": [ 53 | "/lib", 54 | "/messages", 55 | "/npm-shrinkwrap.json", 56 | "/oclif.manifest.json" 57 | ], 58 | "homepage": "https://help.sfdmu.com", 59 | "keywords": [ 60 | "force", 61 | "salesforce", 62 | "salesforcedx", 63 | "sf", 64 | "sf-plugin", 65 | "sfdx", 66 | "sfdx-plugin" 67 | ], 68 | "oclif": { 69 | "commands": "./lib/commands", 70 | "bin": "sfdx", 71 | "topics": { 72 | "sfdmu": { 73 | "description": "The SFDX Data Move Utility (SFDMU) is the most modern and powerful salesforce data migration tool.\n• The SFDMU will help you to quickly populate your Scratch / Dev / Sandbox / Prod org with records imported from another org or CSV files." 74 | } 75 | }, 76 | "devPlugins": [ 77 | "@oclif/plugin-help" 78 | ] 79 | }, 80 | "repository": "forcedotcom/SFDX-Data-Move-Utility", 81 | "addons": { 82 | "run": { 83 | "version": "1.6.0" 84 | } 85 | }, 86 | "scripts": { 87 | "postpack": "rm -f oclif.manifest.json", 88 | "posttest": "tslint -p test -t stylish", 89 | "prepack": "rm -rf lib && tsc -b && oclif-dev manifest && oclif-dev readme", 90 | "test": "nyc --extension .ts mocha --forbid-only \"test/**/*.test.ts\"", 91 | "version": "oclif-dev readme && git add README.md", 92 | "build": "tsc -b", 93 | "typedoc-sfdmu-run-addons": "typedoc --options typedoc-sfdmu-run-addons.json" 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/modules/models/api_models/ApiInfo.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { 9 | MESSAGE_IMPORTANCE, 10 | RESULT_STATUSES, 11 | } from '../../components/common_components/enumerations'; 12 | import ApiResultRecord from './ApiResultRecord'; 13 | 14 | /** 15 | * Holds information about the progress of the executed SFDC API callouts 16 | */ 17 | export default class ApiInfo { 18 | 19 | constructor(init?: Partial) { 20 | Object.assign(this, init); 21 | this.resultRecords = this.resultRecords || new Array(); 22 | } 23 | 24 | sObjectName: string; 25 | strOperation: string; 26 | 27 | contentUrl: string; 28 | 29 | job: any; 30 | jobId: string; 31 | batchId: string; 32 | jobState: "Undefined" | "Info" | "OperationStarted" | "OperationFinished" | "Open" | "Closed" | "Aborted" | "Failed" | "UploadStart" | "UploadComplete" | "InProgress" | "JobComplete" = "Undefined"; 33 | 34 | errorMessage: string; 35 | errorStack: string; 36 | 37 | numberRecordsProcessed: number; 38 | numberRecordsFailed: number; 39 | 40 | resultRecords: Array; 41 | informationMessageData: Array = new Array(); 42 | 43 | get messageImportance(): MESSAGE_IMPORTANCE { 44 | 45 | switch (this.resultStatus) { 46 | 47 | // Silent 48 | default: 49 | return MESSAGE_IMPORTANCE.Silent; 50 | 51 | // Normal 52 | case RESULT_STATUSES.ApiOperationStarted: 53 | case RESULT_STATUSES.ApiOperationFinished: 54 | return MESSAGE_IMPORTANCE.Normal; 55 | 56 | // Low 57 | case RESULT_STATUSES.InProgress: 58 | case RESULT_STATUSES.DataUploaded: 59 | case RESULT_STATUSES.BatchCreated: 60 | case RESULT_STATUSES.JobCreated: 61 | return MESSAGE_IMPORTANCE.Low; 62 | 63 | // Warn 64 | case RESULT_STATUSES.Completed: 65 | if (this.numberRecordsFailed == 0) 66 | return MESSAGE_IMPORTANCE.Normal; 67 | else 68 | return MESSAGE_IMPORTANCE.Warn; 69 | 70 | // Error 71 | case RESULT_STATUSES.ProcessError: 72 | case RESULT_STATUSES.FailedOrAborted: 73 | return MESSAGE_IMPORTANCE.Error; 74 | 75 | } 76 | } 77 | 78 | get resultStatus(): RESULT_STATUSES { 79 | 80 | if (!!this.errorMessage) { 81 | return RESULT_STATUSES.ProcessError; 82 | } 83 | 84 | switch (this.jobState) { 85 | 86 | default: 87 | return RESULT_STATUSES.Undefined; 88 | 89 | case "Info": 90 | return RESULT_STATUSES.Information; 91 | 92 | case "OperationStarted": 93 | return RESULT_STATUSES.ApiOperationStarted; 94 | 95 | case "OperationFinished": 96 | return RESULT_STATUSES.ApiOperationFinished; 97 | 98 | case "Open": 99 | return RESULT_STATUSES.JobCreated; 100 | 101 | case "UploadStart": 102 | return RESULT_STATUSES.BatchCreated; 103 | 104 | case "UploadComplete": 105 | return RESULT_STATUSES.DataUploaded; 106 | 107 | case "InProgress": 108 | case "Closed": 109 | return RESULT_STATUSES.InProgress; 110 | 111 | case "Aborted": 112 | case "Failed": 113 | return RESULT_STATUSES.FailedOrAborted; 114 | 115 | case "JobComplete": 116 | return RESULT_STATUSES.Completed; 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/RecordsFilter/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import SfdmuRunAddonModule from "../../../components/sfdmu-run/sfdmuRunAddonModule"; 9 | import AddonResult from "../../../components/common/addonResult"; 10 | import IAddonContext from "../../../components/common/IAddonContext"; 11 | import { ADDON_EVENTS } from "../../../../modules/components/common_components/enumerations"; 12 | import { SFDMU_RUN_ADDON_MESSAGES } from "../../../messages/sfdmuRunAddonMessages"; 13 | import { BadWordsFilter } from "./BadWordsFilter"; 14 | import { IRecordsFilter, IRecordsFilterArgs } from "./IRecordsFilter"; 15 | 16 | 17 | 18 | // Arguments passed from the export.json 19 | export interface IOnExecuteArguments extends IRecordsFilterArgs { } 20 | 21 | // Supported events 22 | const CONST = { 23 | SUPPORTED_EVENTS: [ 24 | ADDON_EVENTS.filterRecordsAddons 25 | ] 26 | }; 27 | 28 | // The mapping between the string filter type to the proper fitler class type 29 | type Constructor = new (...args: any[]) => T; 30 | 31 | const filterFactory = ( 32 | args: IOnExecuteArguments, 33 | module: SfdmuRunAddonModule, 34 | RecordsFilterConstructor: Constructor 35 | ): T => new RecordsFilterConstructor(args, module); 36 | 37 | 38 | const filterFactoryMap: Map IRecordsFilter> = new Map IRecordsFilter>([ 39 | ['BadWords', (args, module) => filterFactory(args, module, BadWordsFilter)] 40 | ]) 41 | 42 | 43 | // Module 44 | export default class RecordsFilter extends SfdmuRunAddonModule { 45 | 46 | async onExecute(context: IAddonContext, args: IOnExecuteArguments): Promise { 47 | 48 | // Output startup message 49 | this.runtime.logAddonExecutionStarted(this); 50 | 51 | // Verify the current event 52 | if (!this.runtime.validateSupportedEvents(this, CONST.SUPPORTED_EVENTS)) { 53 | this.runtime.logFormattedWarning(this, 54 | SFDMU_RUN_ADDON_MESSAGES.General_EventNotSupported, 55 | context.eventName, 56 | context.moduleDisplayName, 57 | CONST.SUPPORTED_EVENTS.join()); 58 | return null; 59 | } 60 | 61 | // Create filter factory 62 | const filterFactory = filterFactoryMap.get(args.filterType); 63 | if (!filterFactory) { 64 | this.runtime.logFormattedError(this, SFDMU_RUN_ADDON_MESSAGES.FilterUnknown, args.filterType); 65 | return null; 66 | } 67 | 68 | 69 | // Create filter 70 | try { 71 | let filterInstance: IRecordsFilter = filterFactory(args, this); 72 | 73 | // Ensure filter initialized 74 | if (filterInstance.isInitialized) { 75 | // Execute filter 76 | const task = this.runtime.getPluginTask(this); 77 | if (task) { 78 | task.tempRecords = await filterInstance.filterRecords(task.tempRecords); 79 | } 80 | } 81 | } catch (ex: any) { 82 | this.runtime.logFormattedError(this, SFDMU_RUN_ADDON_MESSAGES.FilterOperationError, args.filterType, ex.message); 83 | } 84 | 85 | // Output shoutdown message 86 | this.runtime.logAddonExecutionFinished(this); 87 | 88 | return null; 89 | } 90 | 91 | async onInit(context: IAddonContext, args: IOnExecuteArguments): Promise { 92 | this.context.moduleDisplayName += ':' + (args.filterType || 'UnknownFilter'); 93 | return null; 94 | } 95 | 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/addons/components/sfdmu-run/sfdmuRunAddonTask.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import SfdmuRunAddonTaskData from "./SfdmuRunAddonTaskData"; 9 | 10 | import { MigrationJobTask, ProcessedData, SFieldDescribe } from "../../../modules/models"; 11 | import { OPERATION } from "../../../modules/components/common_components/enumerations"; 12 | import { ISFdmuRunCustomAddonTask } from "../../modules/sfdmu-run/custom-addons/package"; 13 | 14 | 15 | 16 | export default class SfdmuRunAddonTask implements ISFdmuRunCustomAddonTask { 17 | 18 | #migrationJobTask: MigrationJobTask; 19 | #sourceTaskData: SfdmuRunAddonTaskData; 20 | #targetTaskData: SfdmuRunAddonTaskData; 21 | 22 | 23 | constructor(migrationJobTask: MigrationJobTask) { 24 | this.#migrationJobTask = migrationJobTask; 25 | this.#sourceTaskData = new SfdmuRunAddonTaskData(migrationJobTask.sourceData); 26 | this.#targetTaskData = new SfdmuRunAddonTaskData(migrationJobTask.targetData); 27 | } 28 | 29 | 30 | getTargetCSVFilename(operation: OPERATION, fileNameSuffix?: string): string { 31 | return this.#migrationJobTask.data.getTargetCSVFilename(operation, fileNameSuffix); 32 | } 33 | 34 | get sourceCsvFilename(): string { 35 | return this.#migrationJobTask.data.sourceCsvFilename; 36 | } 37 | 38 | get operation(): OPERATION { 39 | return this.#migrationJobTask.operation; 40 | } 41 | 42 | get sObjectName(): string { 43 | return this.#migrationJobTask.sObjectName; 44 | } 45 | 46 | get targetSObjectName(): string { 47 | return this.#migrationJobTask.scriptObject.targetObjectName; 48 | } 49 | 50 | get sourceToTargetRecordMap(): Map { 51 | return this.#migrationJobTask.data.sourceToTargetRecordMap; 52 | } 53 | 54 | get sourceToTargetFieldNameMap(): Map { 55 | return this.#migrationJobTask.scriptObject.sourceToTargetFieldNameMap; 56 | } 57 | 58 | get sourceTaskData(): SfdmuRunAddonTaskData { 59 | return this.#sourceTaskData; 60 | } 61 | 62 | get targetTaskData(): SfdmuRunAddonTaskData { 63 | return this.#targetTaskData; 64 | } 65 | 66 | get processedData(): ProcessedData { 67 | return this.#migrationJobTask.processedData; 68 | } 69 | 70 | get updateMode(): "FIRST_UPDATE" | "SECOND_UPDATE" { 71 | return this.#migrationJobTask.updateMode == 'forwards' ? 'FIRST_UPDATE' : 'SECOND_UPDATE'; 72 | } 73 | 74 | get fieldsInQuery(): string[] { 75 | return this.#migrationJobTask.data.fieldsInQuery; 76 | } 77 | 78 | get fieldsToUpdate(): string[] { 79 | return this.#migrationJobTask.data.fieldsToUpdate; 80 | } 81 | 82 | get fieldsInQueryMap(): Map { 83 | return this.#migrationJobTask.scriptObject.fieldsInQueryMap; 84 | } 85 | 86 | get fieldsToUpdateMap(): Map { 87 | return this.#migrationJobTask.scriptObject.fieldsToUpdateMap; 88 | } 89 | 90 | mapRecords(records: Array): void { 91 | this.#migrationJobTask.mapRecords(records); 92 | } 93 | 94 | get tempRecords(): any[] { 95 | return this.#migrationJobTask.tempRecords; 96 | } 97 | set tempRecords(records: any[]) { 98 | // Preserving the refference to the original array 99 | this.#migrationJobTask.tempRecords = this.#migrationJobTask.tempRecords || []; 100 | this.#migrationJobTask.tempRecords.splice(0, this.#migrationJobTask.tempRecords.length); 101 | this.#migrationJobTask.tempRecords = this.#migrationJobTask.tempRecords.concat(records); 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /src/modules/models/script_models/scriptAddonManifestDefinition.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import * as path from 'path'; 9 | 10 | import { 11 | ISfdmuRunCustomAddonScriptAddonManifestDefinition, 12 | } from '../../../addons/modules/sfdmu-run/custom-addons/package'; 13 | import { ADDON_EVENTS } from '../../components/common_components/enumerations'; 14 | import { CONSTANTS } from '../../components/common_components/statics'; 15 | 16 | /** 17 | * Represent an item of the addons section of the ScriptObject / Script classes 18 | * 19 | * @export 20 | * @class AddonManifestDefinition 21 | * @implements {IAddonManifestDefinition} 22 | */ 23 | export default class ScriptAddonManifestDefinition implements ISfdmuRunCustomAddonScriptAddonManifestDefinition { 24 | 25 | // ------------- JSON -------------- 26 | // Common definitions 27 | command: string = "sfdmu:run"; 28 | path: string; 29 | module: string; 30 | description: string; 31 | excluded: boolean; 32 | args: any; 33 | 34 | // Core definitions 35 | objects: string[]; 36 | 37 | constructor(init: Partial) { 38 | if (init) { 39 | Object.assign(this, init); 40 | } 41 | } 42 | 43 | // ----------------------------------- 44 | basePath: string; 45 | 46 | get isCore(): boolean { 47 | return this.moduleName && this.moduleName.startsWith(CONSTANTS.CORE_ADDON_MODULES_NAME_PREFIX); 48 | } 49 | 50 | get moduleName(): string { 51 | let name = this.module || this.path; 52 | if (name) { 53 | return path.basename(name); 54 | } 55 | } 56 | 57 | get moduleDisplayName(): string { 58 | if (this.moduleName.indexOf(':') >= 0) return this.moduleName; 59 | return this.isCore ? CONSTANTS.CORE_ADDON_MODULES_NAME_PREFIX + this.moduleName 60 | : CONSTANTS.CUSTOM_ADDON_MODULES_NAME_PREFIX + this.moduleName; 61 | } 62 | 63 | get isValid(): boolean { 64 | return !!this.moduleName && this.event != ADDON_EVENTS.none; 65 | } 66 | 67 | event: ADDON_EVENTS = ADDON_EVENTS.none; 68 | 69 | objectName: string = ''; 70 | 71 | get moduleRequirePath(): string { 72 | 73 | if (!this.isValid) { 74 | return null; 75 | } 76 | 77 | let requiredPath = ""; 78 | 79 | if (this.module) { 80 | if (this.module.indexOf(CONSTANTS.CORE_ADDON_MODULES_NAME_PREFIX) >= 0) { 81 | // Core module like ":OnBefore" 82 | let modulePath = CONSTANTS.CORE_ADDON_MODULES_BASE_PATH 83 | + this.command.replace(CONSTANTS.CORE_ADDON_MODULES_FOLDER_SEPARATOR, CONSTANTS.CORE_ADDON_MODULES_FOLDER_NAME_SEPARATOR) + '/' // sfdmu-run/ 84 | + this.module.replace(CONSTANTS.CORE_ADDON_MODULES_NAME_PREFIX, '/'); // /OnBefore 85 | requiredPath = path.normalize(path.resolve(__dirname, modulePath)); 86 | } else { 87 | // NPM module 88 | requiredPath = this.module; 89 | } 90 | } else { 91 | // Module by path 92 | if (!path.isAbsolute(this.path)) { 93 | requiredPath = path.resolve(this.isCore ? __dirname : this.basePath, this.path); 94 | } else { 95 | requiredPath = this.path; 96 | } 97 | } 98 | return requiredPath; 99 | 100 | } 101 | 102 | appliedToObject(objectName: string) { 103 | return this.objectName == objectName 104 | || Array.isArray(this.objects) && (this.objects.length == 0 || this.objects.indexOf(objectName) >= 0); 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonApiService.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISfdmuRunCustomAddonCommandRunInfo, ISfdmuRunCustomAddonContext, ISFdmuRunCustomAddonJob, ISfdmuRunCustomAddonModule, ISfdmuRunCustomAddonProcessedData, ISFdmuRunCustomAddonTask } from "."; 9 | 10 | 11 | 12 | /** 13 | * Provides the Custom Add-On Api functionality. 14 | * 15 | * @export 16 | * @interface ISfdmuRunCustomAddonApiService 17 | */ 18 | export default interface ISfdmuRunCustomAddonApiService { 19 | 20 | /** 21 | * Prints message to the console window and to the .log file (if the the file logging is turned on). 22 | * 23 | * @param {ISfdmuRunCustomAddonModule} module The currently running Add-On module instance. 24 | * @param {string} message The message text or the message template to print. 25 | * @param {("INFO" | "WARNING" | "ERROR" | "JSON")} [messageType] The type of the message. 26 | * @param {...string[]} tokens The optional string tokens to replace in the message template. 27 | *
28 | * For example: 29 | * ```ts 30 | * async onExecute(context: ISfdmuRunCustomAddonContext, args: any): Promise { 31 | * this.runtime.service.log(this, 'My %s template', 'INFO', 'cool'); // Outputs 'My cool template' 32 | * } 33 | * ``` 34 | * @memberof ISfdmuRunCustomAddonApiService 35 | */ 36 | log(module: ISfdmuRunCustomAddonModule, message: string | object, messageType?: "INFO" | "WARNING" | "ERROR" | "JSON", ...tokens: string[]): void; 37 | 38 | 39 | /** 40 | * Returns the data which is currently processed by the Plugin. 41 | * Gives the Add-On module ability to access and modify the data 'on-the-fly'. 42 | *
43 | * @example 44 | * ```ts 45 | * async onExecute(context: ISfdmuRunCustomAddonContext, args: any): Promise { 46 | * 47 | * // Get the data from the Plugin runtime context 48 | * const data = this.runtime.service.getProcessedData(context); 49 | * 50 | * // Modify the records which are about to be inserted into the Target 51 | * data.recordsToInsert.forEach(record => record['TEST__c'] = 'text'); 52 | * } 53 | * ``` 54 | * 55 | * @param {ISfdmuRunCustomAddonContext} context The current instance of the Add-On context. 56 | * @return {ISfdmuRunCustomAddonProcessedData} The data processed by the Plugin in the current runtime context. 57 | * @memberof ISfdmuRunCustomAddonApiService 58 | */ 59 | getProcessedData(context: ISfdmuRunCustomAddonContext): ISfdmuRunCustomAddonProcessedData; 60 | 61 | 62 | /** 63 | * Returns the information about the Sfdmu Plugin and executed CLI command. 64 | *
65 | * @example 66 | * ```ts 67 | * async onExecute(context: ISfdmuRunCustomAddonContext, args: any): Promise { 68 | * console.log(this.runtime.service.getPluginRunInfo().pinfo.pluginName); // Outputs 'sfdmu' 69 | * } 70 | * ``` 71 | * 72 | * @return {ISfdmuRunCustomAddonCommandRunInfo} 73 | * @memberof ISfdmuRunCustomAddonApiService 74 | */ 75 | getPluginRunInfo(): ISfdmuRunCustomAddonCommandRunInfo; 76 | 77 | 78 | /** 79 | * Returns the running SFDMU job. 80 | * 81 | * @return {ISFdmuRunCustomAddonJob} 82 | * @memberof ISfdmuRunCustomAddonApiService 83 | */ 84 | getPluginJob(): ISFdmuRunCustomAddonJob; 85 | 86 | 87 | /** 88 | * Returns the running job task. 89 | * 90 | * @param {ISfdmuRunCustomAddonContext} context The current instance of the Add-On context. 91 | * @return {ISFdmuRunCustomAddonTask} 92 | * @memberof ISfdmuRunCustomAddonApiService 93 | */ 94 | getPluginTask(context: ISfdmuRunCustomAddonContext): ISFdmuRunCustomAddonTask; 95 | 96 | 97 | 98 | } -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/CustomSfdmuRunAddonTemplate/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * -------------------------------------------------------------------------- 3 | * This Add-On module is provided AS IS without any guarantee. 4 | * You can use this example to see how to build your own Add-On modules. 5 | * -------------------------------------------------------------------------- 6 | * 7 | * Copyright (c) 2020, salesforce.com, inc. 8 | * All rights reserved. 9 | * SPDX-License-Identifier: BSD-3-Clause 10 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 11 | */ 12 | 13 | // Imports interfaces for custom Add-On module, context, result, and runtime from a package 14 | import { ISfdmuRunCustomAddonContext, ISfdmuRunCustomAddonModule, ISfdmuRunCustomAddonResult, ISfdmuRunCustomAddonRuntime } from "../package"; 15 | 16 | /** 17 | * Defines a class implementing ISfdmuRunCustomAddonModule for custom manipulations of source records 18 | * before uploading to the Target system. 19 | */ 20 | export default class SfdmuCustomAddOnModule implements ISfdmuRunCustomAddonModule { 21 | 22 | /** 23 | * Constructor that initializes the Add-On module with the provided runtime from the Plugin. 24 | * @param {ISfdmuRunCustomAddonRuntime} runtime - Runtime provided by the Plugin to interact with the module. 25 | */ 26 | constructor(runtime: ISfdmuRunCustomAddonRuntime) { 27 | this.runtime = runtime; // Assigns the passed runtime to this class's runtime property. 28 | } 29 | 30 | /** 31 | * The current instance of the Add-On module runtime to interact with the Salesforce Data Migration Utility (SFDMU) Plugin. 32 | */ 33 | runtime: ISfdmuRunCustomAddonRuntime; 34 | 35 | /** 36 | * The main method that executes the custom logic for manipulating source records. 37 | * @param {ISfdmuRunCustomAddonContext} context - Provides context like module display name and event name. 38 | * @param {any} args - Arbitrary arguments that might be passed to the module for processing. 39 | * @returns {Promise} A promise resolving to null to continue the job. 40 | */ 41 | async onExecute(context: ISfdmuRunCustomAddonContext, args: any): Promise { 42 | 43 | // Logs the start of the module execution 44 | this.runtime.service.log(this, `The Add-On module ${context.moduleDisplayName} has been successfully started. The event ${context.eventName} has been fired.`); 45 | 46 | // Logs a blank line for readability in the log output 47 | this.runtime.service.log(this, ''); // Prints new line 48 | // Logs a message indicating that arguments are about to be displayed 49 | this.runtime.service.log(this, 'The arguments are:'); // Prints string 50 | // Logs the passed arguments as formatted JSON 51 | this.runtime.service.log(this, args, "JSON"); // Prints object as formatted JSON 52 | // Logs another blank line for readability in the log output 53 | this.runtime.service.log(this, ''); // Prints new line 54 | 55 | // Retrieves the processed data relevant to the current context 56 | const data = this.runtime.service.getProcessedData(context); 57 | 58 | // Manipulates the source records based on the retrieved data and arguments 59 | [].concat(data.recordsToInsert, data.recordsToUpdate).forEach(record => { 60 | // Attempts to parse a 'LongText__c' field to JSON, or uses an empty JSON object as fallback 61 | const jsonString = String(record['LongText__c']) || '{}'; 62 | // Parses the JSON string into an object 63 | if (jsonString) { 64 | const obj = JSON.parse(jsonString); 65 | // Assigns a value from the parsed object to 'TEST1__c' field of the record 66 | record['TEST1__c'] = obj['TEST1__c']; 67 | } 68 | // Iterates over args if they exist and assigns their properties to the current record 69 | if (args) { 70 | Object.keys(args).forEach(function (prop) { 71 | record[prop] = args[prop]; 72 | }); 73 | } 74 | }); 75 | 76 | // Logs the completion of the module execution 77 | this.runtime.service.log(this, `The Add-On module ${context.moduleDisplayName} has been successfully completed.`); 78 | 79 | // Returns null to indicate that the migration job should continue 80 | return null; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/custom-addons/package/ISFdmuRunCustomAddonTask.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { ISfdmuRunCustomAddonTaskData } from "."; 9 | import { OPERATION } from "./common"; 10 | 11 | 12 | /** 13 | * The currently running migration task. 14 | * Each SFDMU Job has multiple Tasks, each task dedicated to work with one SF object. 15 | * @see {@link ISFdmuRunCustomAddonJob} 16 | * 17 | * @export 18 | * @interface ISFdmuRunCustomAddonTask 19 | */ 20 | export default interface ISFdmuRunCustomAddonTask { 21 | 22 | /** 23 | * The operation, performed with the current SF object. 24 | * 25 | * @type {OPERATION} 26 | * @memberof ISFdmuRunCustomAddonTask 27 | */ 28 | readonly operation: OPERATION; 29 | 30 | 31 | /** 32 | * The Api name of the processed SF object. 33 | * 34 | * @type {string} 35 | * @memberof ISFdmuRunCustomAddonTask 36 | */ 37 | readonly sObjectName: string; 38 | 39 | 40 | /** 41 | * The target Api name of the SF object in case the Field Mapping feature is active. 42 | * 43 | * @type {string} 44 | * @memberof ISFdmuRunCustomAddonTask 45 | */ 46 | readonly targetSObjectName: string; 47 | 48 | 49 | /** 50 | * The mapping between the record retireved from the Source to the record, retireved from the Target. 51 | * The records are matched using the specified externalId. 52 | * 53 | * @type {Map} 54 | * @memberof ISFdmuRunCustomAddonTask 55 | */ 56 | readonly sourceToTargetRecordMap: Map; 57 | 58 | 59 | /** 60 | * The mapping between the source field api names to the target api names in case the Field Mapping feature is active. 61 | * 62 | * @type {Map} 63 | * @memberof ISFdmuRunCustomAddonTask 64 | */ 65 | readonly sourceToTargetFieldNameMap: Map; 66 | 67 | 68 | /** 69 | * The current update mode. 70 | * Each SF object can be updated twice, for the first time when the records are inserted 71 | * and for the second time when the Plugin populates the missing lookups 72 | * from the previously inserted records. 73 | * 74 | * @type {("FIRST_UPDATE" | "SECOND_UPDATE")} 75 | * @memberof ISFdmuRunCustomAddonTask 76 | */ 77 | readonly updateMode: "FIRST_UPDATE" | "SECOND_UPDATE"; 78 | 79 | 80 | /** 81 | * Returns the task data associated with the data Source of this migration task. 82 | * 83 | * @type {ISfdmuRunCustomAddonTaskData} 84 | * @memberof ISFdmuRunCustomAddonTask 85 | */ 86 | readonly sourceTaskData: ISfdmuRunCustomAddonTaskData; 87 | 88 | 89 | /** 90 | * Returns the task data associated with the data Target of this migration task. 91 | * 92 | * @type {ISfdmuRunCustomAddonTaskData} 93 | * @memberof ISFdmuRunCustomAddonTask 94 | */ 95 | readonly targetTaskData: ISfdmuRunCustomAddonTaskData; 96 | 97 | /** 98 | * The list of the fields included in the sobject's query string. 99 | * 100 | * @type {Array} 101 | * @memberof ISFdmuRunCustomAddonTask 102 | */ 103 | readonly fieldsInQuery: Array; 104 | 105 | /** 106 | * The list of the fields which should be updated with this sobject. 107 | * This property returns the subset of the fields from the {@link ISFdmuRunCustomAddonTask.fieldsInQuery | fieldsInQuery}, 108 | * containing only the fields which can be updated in the Target. 109 | * 110 | * @type {Array} 111 | * @memberof ISFdmuRunCustomAddonTask 112 | */ 113 | readonly fieldsToUpdate: Array; 114 | 115 | /** 116 | * Applies the Values Mapping (if set for the associated sobject) to the passed records. 117 | * The value mapping is defined by the ValueMapping.csv file. 118 | * @see {@link https://help.sfdmu.com/full-documentation/advanced-features/values-mapping | Values Mapping} for the detailed information about this featrue. 119 | * 120 | * @param {Array} records The records to map, e.g. the source records. 121 | * @memberof ISFdmuRunCustomAddonTask 122 | */ 123 | mapRecords(records: Array): void; 124 | 125 | /** 126 | * Get or sets the temporary records data during the records transformation process. 127 | * For example you can use this property to access and to modify the live source records when 128 | * firing the filterRecordsAddons Add-On event. 129 | */ 130 | tempRecords: any[]; 131 | 132 | } 133 | -------------------------------------------------------------------------------- /src/addons/messages/sfdmuRunAddonMessages.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | 10 | export enum SFDMU_RUN_ADDON_MESSAGES { 11 | 12 | // Gneral ------------------------- 13 | General_CheckingArgs = "Checking the args ..", 14 | General_EventNotSupported = "The event %s is not supported by the %s. Only %s event(s) supported. The Add-On execution has been aborted", 15 | General_ArgumentsCannotBeParsed = "Error during parsing the Add-On arguments provided with the Script. The Add-On execution has been aborted", 16 | General_AddOnRuntimeError = "Error occured during executing the Add-On module %s. Check the module source code (if it's a Custom Add-On Module) and the args passed", 17 | General_MissingRequiredArguments = 'Missing required arguments: %s. Can not start the Add-on', 18 | 19 | 20 | // ExportFiles Addon ------------------------- 21 | ExportFiles_Initializing = 'Initializing ..', 22 | ExportFiles_Comparing = 'Comparing records ..', 23 | ExportFiles_RetrievedRecords = 'Fetched %s records', 24 | ExportFiles_RecordsToBeProcessed = 'There are %s records to process', 25 | ExportFiles_ProcessedRecords = 'Total %s records have been processed, %s records failed', 26 | ExportFiles_TotalDataVolume = 'The total volume of the data to process: %s items of total %sMb', 27 | ExportFiles_DataWillBeProcessedInChunksOfSize = 'The processed data was splitted into %s chunks with max size of %sMb each chunk', 28 | ExportFiles_ProcessingChunk = "Processing chunk #%s of %s items", 29 | 30 | ExportFiles_TargetIsFileWarning = 'Cannot process Files on CSV sources or targets. Set a salesforce org as the Source and the Target', 31 | ExportFiles_CouldNotFindObjectToProcessWarning = 'Could not find object data to process', 32 | ExportFiles_ReadonlyOperationWarning = 'Cannot process Files on Readonly objects. Define another operation', 33 | 34 | ExportFiles_ReadTargetContentDocumentLinks = 'Fetching target ContentDocumentLink records ..', 35 | ExportFiles_ReadSourceFeedAttachments = 'Fetching source FeedAttachment records ..', 36 | ExportFiles_ReadTargetFeedAttachments = 'Fetching target FeedAttachment records ..', 37 | ExportFiles_DeleteTargetContentDocuments = 'Deleting target ContentDocument records ..', 38 | ExportFiles_NoSourceRecords = 'There are no linked source records found to process.', 39 | ExportFiles_ReadTargetContentVersions = 'Fetching target ContentVersion records ..', 40 | ExportFiles_ReadSourceContentDocumentLinks = 'Fetching source ContentDocumentLink records ..', 41 | ExportFiles_ReadSourceContentVersions = 'Fetching source ContentVersion records ..', 42 | ExportFiles_ExportingContentVersions = 'Transferring ContentVersion binary data ..', 43 | ExportFiles_ExportingContentDocumentLinks = 'Creating target ContentDocumentLink records ..', 44 | ExportFiles_ExportingFeedAttachments = 'Create target FeedAttachment records ..', 45 | ExportFiles_NothingToProceed = "There are no records to proceed", 46 | ExportFiles_NothingToUpdate = "The target Files were deleted. There is no data to Update. Define another operation", 47 | 48 | 49 | // RecordsTransform Addon ---------------------- 50 | RecordsTransform_SourceTaskNotFound = "[WARN] [Field %s]: The task associated with the sobject %s has not been found. The field will be skipped", 51 | RecordsTransform_SourceFieldNotFound = "[WARN] [Field %s]: This field was not found in the source records of the sobject %s. The field will be skipped", 52 | RecordsTransform_TargetFieldNotFound = "[WARN] [Field %s]: This field was not found in the target records of the sobject %s. The field will be skipped", 53 | 54 | 55 | RecordsTransform_CreatingMappingScheme = "Creating the transformation map ..", 56 | RecordsTransform_Tranforming = "Transforming records ..", 57 | RecordsTransform_TotallyTranformed = "Totally transformed %s records", 58 | RecordsTransform_AppliedValueMapping = "Applying the Values Mapping", 59 | 60 | 61 | // RecordsFilter Addon ------------------------- 62 | FilterUnknown = "Unknown filter %s", 63 | FilteringEnd= "%s records has been filtered (%s remaining)", 64 | FilterOperationError = "Error in the target RecordsFilter module %s: %s", 65 | 66 | // Badword filter *** 67 | BadwordFilter_badwordsDetectStart= "Filtering results on badwords from file %s on %s..", 68 | BadwordFilter_badwordsDetectFileError= "Badwords file not found at location %s", 69 | BadwordFilter_badwordsDetectFileEmptyList= "Badwords file %s must contain a property \"badwords\" with at least one word", 70 | BadwordFilter_badwordsDetectRegex= "Full badwords detection regular expression %s", 71 | BadwordFilter_badwordsDetected= "- %s: %s", 72 | } 73 | -------------------------------------------------------------------------------- /src/modules/components/common_components/mockGenerator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { CONSTANTS } from "./statics"; 9 | 10 | 11 | 12 | /** 13 | * Used to generate random values to mask real data 14 | * 15 | * @export 16 | * @class MockGenerator 17 | */ 18 | export class MockGenerator { 19 | 20 | static counter: any; 21 | 22 | 23 | /** 24 | * When used custom sequental generators the function resets the counters. 25 | * Each object property has its own counter. 26 | * The function resets counters for all properties. 27 | */ 28 | public static resetCounter() { 29 | this.counter = { 30 | counter: {} 31 | }; 32 | } 33 | 34 | /** 35 | * Adds custom generators to the casual engine 36 | * @param casual The casual engine instance 37 | */ 38 | public static createCustomGenerators(casual: any) { 39 | 40 | let self = this; 41 | 42 | casual.define('c_seq_number', function (field: any, prefix: any, from: any, step: any) { 43 | if (!self.counter.counter[field]) { 44 | self.counter.counter[field] = +from || 1; 45 | } else { 46 | self.counter.counter[field] = (+self.counter.counter[field]) + step 47 | } 48 | return prefix + self.counter.counter[field]; 49 | }); 50 | 51 | casual.define('c_set_value', function (field: any, value: any = null, originalValue: any = null) { 52 | if (typeof value == 'string') { 53 | return value.replace(CONSTANTS.MOCK_EXPRESSION_ORIGINAL_VALUE, originalValue); 54 | } 55 | return value; 56 | }); 57 | 58 | casual.define('c_seq_date', function (field: any, from: any, step: any) { 59 | step = step || "d"; 60 | if (!self.counter.counter[field]) { 61 | if (!(from instanceof Date)) { 62 | from = new Date(Date.parse(from)); 63 | } 64 | self.counter.counter[field] = (from instanceof Date ? from : new Date()) 65 | } else { 66 | switch (step) { 67 | case "d": 68 | self.counter.counter[field] = new Date(self.counter.counter[field].setDate(self.counter.counter[field].getDate() + 1)); 69 | break; 70 | case "-d": 71 | self.counter.counter[field] = new Date(self.counter.counter[field].setDate(self.counter.counter[field].getDate() - 1)); 72 | break; 73 | 74 | case "m": 75 | self.counter.counter[field] = new Date(self.counter.counter[field].setMonth(self.counter.counter[field].getMonth() + 1)); 76 | break; 77 | case "-m": 78 | self.counter.counter[field] = new Date(self.counter.counter[field].setMonth(self.counter.counter[field].getMonth() - 1)); 79 | break; 80 | 81 | case "y": 82 | self.counter.counter[field] = new Date(self.counter.counter[field].setFullYear(self.counter.counter[field].getFullYear() + 1)); 83 | break; 84 | case "-y": 85 | self.counter.counter[field] = new Date(self.counter.counter[field].setFullYear(self.counter.counter[field].getFullYear() - 1)); 86 | break; 87 | 88 | case "s": 89 | self.counter.counter[field] = new Date(self.counter.counter[field].setSeconds(self.counter.counter[field].getSeconds() + 1)); 90 | break; 91 | case "-s": 92 | self.counter.counter[field] = new Date(self.counter.counter[field].setSeconds(self.counter.counter[field].getSeconds() - 1)); 93 | break; 94 | 95 | case "ms": 96 | self.counter.counter[field] = new Date(self.counter.counter[field].setMilliseconds(self.counter.counter[field].getMilliseconds() + 1)); 97 | break; 98 | case "-ms": 99 | self.counter.counter[field] = new Date(self.counter.counter[field].setMilliseconds(self.counter.counter[field].getMilliseconds() - 1)); 100 | break; 101 | 102 | 103 | default: 104 | break; 105 | } 106 | } 107 | return new Date(self.counter.counter[field].getTime()); 108 | }); 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Salesforce Open Source Community Code of Conduct 2 | 3 | ## About the Code of Conduct 4 | 5 | Equality is a core value at Salesforce. We believe a diverse and inclusive 6 | community fosters innovation and creativity, and are committed to building a 7 | culture where everyone feels included. 8 | 9 | Salesforce open-source projects are committed to providing a friendly, safe, and 10 | welcoming environment for all, regardless of gender identity and expression, 11 | sexual orientation, disability, physical appearance, body size, ethnicity, nationality, 12 | race, age, religion, level of experience, education, socioeconomic status, or 13 | other similar personal characteristics. 14 | 15 | The goal of this code of conduct is to specify a baseline standard of behavior so 16 | that people with different social values and communication styles can work 17 | together effectively, productively, and respectfully in our open source community. 18 | It also establishes a mechanism for reporting issues and resolving conflicts. 19 | 20 | All questions and reports of abusive, harassing, or otherwise unacceptable behavior 21 | in a Salesforce open-source project may be reported by contacting the Salesforce 22 | Open Source Conduct Committee at ossconduct@salesforce.com. 23 | 24 | ## Our Pledge 25 | 26 | In the interest of fostering an open and welcoming environment, we as 27 | contributors and maintainers pledge to making participation in our project and 28 | our community a harassment-free experience for everyone, regardless of gender 29 | identity and expression, sexual orientation, disability, physical appearance, 30 | body size, ethnicity, nationality, race, age, religion, level of experience, education, 31 | socioeconomic status, or other similar personal characteristics. 32 | 33 | ## Our Standards 34 | 35 | Examples of behavior that contributes to creating a positive environment 36 | include: 37 | 38 | * Using welcoming and inclusive language 39 | * Being respectful of differing viewpoints and experiences 40 | * Gracefully accepting constructive criticism 41 | * Focusing on what is best for the community 42 | * Showing empathy toward other community members 43 | 44 | Examples of unacceptable behavior by participants include: 45 | 46 | * The use of sexualized language or imagery and unwelcome sexual attention or 47 | advances 48 | * Personal attacks, insulting/derogatory comments, or trolling 49 | * Public or private harassment 50 | * Publishing, or threatening to publish, others' private information—such as 51 | a physical or electronic address—without explicit permission 52 | * Other conduct which could reasonably be considered inappropriate in a 53 | professional setting 54 | * Advocating for or encouraging any of the above behaviors 55 | 56 | ## Our Responsibilities 57 | 58 | Project maintainers are responsible for clarifying the standards of acceptable 59 | behavior and are expected to take appropriate and fair corrective action in 60 | response to any instances of unacceptable behavior. 61 | 62 | Project maintainers have the right and responsibility to remove, edit, or 63 | reject comments, commits, code, wiki edits, issues, and other contributions 64 | that are not aligned with this Code of Conduct, or to ban temporarily or 65 | permanently any contributor for other behaviors that they deem inappropriate, 66 | threatening, offensive, or harmful. 67 | 68 | ## Scope 69 | 70 | This Code of Conduct applies both within project spaces and in public spaces 71 | when an individual is representing the project or its community. Examples of 72 | representing a project or community include using an official project email 73 | address, posting via an official social media account, or acting as an appointed 74 | representative at an online or offline event. Representation of a project may be 75 | further defined and clarified by project maintainers. 76 | 77 | ## Enforcement 78 | 79 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 80 | reported by contacting the Salesforce Open Source Conduct Committee 81 | at ossconduct@salesforce.com. All complaints will be reviewed and investigated 82 | and will result in a response that is deemed necessary and appropriate to the 83 | circumstances. The committee is obligated to maintain confidentiality with 84 | regard to the reporter of an incident. Further details of specific enforcement 85 | policies may be posted separately. 86 | 87 | Project maintainers who do not follow or enforce the Code of Conduct in good 88 | faith may face temporary or permanent repercussions as determined by other 89 | members of the project's leadership and the Salesforce Open Source Conduct 90 | Committee. 91 | 92 | ## Attribution 93 | 94 | This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant-home], 95 | version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html. 96 | It includes adaptions and additions from [Go Community Code of Conduct][golang-coc], 97 | [CNCF Code of Conduct][cncf-coc], and [Microsoft Open Source Code of Conduct][microsoft-coc]. 98 | 99 | This Code of Conduct is licensed under the [Creative Commons Attribution 3.0 License][cc-by-3-us]. 100 | 101 | [contributor-covenant-home]: https://www.contributor-covenant.org (https://www.contributor-covenant.org/) 102 | [golang-coc]: https://golang.org/conduct 103 | [cncf-coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md 104 | [microsoft-coc]: https://opensource.microsoft.com/codeofconduct/ 105 | [cc-by-3-us]: https://creativecommons.org/licenses/by/3.0/us/ 106 | -------------------------------------------------------------------------------- /messages/run.json: -------------------------------------------------------------------------------- 1 | { 2 | "commandDescription": "Starts the execution of the migration job using the SFDX Data Move Utility Plugin (SFDMU).", 3 | "commandLongDescription": "Starts execution of the migration job using the SFDX Data Move Utility Plugin (SFDMU).\nRefer https://help.sfdmu.com for the detailed help information.", 4 | 5 | "sourceusernameFlagDescription": "Source org username/alias or 'csvfile' for csv load", 6 | "sourceusernameFlagLongDescription": "Provide a username or alias for the source organization, or use 'csvfile' if CSV files are being used as the data source.", 7 | 8 | "pathFlagDescription": "[Optional] Absolute/relative path to the directory containing export.json file", 9 | "pathFlagLongDescription": "[Optional] The absolute or relative path to the directory that contains the working export.json file. If not provided, the command will search for the file in the current directory.", 10 | 11 | "quietFlagDescription": "[Optional] Suppresses stdout logging", 12 | "quietFlagLongDescription": "[Optional] This flag suppresses the output to the standard output (stdout). If file logging is enabled, the command will still log to the file.", 13 | 14 | "silentFlagDescription": "[Optional] Same as --quiet", 15 | "silentFlagLongDescription": "[Optional] The same as the --quiet flag.", 16 | 17 | "conciseFlagDescription": "[Optional] Emits brief command output to stdout", 18 | "conciseFlagLongDescription": "[Optional] This flag enables the output of only important messages to the standard output (stdout), which are necessary for understanding the command progress.", 19 | 20 | "verboseFlagDescription": "[Optional] Emits full command output to stdout", 21 | "verboseFlagLongDescription": "[Optional] This flag enables the output of all messages to the standard output (stdout).", 22 | 23 | "versionFlagDescription": "[Optional] Prints the current version of the command", 24 | "versionFlagLongDescription": "[Optional] Prints the currently installed or linked version of the command.", 25 | 26 | "apiversionFlagDescription": "[Optional] Overrides the api version set in the export.json definition", 27 | "apiversionFlagLongDescription": "[Optional] If specified, it overrides the apiVersion parameter of the export.json file. This value is used for all API requests made by this command.", 28 | 29 | "filelogFlagDescription": "[Optional] Turns onn/off file logging", 30 | "filelogFlagLongDescription": "[Optional] In addition to logging to the standard output (stdout), this flag controls logging to a file. Set this flag to 1 (or omit this flag) to enable file logging, or set it to 0 to disable file logging.", 31 | 32 | "nopromptFlagDescription": "[Optional] Suppresses prompting the user for input or confirmation", 33 | "nopromptLongFlagDescription": "[Optional] Flag to suppress prompting the user for inputs or confirmation. The command will continue using the default options.", 34 | 35 | "nowarningsFlagDescription": "[Optional] Suppresses all warning messages", 36 | "nowarningsLongFlagDescription": "[Optional] Flag to suppress the output of all warning messages to the standard output (stdout).", 37 | 38 | "jsonFlagDescription": "[Optional] Format the command output as json", 39 | "jsonLongFlagDescription": "[Optional] [Optional] If set to true, the command will return the result as a formatted JSON instead of text to the standard output (stdout). The JSON will be emitted to both the log file and stdout after the command is fully completed. The JSON includes all logged messages during the command execution, along with extended information such as execution start time, end time, and elapsed time.", 40 | 41 | "loglevelFlagDescription": "[Optional] File logging level for this command invocation", 42 | "loglevelLongFlagDescription": "[Optional] Specified the type of messages to be logged to file.\nLog file always contains all messages emitted during execution of the command.", 43 | 44 | "canModifyFlagDescription": "[Optional] Allows modification of target production environment without preliminary prompting the user about it.", 45 | "canModifyFlagLongDescription": "[Optional] When the target org is set to Production and this flag is set to the target domain name (e.g., prod-instance.my.salesforce.com), the command will not prompt the user for approval to make modifications. Otherwise, the user will be prompted to prevent accidental destruction of critical data.", 46 | 47 | "simulationFlagDescription": "[Optional] Turn on simulation mode", 48 | "simulationLongFlagDescription": "[Optional] The simulation mode allows you to check which records will be affected by the export.json configuration without actually updating, deleting, or inserting them.", 49 | 50 | "errorMissingRequiredFlag": "Missing required flag(s): %s", 51 | 52 | "useSfFlagDescription": "[Optional] Enables the usage of modern Salesforce CLI (sf-cli) commands instead of deprecated Salesforce DX CLI (sfdx-cli) commands.", 53 | "useSfLongFlagDescription": "[Optional] If set to 'true', forces the Plugin to use the latest Salesforce CLI (sf-cli) commands instead of Salesforce DX CLI (sfdx-cli) commands for deprecated commands. For example, it will use sf org display instead of sfdx force:org:display. Setting this flag to 'false' will force the Plugin to use the deprecated Salesforce DX CLI (sfdx-cli) commands. If not set, the Plugin will use the latest Salesforce CLI (sf-cli) commands by default. This flag is useful when you have both Salesforce CLI (sf-cli) and Salesforce DX CLI (sfdx-cli) installed and you want to force the Plugin to use a specific CLI.", 54 | 55 | "logfullqueryFlagDescription": "[Optional] Enables logging of full SOQL queries", 56 | "logfullqueryLongFlagDescription": "[Optional] If provided, the command will log full SOQL queries instead of short versions." 57 | } 58 | -------------------------------------------------------------------------------- /src/addons/components/common/addonRuntime.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import { Common } from "../../../modules/components/common_components/common"; 10 | import { Logger, LOG_MESSAGE_TYPE, LOG_MESSAGE_VERBOSITY, RESOURCES } from "../../../modules/components/common_components/logger"; 11 | import { SFDMU_RUN_ADDON_MESSAGES } from "../../messages/sfdmuRunAddonMessages"; 12 | import ICommandRunInfo from "../../../modules/models/common_models/ICommandRunInfo"; 13 | import { ITableMessage } from "../../../modules/models/common_models/helper_interfaces"; 14 | import AddonModule from "./addonModule"; 15 | import { BUILTIN_MESSAGES } from "../../../modules/components/common_components/bulitinMessages"; 16 | import { ADDON_EVENTS } from "../../../modules/components/common_components/enumerations"; 17 | 18 | 19 | export default class AddonRuntime { 20 | 21 | runInfo: ICommandRunInfo; 22 | #logger: Logger; 23 | 24 | /** 25 | * Creates an instance of AddonRuntime. 26 | * @param logger - The logger 27 | * @param runInfo - The run info 28 | */ 29 | constructor(logger: Logger, runInfo: ICommandRunInfo) { 30 | this.#logger = logger; 31 | this.runInfo = runInfo; 32 | } 33 | 34 | createFormattedMessage(module: AddonModule, message: SFDMU_RUN_ADDON_MESSAGES | BUILTIN_MESSAGES | string, ...tokens: string[]): string { 35 | switch (message) { 36 | case BUILTIN_MESSAGES.Break: 37 | return ''; 38 | } 39 | let mess = Common.formatStringLog((message || '').toString(), ...tokens); 40 | return this.#logger.getResourceString(RESOURCES.coreAddonMessageTemplate, 41 | module.context.moduleDisplayName, 42 | module.context.objectDisplayName, 43 | mess); 44 | } 45 | 46 | logFormattedInfo(module: AddonModule, message: SFDMU_RUN_ADDON_MESSAGES | string, ...tokens: string[]): void { 47 | this.log(this.createFormattedMessage(module, message, ...tokens), "INFO"); 48 | } 49 | 50 | logFormattedInfoVerbose(module: AddonModule, message: SFDMU_RUN_ADDON_MESSAGES | string, ...tokens: string[]): void { 51 | this.log(this.createFormattedMessage(module, message, ...tokens), "INFO_VERBOSE"); 52 | } 53 | 54 | logFormattedWarning(module: AddonModule, message: SFDMU_RUN_ADDON_MESSAGES | string, ...tokens: string[]): void { 55 | this.log(this.createFormattedMessage(module, message, ...tokens), "WARNING"); 56 | } 57 | 58 | logFormattedError(module: AddonModule, message: SFDMU_RUN_ADDON_MESSAGES | string, ...tokens: string[]): void { 59 | this.log(this.createFormattedMessage(module, message, ...tokens), "ERROR"); 60 | } 61 | 62 | 63 | logFormatted(module: AddonModule, message: BUILTIN_MESSAGES | string, messageType?: "INFO" | "WARNING" | "ERROR", ...tokens: string[]): void { 64 | 65 | switch (messageType) { 66 | case 'ERROR': 67 | this.logFormattedError(module, String(message), ...tokens); 68 | break; 69 | 70 | case 'WARNING': 71 | this.logFormattedWarning(module, String(message), ...tokens); 72 | break; 73 | 74 | default: 75 | this.logFormattedInfo(module, message, ...tokens); 76 | break; 77 | } 78 | } 79 | 80 | 81 | log(message: string | object | ITableMessage, messageType?: "INFO" | "WARNING" | "ERROR" | "OBJECT" | "TABLE" | "JSON" | "INFO_VERBOSE", ...tokens: string[]): void { 82 | 83 | switch (messageType) { 84 | case "WARNING": 85 | this.#logger.warn(message, ...tokens); 86 | break; 87 | 88 | case "ERROR": 89 | this.#logger.error(message, ...tokens); 90 | break; 91 | 92 | case "OBJECT": 93 | this.#logger.objectNormal(message); 94 | break; 95 | 96 | case "TABLE": 97 | this.#logger.log(message, LOG_MESSAGE_TYPE.TABLE, LOG_MESSAGE_VERBOSITY.NORMAL, ...tokens); 98 | break; 99 | 100 | case "JSON": 101 | this.#logger.log(message, LOG_MESSAGE_TYPE.JSON, LOG_MESSAGE_VERBOSITY.NORMAL, ...tokens); 102 | break; 103 | 104 | case "INFO_VERBOSE": 105 | this.#logger.infoVerbose(message, ...tokens); 106 | break; 107 | 108 | default: 109 | this.#logger.infoNormal(message, ...tokens); 110 | break; 111 | } 112 | } 113 | 114 | 115 | logAddonExecutionStarted(module: AddonModule): void { 116 | this.log(RESOURCES.startAddonExecution.toString(), "INFO", module.context.moduleDisplayName, module.context.objectDisplayName); 117 | } 118 | 119 | validateSupportedEvents(module: AddonModule, supportedEvents: ADDON_EVENTS[]) { 120 | const evt = ADDON_EVENTS[module.context.eventName]; 121 | return supportedEvents.some((x: ADDON_EVENTS) => x == evt); 122 | } 123 | 124 | logAddonExecutionFinished(module: AddonModule) { 125 | this.log(RESOURCES.stopAddonExecution.toString(), "INFO", module.context.moduleDisplayName, module.context.objectDisplayName); 126 | } 127 | 128 | 129 | async parallelExecAsync(fns: ((...args: any[]) => Promise)[], thisArg?: any, maxParallelTasks?: number): Promise { 130 | return await Common.parallelExecAsync(fns, thisArg, maxParallelTasks); 131 | } 132 | 133 | 134 | async serialExecAsync(fns: ((...args: any[]) => Promise)[], thisArg?: any): Promise { 135 | return await Common.serialExecAsync(fns, thisArg); 136 | } 137 | 138 | 139 | deleteFolderRecursive(path: string, throwIOErrors?: boolean): void { 140 | Common.deleteFolderRecursive(path, throwIOErrors); 141 | } 142 | 143 | async readCsvFileAsync(filePath: string, linesToRead?: number, columnDataTypeMap?: Map): Promise { 144 | return await Common.readCsvFileAsync(filePath, linesToRead, columnDataTypeMap); 145 | } 146 | 147 | 148 | async writeCsvFileAsync(filePath: string, records: any[], createEmptyFileOnEmptyArray?: boolean): Promise { 149 | return await Common.writeCsvFileAsync(filePath, records, createEmptyFileOnEmptyArray); 150 | } 151 | 152 | 153 | } 154 | -------------------------------------------------------------------------------- /src/commands/sfdmu/run.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { 9 | flags, 10 | FlagsConfig, 11 | SfdxCommand, 12 | } from '@salesforce/command'; 13 | import { Messages } from '@salesforce/core'; 14 | import { AnyJson } from '@salesforce/ts-types'; 15 | 16 | import { IRunProcess } from '../../modules/commands_processors/IRunProcess'; 17 | import { RunCommand } from '../../modules/commands_processors/runCommand'; 18 | import RunCommandExecutor 19 | from '../../modules/commands_processors/runCommandExecutor'; 20 | import { 21 | IResourceBundle, 22 | IUxLogger, 23 | } from '../../modules/components/common_components/logger'; 24 | import ISfdmuCommand from '../../modules/models/common_models/ISfdxCommand'; 25 | 26 | Messages.importMessagesDirectory(__dirname); 27 | 28 | const commandMessages = Messages.loadMessages('sfdmu', 'run'); 29 | const resources = Messages.loadMessages('sfdmu', 'resources'); 30 | export default class Run extends SfdxCommand implements IRunProcess { 31 | 32 | exportJson: string; 33 | 34 | exitProcess: boolean = true; 35 | 36 | m_flags: any; 37 | m_ux: IUxLogger; 38 | 39 | cmd: ISfdmuCommand; 40 | command: RunCommand; 41 | 42 | commandMessages: IResourceBundle = commandMessages; 43 | resources: IResourceBundle = resources; 44 | 45 | protected static supportsUsername = true; 46 | protected static requiresUsername = false; 47 | protected static varargs = false; 48 | 49 | public static description = commandMessages.getMessage('commandDescription'); 50 | public static longDescription = commandMessages.getMessage('commandLongDescription'); 51 | 52 | public static readonly flagsConfig: FlagsConfig = { 53 | sourceusername: flags.string({ 54 | char: "s", 55 | description: commandMessages.getMessage('sourceusernameFlagDescription'), 56 | longDescription: commandMessages.getMessage('sourceusernameFlagLongDescription'), 57 | default: '' 58 | }), 59 | path: flags.directory({ 60 | char: 'p', 61 | description: commandMessages.getMessage('pathFlagDescription'), 62 | longDescription: commandMessages.getMessage('pathFlagLongDescription'), 63 | default: '' 64 | }), 65 | verbose: flags.builtin({ 66 | description: commandMessages.getMessage('verboseFlagDescription'), 67 | longDescription: commandMessages.getMessage('verboseFlagLongDescription') 68 | }), 69 | concise: flags.builtin({ 70 | description: commandMessages.getMessage('conciseFlagDescription'), 71 | longDescription: commandMessages.getMessage('conciseFlagLongDescription'), 72 | }), 73 | quiet: flags.builtin({ 74 | description: commandMessages.getMessage('quietFlagDescription'), 75 | longDescription: commandMessages.getMessage('quietFlagLongDescription'), 76 | }), 77 | silent: flags.boolean({ 78 | description: commandMessages.getMessage("silentFlagDescription"), 79 | longDescription: commandMessages.getMessage("silentFlagLongDescription") 80 | }), 81 | version: flags.boolean({ 82 | char: "v", 83 | description: commandMessages.getMessage("versionFlagDescription"), 84 | longDescription: commandMessages.getMessage("versionFlagLongDescription") 85 | }), 86 | apiversion: flags.builtin({ 87 | description: commandMessages.getMessage("apiversionFlagDescription"), 88 | longDescription: commandMessages.getMessage("apiversionFlagLongDescription") 89 | }), 90 | filelog: flags.integer({ 91 | char: "l", 92 | description: commandMessages.getMessage("filelogFlagDescription"), 93 | longDescription: commandMessages.getMessage("filelogFlagLongDescription"), 94 | default: 1 95 | }), 96 | noprompt: flags.boolean({ 97 | char: "n", 98 | description: commandMessages.getMessage("nopromptFlagDescription"), 99 | longDescription: commandMessages.getMessage("nopromptLongFlagDescription") 100 | }), 101 | json: flags.boolean({ 102 | description: commandMessages.getMessage("jsonFlagDescription"), 103 | longDescription: commandMessages.getMessage("jsonLongFlagDescription"), 104 | default: false 105 | }), 106 | nowarnings: flags.boolean({ 107 | char: "w", 108 | description: commandMessages.getMessage("nowarningsFlagDescription"), 109 | longDescription: commandMessages.getMessage("nowarningsLongFlagDescription") 110 | }), 111 | canmodify: flags.string({ 112 | char: "c", 113 | description: commandMessages.getMessage('canModifyFlagDescription'), 114 | longDescription: commandMessages.getMessage('canModifyFlagLongDescription'), 115 | default: '' 116 | }), 117 | simulation: flags.boolean({ 118 | char: "m", 119 | description: commandMessages.getMessage("simulationFlagDescription"), 120 | longDescription: commandMessages.getMessage("simulationLongFlagDescription") 121 | }), 122 | loglevel: flags.string({ 123 | description: commandMessages.getMessage('loglevelFlagDescription'), 124 | longDescription: commandMessages.getMessage('loglevelLongFlagDescription'), 125 | default: 'trace', 126 | options: ['info', 'debug', 'warn', 'error', 'fatal', 'trace', 'INFO', 'DEBUG', 'WARN', 'ERROR', 'FATAL', 'TRACE'] 127 | }), 128 | usesf: flags.string({ 129 | description: commandMessages.getMessage("useSfFlagDescription"), 130 | longDescription: commandMessages.getMessage("useSfLongFlagDescription"), 131 | default: "true", 132 | options: ['true', 'false', 'TRUE', 'FALSE'] 133 | }), 134 | logfullquery: flags.boolean({ 135 | description: commandMessages.getMessage("logfullqueryFlagDescription"), 136 | longDescription: commandMessages.getMessage("logfullqueryLongFlagDescription") 137 | }), 138 | }; 139 | 140 | 141 | public async run(): Promise { 142 | 143 | this.ux["isOutputEnabled"] = true; 144 | 145 | this.m_flags = this.flags; 146 | this.m_ux = this.ux; 147 | 148 | await RunCommandExecutor.execute(this); 149 | 150 | return {}; 151 | } 152 | 153 | runCommand(): Promise { 154 | throw new Error('Method not implemented.'); 155 | } 156 | 157 | } 158 | 159 | 160 | -------------------------------------------------------------------------------- /src/modules/models/api_models/helper_interfaces.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { CsvChunks } from '../'; 9 | import { 10 | DATA_CACHE_TYPES, 11 | OPERATION, 12 | } from '../../components/common_components/enumerations'; 13 | import { Logger } from '../../components/common_components/logger'; 14 | import { 15 | IFieldMapping, 16 | IOrgConnectionData, 17 | } from '../common_models/helper_interfaces'; 18 | import { 19 | ApiEngineBase, 20 | ApiInfo, 21 | } from './'; 22 | 23 | /** 24 | * The Api process engine reference 25 | * 26 | * @export 27 | * @interface IApiProcess 28 | */ 29 | export interface IApiEngine { 30 | 31 | /** 32 | * Executes complete api operation 33 | * including api job create and api job execute 34 | * 35 | * @param {Array} allRecords The all source records to process 36 | * @param {Function} progressCallback The progress callback function 37 | * @returns {Promise} Returns null when unresolvable error occured 38 | * @memberof IApiProcess 39 | */ 40 | executeCRUD(allRecords: Array, progressCallback: (progress: ApiInfo) => void): Promise>; 41 | 42 | /** 43 | * Executes complete api operation in several threads in parallel 44 | * including api job create and api job execute 45 | * 46 | * @param {Array} allRecords 47 | * @param {(progress: ApiInfo) => void} progressCallback 48 | * @param {number} threadsCount The maximum threads count 49 | * @return {*} {Promise>} 50 | * @memberof IApiEngine 51 | */ 52 | executeCRUDMultithreaded(allRecords: Array, progressCallback: (progress: ApiInfo) => void, threadsCount: number): Promise>; 53 | 54 | 55 | /** 56 | * Creates api job 57 | * @param {Array} allRecords The all source records to process 58 | * @returns {Promise} 59 | * @memberof IApiProcess 60 | */ 61 | createCRUDApiJobAsync: (allrecords: Array) => Promise; 62 | 63 | /** 64 | * Creates api job in simulation mode 65 | * @param {Array} allRecords The all source records to process 66 | * @returns {Promise} 67 | * @memberof IApiProcess 68 | */ 69 | createCRUDSimulationJobAsync(allRecords: Array): Promise; 70 | 71 | 72 | /** 73 | * Executes previously created api job. 74 | * Returns the same records as the input. 75 | * On Insert operation will add a new Record Id value to each returned record. 76 | * 77 | * @param {Function} progressCallback The progress callback function 78 | * @returns {Promise>} Returns null when unresolvable error occured 79 | * @memberof IApiProcess 80 | */ 81 | processCRUDApiJobAsync: (progressCallback: (progress: ApiInfo) => void) => Promise>; 82 | 83 | /** 84 | * Creates and executes api batch on the given chunk of the input records. 85 | * The function is a part of the api job execution. 86 | * 87 | * @param {ICsvChunk} csvChunk The part of the input 88 | * records to process with the batch 89 | * @param {Function} progressCallback The progress callback function 90 | * @returns {Promise>} Returns null when unresolvable error occured 91 | * @memberof IApiProcess 92 | */ 93 | processCRUDApiBatchAsync(csvChunk: ICsvChunk, progressCallback: (progress: ApiInfo) => void): Promise>; 94 | 95 | /** 96 | * Executes batch in a simulation mode 97 | * Returns the same records as the input. 98 | * On Insert operation will create a new 99 | * random Record Id value to each returned record. 100 | * 101 | * @param {Function} progressCallback The progress callback function 102 | * @returns {Promise>} Returns null when unresolvable error occured 103 | * @memberof IApiProcess 104 | */ 105 | processCRUDSimulationBatchAsync(csvChunk: ICsvChunk, progressCallback: (progress: ApiInfo) => void): Promise>; 106 | 107 | /** 108 | * The name of the current api engine 109 | * 110 | * @returns {string} 111 | * @memberof IApiProcess 112 | */ 113 | getEngineName(): string; 114 | 115 | /** 116 | * Return the name of the current api operation 117 | * 118 | * @returns {string} 119 | * @memberof IApiProcess 120 | */ 121 | getStrOperation(): string; 122 | 123 | /** 124 | * Returns true if this is REST API engine 125 | * 126 | * @return {*} {boolean} 127 | * @memberof IApiEngine 128 | */ 129 | getIsRestApiEngine(): boolean; 130 | 131 | /** 132 | * Returns the runtime type of the current engine 133 | * 134 | * @return {*} {typeof ApiEngineBase} 135 | * @memberof IApiEngine 136 | */ 137 | getEngineClassType(): typeof ApiEngineBase; 138 | 139 | } 140 | 141 | export interface IApiEngineInitParameters { 142 | logger: Logger, 143 | connectionData: IOrgConnectionData, 144 | sObjectName: string, 145 | operation: OPERATION, 146 | pollingIntervalMs: number, 147 | concurrencyMode: string, 148 | updateRecordId: boolean, 149 | targetCSVFullFilename: string, 150 | createTargetCSVFiles: boolean, 151 | bulkApiV1BatchSize?: number, 152 | restApiBatchSize?: number, 153 | allOrNone?: boolean, 154 | targetFieldMapping?: IFieldMapping, 155 | simulationMode?: boolean, 156 | binaryDataCache?: DATA_CACHE_TYPES; 157 | binaryCacheDirectory?: string; 158 | isChildJob?: boolean; 159 | } 160 | 161 | export interface ICsvChunk { 162 | records: Array, 163 | csvString: string 164 | } 165 | 166 | export interface IApiJobCreateResult { 167 | chunks: CsvChunks, 168 | apiInfo: ApiInfo, 169 | connection?: any, 170 | allRecords?: Array 171 | } 172 | 173 | /** 174 | * Holds the meta information about the blob record 175 | */ 176 | export interface IBlobField { 177 | 178 | objectName: string, 179 | fieldName: string, 180 | 181 | /** 182 | * Currently there is only base64 data type, 183 | * but optionally another type can be added. 184 | */ 185 | dataType: 'base64' // | .... 186 | } 187 | 188 | 189 | export interface ICachedRecords { 190 | query: string; 191 | records: any[]; 192 | } 193 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | *This is a suggested `CONTRIBUTING.md` file template for use by open sourced Salesforce projects. The main goal of this file is to make clear the intents and expectations that end-users may have regarding this project and how/if to engage with it. Adjust as needed (especially look for `{project_slug}` which refers to the org and repo name of your project) and remove this paragraph before committing to your repo.* 2 | 3 | # Contributing Guide For SFDX-Data-Move-Utility 4 | 5 | This page lists the operational governance model of this project, as well as the recommendations and requirements for how to best contribute to {PROJECT}. We strive to obey these as best as possible. As always, thanks for contributing – we hope these guidelines make it easier and shed some light on our approach and processes. 6 | 7 | # Governance Model 8 | > Pick the most appropriate one 9 | 10 | ## Community Based 11 | 12 | The intent and goal of open sourcing this project is to increase the contributor and user base. The governance model is one where new project leads (`admins`) will be added to the project based on their contributions and efforts, a so-called "do-acracy" or "meritocracy" similar to that used by all Apache Software Foundation projects. 13 | 14 | > or 15 | 16 | ## Salesforce Sponsored 17 | 18 | The intent and goal of open sourcing this project is to increase the contributor and user base. However, only Salesforce employees will be given `admin` rights and will be the final arbitrars of what contributions are accepted or not. 19 | 20 | > or 21 | 22 | ## Published but not supported 23 | 24 | The intent and goal of open sourcing this project is because it may contain useful or interesting code/concepts that we wish to share with the larger open source community. Although occasional work may be done on it, we will not be looking for or soliciting contributions. 25 | 26 | # Getting started 27 | 28 | Please join the community on {Here list Slack channels, Email lists, Glitter, Discord, etc... links}. Also please make sure to take a look at the project [roadmap](ROADMAP.md) to see where are headed. 29 | 30 | # Issues, requests & ideas 31 | 32 | Use GitHub Issues page to submit issues, enhancement requests and discuss ideas. 33 | 34 | ### Bug Reports and Fixes 35 | - If you find a bug, please search for it in the [Issues](https://github.com/{project_slug}/issues), and if it isn't already tracked, 36 | [create a new issue](https://github.com/{project_slug}/issues/new). Fill out the "Bug Report" section of the issue template. Even if an Issue is closed, feel free to comment and add details, it will still 37 | be reviewed. 38 | - Issues that have already been identified as a bug (note: able to reproduce) will be labelled `bug`. 39 | - If you'd like to submit a fix for a bug, [send a Pull Request](#creating_a_pull_request) and mention the Issue number. 40 | - Include tests that isolate the bug and verifies that it was fixed. 41 | 42 | ### New Features 43 | - If you'd like to add new functionality to this project, describe the problem you want to solve in a [new Issue](https://github.com/{project_slug}/issues/new). 44 | - Issues that have been identified as a feature request will be labelled `enhancement`. 45 | - If you'd like to implement the new feature, please wait for feedback from the project 46 | maintainers before spending too much time writing the code. In some cases, `enhancement`s may 47 | not align well with the project objectives at the time. 48 | 49 | ### Tests, Documentation, Miscellaneous 50 | - If you'd like to improve the tests, you want to make the documentation clearer, you have an 51 | alternative implementation of something that may have advantages over the way its currently 52 | done, or you have any other change, we would be happy to hear about it! 53 | - If its a trivial change, go ahead and [send a Pull Request](#creating_a_pull_request) with the changes you have in mind. 54 | - If not, [open an Issue](https://github.com/{project_slug}/issues/new) to discuss the idea first. 55 | 56 | If you're new to our project and looking for some way to make your first contribution, look for 57 | Issues labelled `good first contribution`. 58 | 59 | # Contribution Checklist 60 | 61 | - [x] Clean, simple, well styled code 62 | - [x] Commits should be atomic and messages must be descriptive. Related issues should be mentioned by Issue number. 63 | - [x] Comments 64 | - Module-level & function-level comments. 65 | - Comments on complex blocks of code or algorithms (include references to sources). 66 | - [x] Tests 67 | - The test suite, if provided, must be complete and pass 68 | - Increase code coverage, not versa. 69 | - Use any of our testkits that contains a bunch of testing facilities you would need. For example: `import com.salesforce.op.test._` and borrow inspiration from existing tests. 70 | - [x] Dependencies 71 | - Minimize number of dependencies. 72 | - Prefer Apache 2.0, BSD3, MIT, ISC and MPL licenses. 73 | - [x] Reviews 74 | - Changes must be approved via peer code review 75 | 76 | # Creating a Pull Request 77 | 78 | 1. **Ensure the bug/feature was not already reported** by searching on GitHub under Issues. If none exists, create a new issue so that other contributors can keep track of what you are trying to add/fix and offer suggestions (or let you know if there is already an effort in progress). 79 | 3. **Clone** the forked repo to your machine. 80 | 4. **Create** a new branch to contain your work (e.g. `git br fix-issue-11`) 81 | 4. **Commit** changes to your own branch. 82 | 5. **Push** your work back up to your fork. (e.g. `git push fix-issue-11`) 83 | 6. **Submit** a Pull Request against the `main` branch and refer to the issue(s) you are fixing. Try not to pollute your pull request with unintended changes. Keep it simple and small. 84 | 7. **Sign** the Salesforce CLA (you will be prompted to do so when submitting the Pull Request) 85 | 86 | > **NOTE**: Be sure to [sync your fork](https://help.github.com/articles/syncing-a-fork/) before making a pull request. 87 | 88 | # Contributor License Agreement ("CLA") 89 | In order to accept your pull request, we need you to submit a CLA. You only need 90 | to do this once to work on any of Salesforce's open source projects. 91 | 92 | Complete your CLA here: 93 | 94 | # Issues 95 | We use GitHub issues to track public bugs. Please ensure your description is 96 | clear and has sufficient instructions to be able to reproduce the issue. 97 | 98 | # Code of Conduct 99 | Please follow our [Code of Conduct](CODE_OF_CONDUCT.md). 100 | 101 | # License 102 | By contributing your code, you agree to license your contribution under the terms of our project [LICENSE](LICENSE.txt) and to sign the [Salesforce CLA](https://cla.salesforce.com/sign-cla) 103 | -------------------------------------------------------------------------------- /src/addons/modules/sfdmu-run/RecordsFilter/BadWordsFilter.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | 9 | import SfdmuRunAddonModule from "../../../components/sfdmu-run/sfdmuRunAddonModule"; 10 | import { SFDMU_RUN_ADDON_MESSAGES } from "../../../messages/sfdmuRunAddonMessages"; 11 | import { IRecordsFilter, IRecordsFilterArgs, IRecordsFilterSetting } from "./IRecordsFilter"; 12 | 13 | import * as fs from "fs"; 14 | import * as path from "path"; 15 | 16 | 17 | // Specific interfaces for this filter 18 | interface IBadwordFilterSettings extends IRecordsFilterSetting { 19 | badwordsFile: string; 20 | detectFields: string[]; 21 | highlightWords: boolean; 22 | outputMatches: boolean; 23 | } 24 | 25 | export interface BadwordFilterArgs extends IRecordsFilterArgs { 26 | settings: IBadwordFilterSettings; 27 | } 28 | 29 | // The filter implementation 30 | export class BadWordsFilter implements IRecordsFilter { 31 | 32 | 33 | isInitialized: boolean; 34 | 35 | _args: BadwordFilterArgs; 36 | _module: SfdmuRunAddonModule; 37 | 38 | _detectFields: string[]; 39 | _badwordsFile: string; 40 | _highlightWords: string | boolean; 41 | _outputMatches: boolean; 42 | 43 | _badwords: string[]; 44 | _badwordsRegex: RegExp; 45 | _filteredNumber = 0; 46 | _keptNumber = 0; 47 | 48 | constructor(args: IRecordsFilterArgs, module: SfdmuRunAddonModule) { 49 | 50 | this._args = args as BadwordFilterArgs; 51 | this._module = module; 52 | 53 | // Checking required arguments 54 | var requiredArgs = [ 55 | '', 56 | '.settings', 57 | '.settings.detectFields']; 58 | 59 | for (let prop of requiredArgs) { 60 | prop = `args${prop}`; 61 | if (!eval(`${prop}`)) { 62 | this._module.runtime.logFormattedError(this._module, SFDMU_RUN_ADDON_MESSAGES.General_MissingRequiredArguments, prop); 63 | return; 64 | } 65 | } 66 | 67 | // Locate the badword file 68 | this._args.settings.badwordsFile = this._args.settings.badwordsFile || "badwords.json"; 69 | this._badwordsFile = path.isAbsolute(this._args.settings.badwordsFile) ? this._args.settings.badwordsFile // Absolute path provided -> as is 70 | : path.join(this._module.runtime.basePath, path.normalize(this._args.settings.badwordsFile)); // Relative to the base folder path -> resolving 71 | if (!fs.existsSync(this._badwordsFile)) { 72 | this._module.runtime.logFormattedError(this._module, SFDMU_RUN_ADDON_MESSAGES.BadwordFilter_badwordsDetectFileError); 73 | return; 74 | } 75 | 76 | // Build badwords regex (and complete with values without special characters) 77 | const badwordsConfig = JSON.parse(fs.readFileSync(this._badwordsFile).toString()); 78 | if (badwordsConfig.badwords == null || badwordsConfig.badwords.length === 0) { 79 | // JSON file property must contain at least one badword 80 | this._module.runtime.logFormattedError(this._module, SFDMU_RUN_ADDON_MESSAGES.BadwordFilter_badwordsDetectFileEmptyList, this._badwordsFile); 81 | return ; 82 | } 83 | this._badwords = badwordsConfig.badwords ; 84 | for (const word of this._badwords) { 85 | const wordWithSpecialChars = word.normalize("NFD").replace(/\p{Diacritic}/gu, ""); 86 | if (!this._badwords.includes(wordWithSpecialChars)) { 87 | this._badwords.push(wordWithSpecialChars); 88 | } 89 | } 90 | const regexString = "\\b(" + this._badwords.join("|") + ")\\b"; 91 | this._module.runtime.logFormattedInfoVerbose(this._module, SFDMU_RUN_ADDON_MESSAGES.BadwordFilter_badwordsDetectRegex, regexString); 92 | this._badwordsRegex = new RegExp(regexString, "gmi"); 93 | this._detectFields = this._args.settings.detectFields; 94 | this._outputMatches = this._args.settings.outputMatches; 95 | this._highlightWords = this._args.settings.highlightWords; 96 | 97 | this.isInitialized = true; 98 | } 99 | 100 | async filterRecords(records: any[]): Promise { 101 | this._module.runtime.logFormattedInfo(this._module, 102 | SFDMU_RUN_ADDON_MESSAGES.BadwordFilter_badwordsDetectStart, 103 | this._badwordsFile, 104 | this._detectFields.length > 0 ? this._detectFields.join(",") : "all fields"); 105 | let filteredRecords = records.filter((record) => this._checkRecord(record)); 106 | if (this._highlightWords) { 107 | const replacement = typeof this._highlightWords === 'string' ? this._highlightWords : `***$1***`; 108 | filteredRecords = filteredRecords.map((record) => this._highlightWordsInRecord(record, replacement)); 109 | } 110 | this._module.runtime.logFormattedInfo(this._module, 111 | SFDMU_RUN_ADDON_MESSAGES.FilteringEnd, 112 | this._filteredNumber.toString(), 113 | this._keptNumber.toString()); 114 | return filteredRecords; 115 | } 116 | 117 | 118 | // ------------ Helper methods --------------------------------- 119 | // Check if a record contains at least one of the bad words 120 | private _checkRecord(record: any) { 121 | const fieldsValues = this._getFieldsValues(record); 122 | // Check regex on each field value 123 | const found = []; 124 | for (const [field, value] of fieldsValues) { 125 | if (this._badwordsRegex.test(value)) { 126 | found.push([field, value]); 127 | } 128 | } 129 | // Manage check result 130 | if (found.length > 0) { 131 | if (this._outputMatches) { 132 | const foundStr = found.map(([field, value]) => `${field}: ${value}` + (value.includes("\n") ? "\n" : '')).join(","); 133 | this._module.runtime.logFormattedInfo(this._module, 134 | SFDMU_RUN_ADDON_MESSAGES.BadwordFilter_badwordsDetected, record.Name, 135 | foundStr); 136 | } 137 | this._keptNumber++; 138 | return true; 139 | } 140 | this._filteredNumber++; 141 | return false 142 | } 143 | 144 | // Apply replacement regex on each matching regex element 145 | private _highlightWordsInRecord(record: any, replacement: string) { 146 | const fieldsValues = this._getFieldsValues(record); 147 | for (const [field, value] of fieldsValues) { 148 | if (typeof record[field] === 'string') { 149 | record[field] = value.replace(this._badwordsRegex, replacement); 150 | } 151 | } 152 | return record 153 | } 154 | 155 | private _getFieldsValues(record: any) { 156 | let fieldsValues = []; 157 | if (this._detectFields.length > 0) { 158 | // Return only fields includes in detectFields 159 | fieldsValues = Object.keys(record) 160 | .filter(field => this._detectFields.includes(field)) 161 | .map(field => [field, record[field]]); 162 | } 163 | else { 164 | // Return all fields values 165 | fieldsValues = Object.keys(record).map(field => [field, record[field]]); 166 | } 167 | return fieldsValues; 168 | } 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | } -------------------------------------------------------------------------------- /src/modules/commands_processors/runCommandExecutor.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { Common } from '../components/common_components/common'; 9 | import { ADDON_EVENTS } from '../components/common_components/enumerations'; 10 | import { 11 | COMMAND_EXIT_STATUSES, 12 | LOG_MESSAGE_TYPE, 13 | LOG_MESSAGE_VERBOSITY, 14 | Logger, 15 | RESOURCES, 16 | } from '../components/common_components/logger'; 17 | import { 18 | CommandAbortedByAddOnError, 19 | CommandAbortedByUserError, 20 | CommandExecutionError, 21 | CommandInitializationError, 22 | OrgMetadataError, 23 | SuccessExit, 24 | UnresolvableWarning, 25 | } from '../models'; 26 | import ISfdmuCommand from '../models/common_models/ISfdxCommand'; 27 | import { IRunProcess } from './IRunProcess'; 28 | import { RunCommand } from './runCommand'; 29 | 30 | export default class RunCommandExecutor { 31 | 32 | static async execute(runProcess: IRunProcess): Promise { 33 | 34 | runProcess.m_flags.verbose = runProcess.m_flags.verbose && !runProcess.m_flags.json; 35 | runProcess.m_flags.quiet = runProcess.m_flags.quiet || runProcess.m_flags.silent || runProcess.m_flags.version; 36 | runProcess.m_flags.filelog = !!runProcess.m_flags.filelog && !runProcess.m_flags.version; 37 | runProcess.m_flags.commandoutput = runProcess.m_flags.version; 38 | 39 | runProcess.cmd = { 40 | statics: runProcess["statics"], 41 | argv: runProcess.argv 42 | } as ISfdmuCommand; 43 | 44 | Common.logger = new Logger( 45 | runProcess.resources, 46 | runProcess.commandMessages, 47 | runProcess.m_ux, 48 | runProcess.cmd, 49 | runProcess.m_flags.loglevel, 50 | runProcess.m_flags.path, 51 | runProcess.m_flags.verbose, 52 | runProcess.m_flags.concise, 53 | runProcess.m_flags.quiet, 54 | runProcess.m_flags.json, 55 | runProcess.m_flags.noprompt, 56 | runProcess.m_flags.nowarnings, 57 | runProcess.m_flags.filelog, 58 | runProcess.m_flags.commandoutput); 59 | 60 | try { 61 | 62 | let pinfo = Common.getPluginInfo(runProcess.cmd); 63 | 64 | // Process --version flag 65 | if (runProcess.m_flags.version) { 66 | 67 | // Exit - success 68 | Common.logger.log( 69 | RESOURCES.pluginVersion, 70 | LOG_MESSAGE_TYPE.STDOUT_ONLY, 71 | LOG_MESSAGE_VERBOSITY.ALWAYS, 72 | pinfo.pluginName, pinfo.version); 73 | Common.logger.commandFinishMessage("", COMMAND_EXIT_STATUSES.SUCCESS); 74 | 75 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.SUCCESS); 76 | // -- 77 | } 78 | 79 | // At least one of the orgs is required to be specified. 80 | // If missing, the second org will be the same one. 81 | if (!runProcess.m_flags.sourceusername && !runProcess.m_flags.targetusername) { 82 | throw new CommandInitializationError(runProcess.commandMessages.getMessage('errorMissingRequiredFlag', ['--sourceusername, --targetusername'])); 83 | } 84 | 85 | if (!runProcess.m_flags.sourceusername) { 86 | runProcess.m_flags.sourceusername = runProcess.m_flags.targetusername; 87 | } 88 | 89 | if (!runProcess.m_flags.targetusername) { 90 | runProcess.m_flags.targetusername = runProcess.m_flags.sourceusername; 91 | } 92 | 93 | let commandResult: any; 94 | runProcess.command = new RunCommand(pinfo, 95 | Common.logger, 96 | runProcess.m_flags.path, 97 | runProcess.m_flags.sourceusername, 98 | runProcess.m_flags.targetusername, 99 | runProcess.m_flags.apiversion, 100 | runProcess.m_flags.canmodify, 101 | runProcess.m_flags.simulation, 102 | runProcess.exportJson, 103 | runProcess.m_flags.usesf == "true", 104 | runProcess.m_flags.logfullquery 105 | ); 106 | 107 | const objectSetsAmount = await runProcess.command.loadAsync(); 108 | 109 | for (let objectSetIndex = 0; objectSetIndex < objectSetsAmount; objectSetIndex++) { 110 | await runProcess.command.setupObjectSetAsync(objectSetIndex); 111 | await runProcess.command.createJobAsync(); 112 | await runProcess.command.processCSVFilesAsync(); 113 | await runProcess.command.prepareJobAsync(); 114 | await runProcess.command.runAddonEventAsync(ADDON_EVENTS.onBefore); 115 | await runProcess.command.executeJobAsync(); 116 | await runProcess.command.runAddonEventAsync(ADDON_EVENTS.onAfter); 117 | } 118 | 119 | // Exit - success 120 | Common.logger.commandFinishMessage( 121 | commandResult || RESOURCES.commandSucceededResult, 122 | COMMAND_EXIT_STATUSES.SUCCESS); 123 | 124 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.SUCCESS); 125 | // -- 126 | 127 | } catch (e: any) { 128 | 129 | // Exit - errors 130 | switch (e.constructor) { 131 | 132 | case SuccessExit: 133 | Common.logger.commandFinishMessage( 134 | RESOURCES.commandSucceededResult, 135 | COMMAND_EXIT_STATUSES.SUCCESS); 136 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.SUCCESS); 137 | 138 | 139 | case CommandInitializationError: 140 | Common.logger.commandFinishMessage( 141 | RESOURCES.commandInitializationErrorResult, 142 | COMMAND_EXIT_STATUSES.COMMAND_INITIALIZATION_ERROR, 143 | e.stack, e.message); 144 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.COMMAND_INITIALIZATION_ERROR); 145 | 146 | 147 | case OrgMetadataError: 148 | Common.logger.commandFinishMessage( 149 | RESOURCES.commandOrgMetadataErrorResult, 150 | COMMAND_EXIT_STATUSES.ORG_METADATA_ERROR, 151 | e.stack, e.message); 152 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.ORG_METADATA_ERROR); 153 | 154 | 155 | case CommandExecutionError: 156 | Common.logger.commandFinishMessage( 157 | RESOURCES.commandExecutionErrorResult, 158 | COMMAND_EXIT_STATUSES.COMMAND_EXECUTION_ERROR, 159 | e.stack, e.message); 160 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.COMMAND_EXECUTION_ERROR); 161 | 162 | 163 | case UnresolvableWarning: 164 | Common.logger.commandFinishMessage( 165 | RESOURCES.commandAbortedDueWarningErrorResult, 166 | COMMAND_EXIT_STATUSES.UNRESOLWABLE_WARNING, e.message); 167 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.UNRESOLWABLE_WARNING); 168 | 169 | 170 | case CommandAbortedByUserError: 171 | Common.logger.commandFinishMessage( 172 | RESOURCES.commandAbortedByUserErrorResult, 173 | COMMAND_EXIT_STATUSES.COMMAND_ABORTED_BY_USER, 174 | e.stack, e.message); 175 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.COMMAND_ABORTED_BY_USER); 176 | 177 | case CommandAbortedByAddOnError: 178 | Common.logger.commandFinishMessage( 179 | RESOURCES.commandAbortedByAddOnErrorResult, 180 | COMMAND_EXIT_STATUSES.COMMAND_ABORTED_BY_ADDON, 181 | e.stack, e.message); 182 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.COMMAND_ABORTED_BY_ADDON); 183 | 184 | 185 | default: 186 | Common.logger.commandFinishMessage( 187 | RESOURCES.commandAbortedDueUnexpectedErrorResult, 188 | COMMAND_EXIT_STATUSES.COMMAND_UNEXPECTED_ERROR, 189 | e.stack, e.message); 190 | runProcess.exitProcess && process.exit(COMMAND_EXIT_STATUSES.COMMAND_UNEXPECTED_ERROR); 191 | 192 | } 193 | // -- 194 | } 195 | 196 | return {}; 197 | } 198 | 199 | } 200 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ![SFDMU](src/images/logo-black.png) SFDX Data Move Utility (SFDMU) 2 | 3 | [![Version](https://img.shields.io/npm/v/sfdmu.svg)](https://npmjs.org/package/sfdmu) 4 | [![Downloads/week](https://img.shields.io/npm/dw/sfdmu.svg)](https://npmjs.org/package/sfdmu) 5 | [![Downloads/total](https://img.shields.io/npm/dt/sfdmu.svg)](https://npmjs.org/package/sfdmu) 6 | [![GitHub stars](https://img.shields.io/github/stars/forcedotcom/SFDX-Data-Move-Utility)](https://gitHub.com/forcedotcom/SFDX-Data-Move-Utility/stargazers/) 7 | [![GitHub contributors](https://img.shields.io/github/contributors/forcedotcom/SFDX-Data-Move-Utility.svg)](https://github.com/forcedotcom/SFDX-Data-Move-Utility/graphs/contributors/) 8 | [![License](https://img.shields.io/npm/l/sfdmu.svg)](https://github.com/forcedotcom/SFDX-Data-Move-Utility/blob/master/LICENSE.txt) 9 | [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) 10 | 11 | 12 | The **SFDX Data Move Utility (SFDMU)** is an advanced SFDX plugin designed to streamline data migration within various Salesforce environments, including scratch, development, sandbox, and production orgs. 13 | 14 | This powerful tool supports migration from other Salesforce orgs or CSV files and efficiently manages various data operations, enabling the migration of **multiple related sObjects in a single run**. 15 | 16 | - **[SFDMU GUI Desktop Application:](https://github.com/forcedotcom/SFDX-Data-Move-Utility-Desktop-App)** **A straightforward and intuitive application for creating, managing, and executing data migrations.** 17 | - [**SFDMU Help Center:**](https://help.sfdmu.com/) Comprehensive documentation available. 18 | - [**User Support Policy:**](https://help.sfdmu.com/full-documentation/additional-information/support_policy) Review guidelines before opening support cases. 19 | - [**Contribution Policy:**](https://help.sfdmu.com/full-documentation/additional-information/code_contribution_policy) Learn how to contribute to our project. 20 | 21 | ## Key Features: 22 | - **Comprehensive Migration Support:** Enables direct Org-to-Org data migration, eliminating the need for CSV intermediates, and supports CRUD operations: Insert, Upsert, Update, Delete. 23 | - **Multiple Objects and Relationships:** Manages migrations involving multiple object sets and handles complex relationships, including [circular references](https://help.sfdmu.com/examples/basic-examples#example-1-handling-circular-references). 24 | - **Ease of Use:** Simplifies the configuration process with a [single export.json file](https://help.sfdmu.com/full-configuration). 25 | - **Secure and Local:** Ensures data security as all operations are performed locally without cloud interactions. 26 | - **High Performance:** Optimizes processing by focusing on necessary data subsets. 27 | - **Extended Functionality:** Provides advanced features such as [custom field mapping](https://help.sfdmu.com/full-documentation/advanced-features/fields-mapping), [data anonymization](https://help.sfdmu.com/full-documentation/advanced-features/data-anonymization), and supports [composite external ID keys](https://help.sfdmu.com/full-documentation/advanced-features/composite-external-id-keys) among others. 28 | 29 | ## Installation Instructions: 30 | 1. **Prepare Environment:** Install the Salesforce CLI following the [official instructions](https://developer.salesforce.com/docs/atlas.en-us.sfdx_setup.meta/sfdx_setup/sfdx_setup_install_cli.htm). 31 | 2. **Plugin Installation:** 32 | 33 | ```bash 34 | # Uninstall old version, if any: 35 | $ sf plugins uninstall sfdmu 36 | 37 | # Install the latest version: 38 | $ sf plugins install sfdmu 39 | ``` 40 | 41 | ## Detailed Setup for Developers: 42 | For developers needing customization or access to the source: 43 | ```bash 44 | # Clone the repository: 45 | $ git clone https://github.com/forcedotcom/SFDX-Data-Move-Utility 46 | # Navigate to the directory and link it: 47 | $ cd SFDX-Data-Move-Utility 48 | $ npm install 49 | $ sf plugins link 50 | ``` 51 | 52 | ## Migration Configuration: 53 | 54 | Set up a migration job by creating an `export.json` file with specific data models and operations, as detailed in the [Full export.json Format Guide](https://help.sfdmu.com/full-configuration). 55 | 56 | Here is a basic `export.json` example for upserting Accounts and their related Contacts, assuming a unique Name for Accounts and a unique LastName for Contacts across source and target orgs: 57 | 58 | ```json 59 | { 60 | "objects": [ 61 | { 62 | "operation": "Upsert", 63 | "externalId": "LastName", 64 | "query": "SELECT FirstName, LastName, AccountId FROM Contact", 65 | "master": false 66 | }, 67 | { 68 | "operation": "Upsert", 69 | "externalId": "Name", 70 | "query": "SELECT Name, Phone FROM Account WHERE Name = 'John Smith'" 71 | } 72 | ] 73 | } 74 | ``` 75 | 76 | ### Description of JSON Content: 77 | 78 | 1. **First Object (Contact)**: 79 | - **Operation:** "Upsert" - Can be "Update", "Upsert", "Insert", "Delete", among others as specified in the documentation. 80 | - **External ID:** "LastName" - Used as the unique identifier for Contacts to support upsert operations. 81 | - **Query:** "SELECT FirstName, LastName, AccountId FROM Contact" - Defines the fields to be transferred from the source to the target during the migration. This ensures that only the specified fields are processed. 82 | - **Master:** `false` - This setting ensures that SFDMU only processes Contact records that are related to the specified Accounts. 83 | 84 | 2. **Second Object (Account)**: 85 | - **Operation:** "Upsert" - Specifies the type of operation for Accounts. 86 | - **External ID:** "Name" - The unique identifier for Accounts, used for upsert operations. 87 | - **Query:** "SELECT Name, Phone FROM Account WHERE Name = 'John Smith'" - Selects specific Accounts by Name for the operation. This ensures that only Accounts with the name "John Smith" are targeted for the upsert. 88 | 89 | ## Migration Execution: 90 | 91 | Navigate to the directory where your `export.json` file is located and execute migrations using commands tailored to your source and target, whether they are Salesforce orgs or CSV files: 92 | 93 | ```bash 94 | # Migrate data from one Salesforce org to another 95 | $ sf sfdmu run --sourceusername source.org.username@name.com --targetusername target.org.username@name.com 96 | 97 | # Export data from a Salesforce org to CSV files 98 | $ sf sfdmu run --sourceusername source.org.username@name.com --targetusername csvfile 99 | 100 | # Import data from CSV files to a Salesforce org 101 | $ sf sfdmu run --sourceusername csvfile --targetusername target.org.username@name.com 102 | ``` 103 | 104 | **Note:** 105 | 106 | When importing or exporting from/to CSV files, ensure that the files are located in the directory containing the `export.json` file. The files should be named according to the API name of the respective sObject, such as `Account.csv`, `Contact.csv`. This naming convention helps in accurately mapping the data to the correct sObjects during the import or export process. 107 | 108 | ## Watch the Demo 109 | 110 | - Experience the plugin in action [here](https://www.youtube.com/watch?v=KI_1vD93prA). 111 | 112 | ## Documentation Links: 113 | - [**Getting Started**](https://help.sfdmu.com/get-started) 114 | - [**Installation Guide**](https://help.sfdmu.com/installation) 115 | - [**Configuration Tips**](https://help.sfdmu.com/configuration) 116 | - [**How to Run Migrations**](https://help.sfdmu.com/running) 117 | 118 | - [**Debugging Steps**](https://help.sfdmu.com/debugging) 119 | - [**Detailed export.json Format**](https://help.sfdmu.com/full-configuration) 120 | - [**SFDMU GUI Application Detailed Documentation**](https://help.sfdmu.com/sfdmu-gui-app) 121 | 122 | ## Additional Notes 123 | 124 | - If you encounter permission issues on MacOS, prepend your commands with `sudo`. Adjust CLI command syntax if using the older SFDX CLI platform. 125 | - To allow SFDMU to connect to your source and target orgs, ensure you have established a local connection to these orgs using the standard `sf org login web` commands, as detailed in the [Authorize an Org Using a Browser](https://developer.salesforce.com/docs/atlas.en-us.sfdx_dev.meta/sfdx_dev/sfdx_dev_auth_web_flow.htm) documentation. 126 | 127 | -------------------------------------------------------------------------------- /src/modules/components/api_engines/bulkApiV1_0Engine.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | import { CsvChunks } from '../../models'; 8 | import { 9 | ApiEngineBase, 10 | ApiInfo, 11 | IApiEngineInitParameters, 12 | } from '../../models/api_models'; 13 | import { 14 | IApiEngine, 15 | IApiJobCreateResult, 16 | ICsvChunk, 17 | } from '../../models/api_models/helper_interfaces'; 18 | import { Common } from '../common_components/common'; 19 | import { OPERATION } from '../common_components/enumerations'; 20 | import { Sfdx } from '../common_components/sfdx'; 21 | import { CONSTANTS } from '../common_components/statics'; 22 | 23 | /** 24 | * Implementation of the Salesforce Bulk API v1.0 25 | * 26 | * @export 27 | * @class BulkApiV1_0sf 28 | */ 29 | // tslint:disable-next-line: class-name 30 | export class BulkApiV1_0Engine extends ApiEngineBase implements IApiEngine { 31 | 32 | constructor(init: IApiEngineInitParameters) { 33 | super(init); 34 | } 35 | 36 | 37 | 38 | 39 | // ----------------------- Interface IApiProcess ---------------------------------- 40 | getEngineName(): string { 41 | return "Bulk API V1.0"; 42 | } 43 | 44 | getEngineClassType(): typeof ApiEngineBase { 45 | return BulkApiV1_0Engine; 46 | } 47 | 48 | async createCRUDApiJobAsync(allRecords: Array): Promise { 49 | 50 | this._fixRecords(allRecords); 51 | 52 | let connection = Sfdx.createOrgConnection(this.connectionData); 53 | connection.bulk.pollTimeout = CONSTANTS.POLL_TIMEOUT; 54 | let job = connection.bulk.createJob( 55 | this.sObjectName, 56 | this.getBulkApiStrOperation(), 57 | { 58 | concurrencyMode: this.concurrencyMode 59 | } 60 | ); 61 | let recordChunks = Common.chunkArray(allRecords, this.bulkApiV1BatchSize); 62 | let chunks = new CsvChunks().fromArrayChunks(recordChunks); 63 | this.apiJobCreateResult = { 64 | chunks, 65 | apiInfo: new ApiInfo({ 66 | jobState: "Undefined", 67 | strOperation: this.getStrOperation(), 68 | sObjectName: this.sObjectName, 69 | job, 70 | jobId: job.id, 71 | }), 72 | allRecords, 73 | connection 74 | }; 75 | return this.apiJobCreateResult; 76 | } 77 | 78 | async processCRUDApiBatchAsync(csvChunk: ICsvChunk, progressCallback: (progress: ApiInfo) => void): Promise> { 79 | 80 | let self = this; 81 | 82 | return new Promise>((resolve, reject) => { 83 | if (progressCallback) { 84 | // Progress message: operation started 85 | progressCallback(new ApiInfo({ 86 | jobState: "OperationStarted" 87 | })); 88 | } 89 | 90 | // Create bulk batch and upload csv *************************** 91 | let pollTimer: any; 92 | let numberBatchRecordsProcessed = 0; 93 | let job = this.apiJobCreateResult.apiInfo.job; 94 | let connection = this.apiJobCreateResult.connection; 95 | let records = csvChunk.records; 96 | let batch = job.createBatch(); 97 | batch.execute(records, { headers: CONSTANTS.SFORCE_API_CALL_HEADERS }); 98 | batch.on("error", function (batchInfo: any) { 99 | if (pollTimer) { 100 | clearInterval(pollTimer); 101 | } 102 | if (progressCallback) { 103 | progressCallback(new ApiInfo({ 104 | jobState: "Failed", 105 | errorMessage: batchInfo.stateMessage, 106 | jobId: job.id, 107 | batchId: batch.id 108 | })); 109 | } 110 | // ERROR RESULT 111 | resolve(null); 112 | return; 113 | }); 114 | batch.on("queue", function (batchInfo: any) { 115 | batch.poll(self.pollingIntervalMs, CONSTANTS.POLL_TIMEOUT); 116 | if (progressCallback) { 117 | // Progress message: job was created 118 | progressCallback(new ApiInfo({ 119 | jobState: "Open", 120 | jobId: job.id 121 | })); 122 | // Progress message: batch was created 123 | progressCallback(new ApiInfo({ 124 | jobState: "UploadStart", 125 | jobId: job.id, 126 | batchId: batch.id 127 | })); 128 | } 129 | pollTimer = setInterval(function () { 130 | connection.bulk.job(job.id).batch(batch.id).check((error: any, results: any) => { 131 | if (error) { 132 | clearInterval(pollTimer); 133 | if (progressCallback) { 134 | progressCallback(new ApiInfo({ 135 | jobState: "Failed", 136 | errorMessage: error, 137 | jobId: job.id, 138 | batchId: batch.id 139 | })); 140 | } 141 | // ERROR RESULT 142 | resolve(null); 143 | return; 144 | } 145 | let processed = +results.numberRecordsProcessed; 146 | let failed = +results.numberRecordsFailed; 147 | if (numberBatchRecordsProcessed != processed) { 148 | if (numberBatchRecordsProcessed == 0) { 149 | // First time 150 | // Progress message: data uploaded 151 | progressCallback(new ApiInfo({ 152 | jobState: "UploadComplete", 153 | jobId: job.id, 154 | batchId: batch.id 155 | })); 156 | } 157 | numberBatchRecordsProcessed = processed; 158 | let progress = new ApiInfo({ 159 | jobState: "InProgress", 160 | numberRecordsProcessed: self.numberJobRecordProcessed + processed, 161 | numberRecordsFailed: self.numberJobRecordsFailed + failed, 162 | jobId: job.id, 163 | batchId: batch.id 164 | }); 165 | if (progressCallback) { 166 | // Progress message: N batch records were processed 167 | progressCallback(progress); 168 | } 169 | } 170 | }); 171 | }, self.pollingIntervalMs); 172 | }); 173 | batch.on("response", async function (resultRecords: any) { 174 | clearInterval(pollTimer); 175 | records.forEach((record, index) => { 176 | if (resultRecords[index].success) { 177 | record[CONSTANTS.ERRORS_FIELD_NAME] = null; 178 | if (self.operation == OPERATION.Insert && self.updateRecordId) { 179 | record["Id"] = resultRecords[index].id; 180 | } 181 | self.numberJobRecordProcessed++; 182 | } else { 183 | if (resultRecords[index].errors) { 184 | record[CONSTANTS.ERRORS_FIELD_NAME] = resultRecords[index].errors.join('; '); 185 | } else { 186 | record[CONSTANTS.ERRORS_FIELD_NAME] = null; 187 | } 188 | self.numberJobRecordsFailed++; 189 | self.numberJobRecordProcessed++; 190 | } 191 | }); 192 | if (progressCallback) { 193 | if (self.numberJobRecordsFailed > 0) { 194 | // Some records are failed 195 | progressCallback(new ApiInfo({ 196 | jobState: "JobComplete", 197 | numberRecordsProcessed: self.numberJobRecordProcessed, 198 | numberRecordsFailed: self.numberJobRecordsFailed, 199 | jobId: job.id, 200 | batchId: batch.id 201 | })); 202 | } 203 | // Progress message: operation finished 204 | progressCallback(new ApiInfo({ 205 | jobState: "OperationFinished", 206 | jobId: job.id, 207 | batchId: batch.id 208 | })); 209 | } 210 | // SUCCESS RESULT 211 | resolve(records); 212 | }); 213 | }); 214 | } 215 | 216 | 217 | // ----------------------- ---------------- ------------------------------------------- 218 | private _fixRecords(allRecords: Array) { 219 | allRecords.forEach(record => { 220 | Object.keys(record).forEach(key => { 221 | if (record[key] === '#N/A') { 222 | record[key] = null; 223 | } 224 | }); 225 | }); 226 | } 227 | 228 | } 229 | -------------------------------------------------------------------------------- /src/modules/components/api_engines/restApiEngine.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { CsvChunks } from '../../models'; 9 | import { 10 | ApiEngineBase, 11 | ApiInfo, 12 | IApiEngineInitParameters, 13 | } from '../../models/api_models'; 14 | import { 15 | IApiEngine, 16 | IApiJobCreateResult, 17 | ICsvChunk, 18 | } from '../../models/api_models/helper_interfaces'; 19 | import { Common } from '../common_components/common'; 20 | import { OPERATION } from '../common_components/enumerations'; 21 | import { RESOURCES } from '../common_components/logger'; 22 | import { Sfdx } from '../common_components/sfdx'; 23 | import { CONSTANTS } from '../common_components/statics'; 24 | 25 | /** 26 | * Implementation of the Salesforce REST Api 27 | * 28 | * @export 29 | * @class BulkApiV1_0sf 30 | */ 31 | export class RestApiEngine extends ApiEngineBase implements IApiEngine { 32 | 33 | constructor(init: IApiEngineInitParameters) { 34 | super(init); 35 | } 36 | 37 | 38 | 39 | // ----------------------- Interface IApiProcess ---------------------------------- 40 | getEngineName(): string { 41 | return "REST API"; 42 | } 43 | 44 | async createCRUDApiJobAsync(allRecords: Array): Promise { 45 | 46 | this._fixRecords(allRecords); 47 | 48 | let connection = Sfdx.createOrgConnection(this.connectionData); 49 | let chunks: CsvChunks; 50 | if (!this.restApiBatchSize) { 51 | chunks = new CsvChunks().fromArray(this.getSourceRecordsArray(allRecords)); 52 | } else { 53 | let recordChunks = Common.chunkArray(this.getSourceRecordsArray(allRecords), this.restApiBatchSize); 54 | chunks = new CsvChunks().fromArrayChunks(recordChunks); 55 | } 56 | this.apiJobCreateResult = { 57 | chunks, 58 | apiInfo: new ApiInfo({ 59 | jobState: "Undefined", 60 | strOperation: this.strOperation, 61 | sObjectName: this.sObjectName, 62 | jobId: "REST", 63 | batchId: "REST" 64 | }), 65 | allRecords, 66 | connection 67 | }; 68 | return this.apiJobCreateResult; 69 | } 70 | 71 | async processCRUDApiBatchAsync(csvChunk: ICsvChunk, progressCallback: (progress: ApiInfo) => void): Promise> { 72 | 73 | let self = this; 74 | 75 | return new Promise>((resolve, reject) => { 76 | 77 | self.loadBinaryDataFromCache(csvChunk.records); 78 | 79 | if (progressCallback) { 80 | // Progress message: operation started 81 | progressCallback(new ApiInfo({ 82 | jobState: "OperationStarted" 83 | })); 84 | } 85 | 86 | let connection = this.apiJobCreateResult.connection; 87 | let apiInfo = this.apiJobCreateResult.apiInfo; 88 | let records = csvChunk.records; 89 | let apiFunctionName: string; 90 | switch (this.operation) { 91 | case OPERATION.Insert: apiFunctionName = "create"; break; 92 | case OPERATION.Update: apiFunctionName = "update"; break; 93 | case OPERATION.Delete: apiFunctionName = "del"; break; 94 | default: 95 | // ERROR RESULT 96 | if (progressCallback) { 97 | progressCallback(new ApiInfo({ 98 | jobState: "Failed", 99 | errorMessage: this.logger.getResourceString(RESOURCES.invalidApiOperation), 100 | jobId: apiInfo.jobId, 101 | batchId: apiInfo.batchId 102 | })); 103 | } 104 | // ERROR RESULT 105 | resolve(null); 106 | return; 107 | } 108 | // Progress message: job was created 109 | progressCallback(new ApiInfo({ 110 | jobState: "Open", 111 | jobId: apiInfo.jobId 112 | })); 113 | connection.sobject(this.sObjectName)[apiFunctionName](records, { 114 | allOrNone: this.allOrNone, 115 | allowRecursive: true, 116 | headers: CONSTANTS.SFORCE_API_CALL_HEADERS 117 | }, async function (error: any, resultRecords: any) { 118 | if (error) { 119 | if (progressCallback) { 120 | progressCallback(new ApiInfo({ 121 | jobState: "Failed", 122 | errorMessage: error.message, 123 | jobId: apiInfo.jobId, 124 | batchId: apiInfo.batchId 125 | })); 126 | } 127 | // ERROR RESULT 128 | resolve(null); 129 | } 130 | records = self.getResultRecordsArray(records); 131 | records.forEach((record, index) => { 132 | if (resultRecords[index].success) { 133 | record[CONSTANTS.ERRORS_FIELD_NAME] = null; 134 | if (self.operation == OPERATION.Insert && self.updateRecordId) { 135 | record["Id"] = resultRecords[index].id; 136 | } 137 | self.numberJobRecordProcessed++; 138 | } else { 139 | record[CONSTANTS.ERRORS_FIELD_NAME] = resultRecords[index].errors[0].message; 140 | self.numberJobRecordsFailed++; 141 | self.numberJobRecordProcessed++; 142 | } 143 | }); 144 | if (progressCallback) { 145 | if (self.numberJobRecordsFailed > 0) { 146 | // Some records are failed 147 | progressCallback(new ApiInfo({ 148 | jobState: "JobComplete", 149 | numberRecordsProcessed: self.numberJobRecordProcessed, 150 | numberRecordsFailed: self.numberJobRecordsFailed, 151 | jobId: apiInfo.jobId, 152 | batchId: apiInfo.batchId 153 | })); 154 | } 155 | // Progress message: operation finished 156 | if (self.numberJobRecordProcessed == self.numberJobTotalRecordsToProcess) { 157 | progressCallback(new ApiInfo({ 158 | jobState: "OperationFinished", 159 | numberRecordsProcessed: self.numberJobRecordProcessed, 160 | numberRecordsFailed: self.numberJobRecordsFailed, 161 | jobId: apiInfo.jobId, 162 | batchId: apiInfo.batchId 163 | })); 164 | } else { 165 | progressCallback(new ApiInfo({ 166 | jobState: "InProgress", 167 | numberRecordsProcessed: self.numberJobRecordProcessed, 168 | numberRecordsFailed: self.numberJobRecordsFailed, 169 | jobId: apiInfo.jobId, 170 | batchId: apiInfo.batchId 171 | })); 172 | } 173 | } 174 | //SUCCESS RESULT 175 | resolve(records); 176 | }); 177 | }); 178 | } 179 | 180 | getEngineClassType(): typeof ApiEngineBase { 181 | return RestApiEngine; 182 | } 183 | 184 | // ----------------------- ---------------- ------------------------------------------- 185 | private _fixRecords(allRecords: Array) { 186 | allRecords.forEach(record => { 187 | Object.keys(record).forEach(key => { 188 | if (record[key] === '#N/A') { 189 | record[key] = null; 190 | } 191 | }); 192 | }); 193 | } 194 | 195 | 196 | 197 | 198 | 199 | } 200 | -------------------------------------------------------------------------------- /src/modules/models/script_models/scriptOrg.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, salesforce.com, inc. 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD-3-Clause 5 | * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause 6 | */ 7 | 8 | import { 9 | OrgInfo, 10 | Script, 11 | SObjectDescribe, 12 | } from '../'; 13 | import { 14 | ISfdmuRunCustomAddonScriptOrg, 15 | } from '../../../addons/modules/sfdmu-run/custom-addons/package'; 16 | import { IAppScriptOrg } from '../../app/appModels'; 17 | import { Common } from '../../components/common_components/common'; 18 | import { 19 | DATA_MEDIA_TYPE, 20 | } from '../../components/common_components/enumerations'; 21 | import { RESOURCES } from '../../components/common_components/logger'; 22 | import { Sfdx } from '../../components/common_components/sfdx'; 23 | import { 24 | CommandAbortedByUserError, 25 | CommandInitializationError, 26 | } from '../common_models/errors'; 27 | import { IOrgConnectionData } from '../common_models/helper_interfaces'; 28 | 29 | /** 30 | * Parsed org object 31 | * from the script file 32 | * 33 | * @export 34 | * @class ScriptOrg 35 | */ 36 | export default class ScriptOrg implements IAppScriptOrg, ISfdmuRunCustomAddonScriptOrg { 37 | 38 | // ------------- JSON -------------- 39 | 40 | /** 41 | * Org Username : if org section is specified in the export.json 42 | * Org username/SFDX user alias : if org section is missing but the SFDX CLI connection is used 43 | */ 44 | name: string = ""; 45 | 46 | /** 47 | * Always the Org Username 48 | */ 49 | orgUserName: string = ""; 50 | 51 | instanceUrl: string = ""; 52 | accessToken: string = ""; 53 | 54 | 55 | // ----------------------------------- 56 | script: Script; 57 | media: DATA_MEDIA_TYPE = DATA_MEDIA_TYPE.Org; 58 | isSource: boolean = false; 59 | isPersonAccountEnabled: boolean = false; 60 | orgDescribe: Map = new Map(); 61 | organizationType: "Developer Edition"; 62 | isSandbox: boolean = false; 63 | 64 | get connectionData(): IOrgConnectionData { 65 | return { 66 | instanceUrl: this.instanceUrl, 67 | accessToken: this.accessToken, 68 | apiVersion: this.script.apiVersion, 69 | proxyUrl: this.script.proxyUrl 70 | }; 71 | } 72 | 73 | get isConnected(): boolean { 74 | return !!this.accessToken; 75 | } 76 | 77 | get isFileMedia(): boolean { 78 | return this.media == DATA_MEDIA_TYPE.File; 79 | } 80 | 81 | get isOrgMedia(): boolean { 82 | return this.media == DATA_MEDIA_TYPE.Org; 83 | } 84 | 85 | get isDescribed(): boolean { 86 | return this.orgDescribe.size > 0; 87 | } 88 | 89 | get objectNamesList(): Array { 90 | return [...this.orgDescribe.keys()]; 91 | } 92 | 93 | get isProduction(): boolean { 94 | return !this.isSandbox && this.organizationType != "Developer Edition"; 95 | } 96 | 97 | get isDeveloper(): boolean { 98 | return this.organizationType == "Developer Edition"; 99 | } 100 | 101 | get instanceDomain(): string { 102 | return Common.extractDomainFromUrlString(this.instanceUrl) || ""; 103 | } 104 | 105 | // ----------------------- Public methods ------------------------------------------- 106 | /** 107 | * Setup this object 108 | * 109 | * @returns {Promise} 110 | * @memberof ScriptOrg 111 | */ 112 | async setupAsync(isSource: boolean): Promise { 113 | // Setup variables 114 | this.isSource = isSource; 115 | 116 | // Setup and verify org connection 117 | await this._setupConnection(); 118 | 119 | // Get org describtion 120 | await this._describeOrg(); 121 | } 122 | 123 | /** 124 | * If it is production environment prompts user for the 125 | * "I know what I do" 126 | * 127 | * @return {*} {Promise} 128 | * @memberof ScriptOrg 129 | */ 130 | async promptUserForProductionModificationAsync(): Promise { 131 | // Prompt user if it is production target 132 | let domain = this.instanceDomain.toLowerCase(); 133 | if ( 134 | !this.isFileMedia // It's Org, not File + 135 | && this.isProduction // It's Production + 136 | && this.script.canModify.toLowerCase() != domain // There is no --canmodify flag passed with the CLI command + 137 | && ( 138 | !this.isSource // It's the Target org ... 139 | || this.isSource && this.script.hasDeleteFromSourceObjectOperation // ... or its the Source org but delete from source is now in progress 140 | ) 141 | ) { 142 | // Prompt the user to allow production modifications 143 | let promptMessage = this.script.logger.getResourceString(RESOURCES.canModifyPrompt, domain); 144 | let response = (await this.script.logger.textPromptAsync(promptMessage)).toLowerCase(); 145 | if (response != domain) { 146 | // Abort the job 147 | throw new CommandAbortedByUserError(this.script.logger.getResourceString(RESOURCES.actionNotPermitted)); 148 | } 149 | } 150 | } 151 | 152 | getConnection(): any { 153 | return Sfdx.createOrgConnection(this.connectionData); 154 | } 155 | 156 | 157 | // ----------------------- Private members ------------------------------------------- 158 | private _parseForceOrgDisplayResult(commandResult: string): OrgInfo { 159 | if (!commandResult) return null; 160 | 161 | const jsonObj = JSON.parse(commandResult.replace(/\n/g, ' ')); 162 | if (jsonObj.status || !jsonObj.result) { 163 | return null; 164 | } 165 | 166 | let output: OrgInfo = new OrgInfo(); 167 | Object.assign(output, { 168 | AccessToken: jsonObj.result.accessToken, 169 | ClientId: jsonObj.result.clientId, 170 | ConnectedStatus: jsonObj.result.connectedStatus, 171 | Status: jsonObj.result.status, 172 | OrgId: jsonObj.result.id, 173 | InstanceUrl: jsonObj.result.instanceUrl, 174 | Username: jsonObj.result.username, 175 | }); 176 | 177 | return output; 178 | }; 179 | 180 | private async _validateOrgAsync(): Promise { 181 | 182 | let apiSf = new Sfdx(this); 183 | if (!this.isFileMedia) { 184 | 185 | // Get org info 186 | try { 187 | let ret = await apiSf.queryOrgAsync("SELECT OrganizationType, IsSandbox FROM Organization LIMIT 1", false); 188 | this.isSandbox = ret[0]["IsSandbox"]; 189 | this.organizationType = ret[0]["OrganizationType"]; 190 | } catch (ex) { 191 | throw new CommandInitializationError(this.script.logger.getResourceString(RESOURCES.accessTokenExpired, this.name)); 192 | } 193 | 194 | // Check person account availability 195 | try { 196 | await apiSf.queryOrgAsync("SELECT IsPersonAccount FROM Account LIMIT 1", false); 197 | this.isPersonAccountEnabled = true; 198 | } catch (ex) { 199 | this.isPersonAccountEnabled = false; 200 | } 201 | } 202 | } 203 | 204 | private async _setupConnection(): Promise { 205 | if (!this.isFileMedia) { 206 | 207 | // By default the org username has the same value as the name 208 | this.orgUserName = this.name; 209 | 210 | if (!this.isConnected) { 211 | // Connect with SFDX/SF 212 | let processResult = ""; 213 | if (this.script.useSf) { 214 | this.script.logger.infoNormal(RESOURCES.connectingToOrgSf, this.name); 215 | processResult = Common.execSf("org display --json", this.name) 216 | } else { 217 | this.script.logger.infoNormal(RESOURCES.connectingToOrg, this.name); 218 | processResult = Common.execSfdx("force:org:display --json", this.name); 219 | } 220 | let orgInfo = this._parseForceOrgDisplayResult(processResult); 221 | if (!orgInfo || !orgInfo.isConnected) { 222 | throw new CommandInitializationError(this.script.logger.getResourceString(RESOURCES.connectingFailed, this.name)); 223 | } else { 224 | Object.assign(this, { 225 | accessToken: orgInfo.AccessToken, 226 | instanceUrl: orgInfo.InstanceUrl, 227 | orgUserName: orgInfo.Username 228 | }); 229 | } 230 | } 231 | 232 | // Validate connection and check person account availability 233 | await this._validateOrgAsync(); 234 | 235 | this.script.logger.infoNormal(RESOURCES.successfullyConnected, this.name); 236 | } 237 | } 238 | 239 | private async _describeOrg(): Promise { 240 | try { 241 | if (this.media == DATA_MEDIA_TYPE.Org) { 242 | let apiSf = new Sfdx(this); 243 | this.orgDescribe = (await apiSf.describeOrgAsync()).reduce((acc, describe) => { 244 | acc.set(describe.name, describe); 245 | return acc; 246 | }, new Map()); 247 | } 248 | } catch (ex) { } 249 | } 250 | 251 | } 252 | 253 | 254 | --------------------------------------------------------------------------------