├── .env-local ├── .gitignore ├── .npmignore ├── LICENSE ├── Procfile ├── README.md ├── app.js ├── app.json ├── bin └── run-sfgit.js ├── package.json ├── sfgit.js └── stuff ├── heroku_scheduler.png ├── heroku_scheduler_addon.png └── log_example.png /.env-local: -------------------------------------------------------------------------------- 1 | SF_API_VERSION=35 2 | SF_USERNAME=your@integrationuser.org 3 | SF_PASSWORD=YourIntegrationUserPassword+Token 4 | GIT_IGNORE=__MACOSX,.DS_Store 5 | REPO_URL=https://username:password@github.com/magicdev/reponame 6 | REPO_COMMIT_MESSAGE=Commit message (defaulted to "Automatic commit (sfgit)") 7 | REPO_USER_NAME=Your Name 8 | REPO_USER_EMAIL=Your Email 9 | REPO_README=Content of the README.MD 10 | EXCLUDE_METADATA=ComaSeparated,ListOf,UnwantedMetadata -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | .DS_Store 11 | .env 12 | 13 | pids 14 | logs 15 | results 16 | 17 | npm-debug.log 18 | node_modules -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/enreeco/sf-git/c319145f9e7cfa7fb1aa9033ee267fe1ffe47780/.npmignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2013 Ricard Aspeljung 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: node app.js -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SF-Git 2 | 3 | Author: Enrico Murru (http://enree.co) 4 | 5 | Blog: http://blog.enree.co/2016/01/salesforce-git-automate-metadata-backup.html 6 | 7 | 8 | Deploy all Salesforce Org's metadata into a git repository 9 | 10 | This app is ready to be deployed in Heroku: 11 | 12 | [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) 13 | 14 | The app is ment to be used as a scheduled job (https://devcenter.heroku.com/articles/scheduler): the web server does anything other then redirecting to this repository for instructions. 15 | 16 | Install the *Heroku Scheduler* add-on and configure a new job: 17 | 18 | ![Heroku Scheduler](https://raw.githubusercontent.com/enreeco/sf-git/master/stuff/heroku_scheduler.png) 19 | 20 | **If you want notifications after a run, you need to implement your own bin's job (see the *doAll* callback)** 21 | 22 | ## App Flow 23 | 24 | The app follows the following flow: 25 | 26 | * Logins with an integration user to your ORG 27 | * Lists all metadata 28 | * Retrieves the zipped unpackaged files 29 | * Clones the remote repository 30 | * Unzips the zipped metadata inside the cloned respository 31 | * Performs *git add -A* 32 | * Performs *git commit* 33 | * Perform *giti push* (if commit has differences) 34 | 35 | ## Env. Variables 36 | 37 | This is the list of the supported environment variabled: 38 | * SF_LOGIN_URL: Salesforce login URL 39 | * SF_API_VERSION: Salesforce API version 40 | * SF_USERNAME: Salesforce integration username 41 | * SF_PASSWORD: Salesforce integration password (+token) 42 | * GIT_IGNORE: coma separated list of items to be ignored by git (es. __MACOSX,.DS_Store) [optional] 43 | * REPO_URL: remote git repo's URL (es. https://username:password@github.com/magicdev/reponame) 44 | * REPO_COMMIT_MESSAGE: commit message (defaulted to "Automatic commit (sfgit)") 45 | * REPO_USER_NAME: git actor's name (user.name) 46 | * REPO_USER_EMAIL: git actor's email (user.email) 47 | * REPO_README: content of the README.MD [optional] 48 | * EXCLUDE_METADATA: coma separated list of unwanted metadata items (es. Scontrol,Settings) [optional] 49 | 50 | ## LICENSE 51 | 52 | See the LICENSE file 53 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* 3 | * This web service is only used to create a web process on Heroku. 4 | * Can be completely removed or simply replaced with anything else. 5 | */ 6 | var express = require('express'); 7 | var app = express(); 8 | app.get('/', function(req,res){ 9 | res.status(200); 10 | res.set("Content-Type","application/json"); 11 | res.send('{"status":"SF-Git is Running", "details":"https://github.com/enreeco/sf-git"}'); 12 | }); 13 | var PORT = process.env.PORT || 5000; 14 | module.exports = app.listen(PORT, function() { 15 | console.log("Listening on " + PORT); 16 | }); 17 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SF-Git", 3 | "description": "Git all Salesforce Metadata", 4 | "repository": "https://github.com/enreeco/sf-git", 5 | "logo": "https://avatars3.githubusercontent.com/u/2393696?v=3&s=460", 6 | "keywords": ["salesforce", "metadata", "git"], 7 | "website": "https://enree.co/", 8 | "env": { 9 | "SF_LOGIN_URL": { 10 | "description": "Salesforce login URL", 11 | "required" : false, 12 | "value":"https://login.salesforce.com" 13 | }, 14 | "SF_API_VERSION": { 15 | "description": "Salesforce API version", 16 | "required" : true, 17 | "value":"35" 18 | }, 19 | "SF_USERNAME": { 20 | "description": "Salesforce integration username", 21 | "required" : true, 22 | "value":"" 23 | }, 24 | "SF_PASSWORD": { 25 | "description": "Salesforce integration password (+token)", 26 | "required" : true, 27 | "value":"" 28 | }, 29 | "REPO_URL": { 30 | "description": "Git remote respository URL (es. https://username:password@github.com/magicdev/reponame)", 31 | "required" : true, 32 | "value":"" 33 | }, 34 | "REPO_USER_NAME": { 35 | "description": "Git commit user's name", 36 | "required" : true, 37 | "value":"" 38 | }, 39 | "REPO_USER_Email": { 40 | "description": "Git commit user's email", 41 | "required" : true, 42 | "value":"" 43 | }, 44 | "GIT_IGNORE": { 45 | "description": "Coma separated list of unwanted file types", 46 | "required" : false, 47 | "value":"__MACOSX,.DS_Store" 48 | }, 49 | "REPO_COMMIT_MESSAGE": { 50 | "description": "Commit's message", 51 | "required" : true, 52 | "value":"Automatic commit (sfgit)" 53 | }, 54 | "REPO_README": { 55 | "description": "Optional README.md content", 56 | "required" : false, 57 | "value":"" 58 | }, 59 | "EXCLUDE_METADATA": { 60 | "description": "Coma separated list of unwanted metadata items (es. Scontrol,Settings,CustomLabels)", 61 | "required" : false, 62 | "value":"" 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /bin/run-sfgit.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* 3 | * Use this script with: 4 | * $ heroku run node bin/run-sfgit.js 5 | * Or add the "Heroku Scheduler" add-on and schedule the "node bin/run-sfgit.js" 6 | * command 7 | */ 8 | var sfgit = require('../sfgit'); 9 | sfgit.doAll(function(err, msg){ 10 | console.log(err, msg); 11 | }); 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SF-Git", 3 | "description": "Git commit your SF org", 4 | "version": "0.1.0", 5 | "homepage": "https://github.com/enreeco/sf-git", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/enreeco/sf-git.git" 9 | }, 10 | "author": { 11 | "name": "Enrico Murru", 12 | "email": "enricomurru@gmail.com", 13 | "url": "http://about.me/enreeco" 14 | }, 15 | "licenses": [ 16 | { 17 | "type": "MIT", 18 | "url": "http://github.com/enreeco/sf-git/raw/master/LICENSE" 19 | } 20 | ], 21 | "engines": { 22 | "node": "8.11.*" 23 | }, 24 | "scripts": { 25 | "start": "node app.js", 26 | "start-local": "nf start -s" 27 | }, 28 | "dependencies": { 29 | "adm-zip": ">=0.4.11", 30 | "async": "2.0.0-rc.6", 31 | "express": "^4.16.3", 32 | "fstream": "1.0.*", 33 | "gift": "0.7.*", 34 | "jsforce": "^1.9.0" 35 | }, 36 | "devDependencies": { 37 | "foreman": "*" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /sfgit.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var git = require('gift'); 3 | var fs = require('fs'); 4 | var fstream = require('fstream'); 5 | var jsforce = require('jsforce'); 6 | var async = require('async'); 7 | var AdmZip = require('adm-zip'); 8 | 9 | //muts all logs 10 | var MUTE = false; 11 | 12 | /* 13 | * Creates the return object for the mainCallback 14 | */ 15 | function createReturnObject(err, msg){ 16 | return { 17 | error: err, 18 | details: msg 19 | }; 20 | } 21 | 22 | /* 23 | * Sync Deletes a folder recursively 24 | * @path: folder path 25 | * @exclude: exclude a certain folder's name 26 | * @doNotDeleteRoot: do not delete root folder 27 | */ 28 | var deleteFolderRecursive = function(path, exclude, doNotDeleteRoot) { 29 | if( fs.existsSync(path) ) { 30 | fs.readdirSync(path).forEach(function(file,index){ 31 | var curPath = path + "/" + file; 32 | if(fs.lstatSync(curPath).isDirectory()) { // recurse 33 | if(!exclude || file != exclude){ 34 | deleteFolderRecursive(curPath, exclude); 35 | } 36 | } else { // delete file 37 | fs.unlinkSync(curPath); 38 | } 39 | }); 40 | if(!doNotDeleteRoot) fs.rmdirSync(path); 41 | } 42 | }; 43 | 44 | 45 | module.exports = { 46 | doAll : function(mainCallback){ 47 | 48 | var loginUrl = process.env.SF_LOGIN_URL || "https://login.salesforce.com"; 49 | //status object 50 | var status = { 51 | tempPath : '/tmp/', 52 | zipPath : "zips/", 53 | repoPath: "repos/", 54 | zipFile : "_MyPackage"+Math.random()+".zip", 55 | sfConnection : (new jsforce.Connection({ 56 | loginUrl: loginUrl 57 | })), 58 | sfLoginResult : null, 59 | types : {}, 60 | }; 61 | if(!MUTE) console.log('SF CONNECTION WITH: ' + loginUrl); 62 | //polling timeout of the SF connection 63 | status.sfConnection.metadata.pollTimeout = process.env.SF_METADATA_POLL_TIMEOUT || 120000; 64 | 65 | //creates all the main folders (temp folder, zip folder and git clone folder) 66 | try{ 67 | if(!fs.existsSync(status.tempPath)){ 68 | fs.mkdirSync(status.tempPath); 69 | } 70 | if (!fs.existsSync(status.tempPath+status.zipPath)){ 71 | fs.mkdirSync(status.tempPath+status.zipPath); 72 | } 73 | if (!fs.existsSync(status.tempPath+status.repoPath)){ 74 | fs.mkdirSync(status.tempPath+status.repoPath); 75 | } 76 | }catch(ex){ 77 | return mainCallback && mainCallback(ex); 78 | } 79 | 80 | //asyncs jobs called sequentially (all the tasks to be done) 81 | async.series({ 82 | //login to SF 83 | sfLogin : function(callback){ 84 | if(!MUTE) console.log('SF LOGIN'); 85 | status.sfConnection.login(process.env.SF_USERNAME, process.env.SF_PASSWORD, function(err, lgnResult) { 86 | status.sfLoginResult = lgnResult; 87 | return callback((err)?createReturnObject(err, 'SF Login failed'):null); 88 | }); 89 | }, 90 | //Describes metadata items 91 | sfDescribeMetadata : function(callback){ 92 | if(!MUTE) console.log('SF DESCRIBE METADATA'); 93 | status.sfConnection.metadata.describe(process.env.SF_API_VERSION+'.0', function(err, describe){ 94 | status.sfDescribe = describe; 95 | return callback((err)?createReturnObject(err, 'SF Describe failed'):null); 96 | }); 97 | }, 98 | //Lists of all metadata details 99 | sfListMetadata : function(callback){ 100 | if(!MUTE) console.log('SF LIST DESCRIBE METADATA ALL'); 101 | var iterations = parseInt(Math.ceil(status.sfDescribe.metadataObjects.length/3.0)); 102 | var excludeMetadata = process.env.EXCLUDE_METADATA || ''; 103 | var excludeMetadataList = excludeMetadata.toLowerCase().split(','); 104 | 105 | var asyncObj = {}; 106 | 107 | function listMetadataBatch(qr){ 108 | return function(cback){ 109 | if(!MUTE) console.log('SF LIST DESCRIBE METADATA: '+JSON.stringify(qr)); 110 | status.sfConnection.metadata.list(qr,process.env.SF_API_VERSION+'.0', function(err, fileProperties){ 111 | if(!err && fileProperties){ 112 | for(var ft = 0; ft < fileProperties.length; ft++){ 113 | if(!status.types[fileProperties[ft].type]){ 114 | status.types[fileProperties[ft].type] = []; 115 | } 116 | status.types[fileProperties[ft].type].push(fileProperties[ft].fullName); 117 | } 118 | } 119 | return cback(err); 120 | }); 121 | } 122 | } 123 | 124 | for(var it = 0; it < iterations; it++){ 125 | var query = []; 126 | for(var i = 0; i < 3; i++){ 127 | var index = it*3+i; 128 | 129 | if(status.sfDescribe.metadataObjects.length > index){ 130 | var metadata = status.sfDescribe.metadataObjects[index]; 131 | if(excludeMetadataList.indexOf((metadata.xmlName||'').toLowerCase()) <0){ 132 | query.push({type: metadata.xmlName, folder: metadata.folderName}); 133 | } 134 | } 135 | } 136 | if(query.length>0){ 137 | asyncObj['fn'+it] = listMetadataBatch(query); 138 | } 139 | } 140 | async.series(asyncObj, function(err, results){ 141 | return callback((err)?createReturnObject(err, 'SF Describe list metadata failed'):null); 142 | }); 143 | 144 | 145 | }, 146 | //Retrieving ZIP file of metadata 147 | sfRetrieveZip : function(callback){ 148 | //should use describe 149 | //retrieve xml 150 | if(!MUTE) console.log('SF RETRIEVE ZIP'); 151 | 152 | var _types = []; 153 | for(var t in status.types){ 154 | _types.push({ 155 | members: status.types[t], 156 | name: t, 157 | }); 158 | } 159 | var stream = status.sfConnection.metadata.retrieve({ 160 | unpackaged: { 161 | types: _types, 162 | version: process.env.SF_API_VERSION, 163 | } 164 | }).stream(); 165 | stream.on('end', function() { 166 | return callback(null); 167 | }); 168 | stream.on('error', function(err){ 169 | return callback((err)?createReturnObject(err, 'SF Retrieving metadata ZIP file failed'):null); 170 | }); 171 | stream.pipe(fs.createWriteStream(status.tempPath+status.zipPath+status.zipFile)); 172 | }, 173 | //Clones original repo 174 | gitClone : function(callback){ 175 | if(!MUTE) console.log('GIT CLONE'); 176 | var folderPath = status.tempPath+status.repoPath+status.zipFile; 177 | 178 | git.clone(process.env.REPO_URL, folderPath, 179 | function(err, _repo){ 180 | status.gitRepo = _repo; 181 | //deletes all cloned files except the .git folder (the ZIP file will be the master) 182 | deleteFolderRecursive(folderPath, '.git', true); 183 | return callback((err)?createReturnObject(err, 'Git clone failed'):null); 184 | }); 185 | }, 186 | //Unzip metadata zip file 187 | unzipFile : function(callback){ 188 | 189 | if(!MUTE) console.log('UNZIP FILE'); 190 | 191 | //create .gitignore 192 | var fs = require('fs'); 193 | 194 | var gitIgnoreBody = '#ignore files'; 195 | if(process.env.GIT_IGNORE){ 196 | var spl = process.env.GIT_IGNORE.split(','); 197 | for(var i in spl){ 198 | if(spl[i]){ 199 | gitIgnoreBody+='\n'+spl[i]; 200 | } 201 | } 202 | } 203 | 204 | var readmeBody = process.env.REPO_README || ""; 205 | fs.writeFile(status.tempPath+status.repoPath+status.zipFile+'/README.md', readmeBody, function(err) { 206 | if(err){ 207 | return callback(createReturnObject(err, 'README.md file creation failed')); 208 | } 209 | fs.writeFile(status.tempPath+status.repoPath+status.zipFile+'/.gitignore', gitIgnoreBody, function(err) { 210 | if(err){ 211 | return callback(createReturnObject(err, '.gitignore file creation failed')); 212 | } 213 | try{ 214 | var zip = new AdmZip(status.tempPath+status.zipPath+status.zipFile); 215 | zip.extractAllTo(status.tempPath+status.repoPath+status.zipFile+'/', true); 216 | return callback(null); 217 | }catch(ex){ 218 | return callback(createReturnObject(ex, 'Unzip failed')); 219 | } 220 | }); 221 | }); 222 | 223 | 224 | }, 225 | //Git add new resources 226 | gitAdd : function(callback){ 227 | if(!MUTE) console.log('GIT ADD'); 228 | 229 | status.gitRepo.add("-A",function(err){ 230 | return callback((err)?createReturnObject(err, 'git add failed'):null); 231 | }); 232 | }, 233 | //Git commit 234 | gitCommit : function(callback){ 235 | if(!MUTE) console.log('GIT COMMIT'); 236 | var userName = process.env.REPO_USER_NAME || "Heroku SFGit"; 237 | var userEmail = process.env.REPO_USER_EMAIL || "sfgit@heroku.com"; 238 | status.gitRepo.identify({"name":userName, "email":userEmail}, function(err, oth){ 239 | var commitMessage = process.env.REPO_COMMIT_MESSAGE || 'Automatic commit (sfgit)'; 240 | status.gitRepo.commit(commitMessage, function(err, oth){ 241 | if(err){ 242 | err.details = oth; 243 | } 244 | return callback((err)?createReturnObject(err, 'git commit failed'):null); 245 | }); 246 | }); 247 | }, 248 | //Git push 249 | gitPush : function(callback){ 250 | if(!MUTE) console.log('GIT PUSH'); 251 | 252 | status.gitRepo.remote_push("origin", "master", function(err, oth){ 253 | if(err){ 254 | err.details = oth; 255 | } 256 | return callback((err)?createReturnObject(err, 'git push failed'):null); 257 | }); 258 | }, 259 | }, 260 | function(err, results){ 261 | //deletes all temp files 262 | deleteFolderRecursive(status.tempPath+status.zipPath+'/'); 263 | deleteFolderRecursive(status.tempPath+status.repoPath+'/'); 264 | 265 | if(err 266 | && err.error.details 267 | && (err.error.details.indexOf("up-to-date")>=0 || err.error.details.indexOf("nothing to commit") >=0)){ 268 | console.log('Success', err.error.details); 269 | return mainCallback && mainCallback(null, err.error.details); 270 | } 271 | 272 | var details = (err && err.error && err.details) || null; 273 | if(err){ 274 | console.log("Error occurred",err); 275 | }else{ 276 | console.log('Success'); 277 | details = 'Success'; 278 | } 279 | return mainCallback && mainCallback(err, details); 280 | }) 281 | }, 282 | } -------------------------------------------------------------------------------- /stuff/heroku_scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/enreeco/sf-git/c319145f9e7cfa7fb1aa9033ee267fe1ffe47780/stuff/heroku_scheduler.png -------------------------------------------------------------------------------- /stuff/heroku_scheduler_addon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/enreeco/sf-git/c319145f9e7cfa7fb1aa9033ee267fe1ffe47780/stuff/heroku_scheduler_addon.png -------------------------------------------------------------------------------- /stuff/log_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/enreeco/sf-git/c319145f9e7cfa7fb1aa9033ee267fe1ffe47780/stuff/log_example.png --------------------------------------------------------------------------------