├── .circleci └── config.yml ├── .eslintrc.json ├── .gitignore ├── LICENSE ├── README.md ├── ditto ├── ditto.js ├── mappers │ ├── map.js │ ├── postmap.js │ └── premap.js └── plugins │ ├── README.md │ ├── base64Encode.js │ ├── cleanEmail.js │ ├── cleanString.js │ ├── cleanURI.js │ ├── concatName.js │ ├── concatString.js │ ├── concatWithComma.js │ ├── createURL.js │ ├── data │ ├── countriesKB.js │ └── languages.json │ ├── extractName.js │ ├── formatDate.js │ ├── generateCleanId.js │ ├── generateFacebookImageLink.js │ ├── generateId.js │ ├── generateIdForLinks.js │ ├── generateIdFromLanguageCode.js │ ├── generateUUID.js │ ├── getCountryCode.js │ ├── getCountryName.js │ ├── getLanguageCode.js │ ├── getLanguageFromCode.js │ ├── getLinkService.js │ ├── getLinkType.js │ ├── getValueAtPath.js │ ├── index.js │ ├── minBy.js │ ├── normalizeString.js │ ├── parseDate.js │ ├── parseString.js │ ├── splitList.js │ └── uniqueArray.js ├── package.json └── test ├── ditto.js ├── index.js ├── mappings ├── services │ ├── facebook.js │ ├── facebook_raw.js │ ├── github.js │ ├── github_raw.js │ ├── google.js │ ├── google_raw.js │ ├── linkedin.js │ └── linkedin_raw.js └── test.js ├── plugins ├── base64Encode.js ├── cleanEmail.js ├── cleanString.js ├── cleanURI.js ├── concatName.js ├── concatString.js ├── concatWithComma.js ├── createURL.js ├── extractName.js ├── formatDate.js ├── generateCleanId.js ├── generateFacebookImageLink.js ├── generateFullExperience.js ├── generateFullLocation.js ├── generateId.js ├── generateIdForLinks.js ├── generateIdFromLanguageCode.js ├── generateUUID.js ├── getLanguageCode.js ├── getLanguageFromCode.js ├── getLinkService.js ├── getLinkType.js ├── getValueAtPath.js ├── minBy.js ├── normalizeString.js ├── parseDate.js ├── parseString.js ├── splitList.js └── uniqueArray.js ├── results ├── services │ ├── facebook.js │ ├── facebook_raw.js │ ├── github.js │ ├── github_raw.js │ ├── google.js │ ├── linkedin.js │ └── linkedin_raw.js └── test.js ├── samples ├── services │ ├── facebook.js │ ├── facebook_raw.js │ ├── github.js │ ├── github_raw.js │ ├── google.js │ ├── linkedin.js │ └── linkedin_raw.js └── test.js └── socialServices.js /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Javascript Node CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-javascript/ for more details 4 | # 5 | version: 2 6 | 7 | defaults: &defaults 8 | working_directory: ~/repo 9 | docker: 10 | - image: circleci/node:7.10 11 | 12 | jobs: 13 | build: 14 | <<: *defaults 15 | environment: 16 | CC_TEST_REPORTER_ID: 571fde0d7c7a5c441d9d768838cf38cc2cb5f1b2b5a6396eb598c957fbd6ce09 17 | steps: 18 | - checkout 19 | 20 | - restore_cache: 21 | keys: 22 | - v1-dependencies-{{ checksum "package.json" }} 23 | - v1-dependencies- 24 | 25 | - run: npm install 26 | 27 | - save_cache: 28 | paths: 29 | - node_modules 30 | key: v1-dependencies-{{ checksum "package.json" }} 31 | test: 32 | <<: *defaults 33 | steps: 34 | - run: 35 | name: Setup Code Climate test-reporter 36 | command: | 37 | curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 38 | chmod +x ./cc-test-reporter 39 | - run: npm test 40 | 41 | - run: 42 | name: Run coverage 43 | command: | 44 | sudo npm install -g istanbul 45 | ./cc-test-reporter before-build 46 | npm run cover 47 | ./cc-test-reporter after-build --coverage-input-type lcov coverage/lcvo.info --exit-code $? 48 | deploy: 49 | <<: *defaults 50 | steps: 51 | - run: 52 | name: NPM Deploy 53 | command: | 54 | echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/repo/.npmrc 55 | npm publish 56 | workflows: 57 | version: 2 58 | untagged-build: 59 | jobs: 60 | - build 61 | - test 62 | tagged-build: 63 | jobs: 64 | - build 65 | - test 66 | - deploy: 67 | filters: 68 | tags: 69 | only: /^v.*/ 70 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "eslint:recommended" 4 | ], 5 | "plugins": [ 6 | ], 7 | "parserOptions": { 8 | "ecmaVersion": 9 9 | }, 10 | "env": { 11 | "es6": true, 12 | "node": true, 13 | "mocha": true 14 | }, 15 | "rules": { 16 | "no-mixed-spaces-and-tabs": 0, 17 | "no-unused-vars": 0, 18 | "no-extra-boolean-cast": 0, 19 | "no-useless-escape": 0, 20 | "no-console": 0, 21 | "no-extra-semi": 0, 22 | "no-irregular-whitespace": 0, 23 | "no-inner-declarations": 0, 24 | "no-unsafe-finally": 0 25 | } 26 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | .DS_Store 5 | .vscode 6 | .vs/ 7 | .github/ 8 | 9 | npm-debug.log* 10 | 11 | # Runtime data 12 | pids 13 | *.pid 14 | *.seed 15 | 16 | # Directory for instrumented libs generated by jscoverage/JSCover 17 | lib-cov 18 | 19 | # Coverage directory used by tools like istanbul 20 | coverage 21 | 22 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 23 | .grunt 24 | 25 | # node-waf configuration 26 | .lock-wscript 27 | 28 | # Compiled binary addons (http://nodejs.org/api/addons.html) 29 | build/Release 30 | 31 | # Dependency directory 32 | node_modules 33 | 34 | # Users Environment Variables 35 | .lock-wscript 36 | 37 | cache 38 | 39 | # Optional npm cache directory 40 | .npm 41 | 42 | # Optional REPL history 43 | .node_repl_history 44 | 45 | # sonar relevant files 46 | .sonar 47 | 48 | coverage/ 49 | 50 | .DS_store 51 | .vscode/ 52 | package-lock.json 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Beamery 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /ditto/ditto.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | let defaultPlugins = require('./plugins/index'); 7 | 8 | class Ditto { 9 | 10 | // TODO: Fix this when we have a supportive Node version for ES6 default parameters 11 | constructor(mappings, plugins) { 12 | 13 | this.mappings = mappings; 14 | this.plugins = plugins ? _.merge(defaultPlugins, plugins) : defaultPlugins; 15 | } 16 | 17 | /** 18 | * @function addPlugins 19 | * 20 | * @description Add extra set of plugins to the default ones 21 | * @param {Object} plugins the extra plugins passed to be added to the default set of ditto plugins 22 | */ 23 | addPlugins(plugins) { 24 | 25 | assert.ok(plugins, 'The ditto function should have a valid array of plugins defined to be set'); 26 | 27 | this.plugins = _.merge(defaultPlugins, plugins); 28 | } 29 | 30 | /** 31 | * @function unify 32 | * 33 | * @description Start the unification process 34 | * @param {Object} document the database document to be unified 35 | * @param {Object} mappings the mapping JSON schema 36 | * @param {Function} callback the callback function to be executed after the unification is done 37 | */ 38 | async unify(document, mappings) { 39 | 40 | const _preMap = require('./mappers/premap'); 41 | const _map = require('./mappers/map'); 42 | const _postMap = require('./mappers/postmap'); 43 | 44 | /** 45 | * Check if we pass arguments to the unify function. This will mean that we have a generic class the 46 | * implements this interface and will be used with various mappers and documents 47 | * We will pass each time we call the unify a new mapping file and a document to be mapped and not rely on the 48 | * global ones defined on initialization 49 | */ 50 | 51 | if (document) { 52 | this.document = document; 53 | } 54 | if (mappings) { 55 | this.mappings = mappings; 56 | } 57 | 58 | assert.ok(this.document, 'The ditto does not have a valid source document to unify'); 59 | assert.ok(this.mappings, 'The ditto does not have a valid mapping set defined'); 60 | 61 | const preMap = await _preMap(this.document); 62 | const map = await _map(preMap, this.mappings, this.plugins); 63 | const postMap = await _postMap(map); 64 | 65 | return postMap; 66 | } 67 | } 68 | 69 | module.exports = Ditto; 70 | -------------------------------------------------------------------------------- /ditto/mappers/map.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | 3 | /** 4 | * @function map 5 | * 6 | * @description Start the unification process based on the manual mapping files 7 | * @param {Object} document the database document to be unified 8 | * @param {Function} callback the callback function to be executed after the unification is done 9 | */ 10 | async function map(document, mappings, plugins) { 11 | 12 | /** 13 | * @function processMappings 14 | * 15 | * @description Process the mappings file and map it to the actual values 16 | * Check if the key is static or dynamic assignment. Static assignment assigns directly the 17 | * the hardcoded value into the result object after removing the extra >> signs 18 | * Otherwise, check if the path of the object exists. If so, check if the values have to be 19 | * added in an array (push) or just simple object assignment 20 | * If a prerequisite is defined then we will only assign the value if the prerequisite has passed 21 | * 22 | * @param {Object} document the object document we want to map 23 | * @param {Object} result the object representing the result file 24 | * @param {Object} mappings the object presenting the mappings between target document and mapped one 25 | */ 26 | function processMappings(mappings, document, result, parentKey) { 27 | 28 | _.each(mappings, function(path, key) { 29 | 30 | key = parentKey ? `${parentKey}.${key}` : key; 31 | 32 | // The path can be multiple definitions for retreiving the same piece of information from multiple places 33 | // here we check for that and construct the appropriate document and mappings objects to deal with it 34 | if (_.isArray(path)) { 35 | _.each(path, function(subPath){ 36 | var subMapping = {}; 37 | subMapping[key] = subPath; 38 | return processMappings(subMapping, document, result); 39 | }); 40 | } 41 | 42 | // Check if the path presents a flat object with a value that can be accessible directly with an _.get 43 | if (!_.isPlainObject(path)) { 44 | let value = applyTransformation(key, path, document.$key, document.$value); 45 | 46 | // here we check if the parent key of the value is not defined and only define it at least one value is there 47 | // this resolves the issue of having an empty object defined e.g. location: {} 48 | // the check if (!_.isUndefined(value)) will make sure we have null values to be picked up 49 | // by our postMapper rather than having if (!_.isUndefined(value) && !!value) :) 50 | if (_.isString(value)) { 51 | if (!!value.length) _.set(result, key, value); 52 | } else if (!_.isUndefined(value) && !_.isNull(value)) _.set(result, key, value); 53 | 54 | } else { 55 | 56 | // Check if the object is a nested object of objects or array of objects or not 57 | if (!path.output) { 58 | // Instantiate the empty object in the desired key and pass it to the recursive function 59 | return processMappings(path, document, result, key); 60 | 61 | } else { 62 | 63 | // Reaching here we now know for sure that we will be creating a set of objects (array or object of objects) 64 | // Assign the result object with its correct type defined in the path output 65 | if (!_.has(result, key)) _.set(result, key, path.output) 66 | 67 | _.each(applyTransformation('', path.innerDocument), function($value, $key) { 68 | 69 | // first we need to check if we will be pushing objects or just plain items to an array 70 | // This can be done by checking if we define a mappings object or not 71 | if (path.mappings) { 72 | 73 | var innerResult = {}; 74 | var processingDocument = path.innerDocument === '!' ? document : _.merge(_.cloneDeep($value), {$value: $value, $key: $key}); 75 | 76 | processMappings(path.mappings, processingDocument, innerResult); 77 | 78 | if (_.isArray(path.required) && 79 | _.find(path.required, requiredPath => _.isNil(_.get(innerResult, requiredPath)))){ 80 | innerResult = null; 81 | } 82 | 83 | parseMappings(result, innerResult, path, key, $value, $key); 84 | 85 | } else { 86 | // reaching here means that we are pushing only to a flat array and not an object 87 | if (_.startsWith(path.value, '@')) { 88 | _.updateWith(result, key, function(theArray){ return applyTransformation(key, path.value) }, []); 89 | return false; 90 | } else return _.updateWith(result, key, function(theArray){ theArray.push($value[path.value]); return theArray }, []); 91 | } 92 | 93 | // here we are breaking out of the each if we have defined the innerDocument as the parent document 94 | if (path.innerDocument === '!') return false; 95 | }); 96 | 97 | function parseMappings(result, innerResult, path, key, $value, $key) { 98 | // based on the type of the result [] or {} we will either push or assign with key 99 | if (!!innerResult) { 100 | if (_.isArray(_.get(result,key))) { 101 | if (!!path.prerequisite) { 102 | if (!!eval(path.prerequisite)) _.updateWith(result, key, function(theArray){ theArray.push(innerResult); return theArray }, []); 103 | } else _.updateWith(result, key, function(theArray){ theArray.push(innerResult); return theArray }, []); 104 | } else { 105 | let fullPath = `${key}.${applyTransformation(key, path['key'], $key, $value)}`; 106 | if (!!path.prerequisite) { 107 | if (!!eval(path.prerequisite)) _.set(result, fullPath, innerResult); 108 | } else _.set(result, fullPath, innerResult); 109 | } 110 | } 111 | 112 | // After assigning the innerResult we need to make sure that there are no requirements on it (e.g., unique array) 113 | if (!!path.requirements) { 114 | _.each(path.requirements, function(requirement){ 115 | _.set(result, key, applyTransformation(key, requirement, $key, $value)); 116 | }); 117 | } 118 | return; 119 | } 120 | } 121 | } 122 | }); 123 | 124 | /** @function applyTransformation 125 | * @description Apply a tranformation function on a path 126 | * 127 | * @param {String} path the path to pass for the _.get to retrieve the value 128 | * @param {String} key the key of the result object that will contain the new mapped value 129 | */ 130 | function applyTransformation(key, path, $key, $value) { 131 | 132 | if (path.includes('??')) { 133 | return getValue(path, $value); 134 | } else if (_.startsWith(path, '$')) { 135 | return eval(path); 136 | } else if (_.startsWith(path, '@!')) { 137 | return eval(path.replace('@!', '')); 138 | } else if (_.startsWith(path, '@')) { 139 | 140 | /** 141 | * The parts in the string function are split into: 142 | * before the @ is the first paramteres passed to the function 143 | * the function name is anything after the @ and the () if exists 144 | * the paramteres are anything inside the () separated by a | 145 | */ 146 | let paramteresArray, paramteresValues = []; 147 | 148 | // Regular expression to extract any text between () 149 | let functionParameteres = path.match(/.+?\((.*)\)/); 150 | let functionCall = path.split('(')[0].replace('@', ''); 151 | 152 | // Now we want to split the paramteres by a | in case we pass more than one param 153 | // We also need to check if we are assigning a default value for that function denoted by a * 154 | if (!!functionParameteres) { 155 | 156 | // We need to check if the function parameters have inlined functions that we need to execute 157 | paramteresArray = _.compact(functionParameteres[1].split('|')); 158 | 159 | if (_.last(paramteresArray).includes('*') && !! applyTransformation(key, _.last(paramteresArray).replace('*', '').replace(',', '|'), $key, $value)) { 160 | return applyTransformation(key, _.last(paramteresArray).replace('*', '').replace(',', '|'), $key, $value) 161 | } else { 162 | // we compact the array here to remove any undefined objects that are not caught by the _.get in the map function 163 | paramteresValues = _.union(paramteresValues, _.map(paramteresArray, function(param){ return _.startsWith(param, '$') ? eval(param) : applyTransformation(key, param.replace(',', '|'), $key, $value) })); 164 | } 165 | } 166 | 167 | // Extract the function call and the first parameter 168 | // Now the paramteres array contains the PATH for each element we want to pass to the @function 169 | // We need now to actually get the actual values of these paths 170 | // If the getValues fails that means that we are passing a hardocded value, so pass it as is 171 | 172 | // Only execute the function if the parameters array is not empty 173 | if (!!_.compact(paramteresValues).length && plugins[functionCall]) { 174 | return plugins[functionCall].apply(null, paramteresValues); 175 | } 176 | 177 | } else return getValue(path, $value); 178 | } 179 | 180 | /** 181 | * @function getValue 182 | * @description This function will get the value using the _.get by inspecting the scope of the get 183 | * The existence of the ! will define a local scope in the result object rather than the document 184 | * 185 | * The Flat structure can contain the following cases: 186 | * - starts with !: This will denote that the contact on which the _.get will be on a previously extracted 187 | * value in the result file 188 | * - starts with >>: This means a hardcoded value e.g., >>test -> test 189 | * - contains @: This means that a function will be applied on the value before the @ sign 190 | * The functions first parameter will be the value before the @ and any additional parameters will be defined 191 | * between the parenthesis e.g., name@strip(','') -- will call --> strip(name, ',') 192 | * - contains %: This will denote a casting function to the value using eval 193 | * e.g., >>%true -> will be a true as a boolean and not as a string 194 | * - contains ?? means that a condition has to be applied before assigning the value. The condition to apply 195 | * is the one defined after the -> 196 | * 197 | * @param {String} path the path to pass for the _.get to retrieve the value 198 | * @return {Object} result the object representing the result file 199 | */ 200 | function getValue(path, subDocument) { 201 | 202 | if (!path) return; 203 | 204 | if (path === '!') return document; 205 | 206 | if (_.startsWith(path, '>>')) { 207 | return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); 208 | } else if (_.startsWith(path, '!')) { 209 | return _.get(result, path.replace('!', '')); 210 | } else if (/\|\|/.test(path) && !path.includes('??') ) { 211 | let pathWithDefault = path.split(/\|\|/); 212 | return getValue(pathWithDefault[0], subDocument) || getValue(`${pathWithDefault[1]}`); 213 | } else if (path.includes('??') ){ 214 | // First we need to get the value the condition is checking against .. and get the main value only if it is truthy 215 | let parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], 216 | path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); 217 | 218 | // Run a comparison between the values, and if fails skip the current data 219 | const firstValue = applyTransformation('', parameters.comparator, '', JSON.stringify(subDocument)); 220 | const secondValue = applyTransformation('', parameters.condition, '', JSON.stringify(subDocument)) 221 | let isValidValue = operation(parameters.comparison, firstValue, secondValue); 222 | 223 | return isValidValue ? applyTransformation(null, parameters.targetValue, null, subDocument) : null; 224 | } else { 225 | // Here we check if the subDocument is a string (which indicates we need to get the value from a sub-path) 226 | // We check for it to be a string because otherwise it can be the index of an array in the _.map() 227 | return _.isPlainObject(subDocument) ? _.get(subDocument, path) : _.get(document, path); 228 | } 229 | 230 | /** 231 | * Runs a comparison ( === , ==, !==, != ) against the given values 232 | * @param {string} op 233 | * @param {*} value1 234 | * @param {*} value2 235 | */ 236 | function operation(op, value1, value2){ 237 | switch(op){ 238 | case '===': 239 | return value1 === value2; 240 | case '==': 241 | return value1 == value2; 242 | case '!==': 243 | return value1 !== value2; 244 | case '!=': 245 | return value1 != value2; 246 | } 247 | return false; 248 | } 249 | } 250 | } 251 | 252 | var output = {} 253 | processMappings(mappings, document, output); 254 | return output; 255 | } 256 | 257 | module.exports = map; -------------------------------------------------------------------------------- /ditto/mappers/postmap.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | 3 | /** 4 | * @function postMap 5 | * 6 | * @description Start the post-mapping process which will run after the main mapping is done 7 | * @param {Object} document the object document we want to map 8 | * @param {Object} result the object representing the result file 9 | */ 10 | async function postMap(result) { 11 | function isEmptyObject(value, key) { 12 | return _.isPlainObject(value) && _.isEmpty(value) ? true : false; 13 | } 14 | return _(result).omitBy(_.isUndefined).omitBy(_.isNull).omitBy(isEmptyObject).value(); 15 | } 16 | 17 | module.exports = postMap; -------------------------------------------------------------------------------- /ditto/mappers/premap.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @function preMap 3 | * 4 | * @description Start the pre-mapping process which runs before the main mapping process 5 | * @param {Object} document the object document we want to map 6 | * @param {Object} result the object representing the result file 7 | */ 8 | async function preMap(document) { 9 | return document 10 | } 11 | 12 | module.exports = preMap; -------------------------------------------------------------------------------- /ditto/plugins/base64Encode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * @function base64Encode 7 | * @description base64 encode a text string 8 | * @param {String} input 9 | * @returns {String} 10 | */ 11 | function base64Encode(input) { 12 | if (_.isString(input) && !!input) { 13 | return Buffer.from(input).toString('base64') 14 | } else return null; 15 | } 16 | 17 | module.exports = base64Encode; 18 | -------------------------------------------------------------------------------- /ditto/plugins/cleanEmail.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * @function cleanEmail 7 | * @description normalize an email by converting it into an all lowercase 8 | * This will be extended in te future by doing more robust email validation 9 | * 10 | * @param {String} source the string we want to clean out 11 | * @returns {String} 12 | */ 13 | function cleanEmail(source) { 14 | if (_.isString(source) && !!source) { 15 | const cleanedEmail = source.toLowerCase().replace(/\s\s+/g, "").trim(); 16 | return !cleanedEmail.length ? null : cleanedEmail; 17 | } else return null; 18 | } 19 | 20 | module.exports = cleanEmail; 21 | -------------------------------------------------------------------------------- /ditto/plugins/cleanString.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * @function concatName 7 | * @description Clean a string from special characters, HTML tags and trailing dots and commas 8 | * 9 | * @param {String} source the string we want to clean out 10 | * @returns {String} 11 | */ 12 | function cleanString(source) { 13 | 14 | if (_.isString(source) && !!source) { 15 | const cleanedString = source.toString().replace(/[`~$^*¦|+\=?;,:<>\{\}\[\]\\\/]/gi, '') 16 | .replace(/^(\.+)/g, "") 17 | .replace(/\.(?=\.+$)/g, "") 18 | .replace(/\s\s+/g, "") 19 | .trim(); 20 | return !cleanedString.length ? null : cleanedString; 21 | } else return null; 22 | } 23 | 24 | module.exports = cleanString; 25 | -------------------------------------------------------------------------------- /ditto/plugins/cleanURI.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const URL = require('url'); 4 | const urlUtils = require('beam-uri'); 5 | 6 | const BANNED_URLS = [ 7 | "linkedin.com/profile/view", 8 | "linkedin.com/profile" 9 | ] 10 | /** 11 | * @function cleanURI 12 | * @description Clean a URI from special characters, HTML tags and trailing dots and commas 13 | * 14 | * @param {String} source the URI we want to clean out 15 | * @returns {String} 16 | */ 17 | function cleanURI(source, service) { 18 | 19 | if (!!source && typeof(source) === "string") { 20 | 21 | // Replace all spaces with nothing, and truncate to 400 chars, 22 | // or URL.parse will blow out and loop on 100% cpu :( 23 | source = source.replace(/\s/g, '').trim().substr(0,400); 24 | 25 | // We should not process messaging links (e.g., Skype) , as they don't need http 26 | if(service === 'messaging') return source; 27 | 28 | try { 29 | // Check if the source is a twitter username 30 | if (/(^|[^@\w])@(\w{1,15})\b/g.test(source)) source = `http://twitter.com/${source.replace('@', '')}` 31 | // Check if the url passed does not contain http:// 32 | if (URL.parse(source).protocol === null) source = "http://" + source; 33 | // Check if its a Linkedin URI and get the canonical URI 34 | if (!!urlUtils.getDomain(source) && urlUtils.getDomainName(source).toLowerCase() == "linkedin") { 35 | source = urlUtils.getCanonicalLinkedinUrl(source); 36 | // Handle edge case where linkedin profile url is malformed 37 | if (BANNED_URLS.includes(source.replace(/^(https?|ftp):\/\/(www.)?/, ""))){ 38 | source = null; 39 | } 40 | } 41 | } finally { 42 | // Check if the URI is not validated and remove it ! 43 | if (urlUtils.isValidURI(source)) { 44 | try { 45 | return decodeURI(source); 46 | } catch(err) { return null; } 47 | } else return null; 48 | } 49 | } else return null; 50 | 51 | } 52 | 53 | module.exports = cleanURI; 54 | -------------------------------------------------------------------------------- /ditto/plugins/concatName.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require("lodash"); 4 | 5 | /** 6 | * @function concatName 7 | * @description Concatinate a string with one or more other strings separated by a space 8 | * Since we might be passing one or more (n) strings, we will use `arguments` 9 | * @returns {String} 10 | */ 11 | function concatName() { 12 | 13 | let name = _.remove(_.flatten(_.values(arguments)), _.isString); 14 | return !!name && !!name.length ? name.join(' ').replace(/\s\s+/g,' ').trim() : null; 15 | } 16 | 17 | module.exports = concatName; 18 | -------------------------------------------------------------------------------- /ditto/plugins/concatString.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require("lodash"); 4 | 5 | /** 6 | * @function concatString 7 | * @description Concatinate a string with one or more other strings 8 | * @returns {String} 9 | */ 10 | function concatString() { 11 | 12 | let string = _.remove(_.flatten(_.values(arguments)), _.isString); 13 | return !!string && !!string.length ? string.join('').trim() : null; 14 | } 15 | 16 | module.exports = concatString; 17 | -------------------------------------------------------------------------------- /ditto/plugins/concatWithComma.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require("lodash"); 4 | 5 | /** 6 | * @description A string is considered valid if is a string and is not empty 7 | * @param {String} str 8 | */ 9 | const isValidString = (str) => { 10 | return _.isString(str) && !_.isEmpty(str); 11 | }; 12 | 13 | /** 14 | * @function concatWithComma 15 | * @description Concatinate a string with one or more other strings and join 16 | * them using comma and space. 17 | * @returns {String} 18 | */ 19 | function concatWithComma() { 20 | const strings = _.filter(_.flatten(_.values(arguments)), isValidString); 21 | return _.join(strings, ', ').trim() || null; 22 | } 23 | 24 | module.exports = concatWithComma; -------------------------------------------------------------------------------- /ditto/plugins/createURL.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @function createURL 5 | * @description Create a url from passed parameteres 6 | * 7 | * @param {String} url the main url base 8 | * @param {String} source the string to concatinate to the base url 9 | * @returns {String} 10 | */ 11 | function createURL(url, source) { 12 | if (!!url && typeof(url) === "string") { 13 | return !!source && typeof(source) === "string" ? (url + source).replace(/\s/g, "").toLowerCase().trim() : url.replace(/\s/g, "").toLowerCase().trim(); 14 | } else return null; 15 | } 16 | 17 | module.exports = createURL; 18 | -------------------------------------------------------------------------------- /ditto/plugins/data/languages.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"code":"ab","name":"Abkhaz","nativeName":"аҧсуа"}, 3 | {"code":"aa","name":"Afar","nativeName":"Afaraf"}, 4 | {"code":"af","name":"Afrikaans","nativeName":"Afrikaans"}, 5 | {"code":"ak","name":"Akan","nativeName":"Akan"}, 6 | {"code":"sq","name":"Albanian","nativeName":"Shqip"}, 7 | {"code":"am","name":"Amharic","nativeName":"አማርኛ"}, 8 | {"code":"ar","name":"Arabic","nativeName":"العربية"}, 9 | {"code":"an","name":"Aragonese","nativeName":"Aragonés"}, 10 | {"code":"hy","name":"Armenian","nativeName":"Հայերեն"}, 11 | {"code":"as","name":"Assamese","nativeName":"অসমীয়া"}, 12 | {"code":"av","name":"Avaric","nativeName":"авар мацӀ, магӀарул мацӀ"}, 13 | {"code":"ae","name":"Avestan","nativeName":"avesta"}, 14 | {"code":"ay","name":"Aymara","nativeName":"aymar aru"}, 15 | {"code":"az","name":"Azerbaijani","nativeName":"azərbaycan dili"}, 16 | {"code":"bm","name":"Bambara","nativeName":"bamanankan"}, 17 | {"code":"ba","name":"Bashkir","nativeName":"башҡорт теле"}, 18 | {"code":"eu","name":"Basque","nativeName":"euskara, euskera"}, 19 | {"code":"be","name":"Belarusian","nativeName":"Беларуская"}, 20 | {"code":"bn","name":"Bengali","nativeName":"বাংলা"}, 21 | {"code":"bh","name":"Bihari","nativeName":"भोजपुरी"}, 22 | {"code":"bi","name":"Bislama","nativeName":"Bislama"}, 23 | {"code":"bs","name":"Bosnian","nativeName":"bosanski jezik"}, 24 | {"code":"br","name":"Breton","nativeName":"brezhoneg"}, 25 | {"code":"bg","name":"Bulgarian","nativeName":"български език"}, 26 | {"code":"my","name":"Burmese","nativeName":"ဗမာစာ"}, 27 | {"code":"ca","name":"Catalan; Valencian","nativeName":"Català"}, 28 | {"code":"ch","name":"Chamorro","nativeName":"Chamoru"}, 29 | {"code":"ce","name":"Chechen","nativeName":"нохчийн мотт"}, 30 | {"code":"ny","name":"Chichewa; Chewa; Nyanja","nativeName":"chiCheŵa, chinyanja"}, 31 | {"code":"zh","name":"Chinese","nativeName":"中文 (Zhōngwén), 汉语, 漢語"}, 32 | {"code":"cv","name":"Chuvash","nativeName":"чӑваш чӗлхи"}, 33 | {"code":"kw","name":"Cornish","nativeName":"Kernewek"}, 34 | {"code":"co","name":"Corsican","nativeName":"corsu, lingua corsa"}, 35 | {"code":"cr","name":"Cree","nativeName":"ᓀᐦᐃᔭᐍᐏᐣ"}, 36 | {"code":"hr","name":"Croatian","nativeName":"hrvatski"}, 37 | {"code":"cs","name":"Czech","nativeName":"česky, čeština"}, 38 | {"code":"da","name":"Danish","nativeName":"dansk"}, 39 | {"code":"dv","name":"Divehi; Dhivehi; Maldivian;","nativeName":"ދިވެހި"}, 40 | {"code":"nl","name":"Dutch","nativeName":"Nederlands, Vlaams"}, 41 | {"code":"en","name":"English","nativeName":"English"}, 42 | {"code":"eo","name":"Esperanto","nativeName":"Esperanto"}, 43 | {"code":"et","name":"Estonian","nativeName":"eesti, eesti keel"}, 44 | {"code":"ee","name":"Ewe","nativeName":"Eʋegbe"}, 45 | {"code":"fo","name":"Faroese","nativeName":"føroyskt"}, 46 | {"code":"fj","name":"Fijian","nativeName":"vosa Vakaviti"}, 47 | {"code":"fi","name":"Finnish","nativeName":"suomi, suomen kieli"}, 48 | {"code":"fr","name":"French","nativeName":"français, langue française"}, 49 | {"code":"ff","name":"Fula; Fulah; Pulaar; Pular","nativeName":"Fulfulde, Pulaar, Pular"}, 50 | {"code":"gl","name":"Galician","nativeName":"Galego"}, 51 | {"code":"ka","name":"Georgian","nativeName":"ქართული"}, 52 | {"code":"de","name":"German","nativeName":"Deutsch"}, 53 | {"code":"el","name":"Greek, Modern","nativeName":"Ελληνικά"}, 54 | {"code":"gn","name":"Guaraní","nativeName":"Avañeẽ"}, 55 | {"code":"gu","name":"Gujarati","nativeName":"ગુજરાતી"}, 56 | {"code":"ht","name":"Haitian; Haitian Creole","nativeName":"Kreyòl ayisyen"}, 57 | {"code":"ha","name":"Hausa","nativeName":"Hausa, هَوُسَ"}, 58 | {"code":"he","name":"Hebrew (modern)","nativeName":"עברית"}, 59 | {"code":"hz","name":"Herero","nativeName":"Otjiherero"}, 60 | {"code":"hi","name":"Hindi","nativeName":"हिन्दी, हिंदी"}, 61 | {"code":"ho","name":"Hiri Motu","nativeName":"Hiri Motu"}, 62 | {"code":"hu","name":"Hungarian","nativeName":"Magyar"}, 63 | {"code":"ia","name":"Interlingua","nativeName":"Interlingua"}, 64 | {"code":"id","name":"Indonesian","nativeName":"Bahasa Indonesia"}, 65 | {"code":"ie","name":"Interlingue","nativeName":"Originally called Occidental; then Interlingue after WWII"}, 66 | {"code":"ga","name":"Irish","nativeName":"Gaeilge"}, 67 | {"code":"ig","name":"Igbo","nativeName":"Asụsụ Igbo"}, 68 | {"code":"ik","name":"Inupiaq","nativeName":"Iñupiaq, Iñupiatun"}, 69 | {"code":"io","name":"Ido","nativeName":"Ido"}, 70 | {"code":"is","name":"Icelandic","nativeName":"Íslenska"}, 71 | {"code":"it","name":"Italian","nativeName":"Italiano"}, 72 | {"code":"iu","name":"Inuktitut","nativeName":"ᐃᓄᒃᑎᑐᑦ"}, 73 | {"code":"ja","name":"Japanese","nativeName":"日本語 (にほんご／にっぽんご)"}, 74 | {"code":"jv","name":"Javanese","nativeName":"basa Jawa"}, 75 | {"code":"kl","name":"Kalaallisut, Greenlandic","nativeName":"kalaallisut, kalaallit oqaasii"}, 76 | {"code":"kn","name":"Kannada","nativeName":"ಕನ್ನಡ"}, 77 | {"code":"kr","name":"Kanuri","nativeName":"Kanuri"}, 78 | {"code":"ks","name":"Kashmiri","nativeName":"कश्मीरी, كشميري‎"}, 79 | {"code":"kk","name":"Kazakh","nativeName":"Қазақ тілі"}, 80 | {"code":"km","name":"Khmer","nativeName":"ភាសាខ្មែរ"}, 81 | {"code":"ki","name":"Kikuyu, Gikuyu","nativeName":"Gĩkũyũ"}, 82 | {"code":"rw","name":"Kinyarwanda","nativeName":"Ikinyarwanda"}, 83 | {"code":"ky","name":"Kirghiz, Kyrgyz","nativeName":"кыргыз тили"}, 84 | {"code":"kv","name":"Komi","nativeName":"коми кыв"}, 85 | {"code":"kg","name":"Kongo","nativeName":"KiKongo"}, 86 | {"code":"ko","name":"Korean","nativeName":"한국어 (韓國語), 조선말 (朝鮮語)"}, 87 | {"code":"ku","name":"Kurdish","nativeName":"Kurdî, كوردی‎"}, 88 | {"code":"kj","name":"Kwanyama, Kuanyama","nativeName":"Kuanyama"}, 89 | {"code":"la","name":"Latin","nativeName":"latine, lingua latina"}, 90 | {"code":"lb","name":"Luxembourgish, Letzeburgesch","nativeName":"Lëtzebuergesch"}, 91 | {"code":"lg","name":"Luganda","nativeName":"Luganda"}, 92 | {"code":"li","name":"Limburgish, Limburgan, Limburger","nativeName":"Limburgs"}, 93 | {"code":"ln","name":"Lingala","nativeName":"Lingála"}, 94 | {"code":"lo","name":"Lao","nativeName":"ພາສາລາວ"}, 95 | {"code":"lt","name":"Lithuanian","nativeName":"lietuvių kalba"}, 96 | {"code":"lu","name":"Luba-Katanga","nativeName":""}, 97 | {"code":"lv","name":"Latvian","nativeName":"latviešu valoda"}, 98 | {"code":"gv","name":"Manx","nativeName":"Gaelg, Gailck"}, 99 | {"code":"mk","name":"Macedonian","nativeName":"македонски јазик"}, 100 | {"code":"mg","name":"Malagasy","nativeName":"Malagasy fiteny"}, 101 | {"code":"ms","name":"Malay","nativeName":"bahasa Melayu, بهاس ملايو‎"}, 102 | {"code":"ml","name":"Malayalam","nativeName":"മലയാളം"}, 103 | {"code":"mt","name":"Maltese","nativeName":"Malti"}, 104 | {"code":"mi","name":"Māori","nativeName":"te reo Māori"}, 105 | {"code":"mr","name":"Marathi (Marāṭhī)","nativeName":"मराठी"}, 106 | {"code":"mh","name":"Marshallese","nativeName":"Kajin M̧ajeļ"}, 107 | {"code":"mn","name":"Mongolian","nativeName":"монгол"}, 108 | {"code":"na","name":"Nauru","nativeName":"Ekakairũ Naoero"}, 109 | {"code":"nv","name":"Navajo, Navaho","nativeName":"Diné bizaad, Dinékʼehǰí"}, 110 | {"code":"nb","name":"Norwegian Bokmål","nativeName":"Norsk bokmål"}, 111 | {"code":"nd","name":"North Ndebele","nativeName":"isiNdebele"}, 112 | {"code":"ne","name":"Nepali","nativeName":"नेपाली"}, 113 | {"code":"ng","name":"Ndonga","nativeName":"Owambo"}, 114 | {"code":"nn","name":"Norwegian Nynorsk","nativeName":"Norsk nynorsk"}, 115 | {"code":"no","name":"Norwegian","nativeName":"Norsk"}, 116 | {"code":"ii","name":"Nuosu","nativeName":"ꆈꌠ꒿ Nuosuhxop"}, 117 | {"code":"nr","name":"South Ndebele","nativeName":"isiNdebele"}, 118 | {"code":"oc","name":"Occitan","nativeName":"Occitan"}, 119 | {"code":"oj","name":"Ojibwe, Ojibwa","nativeName":"ᐊᓂᔑᓈᐯᒧᐎᓐ"}, 120 | {"code":"cu","name":"Old Church Slavonic, Church Slavic, Church Slavonic, Old Bulgarian, Old Slavonic","nativeName":"ѩзыкъ словѣньскъ"}, 121 | {"code":"om","name":"Oromo","nativeName":"Afaan Oromoo"}, 122 | {"code":"or","name":"Oriya","nativeName":"ଓଡ଼ିଆ"}, 123 | {"code":"os","name":"Ossetian, Ossetic","nativeName":"ирон æвзаг"}, 124 | {"code":"pa","name":"Panjabi, Punjabi","nativeName":"ਪੰਜਾਬੀ, پنجابی‎"}, 125 | {"code":"pi","name":"Pāli","nativeName":"पाऴि"}, 126 | {"code":"fa","name":"Persian","nativeName":"فارسی"}, 127 | {"code":"pl","name":"Polish","nativeName":"polski"}, 128 | {"code":"ps","name":"Pashto, Pushto","nativeName":"پښتو"}, 129 | {"code":"pt","name":"Portuguese","nativeName":"Português"}, 130 | {"code":"qu","name":"Quechua","nativeName":"Runa Simi, Kichwa"}, 131 | {"code":"rm","name":"Romansh","nativeName":"rumantsch grischun"}, 132 | {"code":"rn","name":"Kirundi","nativeName":"kiRundi"}, 133 | {"code":"ro","name":"Romanian, Moldavian, Moldovan","nativeName":"română"}, 134 | {"code":"ru","name":"Russian","nativeName":"русский язык"}, 135 | {"code":"sa","name":"Sanskrit (Saṁskṛta)","nativeName":"संस्कृतम्"}, 136 | {"code":"sc","name":"Sardinian","nativeName":"sardu"}, 137 | {"code":"sd","name":"Sindhi","nativeName":"सिन्धी, سنڌي، سندھی‎"}, 138 | {"code":"se","name":"Northern Sami","nativeName":"Davvisámegiella"}, 139 | {"code":"sm","name":"Samoan","nativeName":"gagana faa Samoa"}, 140 | {"code":"sg","name":"Sango","nativeName":"yângâ tî sängö"}, 141 | {"code":"sr","name":"Serbian","nativeName":"српски језик"}, 142 | {"code":"gd","name":"Scottish Gaelic; Gaelic","nativeName":"Gàidhlig"}, 143 | {"code":"sn","name":"Shona","nativeName":"chiShona"}, 144 | {"code":"si","name":"Sinhala, Sinhalese","nativeName":"සිංහල"}, 145 | {"code":"sk","name":"Slovak","nativeName":"slovenčina"}, 146 | {"code":"sl","name":"Slovene","nativeName":"slovenščina"}, 147 | {"code":"so","name":"Somali","nativeName":"Soomaaliga, af Soomaali"}, 148 | {"code":"st","name":"Southern Sotho","nativeName":"Sesotho"}, 149 | {"code":"es","name":"Spanish; Castilian","nativeName":"español, castellano"}, 150 | {"code":"su","name":"Sundanese","nativeName":"Basa Sunda"}, 151 | {"code":"sw","name":"Swahili","nativeName":"Kiswahili"}, 152 | {"code":"ss","name":"Swati","nativeName":"SiSwati"}, 153 | {"code":"sv","name":"Swedish","nativeName":"svenska"}, 154 | {"code":"ta","name":"Tamil","nativeName":"தமிழ்"}, 155 | {"code":"te","name":"Telugu","nativeName":"తెలుగు"}, 156 | {"code":"tg","name":"Tajik","nativeName":"тоҷикӣ, toğikī, تاجیکی‎"}, 157 | {"code":"th","name":"Thai","nativeName":"ไทย"}, 158 | {"code":"ti","name":"Tigrinya","nativeName":"ትግርኛ"}, 159 | {"code":"bo","name":"Tibetan Standard, Tibetan, Central","nativeName":"བོད་ཡིག"}, 160 | {"code":"tk","name":"Turkmen","nativeName":"Türkmen, Түркмен"}, 161 | {"code":"tl","name":"Tagalog","nativeName":"Wikang Tagalog, ᜏᜒᜃᜅ᜔ ᜆᜄᜎᜓᜄ᜔"}, 162 | {"code":"tn","name":"Tswana","nativeName":"Setswana"}, 163 | {"code":"to","name":"Tonga (Tonga Islands)","nativeName":"faka Tonga"}, 164 | {"code":"tr","name":"Turkish","nativeName":"Türkçe"}, 165 | {"code":"ts","name":"Tsonga","nativeName":"Xitsonga"}, 166 | {"code":"tt","name":"Tatar","nativeName":"татарча, tatarça, تاتارچا‎"}, 167 | {"code":"tw","name":"Twi","nativeName":"Twi"}, 168 | {"code":"ty","name":"Tahitian","nativeName":"Reo Tahiti"}, 169 | {"code":"ug","name":"Uighur, Uyghur","nativeName":"Uyƣurqə, ئۇيغۇرچە‎"}, 170 | {"code":"uk","name":"Ukrainian","nativeName":"українська"}, 171 | {"code":"ur","name":"Urdu","nativeName":"اردو"}, 172 | {"code":"uz","name":"Uzbek","nativeName":"zbek, Ўзбек, أۇزبېك‎"}, 173 | {"code":"ve","name":"Venda","nativeName":"Tshivenḓa"}, 174 | {"code":"vi","name":"Vietnamese","nativeName":"Tiếng Việt"}, 175 | {"code":"vo","name":"Volapük","nativeName":"Volapük"}, 176 | {"code":"wa","name":"Walloon","nativeName":"Walon"}, 177 | {"code":"cy","name":"Welsh","nativeName":"Cymraeg"}, 178 | {"code":"wo","name":"Wolof","nativeName":"Wollof"}, 179 | {"code":"fy","name":"Western Frisian","nativeName":"Frysk"}, 180 | {"code":"xh","name":"Xhosa","nativeName":"isiXhosa"}, 181 | {"code":"yi","name":"Yiddish","nativeName":"ייִדיש"}, 182 | {"code":"yo","name":"Yoruba","nativeName":"Yorùbá"}, 183 | {"code":"za","name":"Zhuang, Chuang","nativeName":"Saɯ cueŋƅ, Saw cuengh"} 184 | ] -------------------------------------------------------------------------------- /ditto/plugins/extractName.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const human = require('humanparser'); 4 | 5 | /** 6 | * @function extractName 7 | * @description Extract the first name of a contact as it is a required field 8 | * 9 | * @param {String} fullName the contact fullname 10 | * @param {String} position the position of the name to extract (firstName, lastName, middleName) 11 | * @returns {String/Array} Returns the extracted firstName or lastName as Strings or the middleName(s) as an array 12 | */ 13 | 14 | function extractName(fullName, position) { 15 | 16 | const ALLOWED_POSITIONS = ["firstName", "middleName", "lastName"]; 17 | 18 | if (!!fullName && typeof(fullName) === "string" 19 | && !!position && ALLOWED_POSITIONS.indexOf(position) > -1) { 20 | 21 | const name = human.parseName(fullName); 22 | 23 | switch(position) { 24 | case "firstName": return name.firstName; 25 | case "lastName": return name.lastName; 26 | case "middleName": return name.middleName ? name.middleName.split(' ') : [] ; 27 | } 28 | } else return null; 29 | } 30 | 31 | module.exports = extractName; 32 | -------------------------------------------------------------------------------- /ditto/plugins/formatDate.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const moment = require('moment'); 5 | 6 | /** 7 | * @function formatDate 8 | * @description Format a date according to parameters 9 | * @param {Date} date 10 | * @param {String} [format] Format of the date. 11 | * @param {Boolean} [isUtc=true] If timezone should be utc or not 12 | * @returns {String} 13 | */ 14 | function formatDate(date, format, isUtc=true) { 15 | const convertedDate = moment(date); 16 | if (convertedDate.isValid() && date) { 17 | if (isUtc) return convertedDate.utc().format(format); 18 | return convertedDate.format(format); 19 | } 20 | return date; 21 | } 22 | 23 | module.exports = formatDate; 24 | -------------------------------------------------------------------------------- /ditto/plugins/generateCleanId.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const crypto = require('crypto'); 4 | const _ = require('lodash'); 5 | 6 | /** 7 | * @function generateCleanId 8 | * @description Create an md5 hash based on concatentating passed String Values 9 | * Since we might be passing one or more (n) strings, we will use `arguments` 10 | * 11 | * @returns {String} result the concatenated cleaned string 12 | */ 13 | function generateCleanId() { 14 | 15 | let idFields = _.remove(_.flatten(_.values(arguments)), _.isString); 16 | 17 | if (!!idFields && !!idFields.length) { 18 | const cleanedSourceString = idFields.join(' ') 19 | .toString().replace(/[`~$^*¦_|+\=?;:'"<>\{\}\[\]\\\/]/gi, '') 20 | .replace(/[\n\t\r]/g,"") 21 | .replace(/^[,.\s]+|[,.\s]+$/g, "") 22 | .replace(/\s\s+/g,' ') 23 | .toLowerCase().trim(); 24 | return crypto.createHash('md5').update(cleanedSourceString).digest("hex") 25 | } else return null; 26 | } 27 | 28 | module.exports = generateCleanId; 29 | -------------------------------------------------------------------------------- /ditto/plugins/generateFacebookImageLink.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @function generateFacebookImageLink 5 | * @description Generate a link for the Facebook profile photo based on the facebook ID 6 | * 7 | * @param {String} Facebook profile ID 8 | * @returns {String} 9 | */ 10 | function generateFacebookImageLink(id) { 11 | if (!!id && typeof(id) === "string") { 12 | return "http://graph.facebook.com/" + id + "/picture?type=large" 13 | } else return null; 14 | } 15 | 16 | module.exports = generateFacebookImageLink; 17 | -------------------------------------------------------------------------------- /ditto/plugins/generateId.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const crypto = require('crypto'); 5 | 6 | /** 7 | * @function generateId 8 | * @description Create an md5 hash based on concatentating passed String Values 9 | * This function will take multiple arguments that will be extracted via the `arguments` keyword 10 | * 11 | * @returns {String} 12 | */ 13 | function generateId() { 14 | 15 | const idFields = _.remove(_.flatten(_.values(arguments)), _.isString); 16 | 17 | // TODO: re-add the regex .replace(/\s\s+/g,' ') to the chain to make sure we have a properly cleaned string 18 | return !!idFields && !!idFields.length ? crypto.createHash('md5').update(idFields.join(' ').toLowerCase().trim()).digest("hex") : null; 19 | } 20 | 21 | module.exports = generateId; 22 | -------------------------------------------------------------------------------- /ditto/plugins/generateIdForLinks.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const crypto = require('crypto'); 4 | 5 | const cleanUri = require('./cleanURI') 6 | /** 7 | * @function generateIdForLinks 8 | * @description Create an md5 hash based on concatentating passed String Values for links 9 | * The function cleans the URIs before creating the MD5 hash 10 | * 11 | * @param {String} source the URI we want to clean out 12 | * @returns {String} 13 | */ 14 | function generateIdForLinks(source, type) { 15 | 16 | let cleanedSource; 17 | 18 | try { 19 | cleanedSource = cleanUri(source, type); 20 | } finally { 21 | // Check if the URI is not validated and remove it ! 22 | if (!!cleanedSource) { 23 | return crypto.createHash('md5').update(cleanedSource.toLowerCase()).digest("hex"); 24 | } else { 25 | return null; 26 | } 27 | } 28 | 29 | } 30 | 31 | module.exports = generateIdForLinks; 32 | -------------------------------------------------------------------------------- /ditto/plugins/generateIdFromLanguageCode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const generateId = require('./generateId'); 4 | const getLanguageFromCode = require('./getLanguageFromCode'); 5 | 6 | /** 7 | * @function generateIdFromLanguageCode 8 | * @description Lanaugage id generation is done on the value of the language. This function will generate the id from a language ISO code 9 | * by doing a lookup first on the language valuye then generate the id from that one 10 | * 11 | * @param {String} languageCode The language code 12 | * @return {String} 13 | */ 14 | 15 | function generateIdFromLanguageCode(languageCode) { 16 | 17 | if (!languageCode || typeof(languageCode) !== "string") return null; 18 | 19 | return generateId(getLanguageFromCode(languageCode)); 20 | } 21 | 22 | module.exports = generateIdFromLanguageCode; 23 | -------------------------------------------------------------------------------- /ditto/plugins/generateUUID.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const uuid = require('uuid'); 4 | 5 | /** 6 | * @function generateUUID 7 | * @description Create an random UUID value 8 | * @returns {String} 9 | */ 10 | function generateUUID() { 11 | return uuid.v4(); 12 | } 13 | 14 | module.exports = generateUUID; 15 | -------------------------------------------------------------------------------- /ditto/plugins/getCountryCode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * @function getCountryCode 7 | * @description Get the language code and normalize as the well the displayName of the language 8 | * 9 | * @param {String} countryCode the ISO2 country code 10 | * @param {String} country the country name 11 | * @returns {String} the ISO3 country code 12 | */ 13 | function getCountryCode(countryCode, country) { 14 | 15 | if (!countryCode && !country) return null; 16 | 17 | const countries = require('./data/countriesKB') 18 | 19 | // Check if we don't have the ISO2 country code but have a country 20 | if (!countryCode) { 21 | return _.get(_.filter(countries, _country => { 22 | return _country.name.toLowerCase() === country.toLowerCase() 23 | })[0], 'alpha3Code', null); 24 | } 25 | 26 | return _.get(_.filter(countries, _country => { 27 | return _country.alpha2Code.toLowerCase() === countryCode.toLowerCase() 28 | })[0], 'alpha3Code', null); 29 | 30 | } 31 | 32 | module.exports = getCountryCode; 33 | -------------------------------------------------------------------------------- /ditto/plugins/getCountryName.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * @function getCountryName 7 | * @description Get the country name given the country ISO3 code provided 8 | * 9 | * @param {String} countryCode The ISO3 Country Code 10 | * @returns {String} The country name 11 | */ 12 | function getCountryName(countryCode) { 13 | 14 | if (!countryCode) return null; 15 | 16 | const countries = require('./data/countriesKB') 17 | 18 | return _.get(_.filter(countries, _country => { 19 | return (_.has(_country, 'alpha3Code') && _country.alpha3Code.toLowerCase() === countryCode.toLowerCase()) 20 | })[0], 'name', null); 21 | 22 | } 23 | 24 | module.exports = getCountryName; 25 | -------------------------------------------------------------------------------- /ditto/plugins/getLanguageCode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * @function getLanguageCode 7 | * @description Get the language code and normalize as the well the displayName of the language 8 | * 9 | * @param {String} source the language display name 10 | * @returns {String} the langauage ISO code 11 | */ 12 | function getLanguageCode(source) { 13 | 14 | if (!source || typeof(source) !== "string") return null; 15 | 16 | const languages = require('./data/languages.json') 17 | 18 | let languageCode = null; 19 | 20 | _.filter(languages, function(languageObject){ 21 | _.each(_.words(languageObject.name), function(language){ 22 | if (language.toLowerCase() === source.replace(/\s\s+/g,"").trim().toLowerCase()) { 23 | languageCode = languageObject.code; 24 | return; 25 | } 26 | }); 27 | }); 28 | return languageCode; 29 | } 30 | 31 | module.exports = getLanguageCode; 32 | -------------------------------------------------------------------------------- /ditto/plugins/getLanguageFromCode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | const languages = require('./data/languages.json') 6 | 7 | /** 8 | * @function getLanguageFromCode 9 | * @description Get the language displayName from Code 10 | * 11 | * @param {String} source the langauage code 12 | * @returns {String} 13 | */ 14 | function getLanguageFromCode(source) { 15 | 16 | if (!source || typeof(source) !== "string") return null; 17 | 18 | const language = languages.find($ => $.code === source.replace(/\s\s+/g,"").trim().toLowerCase()) || { code:'', name: null, nativeName: '' }; 19 | return language.name; 20 | } 21 | 22 | module.exports = getLanguageFromCode; 23 | -------------------------------------------------------------------------------- /ditto/plugins/getLinkService.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Joi = require('joi'); 4 | const URL = require('url'); 5 | const urlUtils = require('beam-uri'); 6 | 7 | /** 8 | * @function getLinkService 9 | * @description Identify if the service provider of the link 10 | * 11 | * @param {String} source the link URI we wish to examine 12 | * @param {String} service the link service name 13 | * @returns {String} 14 | */ 15 | 16 | function getLinkService(source, service) { 17 | 18 | if (!source || typeof(source) !== "string" || !Joi.validate(source, Joi.string().email({errorLevel: true})).error ) return null; 19 | if (!!service) return service.replace(/\s\s+/g, "").trim().toLowerCase(); 20 | 21 | // Check if the source is a twitter username 22 | if (URL.parse(source).protocol === null && /(^|[^@\w])@(\w{1,15})\b/g.test(source)) source = `http://twitter.com/${source.replace('@', '')}` 23 | // Check if the url passed does not contain http:// 24 | if (URL.parse(source).protocol === null) source = "http://" + source; 25 | return urlUtils.getDomainName(source.replace(/\s\s+/g, "").trim().toLowerCase()); 26 | } 27 | 28 | module.exports = getLinkService; 29 | -------------------------------------------------------------------------------- /ditto/plugins/getLinkType.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const URL = require('url'); 5 | const urlUtils = require('beam-uri'); 6 | 7 | /** 8 | * @function getLinkType 9 | * @description Identify if the link is for a social website 10 | * 11 | * @param {String} source the link URI we wish to examine 12 | * @returns {String} 13 | */ 14 | 15 | const SOCIAL_SERVICES_DOMAIN = ["12seconds.tv", "4travel.jp", "advogato.org", "ameba.jp", "anobii.com", "asmallworld.net", "backtype.com", "badoo.com", "bebo.com", "bigadda.com", "bigtent.com", "biip.no", "blackplanet.com", "blog.seesaa.jp", "blogspot.com", "blogster.com", "blomotion.jp", "bolt.com", "brightkite.com", "buzznet.com", "cafemom.com", "care2.com", "classmates.com", "cloob.com", "collegeblender.com", "cyworld.co.kr", "cyworld.com.cn", "dailymotion.com", "delicious.com", "deviantart.com", "digg.com", "diigo.com", "disqus.com", "draugiem.lv", "facebook.com", "faceparty.com", "fc2.com", "flickr.com", "flixster.com", "fotolog.com", "foursquare.com", "friendfeed.com", "friendsreunited.com", "friendster.com", "fubar.com", "gaiaonline.com", "geni.com", "goodreads.com", "grono.net", "habbo.com", "hatena.ne.jp", "hi5.com", "hotnews.infoseek.co.jp", "hyves.nlibibo.comidenti.ca", "imeem.com", "intensedebate.com", "irc-galleria.net", "iwiw.hu", "jaiku.com", "jp.myspace.com", "kaixin001.com", "kaixin002.com", "kakaku.com", "kanshin.com", "kozocom.com", "last.fm", "linkedin.com", "livejournal.com", "matome.naver.jp", "me2day.net", "meetup.com", "mister-wong.com", "mixi.jp", "mixx.com", "mouthshut.com", "multiply.com", "myheritage.com", "mylife.com", "myspace.com", "myyearbook.com", "nasza-klasa.pl", "netlog.com", "nettby.no", "netvibes.com", "nicovideo.jp", "ning.com", "odnoklassniki.ru", "orkut.com", "pakila.jp", "photobucket.com", "pinterest.com", "plaxo.com", "plurk.com", "plus.google.com", "reddit.com", "renren.com", "skyrock.com", "slideshare.net", "smcb.jp", "smugmug.com", "sonico.com", "studivz.net", "stumbleupon.com", "t.163.com", "t.co", "t.hexun.com", "t.ifeng.com", "t.people.com.cn", "t.qq.com", "t.sina.com.cn", "t.sohu.com", "tabelog.com", "tagged.com", "taringa.net", "thefancy.com", "tripit.com", "trombi.com", "trytrend.jp", "tuenti.com", "tumblr.com", "twine.com", "twitter.com", "uhuru.jp", "viadeo.comvimeo.com", "vk.com", "vox.com", "wayn.com", "weibo.com", "weourfamily.com", "wer-kennt-wen.de", "wordpress.com", "xanga.com", "xing.com", "yammer.com", "yaplog.jp", "yelp.com", "youku.com", "youtube.com", "yozm.daum.net", "yuku.com", "zooomr.com", "soundcloud.com", "vevo.com"]; 16 | 17 | const MESSAGING_SERVICES = ["blackberry messenger", "blackberry", "aim", "xmpp", "ebuddy", "imessage", "facebook messenger", "windows live messenger", "yahoo messenger", "qq", "ibm sametime", "sametime", "skype", "mxit", "xfire", "gadu-gadu", "gadu gadu", "icq", "paltalk", "imvu", "gitter", "google hangouts", "hangouts", "hipChat", "meebo", "slack", "streamup", "talkomatic", "yinychat", "yokbox", "xfire"]; 18 | 19 | function getLinkType(source, service) { 20 | 21 | if (!source || typeof(source) !== "string") return null; 22 | 23 | if(!!service && MESSAGING_SERVICES.find(item => item.toLowerCase().trim() === service.toLowerCase().trim())){ 24 | return "messaging"; 25 | } 26 | 27 | source = source.replace(/\s\s+/g,"").trim().toLowerCase(); 28 | // Check if the source is a twitter username 29 | if (/(^|[^@\w])@(\w{1,15})\b/g.test(source)) source = `http://twitter.com/${source.replace('@', '')}` 30 | // Check if the url passed does not contain http:// 31 | if (URL.parse(source).protocol === null) source = "http://" + source; 32 | 33 | source = urlUtils.getDomain(source) || source; 34 | return SOCIAL_SERVICES_DOMAIN.indexOf(source.toLowerCase()) > -1 ? "social" : "website"; 35 | 36 | } 37 | 38 | module.exports = getLinkType; 39 | -------------------------------------------------------------------------------- /ditto/plugins/getValueAtPath.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * Simple wrapper for lodash `get`. 7 | * 8 | * @example {a: {b: 1}} => ['a', 'b'] => 1 9 | * 10 | * @param {Object} object The object to query. 11 | * @param {Array|String} path Path of the property to get. 12 | * @returns {any} The value returned or undefined. 13 | */ 14 | 15 | function getValueAtPath(object, path) { 16 | return _.get(object, path); 17 | } 18 | 19 | module.exports = getValueAtPath; 20 | -------------------------------------------------------------------------------- /ditto/plugins/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | base64Encode: require('./base64Encode'), 5 | cleanString: require('./cleanString'), 6 | cleanEmail: require('./cleanEmail'), 7 | cleanURI: require('./cleanURI'), 8 | concatName: require('./concatName'), 9 | concatString: require('./concatString'), 10 | concatWithComma: require('./concatWithComma'), 11 | createURL: require('./createURL'), 12 | extractName: require('./extractName'), 13 | generateCleanId: require('./generateCleanId'), 14 | generateFacebookImageLink: require('./generateFacebookImageLink'), 15 | generateId: require('./generateId'), 16 | generateIdForLinks: require('./generateIdForLinks'), 17 | generateIdFromLanguageCode: require('./generateIdFromLanguageCode'), 18 | generateUUID: require('./generateUUID'), 19 | getCountryCode: require('./getCountryCode'), 20 | getCountryName: require('./getCountryName'), 21 | getLanguageCode: require('./getLanguageCode'), 22 | getLanguageFromCode: require('./getLanguageFromCode'), 23 | getLinkService: require('./getLinkService'), 24 | getLinkType: require('./getLinkType'), 25 | getValueAtPath: require('./getValueAtPath'), 26 | minBy: require('./minBy'), 27 | normalizeString: require('./normalizeString'), 28 | parseDate: require('./parseDate'), 29 | parseString: require('./parseString'), 30 | splitList: require('./splitList'), 31 | uniqueArray: require('./uniqueArray'), 32 | formatDate: require('./formatDate') 33 | }; 34 | -------------------------------------------------------------------------------- /ditto/plugins/minBy.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * Return the min value (numerical or by character code) in array. 7 | * If only array is passed it is assumed that array is of numbers or strings. 8 | * If path is passed it is assumed that array is of objects, and value that 9 | * path resolves to is used. 10 | * 11 | * @example [1,2] => 1 12 | * 13 | * @param {Array} array - Description. 14 | * @param {string=} path - Path to prop in object. 15 | * @returns {any} Min value or undefined. 16 | */ 17 | function minBy(array, path = undefined) { 18 | if (!_.isArray(array)) { 19 | return undefined; 20 | } 21 | 22 | return _.minBy(array, path); 23 | } 24 | 25 | module.exports = minBy; 26 | -------------------------------------------------------------------------------- /ditto/plugins/normalizeString.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | const cleanString = require('./cleanString'); 6 | 7 | /** 8 | * @function normalizeString 9 | * @description normalizeString a string from special characters, HTML tags and trailing dots and commas and lower case it 10 | * 11 | * @param {String} source the string we want to clean out 12 | * @returns {String} 13 | */ 14 | function normalizeString(source) { 15 | 16 | if (_.isString(source) && !!source) { 17 | return !!cleanString(source) ? cleanString(source).toLowerCase() : null; 18 | } else return null; 19 | } 20 | 21 | module.exports = normalizeString; 22 | -------------------------------------------------------------------------------- /ditto/plugins/parseDate.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @function parseDate 5 | * @description Accepts a string or a Date object as input, 6 | * check it's validity, and either return it as Date object, or returns null 7 | * 8 | * @param {String} date the date we wish to transform 9 | * @param {String} month the month if found to be added to the parsed date 10 | * @param {String} day the day if found to be added to the parsed date 11 | * @returns {Date} 12 | */ 13 | 14 | function isValid(value) { 15 | return !!value; 16 | } 17 | 18 | function parseDate(date, month, day) { 19 | 20 | if (!date || date ==='notKnown') return null; 21 | 22 | if (month) { 23 | date = [ date, month, day ].filter(isValid).join('-').concat(' GMT'); 24 | } 25 | // Try to parse the date, and get the timestamp 26 | const timestamp = Date.parse(date); 27 | return !!isNaN(timestamp) ? null : new Date(timestamp) 28 | } 29 | 30 | module.exports = parseDate; 31 | -------------------------------------------------------------------------------- /ditto/plugins/parseString.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @function parseString 5 | * @description Convert a value into a string by concatenating it with an empty space 6 | * Known issue is that we will lose double precision when converting to string (check the tests) 7 | * 8 | * @param {String} source the string we wish to transform 9 | * @returns {String} 10 | */ 11 | function parseString(source) { 12 | 13 | return (typeof(source) === "boolean" || !!source) ? "" + source : null; 14 | 15 | } 16 | 17 | module.exports = parseString; 18 | -------------------------------------------------------------------------------- /ditto/plugins/splitList.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * @function splitList 7 | * @description Split a list of items into an array 8 | * 9 | * @param {String|null} input string to split 10 | * @returns {String|null} 11 | */ 12 | function splitList(input) { 13 | if (typeof input === 'string') { 14 | return input.split(/\s*(?:,|\sand\s)\s*/gi); 15 | } else { 16 | return null; 17 | } 18 | } 19 | 20 | module.exports = splitList; 21 | -------------------------------------------------------------------------------- /ditto/plugins/uniqueArray.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | /** 6 | * @function uniqueArray 7 | * @description Ensure array elements are unique 8 | * 9 | * @param {Array} target the target Array we will ensure its uniqueness 10 | * @returns {Array} 11 | */ 12 | function uniqueArray(target, key) { 13 | 14 | if (!target || !target.length) return []; 15 | 16 | return !!key ? _.uniqBy(target, key) : _.uniq(target); 17 | } 18 | 19 | module.exports = uniqueArray; 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "json-ditto", 3 | "version": "1.2.9", 4 | "description": "JSON shape-shifter 👻 | Easily map your JSON formats", 5 | "main": "ditto/ditto.js", 6 | "scripts": { 7 | "test": "mocha test/index.js", 8 | "performance": "node --allow-natives-syntax ./node_modules/mocha/bin/_mocha --reporter mocha-performance ./test/index.js", 9 | "cover": "NODE_ENV=test istanbul cover _mocha -- -R spec test/index.js", 10 | "lint": "eslint ." 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "git+https://github.com:BeameryHQ/Ditto.git" 15 | }, 16 | "license": "ISC", 17 | "dependencies": { 18 | "beam-uri": "^2.1.1", 19 | "humanparser": "^1.10.0", 20 | "joi": "^14.3.1", 21 | "lodash": "^4.5.1", 22 | "moment": "^2.15.0", 23 | "uuid": "^3.3.2" 24 | }, 25 | "devDependencies": { 26 | "debug": "^4.1.1", 27 | "eslint": "^5.15.3", 28 | "mocha": "^6.0.2" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /test/ditto.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | const crypto = require('crypto'); 6 | const fs = require('fs'); 7 | const path = require('path'); 8 | 9 | const ditto = require('../ditto/ditto'); 10 | 11 | describe('ditto Interface', function(){ 12 | 13 | let dummySample = require('./samples/test'); 14 | let dummyResult = require('./results/test'); 15 | let dummyMappings = require('./mappings/test'); 16 | 17 | let dummyPlugin = { 18 | transformTwitterHandle: function transformTwitterHandle(target){ 19 | _.each(target, function(link){ 20 | link.value = "@ahmadaassaf" 21 | }); 22 | return target; 23 | } 24 | }; 25 | 26 | before(function(){ 27 | return new ditto(dummyMappings, dummyPlugin).unify(dummySample).then((result) => { 28 | this.result = result; 29 | }); 30 | }); 31 | 32 | it('should be able to map an object with a direct flat mapping', function(){ 33 | assert.strictEqual(this.result.name, dummyResult.name); 34 | }); 35 | 36 | it('should be able to map an object with a default hardcoded value if a mapping value is not found', function(){ 37 | assert.strictEqual(this.result.nickname, "nickname_not_found"); 38 | assert.strictEqual(this.result.isNickNameFound, false); 39 | }); 40 | 41 | it('should be able to map an object with a default value extracted from the mapping if a mapping value is not found', function(){ 42 | assert.strictEqual(this.result.isDynamicDefault, this.result.name); 43 | }); 44 | 45 | it('should be able to combine two or more strings', function(){ 46 | assert.strictEqual(this.result.fullName, dummyResult.fullName); 47 | }); 48 | 49 | it('should be able to combine two or more strings one of which is an already extracted value', function(){ 50 | assert.strictEqual(this.result.completeName, dummyResult.completeName); 51 | }); 52 | 53 | it('should be able to assign a new value based on an already mapped one', function(){ 54 | assert.strictEqual(this.result.displayName, "Ahmad Assaf"); 55 | }); 56 | 57 | it('should be able to apply a condition on an output path', function(){ 58 | assert.equal(this.result.social_media_addresses.length, 1); 59 | }); 60 | 61 | it('should be able to apply multiple conditions on an output path', function(){ 62 | assert.equal(this.result.social_media_addresses.length, 1); 63 | assert.deepStrictEqual(this.result.social_media_addresses[0], {"value": "@ahmadaassaf"}); 64 | }); 65 | 66 | it('should be able to map nested/sub object', function(){ 67 | assert.deepStrictEqual(this.result.email, dummyResult.email); 68 | }); 69 | 70 | it('should be able to assign a value based on the value of an iterating object', function(){ 71 | assert.strictEqual(this.result.social_links[0].value, "http://a.com"); 72 | }); 73 | 74 | it('should be able to assign a value based on the key of an iterating object', function(){ 75 | assert.strictEqual(this.result.social_links[0].type, "test"); 76 | }); 77 | 78 | it('should be able to assign a value based on a hard-coded string', function(){ 79 | assert.strictEqual(this.result.social_links[0].order, 0); 80 | }); 81 | 82 | it('should be able to assign a boolean value based on a hard-coded boolean', function(){ 83 | assert.strictEqual(this.result.social_links[0].social, true); 84 | }); 85 | 86 | it('should be able to assign a default hardcoded value based if an extracted value from a path does not exist', function(){ 87 | assert.strictEqual(this.result.default_name, "this_should_be_the_firstName"); 88 | }); 89 | 90 | it('should be able to assign a value only if a condition is met', function(){ 91 | assert.strictEqual(this.result.social_media_addresses[0].value, "@ahmadaassaf"); 92 | assert.strictEqual(this.result.website_addresses_keyless.length, 3); 93 | }); 94 | 95 | it('should be able to assign a value only if the prerequisite value is valid', function(){ 96 | assert.strictEqual(this.result.website_addresses["f5e32a6faaa7ead6ba201e8fa25733ee"].value, "http://klout.com/ahmadassaf"); 97 | }); 98 | 99 | it('should be able to map an object based on iterating one or more target paths', function(){ 100 | assert.deepStrictEqual(this.result.social_links, dummyResult.social_links); 101 | }); 102 | 103 | it('should be able to map an object based on iterating one or more target paths with the id generated on elements in the root document', function(){ 104 | assert.deepStrictEqual(this.result.experience_primary, dummyResult.experience_primary); 105 | }); 106 | 107 | it('should be able to map an array of objects based on iterating over an already mapped set of values', function(){ 108 | assert.deepStrictEqual(this.result.social_links, dummyResult.social_links); 109 | }); 110 | 111 | it('should be able hash a defined value as the key of a mapped object', function(){ 112 | assert.strictEqual(_.keys(this.result.social_links_objects)[0] 113 | , "388495550d157e5c5d299b7ecfeb1c2d"); 114 | }); 115 | 116 | it('should be able to map an object of objects based on iterating over an already mapped set of values', function(){ 117 | assert.deepStrictEqual(this.result.social_links_objects, dummyResult.social_links_objects); 118 | }); 119 | 120 | it('should be able to apply a function on a nested object', function(){ 121 | assert.equal(this.result.experience_object.values["894b6152a9dde92713a40590f6f4d5b8"].id, "894b6152a9dde92713a40590f6f4d5b8"); 122 | }); 123 | 124 | it('should return a default value if found without executing the function defined', function(){ 125 | assert.strictEqual(this.result.fullNameDefault, "Ahmad Assaf"); 126 | assert.strictEqual(this.result.fullNameDefaultHardcoded, "default"); 127 | }); 128 | 129 | it('should be able to create an array of values by iterating over a set of objects', function(){ 130 | assert.equal(this.result.experiences.length, 2); 131 | assert.equal(this.result.experiences[0], 'beamery'); 132 | assert.equal(this.result.experiences[1], 'SAP'); 133 | }); 134 | 135 | it('should be able hash a defined value as they key of a mapped object based on multiple values', function(){ 136 | assert.strictEqual(_.keys(this.result.experience_object.values)[0] 137 | , "894b6152a9dde92713a40590f6f4d5b8"); 138 | }); 139 | 140 | it('should be able to call nested function calls', function(){ 141 | assert.deepStrictEqual(this.result.messaging, dummyResult.messaging); 142 | assert.deepStrictEqual(this.result.messaging[0].type, "messaging"); 143 | }); 144 | 145 | it('should be able to assign a value based on a condition that includes nested function calls', function(){ 146 | assert.deepStrictEqual(this.result.messaging[0].value, "ahmad.a.assaf"); 147 | assert.deepStrictEqual(this.result.messaging.length, 1); 148 | }); 149 | 150 | it('should be able to create an array of objects by iterating over a nested/sub object', function(){ 151 | assert.deepStrictEqual(this.result.education, dummyResult.education); 152 | }); 153 | 154 | it('should be able to create an array of objects by iterating over multiple objects in the source document', function(){ 155 | assert.equal(this.result.social_links.length, 5); 156 | }); 157 | 158 | it('should be able to ignore parsing objects that have a false/null/undefined values', function(){ 159 | assert.equal(this.result.social_links.length, 5); 160 | }); 161 | 162 | it('should be able to create an object value based on the iterating key of an object', function(){ 163 | assert.strictEqual(this.result.education[0].universityName, "telecomParisTech"); 164 | }); 165 | 166 | it('should be able to create an object where they key is a combination of a previous object key and other value(s)', function(){ 167 | assert.strictEqual(_.keys(this.result.education_object)[0], "6ea8c4cf5c35861ba3b690f93828e44f"); 168 | }); 169 | 170 | it('should be able to create a value by combining a string with an exsiting mapped value (string templates)', function(){ 171 | assert.strictEqual(this.result.primaryPhoto, "http://photo.com/ahmadassaf"); 172 | }); 173 | 174 | it('should be able to create an object based on selecting an array element from an existing object', function(){ 175 | assert.deepEqual(this.result.primaryExperience, dummyResult.primaryExperience); 176 | }); 177 | 178 | it('should cover all the cases defined by the ditto interface', function(){ 179 | assert.deepStrictEqual(this.result, dummyResult); 180 | }); 181 | 182 | it('should be able to parse a unicode string', function(){ 183 | const badObj = { data : { 184 | links: { 185 | values: { 186 | ciao: { 187 | id: "ciao", 188 | bio:`
`, 189 | value:" this is a value", 190 | type: "social" 191 | } 192 | } 193 | } 194 | }}; 195 | 196 | return new ditto({ 197 | "social_media_addresses": { 198 | "output": [], 199 | "innerDocument": "data.links.values", 200 | "prerequisite": "!!innerResult.value", 201 | "required": ["value"], 202 | "mappings": { 203 | "value": "value??type#==#>>social" 204 | } 205 | } 206 | }).unify(badObj).then((result) => { 207 | assert.deepEqual(result, { social_media_addresses: [ { value: ' this is a value' } ] }); 208 | }); 209 | 210 | 211 | }); 212 | 213 | const plugins = require('../ditto/plugins/index'); 214 | 215 | describe('ditto Plugins', function(){ 216 | _.each(plugins, function(plugin, pluginName) { 217 | const plugintTestPath = path.join(__dirname, `./plugins/${pluginName}.js`); 218 | if (!fs.existsSync(plugintTestPath)) { 219 | console.error(`\x1b[31m No valid test is found for ditto Plugin: ${pluginName}`); 220 | } else { 221 | require(plugintTestPath); 222 | } 223 | }); 224 | }); 225 | }); 226 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | require('./ditto'); 2 | require('./socialServices'); -------------------------------------------------------------------------------- /test/mappings/services/facebook.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /* 4 | * @description The mapping file between the Facebook API and the sherlock model 5 | * The key represents the Facebook field name and the value is the mapped sherlock field name 6 | * If the value is an object this means that the Facebook field is an object and we need to map 7 | * the internal fields as well 8 | * @returns {Object} 9 | */ 10 | 11 | module.exports = { 12 | "id" : "@generateUUID(>>id)", 13 | "firstName" : "data.profile.name.givenName", 14 | "lastName" : "data.profile.name.familyName", 15 | "fullName" : "@concatName(data.profile.name.givenName|data.profile.name.familyName)", 16 | "gender" : "data.profile.gender", 17 | "birthday" : "data.profile.birthday", 18 | "links" : { 19 | "values" : { 20 | "output": {}, 21 | "key": "@generateIdForLinks(data.profile.profileUrl)", 22 | "innerDocument": "!", 23 | "required": ["value"], 24 | "mappings" : { 25 | "id" : "@generateIdForLinks(data.profile.profileUrl)", 26 | "type" : ">>social", 27 | "value" : "@cleanURI(data.profile.profileUrl|@getLinkType(value,@getLinkService(value,service)))", 28 | "service" : ">>facebook", 29 | "username": "data.profile.id", 30 | "userId" : "data.profile.id" 31 | } 32 | }, 33 | "keys" : { 34 | "output": [], 35 | "innerDocument": "!links.values", 36 | "value": "id" 37 | } 38 | }, 39 | "emails" : { 40 | "values" : { 41 | "output": {}, 42 | "key": "@generateId(value)", 43 | "innerDocument": "data.profile.emails", 44 | "required": ["value"], 45 | "mappings" : { 46 | "id" : "@generateId(value)", 47 | "value": "@cleanEmail(value)", 48 | "type" : ">>facebook" 49 | } 50 | }, 51 | "keys" : { 52 | "output": [], 53 | "innerDocument": "!emails.values", 54 | "value": "id" 55 | } 56 | }, 57 | "photos" : { 58 | "values" : { 59 | "output": {}, 60 | "key": "@generateId(data.profile.id)", 61 | "innerDocument": "!", 62 | "required": ["value"], 63 | "mappings" : { 64 | "id" : "@generateId(data.profile.id)", 65 | "type" : ">>facebook", 66 | "value" : "@generateFacebookImageLink(data.profile.id)" 67 | } 68 | }, 69 | "keys" : { 70 | "output": [], 71 | "innerDocument": "!photos.values", 72 | "value": "id" 73 | } 74 | }, 75 | "experience": { 76 | "values" : { 77 | "output" : {}, 78 | "key" : "@generateId(position.name|employer.name)", 79 | "innerDocument": "data.profile._json.work", 80 | "required" : ["id"], 81 | "mappings" : { 82 | "id" : "@generateId(position.name|employer.name)", 83 | "startDate" : "@parseDate(start_date)", 84 | "endDate" : "@parseDate(end_date)", 85 | "organisationName": "@cleanString(employer.name)", 86 | "role" : "@cleanString(position.name)" 87 | } 88 | }, 89 | "keys" : { 90 | "output": [], 91 | "innerDocument": "!experience.values", 92 | "value": "id" 93 | } 94 | }, 95 | "education" : { 96 | "values" : { 97 | "output" : {}, 98 | "key" : "@generateId(school.name|type)", 99 | "innerDocument": "data.profile._json.education", 100 | "required" : ["id"], 101 | "mappings" : { 102 | "id" : "@generateId(school.name|type)", 103 | "organisationName": "@cleanString(school.name)", 104 | "level" : "@cleanString(type)", 105 | "degree" : "@cleanString(type)", 106 | "endDate" : "@parseDate(year.name)" 107 | } 108 | }, 109 | "keys" : { 110 | "output": [], 111 | "innerDocument": "!education.values", 112 | "value": "id" 113 | } 114 | }, 115 | "createdAt": "@!new Date()", 116 | "creationSource": { 117 | "value": "source" 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /test/mappings/services/facebook_raw.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /* 4 | * @description The mapping file between the Facebook API and the sherlock model 5 | * The key represents the Facebook field name and the value is the mapped sherlock field name 6 | * If the value is an object this means that the Facebook field is an object and we need to map 7 | * the internal fields as well 8 | * 9 | * SPECIAL: keys that start with ^^ mean that they do not exist in Facebook but will be hardoded 10 | * in sherlock e.g. type field for social links, chat and websites 11 | * @returns {Object} 12 | */ 13 | 14 | module.exports = { 15 | "id" : "@generateUUID(>>id)", 16 | "fullName" : "@concatName(data.first_name|data.last_name)", 17 | "firstName" : "data.first_name", 18 | "lastName" : "data.last_name", 19 | "gender" : "data.gender", 20 | "birthday" : "data.birthday", 21 | "links" : { 22 | "values" : { 23 | "output": {}, 24 | "key": "@generateIdForLinks(data.link)", 25 | "innerDocument": "!", 26 | "required": ["value"], 27 | "mappings" : { 28 | "id" : "@generateIdForLinks(data.link)", 29 | "type" : ">>social", 30 | "value" : "data.link", 31 | "service" : ">>facebook", 32 | "username": "data.id", 33 | "userId" : "data.id" 34 | } 35 | }, 36 | "keys" : { 37 | "output": [], 38 | "innerDocument": "!links.values", 39 | "value": "id" 40 | } 41 | }, 42 | "emails" : { 43 | "values" : { 44 | "output": {}, 45 | "key": "@generateId(data.email)", 46 | "innerDocument": "!", 47 | "required": ["value"], 48 | "mappings" : { 49 | "id" : "@generateId(data.email)", 50 | "value": "@cleanEmail(data.email)", 51 | "type" : ">>facebook" 52 | } 53 | }, 54 | "keys" : { 55 | "output": [], 56 | "innerDocument": "!emails.values", 57 | "value": "id" 58 | } 59 | }, 60 | "experience": { 61 | "values" : { 62 | "output" : {}, 63 | "key" : "@generateId(position.name|employer.name)", 64 | "innerDocument": "data.work", 65 | "required" : ["id"], 66 | "mappings" : { 67 | "id" : "@generateId(position.name|employer.name)", 68 | "startDate" : "@parseDate(start_date)", 69 | "endDate" : "@parseDate(end_date)", 70 | "organisationName": "@cleanString(employer.name)", 71 | "location" : "location.name", 72 | "role" : "@cleanString(position.name)" 73 | } 74 | }, 75 | "keys" : { 76 | "output": [], 77 | "innerDocument": "!experience.values", 78 | "value": "id" 79 | } 80 | }, 81 | "education" : { 82 | "values" : { 83 | "output" : {}, 84 | "key" : "@generateId(school.name|type)", 85 | "innerDocument": "data.education", 86 | "required" : ["id"], 87 | "mappings" : { 88 | "id" : "@generateId(school.name|type)", 89 | "organisationName": "@cleanString(school.name)", 90 | "endDate" : "@parseDate(year.name)", 91 | "level" : "@cleanString(type)", 92 | "degree" : "@cleanString(type)" 93 | } 94 | }, 95 | "keys" : { 96 | "output": [], 97 | "innerDocument": "!education.values", 98 | "value": "id" 99 | } 100 | }, 101 | "photos" : { 102 | "values" : { 103 | "output" : {}, 104 | "innerDocument": "!", 105 | "key" : "@generateId(data.picture.data.url)", 106 | "required" : ["value"], 107 | "mappings" : { 108 | "id" : "@generateId(data.picture.data.url)", 109 | "value": "data.picture.data.url" 110 | } 111 | }, 112 | "keys" : { 113 | "output": [], 114 | "innerDocument": "!photos.values", 115 | "value": "id" 116 | } 117 | }, 118 | "createdAt": "@!new Date()", 119 | "creationSource": { 120 | "value": "source" 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /test/mappings/services/github.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /* 4 | * @description The mapping file between the Github API and the sherlock model 5 | * The key represents the Github field name and the value is the mapped sherlock field name 6 | * If the value is an object this means that the Github field is an object and we need to map 7 | * the internal fields as well 8 | * 9 | * SPECIAL: keys that start with ^^ mean that they do not exist in Github but will be hardoded 10 | * in sherlock e.g. type field for social links, chat and websites 11 | * @returns {Object} 12 | */ 13 | 14 | module.exports = { 15 | "id" : "@generateUUID(>>id)", 16 | "fullName" : "data.profile.displayName", 17 | "firstName": "@extractName(data.profile.displayName|>>firstName)", 18 | "links" : { 19 | "values" : [{ 20 | "output": {}, 21 | "key": "@generateIdForLinks(data.profile.profileUrl)", 22 | "innerDocument": "!", 23 | "required": ["value"], 24 | "mappings" : { 25 | "id" : "@generateIdForLinks(data.profile.profileUrl)", 26 | "type" : ">>social", 27 | "value" : "@cleanURI(data.profile.profileUrl|@getLinkType(value,@getLinkService(value,service)))", 28 | "service" : ">>github", 29 | "username" : "data.profile.username", 30 | "userId" : "data.profile.id", 31 | "following": "data.profile._json.following", 32 | "followers": "data.profile._json.followers" 33 | } 34 | },{ 35 | "output": {}, 36 | "key": "@generateId(data.profile._json.blog)", 37 | "innerDocument": "!", 38 | "required": ["value"], 39 | "mappings" : { 40 | "id" : "@generateId(data.profile._json.blog)", 41 | "type" : ">>website", 42 | "value" : "data.profile._json.blog" 43 | } 44 | }], 45 | "keys" : { 46 | "output": [], 47 | "innerDocument": "!links.values", 48 | "value": "id" 49 | } 50 | }, 51 | "experience": { 52 | "values" : { 53 | "output": {}, 54 | "key": "@generateId(data.profile._json.company)", 55 | "innerDocument": "!", 56 | "required": ["organisationName"], 57 | "mappings" : { 58 | "id" : "@generateId(data.profile._json.company)", 59 | "organisationName": "@cleanString(data.profile._json.company)" 60 | } 61 | }, 62 | "keys" : { 63 | "output": [], 64 | "innerDocument": "!experience.values", 65 | "value": "id" 66 | } 67 | }, 68 | "emails" : { 69 | "values" : { 70 | "output": {}, 71 | "key": "@generateId(data.profile._json.email)", 72 | "innerDocument": "!", 73 | "required": ["value"], 74 | "mappings" : { 75 | "id" : "@generateId(data.profile._json.email)", 76 | "value": "@cleanEmail(data.profile._json.email)", 77 | "type" : ">>github" 78 | } 79 | }, 80 | "keys" : { 81 | "output": [], 82 | "innerDocument": "!emails.values", 83 | "value": "id" 84 | } 85 | }, 86 | "location": { 87 | "address": "data.profile._json.location" 88 | }, 89 | "createdAt": "@!new Date()", 90 | "creationSource": { 91 | "value": "source" 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /test/mappings/services/github_raw.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /* 4 | * @description The mapping file between the Github API and the sherlock model 5 | * The key represents the Github field name and the value is the mapped sherlock field name 6 | * If the value is an object this means that the Github field is an object and we need to map 7 | * the internal fields as well 8 | * 9 | * SPECIAL: keys that start with ^^ mean that they do not exist in Github but will be hardoded 10 | * in sherlock e.g. type field for social links, chat and websites 11 | * @returns {Object} 12 | */ 13 | 14 | module.exports = { 15 | "id" : "@generateUUID(>>id)", 16 | "fullName" : "data.name", 17 | "firstName": "@extractName(data.name|>>firstName)", 18 | "links" : { 19 | "values" : [{ 20 | "output": {}, 21 | "key": "@generateIdForLinks(data.html_url)", 22 | "innerDocument": "!", 23 | "required": ["value"], 24 | "mappings" : { 25 | "id" : "@generateIdForLinks(data.html_url)", 26 | "type" : ">>social", 27 | "value" : "@cleanURI(data.html_url|@getLinkType(value,@getLinkService(value,service)))", 28 | "service" : ">>github", 29 | "username" : "data.login", 30 | "userId" : "@parseString(data.id)", 31 | "following": "data.following", 32 | "followers": "data.followers" 33 | } 34 | },{ 35 | "output": {}, 36 | "innerDocument": "!", 37 | "key": "@generateId(data.blog)", 38 | "required": ["value"], 39 | "mappings" : { 40 | "id" : "@generateId(data.blog)", 41 | "type" : ">>website", 42 | "value" : "data.blog" 43 | } 44 | }], 45 | "keys" : { 46 | "output": [], 47 | "innerDocument": "!links.values", 48 | "value": "id" 49 | } 50 | }, 51 | "experience": { 52 | "values" : { 53 | "output": {}, 54 | "key": "@generateId(data.company)", 55 | "innerDocument": "!", 56 | "required": ["organisationName"], 57 | "mappings" : { 58 | "id" : "@generateId(data.company)", 59 | "organisationName": "@cleanString(data.company)" 60 | } 61 | }, 62 | "keys" : { 63 | "output": [], 64 | "innerDocument": "!experience.values", 65 | "value": "id" 66 | } 67 | }, 68 | "emails" : { 69 | "values" : { 70 | "output": {}, 71 | "key": "@generateId(data.email)", 72 | "innerDocument": "!", 73 | "required": ["value"], 74 | "mappings" : { 75 | "id" : "@generateId(data.email)", 76 | "value": "@cleanEmail(data.email)", 77 | "type" : ">>github" 78 | } 79 | }, 80 | "keys" : { 81 | "output": [], 82 | "innerDocument": "!emails.values", 83 | "value": "id" 84 | } 85 | }, 86 | "photos" : { 87 | "values" : { 88 | "output": {}, 89 | "key": "@generateId(data.avatar_url)", 90 | "innerDocument": "!", 91 | "required": ["value"], 92 | "mappings" : { 93 | "id" : "@generateId(data.avatar_url)", 94 | "value": "data.avatar_url" 95 | } 96 | }, 97 | "keys" : { 98 | "output": [], 99 | "innerDocument": "!photos.values", 100 | "value": "id" 101 | } 102 | }, 103 | "location": { 104 | "address": "data.location" 105 | }, 106 | "createdAt": "@!new Date()", 107 | "creationSource": { 108 | "value": "source" 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /test/mappings/services/google.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /* 4 | * @description The mapping file between the Google API and the sherlock model 5 | * The key represents the Google field name and the value is the mapped sherlock field name 6 | * If the value is an object this means that the Google field is an object and we need to map 7 | * the internal fields as well 8 | * 9 | * SPECIAL: keys that start with ^^ mean that they do not exist in Google but will be hardoded 10 | * in sherlock e.g. type field for social links, chat and websites 11 | * @returns {Object} 12 | */ 13 | 14 | module.exports = { 15 | "id" : "@generateUUID(>>id)", 16 | "fullName" : "data.profile.displayName", 17 | "firstName" : "data.profile.name.givenName", 18 | "lastName" : "data.profile.name.familyName", 19 | "gender" : "data.profile._json.gender", 20 | "links" : { 21 | "values" : { 22 | "output": {}, 23 | "key": "@generateIdForLinks(data.profile._json.link)", 24 | "innerDocument": "!", 25 | "required": ["value"], 26 | "mappings" : { 27 | "id" : "@generateIdForLinks(data.profile._json.link)", 28 | "type" : ">>social", 29 | "username": "data.profile._json.id", 30 | "userId" : "data.profile.id", 31 | "value" : "@cleanURI(data.profile._json.link|@getLinkType(value,@getLinkService(value,service)))", 32 | "service" : ">>google" 33 | } 34 | }, 35 | "keys" : { 36 | "output": [], 37 | "innerDocument": "!links.values", 38 | "value": "id" 39 | } 40 | }, 41 | "photos" : { 42 | "values" : { 43 | "output": {}, 44 | "key": "@generateId(data.profile._json.picture)", 45 | "innerDocument": "!", 46 | "required": ["value"], 47 | "mappings" : { 48 | "id" : "@generateId(data.profile._json.picture)", 49 | "value": "data.profile._json.picture", 50 | "type" : ">>google" 51 | } 52 | }, 53 | "keys" : { 54 | "output": [], 55 | "innerDocument": "!photos.values", 56 | "value": "id" 57 | } 58 | }, 59 | "emails" : { 60 | "values" : { 61 | "output": {}, 62 | "key": "@generateId(value)", 63 | "innerDocument": "data.profile.emails", 64 | "required": ["value"], 65 | "mappings" : { 66 | "id" : "@generateId(value)", 67 | "value": "@cleanEmail(value)", 68 | "type" : ">>google" 69 | } 70 | }, 71 | "keys" : { 72 | "output": [], 73 | "innerDocument": "!emails.values", 74 | "value": "id" 75 | } 76 | }, 77 | "createdAt": "@!new Date()", 78 | "creationSource": { 79 | "value": "source" 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /test/mappings/services/google_raw.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /* 4 | * @description The mapping file between the Google API and the sherlock model 5 | * The key represents the Google field name and the value is the mapped sherlock field name 6 | * If the value is an object this means that the Google field is an object and we need to map 7 | * the internal fields as well 8 | * 9 | * SPECIAL: keys that start with ^^ mean that they do not exist in Google but will be hardoded 10 | * in sherlock e.g. type field for social links, chat and websites 11 | * @returns {Object} 12 | */ 13 | 14 | /* 15 | * The Google API call to get response with current structure 16 | * https://www.googleapis.com/oauth2/v1/userinfo 17 | */ 18 | 19 | module.exports = { 20 | "id" : "@generateUUID(>>id)", 21 | "fullName" : "name", 22 | "firstName" : "data.given_name", 23 | "lastName" : "data.family_name", 24 | "gender" : "data.gender", 25 | "links" : { 26 | "values" : { 27 | "output": {}, 28 | "key": "@generateIdForLinks(data.link)", 29 | "innerDocument": "!", 30 | "required": ["value"], 31 | "mappings" : { 32 | "id" : "@generateIdForLinks(data.link)", 33 | "type" : ">>social", 34 | "username": "data.id", 35 | "userId" : "data.id", 36 | "value" : "@cleanURI(data.link|@getLinkType(value,@getLinkService(value,service)))", 37 | "service" : ">>google" 38 | } 39 | }, 40 | "keys" : { 41 | "output": [], 42 | "innerDocument": "!links.values", 43 | "value": "id" 44 | } 45 | }, 46 | "photos" : { 47 | "values" : { 48 | "output": {}, 49 | "key": "@generateId(data.picture)", 50 | "innerDocument": "!", 51 | "required": ["value"], 52 | "mappings" : { 53 | "id" : "@generateId(data.picture)", 54 | "value": "data.picture", 55 | "type" : ">>google" 56 | } 57 | }, 58 | "keys" : { 59 | "output": [], 60 | "innerDocument": "!photos.values", 61 | "value": "id" 62 | } 63 | }, 64 | "emails" : { 65 | "values" : { 66 | "output": {}, 67 | "key": "@generateId(data.email)", 68 | "innerDocument": "!", 69 | "required": ["value"], 70 | "mappings" : { 71 | "id" : "@generateId(data.email)", 72 | "value": "@cleanEmail(data.email)", 73 | "type" : ">>google" 74 | } 75 | }, 76 | "keys" : { 77 | "output": [], 78 | "innerDocument": "!emails.values", 79 | "value": "id" 80 | } 81 | }, 82 | "createdAt": "@!new Date()", 83 | "creationSource": { 84 | "value": "source" 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /test/mappings/services/linkedin.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /* 4 | * @description The mapping file between the Linkedin API and the sherlock model 5 | * The key represents the Linkedin field name and the value is the mapped sherlock field name 6 | * If the value is an object this means that the Linkedin field is an object and we need to map 7 | * the internal fields as well 8 | * 9 | * SPECIAL: keys that start with ^^ mean that they do not exist in Linkedin but will be hardoded 10 | * in sherlock e.g. type field for social links, chat and websites 11 | * @returns {Object} 12 | */ 13 | 14 | module.exports = { 15 | "id" : "@generateUUID(>>id)", 16 | "fullName" : "@concatName(data.profile.name.givenName|data.profile.name.familyName)", 17 | "firstName" : "data.profile.name.givenName", 18 | "lastName" : "data.profile.name.familyName", 19 | "links" : { 20 | "values" : { 21 | "output": {}, 22 | "key": "@generateIdForLinks(data.profile._json.publicProfileUrl)", 23 | "innerDocument": "!", 24 | "required": ["value"], 25 | "mappings" : { 26 | "id" : "@generateIdForLinks(data.profile._json.publicProfileUrl)", 27 | "type" : ">>social", 28 | "value" : "@cleanURI(data.profile._json.publicProfileUrl|type)", 29 | "service" : ">>linkedin", 30 | "username" : "data.profile.id", 31 | "userId" : "data.profile.id", 32 | "bio" : "data.profile._json.summary", 33 | "following": "data.profile._json.numConnections" 34 | } 35 | }, 36 | "keys" : { 37 | "output": [], 38 | "innerDocument": "!links.values", 39 | "value": "id" 40 | } 41 | }, 42 | "emails" : { 43 | "values" : { 44 | "output": {}, 45 | "key": "@generateId(value)", 46 | "innerDocument": "data.profile.emails", 47 | "required": ["value"], 48 | "mappings" : { 49 | "id" : "@generateId(value)", 50 | "value": "@cleanEmail(value)", 51 | "type" : ">>linkedin" 52 | } 53 | }, 54 | "keys" : { 55 | "output": [], 56 | "innerDocument": "!emails.values", 57 | "value": "id" 58 | } 59 | }, 60 | "location": { 61 | "address" : "data.profile._json.location.name", 62 | "countryCode": "data.profile._json.location.country.code" 63 | }, 64 | "experience": { 65 | "values" : { 66 | "output" : {}, 67 | "key" : "@generateId(title|company.name)", 68 | "innerDocument": "data.profile._json.positions.values", 69 | "required" : ["id"], 70 | "mappings" : { 71 | "id" : "@generateId(title|company.name)", 72 | "startDate" : "@parseDate(startDate.year|startDate.month)", 73 | "endDate" : "@parseDate(endDate.year|endDate.month)", 74 | "organisationName": "@cleanString(company.name)", 75 | "description" : "summary", 76 | "role" : "@cleanString(title)", 77 | "current" : "isCurrent" 78 | } 79 | }, 80 | "keys" : { 81 | "output": [], 82 | "innerDocument": "!experience.values", 83 | "value": "id" 84 | } 85 | }, 86 | "education" : { 87 | "values" : { 88 | "output" : {}, 89 | "key" : "@generateId(schoolName|type|fieldOfStudy)", 90 | "innerDocument": "data.profile._json.educations.values", 91 | "required" : ["id"], 92 | "mappings" : { 93 | "id" : "@generateId(schoolName|type|fieldOfStudy)", 94 | "startDate" : "@parseDate(startDate.year|startDate.month)", 95 | "endDate" : "@parseDate(endDate.year|endDate.month)", 96 | "degree" : "@cleanString(degree)", 97 | "major" : "@cleanString(fieldOfStudy)", 98 | "description" : "notes", 99 | "level" : "@cleanString(type)", 100 | "organisationName": "@cleanString(schoolName)" 101 | } 102 | }, 103 | "keys" : { 104 | "output": [], 105 | "innerDocument": "!education.values", 106 | "value": "id" 107 | } 108 | }, 109 | "languages" : { 110 | "values" : { 111 | "output": {}, 112 | "key": "@generateId(language.name)", 113 | "innerDocument": "data.profile._json.languages.values", 114 | "required": ["value"], 115 | "mappings" : { 116 | "id" : "@generateId(language.name)", 117 | "value": "language.name" 118 | } 119 | }, 120 | "keys" : { 121 | "output": [], 122 | "innerDocument": "!languages.values", 123 | "value": "id" 124 | } 125 | }, 126 | "photos" : { 127 | "values" : { 128 | "output": {}, 129 | "key": "@generateId(data.profile._json.pictureUrl)", 130 | "innerDocument": "!", 131 | "required": ["value"], 132 | "mappings" : { 133 | "id" : "@generateId(data.profile._json.pictureUrl)", 134 | "value": "data.profile._json.pictureUrl" 135 | } 136 | }, 137 | "keys" : { 138 | "output": [], 139 | "innerDocument": "!photos.values", 140 | "value": "id" 141 | } 142 | }, 143 | "createdAt": "@!new Date()", 144 | "creationSource": { 145 | "value": "source" 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /test/mappings/services/linkedin_raw.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /* 4 | * @description The mapping file between the Linkedin API and the sherlock model 5 | * The key represents the Linkedin field name and the value is the mapped sherlock field name 6 | * If the value is an object this means that the Linkedin field is an object and we need to map 7 | * the internal fields as well 8 | * 9 | * SPECIAL: keys that start with ^^ mean that they do not exist in Linkedin but will be hardoded 10 | * in sherlock e.g. type field for social links, chat and websites 11 | * @returns {Object} 12 | */ 13 | 14 | module.exports = { 15 | "id" : "@generateUUID(>>id)", 16 | "fullName" : "data.formattedName", 17 | "firstName": "data.firstName", 18 | "lastName": "data.lastName", 19 | "links" : { 20 | "values" : { 21 | "output": {}, 22 | "key": "@generateIdForLinks(data.publicProfileUrl)", 23 | "innerDocument": "!", 24 | "required": ["value"], 25 | "mappings" : { 26 | "id" : "@generateIdForLinks(data.publicProfileUrl)", 27 | "type" : ">>social", 28 | "value" : "@cleanURI(data.publicProfileUrl|@getLinkType(value,@getLinkService(value,service)))", 29 | "service" : ">>linkedin", 30 | "username" : "data.id", 31 | "userId" : "data.id", 32 | "bio" : "data.summary", 33 | "following": "data.numConnections" 34 | } 35 | }, 36 | "keys" : { 37 | "output": [], 38 | "innerDocument": "!links.values", 39 | "value": "id" 40 | } 41 | }, 42 | "emails" : { 43 | "values" : { 44 | "output": {}, 45 | "key": "@generateId(data.emailAddress)", 46 | "innerDocument": "!", 47 | "required": ["value"], 48 | "mappings" : { 49 | "id" : "@generateId(data.emailAddress)", 50 | "value": "@cleanEmail(data.emailAddress)", 51 | "type" : ">>linkedin" 52 | } 53 | }, 54 | "keys" : { 55 | "output": [], 56 | "innerDocument": "!emails.values", 57 | "value": "id" 58 | } 59 | }, 60 | "location": { 61 | "address" : "data.location.name", 62 | "countryCode": "data.location.country.code" 63 | }, 64 | "experience": { 65 | "values" : { 66 | "output" : {}, 67 | "innerDocument": "data.positions.values", 68 | "key" : "@generateId(title|company.name)", 69 | "required" : ["id"], 70 | "mappings" : { 71 | "id" : "@generateId(title|company.name)", 72 | "startDate" : "@parseDate(startDate.year|startDate.month)", 73 | "endDate" : "@parseDate(endDate.year|endDate.month)", 74 | "description" : "summary", 75 | "role" : "@cleanString(title)", 76 | "current" : "isCurrent", 77 | "organisationName": "@cleanString(company.name)" 78 | } 79 | }, 80 | "keys" : { 81 | "output": [], 82 | "innerDocument": "!experience.values", 83 | "value": "id" 84 | } 85 | }, 86 | "education" : { 87 | "values" : { 88 | "output" : {}, 89 | "key" : "@generateId(schoolName|type|fieldOfStudy)", 90 | "innerDocument": "data.educations.values", 91 | "required" : ["id"], 92 | "mappings" : { 93 | "id" : "@generateId(schoolName|type|fieldOfStudy)", 94 | "startDate" : "@parseDate(startDate.year|startDate.month)", 95 | "endDate" : "@parseDate(endDate.year|endDate.month)", 96 | "degree" : "@cleanString(degree)", 97 | "major" : "fieldOfStudy", 98 | "description" : "@cleanString(notes)", 99 | "level" : "@cleanString(type)", 100 | "organisationName": "@cleanString(schoolName)" 101 | } 102 | }, 103 | "keys" : { 104 | "output": [], 105 | "innerDocument": "!education.values", 106 | "value": "id" 107 | } 108 | }, 109 | "languages" : { 110 | "values" : { 111 | "output": {}, 112 | "key": "@generateId(language.name)", 113 | "innerDocument": "data.languages.values", 114 | "required": ["value"], 115 | "mappings" : { 116 | "id" : "@generateId(language.name)", 117 | "value": "language.name" 118 | } 119 | }, 120 | "keys" : { 121 | "output": [], 122 | "innerDocument": "!languages.values", 123 | "value": "id" 124 | } 125 | }, 126 | "photos" : { 127 | "values" : { 128 | "output": {}, 129 | "key": "@generateId($value)", 130 | "innerDocument": "data.pictureUrls.values", 131 | "required": ["value"], 132 | "mappings" : { 133 | "id" : "@generateId($value)", 134 | "value": "$value" 135 | } 136 | }, 137 | "keys" : { 138 | "output": [], 139 | "innerDocument": "!photos.values", 140 | "value": "id" 141 | } 142 | }, 143 | "createdAt": "@!new Date()", 144 | "creationSource": { 145 | "value": "source" 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /test/mappings/test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "name" : "firstName", 5 | "default_name" : "nonExistingProperty||>>this_should_be_the_firstName", 6 | "nickname" : "nickname||>>nickname_not_found", 7 | "isNickNameFound" : "nickname||>>%false", 8 | "isDynamicDefault" : "nickname||firstName", 9 | "fullName" : "@concatName(firstName|lastName)", 10 | "fullNameDefault" : "@concatName(firstName|*!fullName)", 11 | "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", 12 | "completeName" : "@concatName(firstName|!fullName)", 13 | "displayName" : "!fullName", 14 | "email": { 15 | "value": "email" 16 | }, 17 | "links": "links", 18 | "social_links": [{ 19 | "output" : [], 20 | "innerDocument": "!links", 21 | "required": ["value"], 22 | "mappings" : { 23 | "value": "$value", 24 | "type": ">>test", 25 | "order": "$key", 26 | "social": ">>%true" 27 | } 28 | },{ 29 | "output" : [], 30 | "innerDocument": "social", 31 | "required": ["value"], 32 | "mappings" : { 33 | "value" : "value", 34 | "service": "service", 35 | "type": ">>social" 36 | } 37 | }], 38 | "website_addresses_keyless": { 39 | "output": [], 40 | "innerDocument": "linksv2.values", 41 | "prerequisite": "!!innerResult.value", 42 | "mappings": { 43 | "value": "value??type#==#>>website", 44 | "type": ">>other", 45 | } 46 | }, 47 | "website_addresses": { 48 | "output": {}, 49 | "innerDocument": "linksv2.values", 50 | "key": "id", 51 | "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", 52 | "mappings": { 53 | "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", 54 | "type": ">>other", 55 | "keys": "keys" 56 | } 57 | }, 58 | "social_media_addresses": { 59 | "output": [], 60 | "innerDocument": "linksv2.values", 61 | "prerequisite": "!!innerResult.value", 62 | "requirements": ["@uniqueArray(!social_media_addresses|>>value)", "@transformTwitterHandle(!social_media_addresses)"], 63 | "mappings": { 64 | "value": "value??type#==#>>social" 65 | } 66 | }, 67 | "messaging": { 68 | "output": [], 69 | "innerDocument": "linksv2.values", 70 | "prerequisite": "!!innerResult.value", 71 | "mappings": { 72 | "service": "@getLinkService(value|service)", 73 | "type" : "@getLinkType(value|@getLinkService(value,service))", 74 | "value" : "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" 75 | } 76 | }, 77 | "social_links_objects": { 78 | "output" : {}, 79 | "innerDocument": "!links", 80 | "key": "@generateId($value)", 81 | "mappings" : { 82 | "value": "$value" 83 | } 84 | }, 85 | "experience_primary": { 86 | "values": { 87 | "output" : {}, 88 | "innerDocument": "!", 89 | "key" : "@generateId(title|company)", 90 | "mappings" : { 91 | "id" : "@generateId(title|company)", 92 | "role" : "title", 93 | "organisationName": "company" 94 | } 95 | } 96 | }, 97 | "experience": { 98 | "output": [], 99 | "innerDocument": "work", 100 | "mappings": { 101 | "name" : "companyName", 102 | "role" : "title", 103 | "startDate": "@parseDate(startDate)", 104 | "current" : "current" 105 | } 106 | }, 107 | "primaryExperience": "!experience[0]", 108 | "primaryRole": "!experience[0].role", 109 | "experiences": { 110 | "output": [], 111 | "innerDocument": "work", 112 | "value": "companyName" 113 | }, 114 | "experience_object": { 115 | "values": { 116 | "output": {}, 117 | "innerDocument": "work", 118 | "key": "@generateId(companyName|title)", 119 | "mappings": { 120 | "id": "@generateId(companyName|title)", 121 | "name": "companyName", 122 | "role": "title", 123 | "startDate": "startDate", 124 | "current": "current" 125 | } 126 | } 127 | }, 128 | "education": { 129 | "output": [], 130 | "innerDocument": "json.education", 131 | "mappings": { 132 | "universityName": "$key", 133 | "degree": "degree", 134 | "location": "location" 135 | } 136 | }, 137 | "education_object": { 138 | "output": {}, 139 | "key": "@generateId($key|degree)", 140 | "innerDocument": "json.education", 141 | "mappings": { 142 | "degree": "degree", 143 | "location": "location", 144 | "universityName": "universityName" 145 | } 146 | }, 147 | "primaryPhoto": "@createURL(>>http://photo.com/|!fullName)" 148 | } 149 | -------------------------------------------------------------------------------- /test/plugins/base64Encode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('base64Encode plugin', function(){ 8 | 9 | it('should encode strings', function() { 10 | assert.equal(plugins.base64Encode("leandro"), "bGVhbmRybw=="); 11 | assert.equal(plugins.base64Encode(null), null); 12 | assert.equal(plugins.base64Encode('áccẽntz̧'), "w6FjY+G6vW50esyn"); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /test/plugins/cleanEmail.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('CleanEmail plugin', function(){ 9 | 10 | it('should correctly clean an email and strip any un-needed spaces and lowercase it', function(){ 11 | assert.equal(plugins.cleanEmail("AHMAD.A.assaf@gmail.com"), "ahmad.a.assaf@gmail.com"); 12 | assert.equal(plugins.cleanEmail(" AHMAD.A.assaf@gmail.com"), "ahmad.a.assaf@gmail.com"); 13 | assert.equal(plugins.cleanEmail("AHMAD.A.assaf@gmail.com "), "ahmad.a.assaf@gmail.com"); 14 | assert.equal(plugins.cleanEmail(" AHMAD.A.assaf@gmail.com "), "ahmad.a.assaf@gmail.com"); 15 | assert.equal(plugins.cleanEmail("A HMA D.A.a ssaf@g mail.com"), "ahmad.a.assaf@gmail.com"); 16 | }); 17 | 18 | it('should return null if no valid string is found', function(){ 19 | assert.equal(plugins.cleanEmail(" "), null); 20 | }); 21 | 22 | it('should return null if no valid string is passed as a parameter', function(){ 23 | assert.equal(plugins.cleanEmail(true), null); 24 | assert.equal(plugins.cleanEmail({}), null); 25 | assert.equal(plugins.cleanEmail(["test"]), null); 26 | assert.equal(plugins.cleanEmail(undefined), null); 27 | }); 28 | 29 | }); 30 | -------------------------------------------------------------------------------- /test/plugins/cleanString.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('CleanString plugin', function(){ 9 | 10 | it('should correctly clean a string and strip any un-needed characters', function(){ 11 | assert.equal(plugins.cleanString("...///T<<><>"), "T"); 12 | assert.equal(plugins.cleanString(".../// tEsT <<><>"), "tEsT"); 13 | assert.equal(plugins.cleanString(",,Test......"), "Test."); 14 | assert.equal(plugins.cleanString(".Test,,,,....."), "Test."); 15 | assert.equal(plugins.cleanString(" Test "), "Test"); 16 | }); 17 | 18 | it('should return null if no valid string is found', function(){ 19 | assert.equal(plugins.cleanString("...///<<><>"), null); 20 | }); 21 | 22 | it('should return null if no valid string is passed as a parameter', function(){ 23 | assert.equal(plugins.cleanString(true), null); 24 | assert.equal(plugins.cleanString({}), null); 25 | assert.equal(plugins.cleanString(["test"]), null); 26 | assert.equal(plugins.cleanString(undefined), null); 27 | }); 28 | 29 | }); 30 | -------------------------------------------------------------------------------- /test/plugins/cleanURI.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('CleanURI plugin', function(){ 9 | 10 | it('should return null if the URI is not valid', function(){ 11 | assert.equal(plugins.cleanURI("http://my:::test.asp?name=st%C3%A5le&car=saab"), null); 12 | assert.equal(plugins.cleanURI("...///<<><>"), null); 13 | assert.equal(plugins.cleanURI("https://www.linkedin.com/profile/view"), null); 14 | assert.equal(plugins.cleanURI("http://www.linkedin.com/profile/view"), null); 15 | assert.equal(plugins.cleanURI("https://linkedin.com/profile/view"), null); 16 | assert.equal(plugins.cleanURI("http://linkedin.com/profile/view"), null); 17 | assert.equal(plugins.cleanURI("http://www.linkedin.com/profile?viewProfile=&key=38795600&authToken=TlUF&authType=name&trk=api*a102401*s102401"), null); 18 | 19 | assert.equal(plugins.cleanURI("linkedin.com/profile/view"), null); 20 | }); 21 | 22 | it('should correctly clean a URI and decode it', function(){ 23 | assert.equal(plugins.cleanURI("http://mytest.asp?name=st%C3%A5le&car=saab"), "http://mytest.asp?name=ståle&car=saab"); 24 | }); 25 | 26 | it('should not blow out the cpu when parsing too long content', function(){ 27 | assert.ok(plugins.cleanURI(`I think I will be a good fit for this role as I already work within a fellowship and liaise day to day 28 | with fellows and colleagues. I have a strong personality which I channel to build a strong rapport with the surrounding 29 | teams and external associates. I have strong organisational skills I am pro-active and happy to take decisions independently. 30 | I thrive on getting a task done to the best of my abilities even when under pressure. I currently organise all of our Panel meetings 31 | there are 23 across the year as well as this any other important committee meetings in the academy calendar these include 32 | the MacRobert Awards committee, The Awards Committee, The Operating Committees Group, the Membership Committee as well as assisting 33 | with the organisation of the biggest event in the academy calendar the annual Awards dinner, where I work very closely with our events 34 | team. I am a team player and I use this skill when working closely with others including finance our programmes teams, events, policy, 35 | and the majority of the office. I am wanted to progress within the role that I am currently in and this role of Fellowship Operations 36 | assistant would enable me to do this effectively.`)); 37 | 38 | }) 39 | 40 | it('should correctly detect missing HTTP protocol and add it', function(){ 41 | assert.equal(plugins.cleanURI("mytest.asp?name=st%C3%A5le&car=saab"), "http://mytest.asp?name=ståle&car=saab"); 42 | assert.equal(plugins.cleanURI("mytest?name=st%C3%A5le&car=saab"), null); 43 | }); 44 | 45 | it('should correctly detect a Twitter handle and correct the url', function(){ 46 | assert.equal(plugins.cleanURI("@ahmadaassaf"), "http://twitter.com/ahmadaassaf"); 47 | assert.equal(plugins.cleanURI("ahmadaassaf"), null); 48 | }); 49 | 50 | it('should correctly detect a Skype handle and correct the url', function(){ 51 | assert.equal(plugins.cleanURI("ahmad.a.assaf", "messaging"), "ahmad.a.assaf"); 52 | assert.equal(plugins.cleanURI(" ahmadaassaf ", "messaging"), "ahmadaassaf"); 53 | }); 54 | 55 | it('should correctly detect a linkedin URL and canonicalize it', function(){ 56 | assert.equal(plugins.cleanURI("linkedin.com/pub/ahmadassaf?refId=extension"), "https://www.linkedin.com/in/ahmadassaf"); 57 | assert.equal(plugins.cleanURI("http://LINKEDIN.com/pub/koko-klai/a7/576/b50?trk=biz_employee_pub"), "https://www.linkedin.com/in/koko-klai-b50576a7"); 58 | assert.equal(plugins.cleanURI("https://sy.linkedin.com/pub/koko-klai/a7/576/b50?trk=biz_employee_pub"), "https://www.linkedin.com/in/koko-klai-b50576a7"); 59 | }); 60 | 61 | it('should return null if no valid string is passed as a parameter', function(){ 62 | assert.equal(plugins.cleanURI(true), null); 63 | assert.equal(plugins.cleanURI({}), null); 64 | assert.equal(plugins.cleanURI(["test"]), null); 65 | assert.equal(plugins.cleanURI(undefined), null); 66 | }); 67 | 68 | }); 69 | -------------------------------------------------------------------------------- /test/plugins/concatName.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('concatName plugin', function(){ 9 | 10 | it('should concatinate two or more strings', function(){ 11 | assert.equal(plugins.concatName("String 1", "String 2"), "String 1 String 2"); 12 | assert.equal(plugins.concatName("String 1", " ", "String 2"), "String 1 String 2"); 13 | assert.equal(plugins.concatName(["test"]), "test"); 14 | assert.equal(plugins.concatName("test"), "test"); 15 | assert.equal(plugins.concatName("test", {}, undefined), "test"); 16 | assert.equal(plugins.concatName({}, undefined, "test"), "test"); 17 | }); 18 | 19 | it('should conserve inner spaces in the strings but trim extra spaces on the begining and end', function(){ 20 | assert.equal(plugins.concatName("String 1", "String 2"), "String 1 String 2"); 21 | assert.equal(plugins.concatName(" String 1", "String 2 "), "String 1 String 2"); 22 | assert.equal(plugins.concatName("String 1", "String 2"), "String 1 String 2"); 23 | assert.equal(plugins.concatName(" String 1 ", " String 2 "), "String 1 String 2"); 24 | assert.equal(plugins.concatName("String 1", " ", " String 2"), "String 1 String 2"); 25 | }); 26 | 27 | it('should accept an array as one the arguments passed', function(){ 28 | assert.equal(plugins.concatName("String 1", ["middle string 1", "middle string2"], "String 2"), "String 1 middle string 1 middle string2 String 2"); 29 | assert.equal(plugins.concatName("String 1", [" middle string 1", " middle string 2 "], "String 2"), "String 1 middle string 1 middle string 2 String 2"); 30 | assert.equal(plugins.concatName("String 1", " ", ["middle string 1", " ", "middle string2"], " ", ["String 2"]), "String 1 middle string 1 middle string2 String 2"); 31 | assert.equal(plugins.concatName("String 1", " ", ["middle string 1", "middle string 2"], "String 2"), "String 1 middle string 1 middle string 2 String 2"); 32 | }); 33 | 34 | it('should return null if no valid arguments are passed as a parameter', function(){ 35 | assert.equal(plugins.concatName(true), null); 36 | assert.equal(plugins.concatName({}, {}, undefined), null); 37 | assert.equal(plugins.concatName(undefined), null); 38 | assert.equal(plugins.concatName(undefined, undefined), null); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /test/plugins/concatString.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('concatString plugin', function(){ 8 | 9 | it('should encode strings', function() { 10 | assert.equal(plugins.concatString("leandro", "1", "2"), "leandro12"); 11 | assert.equal(plugins.concatString(null), null); 12 | assert.equal(plugins.concatString("", "", ""), ""); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /test/plugins/concatWithComma.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('concatWithComma plugin', function(){ 8 | 9 | it('should encode strings', function() { 10 | assert.equal(plugins.concatWithComma('leandro', '1', '2'), 'leandro, 1, 2'); 11 | assert.equal(plugins.concatWithComma(null), null); 12 | assert.equal(plugins.concatWithComma('', '', ''), null); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /test/plugins/createURL.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('createURL plugin', function(){ 9 | 10 | it('should create a url from passed parameters', function(){ 11 | assert.equal(plugins.createURL("http://linkedin.com", "/in/ahmadassaf"), "http://linkedin.com/in/ahmadassaf"); 12 | assert.equal(plugins.createURL("http://linkedin.com", "/in/AHMADASSAF"), "http://linkedin.com/in/ahmadassaf"); 13 | assert.equal(plugins.createURL("http://LinKEDIN.com", " /in/AHMADASSAF "), "http://linkedin.com/in/ahmadassaf"); 14 | assert.equal(plugins.createURL("http://LinKEDIN.com"), "http://linkedin.com"); 15 | assert.equal(plugins.createURL("http://LinKEDIN.com "), "http://linkedin.com"); 16 | }); 17 | 18 | it('should return null if no valid string is passed as a parameter', function(){ 19 | assert.equal(plugins.createURL(), null); 20 | assert.equal(plugins.createURL(true), null); 21 | assert.equal(plugins.createURL({}), null); 22 | assert.equal(plugins.createURL(["test"]), null); 23 | assert.equal(plugins.createURL(undefined), null); 24 | }); 25 | 26 | }); 27 | -------------------------------------------------------------------------------- /test/plugins/extractName.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('extractName plugin', function(){ 9 | 10 | it('should extract correctly a first name', function(){ 11 | assert.equal(plugins.extractName("ahmad abdel mouti mohammad assaf", "firstName"), "ahmad"); 12 | assert.equal(plugins.extractName(" ahmad abdel mouti mohammad assaf", "firstName"), "ahmad"); 13 | assert.equal(plugins.extractName(" Ahmad abdel mouti mohammad assaf", "firstName"), "Ahmad"); 14 | }); 15 | 16 | it('should extract correctly a last name', function(){ 17 | assert.equal(plugins.extractName("ahmad abdel mouti mohammad assaf", "lastName"), "assaf"); 18 | assert.equal(plugins.extractName(" ahmad abdel mouti mohammad assaf ", "lastName"), "assaf"); 19 | assert.equal(plugins.extractName(" ahmad abdel mouti mohammad Assaf ", "lastName"), "Assaf"); 20 | }); 21 | 22 | it('should extract correctly middle names', function(){ 23 | assert.deepEqual(plugins.extractName("ahmad abdelmouti mohammad assaf", "middleName"), ["abdelmouti", "mohammad"]); 24 | assert.deepEqual(plugins.extractName(" ahmad abdelmouti mohammad assaf", "middleName"), ["abdelmouti", "mohammad"]); 25 | assert.deepEqual(plugins.extractName(" ahmad abdelmOUti Mohammad assaf", "middleName"), ["abdelmOUti", "Mohammad"]); 26 | assert.deepEqual(plugins.extractName(" ahmad abdelmOUti assaf", "middleName"), ["abdelmOUti"]); 27 | }); 28 | 29 | it('should return null if no valid arguments are passed', function(){ 30 | assert.equal(plugins.extractName(), null); 31 | assert.equal(plugins.extractName(true), null); 32 | assert.equal(plugins.extractName({}), null); 33 | assert.equal(plugins.extractName(["test"]), null); 34 | assert.equal(plugins.extractName("ahmad abdel mouti mohammad assaf"), null); 35 | assert.equal(plugins.extractName("ahmad abdel mouti mohammad assaf", "first"), null); 36 | assert.equal(plugins.extractName("ahmad abdel mouti mohammad assaf", undefined), null); 37 | assert.equal(plugins.extractName("ahmad abdel mouti mohammad assaf", null), null); 38 | assert.equal(plugins.extractName("ahmad abdel mouti mohammad assaf", ["firstName"]), null); 39 | assert.equal(plugins.extractName(undefined), null); 40 | }); 41 | 42 | }); 43 | -------------------------------------------------------------------------------- /test/plugins/formatDate.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const assert = require('assert'); 5 | const moment = require('moment'); 6 | 7 | const plugins = require('../../ditto/plugins/index'); 8 | 9 | describe('formatDate plugin', function () { 10 | 11 | it('should convert dates to strings without format', function () { 12 | const dateString = '2018-01-01'; 13 | assert.equal(plugins.formatDate(new Date(dateString)), '2018-01-01T00:00:00Z'); 14 | assert.equal(plugins.formatDate(moment(dateString)), '2018-01-01T00:00:00Z'); 15 | assert.equal(plugins.formatDate(dateString), '2018-01-01T00:00:00Z'); 16 | }); 17 | 18 | it('should convert dates to strings depending on format', function () { 19 | const format = 'YYYY/MM/DD'; 20 | const dateString = '2018-01-01'; 21 | assert.equal(plugins.formatDate(new Date(dateString), format), '2018/01/01'); 22 | assert.equal(plugins.formatDate(moment(dateString), format), '2018/01/01'); 23 | assert.equal(plugins.formatDate(dateString, format), '2018/01/01'); 24 | }); 25 | 26 | it('should return itself if no valid date is found', function () { 27 | assert.equal(plugins.formatDate(''), ''); 28 | assert.equal(plugins.formatDate(null), null); 29 | assert.equal(plugins.formatDate(undefined), undefined); 30 | assert.equal(plugins.formatDate('2018-50-50'), '2018-50-50'); 31 | }); 32 | 33 | it('should convert dates to string when is not UTC', function () { 34 | const format = null; 35 | const dateString = '2018-01-01'; 36 | const isUtc = false; 37 | assert.equal(plugins.formatDate(new Date(dateString), format, isUtc), '2018-01-01T00:00:00+00:00'); 38 | assert.equal(plugins.formatDate(new Date(dateString), format, isUtc), '2018-01-01T00:00:00+00:00'); 39 | assert.equal(plugins.formatDate(new Date(dateString), format, isUtc), '2018-01-01T00:00:00+00:00'); 40 | assert.equal(plugins.formatDate(new Date(dateString), format, isUtc), '2018-01-01T00:00:00+00:00'); 41 | }); 42 | 43 | }); 44 | -------------------------------------------------------------------------------- /test/plugins/generateCleanId.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('generateCleanId plugin', function(){ 8 | 9 | it('should generate a valid clean id', function() { 10 | assert.equal(plugins.generateCleanId("ahmad abdel mouti mohammad assaf", "firstName"), "0507d1431d3e9ad5b16e32ec64495df8"); 11 | assert.equal(plugins.generateCleanId("ahmad AbdEl mouti MOHAMMAD assaf", "fiRSTName"), "0507d1431d3e9ad5b16e32ec64495df8"); 12 | assert.equal(plugins.generateCleanId("ahmad ^^$ abdel mouti mohammad assaf ", "FIRstNAME"), "0507d1431d3e9ad5b16e32ec64495df8"); 13 | assert.equal(plugins.generateCleanId(["test"]), "098f6bcd4621d373cade4e832627b4f6"); 14 | assert.equal(plugins.generateCleanId("ahmad abdel mouti mohammad assaf"), "b55969000c21c7448df5f57ae51de823"); 15 | assert.equal(plugins.generateCleanId("ahmad abdel mouti mohammad assaf", undefined), "b55969000c21c7448df5f57ae51de823"); 16 | assert.equal(plugins.generateCleanId("ahmad abdel mouti mohammad assaf", null), "b55969000c21c7448df5f57ae51de823"); 17 | assert.equal(plugins.generateCleanId("ahmad abdel mouti mohammad assaf", ["firstName"]), "0507d1431d3e9ad5b16e32ec64495df8"); 18 | }); 19 | 20 | it('should return null if no valid arguments are passed', function(){ 21 | assert.equal(plugins.generateCleanId(), null); 22 | assert.equal(plugins.generateCleanId(true), null); 23 | assert.equal(plugins.generateCleanId({}), null); 24 | assert.equal(plugins.generateCleanId(undefined), null); 25 | }); 26 | }); 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /test/plugins/generateFacebookImageLink.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('generateFacebookImageLink plugin', function(){ 8 | 9 | it('should generate a valid facebook image link', function() { 10 | assert.equal(plugins.generateFacebookImageLink("0507d1431d3e9ad5b16e32ec64495df8"), "http://graph.facebook.com/0507d1431d3e9ad5b16e32ec64495df8/picture?type=large"); 11 | assert.equal(plugins.generateFacebookImageLink("ahmad.a.assaf"), "http://graph.facebook.com/ahmad.a.assaf/picture?type=large"); 12 | assert.equal(plugins.generateFacebookImageLink("0507d1431d3e9ad5b16e32ec64495df8", "firstName"), "http://graph.facebook.com/0507d1431d3e9ad5b16e32ec64495df8/picture?type=large"); 13 | }); 14 | 15 | it('should return null if no valid arguments are passed', function(){ 16 | assert.equal(plugins.generateFacebookImageLink(), null); 17 | assert.equal(plugins.generateFacebookImageLink(true), null); 18 | assert.equal(plugins.generateFacebookImageLink({}), null); 19 | assert.equal(plugins.generateFacebookImageLink(undefined), null); 20 | }); 21 | }); 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /test/plugins/generateFullExperience.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | const _ = require('lodash'); 5 | const debug = require('debug')('tests:generateFullExperience') 6 | const dittoInterface = require('../../ditto/ditto'); 7 | const plugins = require('../../ditto/plugins/index'); 8 | 9 | describe('generateFullExperience', function(){ 10 | const testMappings = { 11 | "experience": "@generateFullExperience(data.experience)" 12 | 13 | }; 14 | 15 | 16 | it('should reset the "organisationId" if "organisationName" is present in the input', function() { 17 | const testData = { 18 | data: { 19 | experience:{ 20 | organisationName: 'Microsoft' 21 | } 22 | } 23 | }; 24 | return new dittoInterface().unify(testData, testMappings).then((result) => { 25 | debug({result}); 26 | assert.equal(result.organisationId, null); 27 | }); 28 | }); 29 | 30 | 31 | it('should NOT reset the "organisationId" if "organisationName" is NOT present in the input', function() { 32 | const testData = { 33 | data: { 34 | experience:{ 35 | role: 'CEO' 36 | } 37 | } 38 | }; 39 | return new dittoInterface().unify(testData, testMappings).then((result) => { 40 | debug({result}); 41 | assert.equal(result.organisationName, void 0); 42 | assert.equal(result.organisationId, void 0); 43 | }); 44 | }); 45 | 46 | 47 | }); 48 | -------------------------------------------------------------------------------- /test/plugins/generateFullLocation.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | const _ = require('lodash'); 5 | const debug = require('debug')('tests:generateFullLocation') 6 | const dittoInterface = require('../../ditto/ditto'); 7 | const plugins = require('../../ditto/plugins/index'); 8 | 9 | describe('generateFullLocation', function(){ 10 | const testMappings = { 11 | "location": "@generateFullLocation(data.location)" 12 | 13 | }; 14 | 15 | /** 16 | * Helper function to ensure that in every case we set 'null' values on the values we can't provide, 17 | * to be sure to keep the location object coherent, in case this was going to update data set from the autocomplete, 18 | * which also provides the below keys 19 | */ 20 | const assertNullValues = ( location, propertyToBeNull ) => propertyToBeNull.forEach( prop => { 21 | assert.ok(location[prop] === null, `${prop} is not "null"`) 22 | }); 23 | 24 | it('should generate a valid address object with only country', function() { 25 | const testData = { 26 | data: { 27 | location:{ 28 | country: 'Spain' 29 | } 30 | } 31 | }; 32 | return new dittoInterface().unify(testData, testMappings).then((result) => { 33 | debug({result}); 34 | assert.equal(result.location.address, `${testData.data.location.country}`); 35 | assertNullValues(result.location, [ 'geometry', 'postalCode' ]); 36 | }); 37 | }); 38 | 39 | it('should generate a valid address object with city and country', function() { 40 | 41 | const testData = { 42 | data: { 43 | location:{ 44 | city: 'Barcelone', 45 | country: 'Spain' 46 | } 47 | } 48 | }; 49 | return new dittoInterface().unify(testData, testMappings).then((result) => { 50 | debug(result); 51 | assert.equal(result.location.address, `${testData.data.location.city}, ${testData.data.location.country}`); 52 | assertNullValues(result.location, [ 'geometry', 'postalCode' ]); 53 | 54 | }); 55 | }); 56 | 57 | it('should extract a valid country code', function() { 58 | 59 | const testData = { 60 | data: { 61 | location:{ 62 | city: 'Barcelone', 63 | country: 'Spain' 64 | } 65 | } 66 | }; 67 | return new dittoInterface().unify(testData, testMappings).then((result) => { 68 | debug(result); 69 | assert.equal(result.location.address, `${testData.data.location.city}, ${testData.data.location.country}`); 70 | assert.equal(result.location.countryCode, 'es'); 71 | 72 | }); 73 | }); 74 | 75 | it('should generate a blank address object with only city', function() { 76 | const testData = { 77 | data: { 78 | location:{ 79 | city: 'Barcelone' 80 | } 81 | } 82 | }; 83 | return new dittoInterface().unify(testData, testMappings).then((result) => { 84 | debug(result); 85 | assert.equal(Object.keys(result).length, 0); 86 | }); 87 | }); 88 | }); 89 | -------------------------------------------------------------------------------- /test/plugins/generateId.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('generateId plugin', function(){ 8 | 9 | it('should generate a valid id', function() { 10 | assert.equal(plugins.generateId("ahmad abdel mouti mohammad assaf", "firstName"), "0507d1431d3e9ad5b16e32ec64495df8"); 11 | assert.equal(plugins.generateId("ahmad AbdEl mouti MOHAMMAD assaf", "fiRSTName"), "0507d1431d3e9ad5b16e32ec64495df8"); 12 | assert.equal(plugins.generateId(["test"]), "098f6bcd4621d373cade4e832627b4f6"); 13 | assert.equal(plugins.generateId("ahmad abdel mouti mohammad assaf"), "b55969000c21c7448df5f57ae51de823"); 14 | assert.equal(plugins.generateId("ahmad abdel mouti mohammad assaf", undefined), "b55969000c21c7448df5f57ae51de823"); 15 | assert.equal(plugins.generateId("ahmad abdel mouti mohammad assaf", null), "b55969000c21c7448df5f57ae51de823"); 16 | assert.equal(plugins.generateId("ahmad abdel mouti mohammad assaf", ["firstName"]), "0507d1431d3e9ad5b16e32ec64495df8"); 17 | }); 18 | 19 | it('should return null if no valid arguments are passed', function(){ 20 | assert.equal(plugins.generateId(), null); 21 | assert.equal(plugins.generateId(true), null); 22 | assert.equal(plugins.generateId({}), null); 23 | assert.equal(plugins.generateId(undefined), null); 24 | }); 25 | }); 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /test/plugins/generateIdForLinks.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('generateIdForLinks plugin', function(){ 8 | 9 | it('should generate a valid link id', function() { 10 | assert.equal(plugins.generateIdForLinks("http://ahmadassaf.com"), "05d6c6c022a21e9bb169754b78c1e289"); 11 | assert.equal(plugins.generateIdForLinks("http://ahmadassaf.com "), "05d6c6c022a21e9bb169754b78c1e289"); 12 | assert.equal(plugins.generateIdForLinks(" http://ahmadassaf.com "), "05d6c6c022a21e9bb169754b78c1e289"); 13 | assert.equal(plugins.generateIdForLinks(" http://ahmadassaf. com "), "05d6c6c022a21e9bb169754b78c1e289"); 14 | assert.equal(plugins.generateIdForLinks("http://AHMADASSAF.com"), "05d6c6c022a21e9bb169754b78c1e289"); 15 | assert.equal(plugins.generateIdForLinks("http://AHMADAssAF.com"), "05d6c6c022a21e9bb169754b78c1e289"); 16 | assert.equal(plugins.generateIdForLinks("http://ahmadassaf.com?refid=extension"), "439b66339ad38ff4064d1d0e9eff1ae1"); 17 | }); 18 | 19 | it('should generate a valid link id for a link with missing HTTP protocol', function() { 20 | assert.equal(plugins.generateIdForLinks("ahmadassaf.com"), "05d6c6c022a21e9bb169754b78c1e289"); 21 | assert.equal(plugins.generateIdForLinks("http://ahmadassaf.com "), "05d6c6c022a21e9bb169754b78c1e289"); 22 | assert.equal(plugins.generateIdForLinks("ahmadassaf.com?refid=extension"), "439b66339ad38ff4064d1d0e9eff1ae1"); 23 | }); 24 | 25 | it('should generate a valid link id for a canonical linkeidn url', function() { 26 | assert.equal(plugins.generateIdForLinks("http://linkedin.com/pub/ahmadassaf"), "eff585427879a3fd62cfe884fa5d8068"); 27 | assert.equal(plugins.generateIdForLinks("http://linkedin.com/pub/ahmadassaf?ref=extension"), "eff585427879a3fd62cfe884fa5d8068"); 28 | assert.equal(plugins.generateIdForLinks("linkedin.com/pub/ahmadassaf"), "eff585427879a3fd62cfe884fa5d8068"); 29 | }); 30 | 31 | 32 | it('should generate a valid id for a twitter handle', function() { 33 | assert.equal(plugins.generateIdForLinks("@ahmadaassaf"), "c1c9635aac4c64af6bc2d4e688f5bbf4"); 34 | assert.equal(plugins.generateIdForLinks("@ahmadaassaf "), "c1c9635aac4c64af6bc2d4e688f5bbf4"); 35 | assert.equal(plugins.generateIdForLinks(" @ahmadaassaf "), "c1c9635aac4c64af6bc2d4e688f5bbf4"); 36 | }); 37 | 38 | it('should return null if no valid arguments are passed', function(){ 39 | assert.equal(plugins.generateIdForLinks(), null); 40 | assert.equal(plugins.generateIdForLinks(true), null); 41 | assert.equal(plugins.generateIdForLinks(null), null); 42 | assert.equal(plugins.generateIdForLinks({}), null); 43 | assert.equal(plugins.generateIdForLinks(undefined), null); 44 | assert.equal(plugins.generateIdForLinks("ahmadassaf"), null); 45 | }); 46 | }); 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /test/plugins/generateIdFromLanguageCode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('generateIdFromLanguageCode plugin', function(){ 8 | 9 | it('should return null if no valid arguments are passed', function(){ 10 | assert.equal(plugins.generateIdFromLanguageCode("en"), "ba0a6ddd94c73698a3658f92ac222f8a"); 11 | assert.equal(plugins.generateIdFromLanguageCode("AS"), "4e9c612f638bb88514ef8327b6bc02af"); 12 | assert.equal(plugins.generateIdFromLanguageCode("La"), "f4b418eba9c0b3d4901880b743cbbaff"); 13 | }); 14 | 15 | it('should return null if no valid language code is found', function(){ 16 | assert.equal(plugins.generateIdFromLanguageCode("qq"), null); 17 | assert.equal(plugins.generateIdFromLanguageCode("311"), null); 18 | assert.equal(plugins.generateIdFromLanguageCode(999), null); 19 | }); 20 | 21 | it('should return null if no valid arguments are passed', function(){ 22 | assert.equal(plugins.generateIdFromLanguageCode(), null); 23 | assert.equal(plugins.generateIdFromLanguageCode(true), null); 24 | assert.equal(plugins.generateIdFromLanguageCode(null), null); 25 | assert.equal(plugins.generateIdFromLanguageCode({}), null); 26 | assert.equal(plugins.generateIdFromLanguageCode(undefined), null); 27 | }); 28 | }); 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /test/plugins/generateUUID.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('generateUUID plugin', function(){ 8 | 9 | it('should generate a valid UUID', function() { 10 | assert.ok(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/.test(plugins.generateUUID())); 11 | }); 12 | }); 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /test/plugins/getLanguageCode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('getLanguageCode plugin', function(){ 8 | 9 | it('should return correct language code', function(){ 10 | assert.equal(plugins.getLanguageCode("ENGLISH"), "en"); 11 | assert.equal(plugins.getLanguageCode(" AssaMesE"), "as"); 12 | assert.equal(plugins.getLanguageCode(" latin "), "la"); 13 | assert.ok(plugins.getLanguageCode("atin") !== "la"); 14 | }); 15 | 16 | it('should return null if no valid language code is found', function(){ 17 | assert.equal(plugins.getLanguageCode("amazonas"), null); 18 | assert.equal(plugins.getLanguageCode("blabloom"), null); 19 | }); 20 | 21 | it('should return null if no valid arguments are passed', function(){ 22 | assert.equal(plugins.getLanguageCode(), null); 23 | assert.equal(plugins.getLanguageCode(true), null); 24 | assert.equal(plugins.getLanguageCode(null), null); 25 | assert.equal(plugins.getLanguageCode({}), null); 26 | assert.equal(plugins.getLanguageCode(undefined), null); 27 | }); 28 | }); 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /test/plugins/getLanguageFromCode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('getLanguageFromCode plugin', function(){ 8 | 9 | it('should return correct language code', function(){ 10 | assert.equal(plugins.getLanguageFromCode("en"), "English"); 11 | assert.equal(plugins.getLanguageFromCode(" As"), "Assamese"); 12 | assert.equal(plugins.getLanguageFromCode(" LA "), "Latin"); 13 | }); 14 | 15 | it('should return null if no valid language code is found', function(){ 16 | assert.equal(plugins.getLanguageFromCode("amazonas"), null); 17 | assert.equal(plugins.getLanguageFromCode("blabloom"), null); 18 | }); 19 | 20 | it('should return null if no valid arguments are passed', function(){ 21 | assert.equal(plugins.getLanguageFromCode(), null); 22 | assert.equal(plugins.getLanguageFromCode(true), null); 23 | assert.equal(plugins.getLanguageFromCode(null), null); 24 | assert.equal(plugins.getLanguageFromCode({}), null); 25 | assert.equal(plugins.getLanguageFromCode(undefined), null); 26 | }); 27 | }); 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /test/plugins/getLinkService.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('getLinkService plugin', function(){ 8 | 9 | it('should generate a valid link service', function() { 10 | assert.equal(plugins.getLinkService("http://globo.com/eget/eros/elementum.js?ut=ut&odio=at&cras=dolor&mi=quis&pede=odio&malesuada=consequat&in=varius&imperdiet=integer&et=ac&commodo=leo&vulputate=pellentesque&justo=ultrices&in=mattis&blandit=odio&ultrices=donec&enim=vitae&lorem=nisi&ipsum=nam&dolor=ultrices&sit=libero&amet=non"), "globo"); 11 | assert.equal(plugins.getLinkService("http://Facebook.com/ahmad.a.assaf "), "facebook"); 12 | assert.equal(plugins.getLinkService("FACEBOOK.com?refid=extension"), "facebook"); 13 | }); 14 | 15 | it('should generate a valid link service overriding the extraction from the source', function() { 16 | assert.equal(plugins.getLinkService("http://ahmadassaf.com", "facebook"), "facebook"); 17 | assert.equal(plugins.getLinkService("http://Facebook.com/ahmad.a.assaf ", "linkEDIN"), "linkedin"); 18 | assert.equal(plugins.getLinkService("FACEBOOK.com?refid=extension", "FACEBOOK"), "facebook"); 19 | }); 20 | 21 | it('should generate a valid link service for a link with missing HTTP protocol', function() { 22 | assert.equal(plugins.getLinkService("ahmadassaf.com"), "ahmadassaf"); 23 | assert.equal(plugins.getLinkService("http://ahmadassaf.com "), "ahmadassaf"); 24 | assert.equal(plugins.getLinkService("ahmadassaf.com?refid=extension"), "ahmadassaf"); 25 | }); 26 | 27 | it('should generate a valid link id for a canonical linkeidn url', function() { 28 | assert.equal(plugins.getLinkService("http://linkedin.com/pub/ahmadassaf"), "linkedin"); 29 | assert.equal(plugins.getLinkService("http://linkedin.com/pub/ahmadassaf?ref=extension"), "linkedin"); 30 | assert.equal(plugins.getLinkService("linkedin.com/pub/ahmadassaf"), "linkedin"); 31 | }); 32 | 33 | 34 | it('should generate a valid id for a twitter handle', function() { 35 | assert.equal(plugins.getLinkService("@ahmadaassaf"), "twitter"); 36 | assert.equal(plugins.getLinkService("@ahmadaassaf "), "twitter"); 37 | assert.equal(plugins.getLinkService(" @ahmadaassaf "), "twitter"); 38 | }); 39 | 40 | it('should return null if called on an email address', function() { 41 | assert.equal(plugins.getLinkService("cocoagems@github.io"), null ) 42 | assert.equal(plugins.getLinkService("http://github.io/user:@ahmadassaf"), "github" ) 43 | }); 44 | 45 | it('should return null if no valid arguments are passed', function(){ 46 | assert.equal(plugins.getLinkService(), null); 47 | assert.equal(plugins.getLinkService(true), null); 48 | assert.equal(plugins.getLinkService(null), null); 49 | assert.equal(plugins.getLinkService({}), null); 50 | assert.equal(plugins.getLinkService(undefined), null); 51 | assert.equal(plugins.getLinkService("ahmadassaf"), null); 52 | }); 53 | }); 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /test/plugins/getLinkType.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const dittoInterface = require('../../ditto/ditto'); 7 | const plugins = require('../../ditto/plugins/index'); 8 | 9 | describe('getLinkType plugin', function(){ 10 | 11 | it('should return a correct link type from a link URI', function(){ 12 | assert.equal(plugins.getLinkType("http://facebook.com/ahmad.a.assaf"), "social"); 13 | assert.equal(plugins.getLinkType("https://linkedin.com/in/ahmadassaf"), "social"); 14 | assert.equal(plugins.getLinkType("https://wwww.linkedin.com/in/ahmadassaf"), "social"); 15 | assert.equal(plugins.getLinkType("https://twitter.com/ahmadaassaf"), "social"); 16 | assert.equal(plugins.getLinkType("https://google.com/ahmadaassaf"), "website"); 17 | assert.equal(plugins.getLinkType("https://ahmadassaf.com/"), "website"); 18 | }); 19 | 20 | it('should return a correct link type from a twitter handle', function(){ 21 | assert.equal(plugins.getLinkType("@ahmadaassaf"), "social"); 22 | assert.equal(plugins.getLinkType(" @ahmadaassaf "), "social"); 23 | }); 24 | 25 | it('should return a correct link type of messaging when a service is passed', function(){ 26 | assert.equal(plugins.getLinkType("ahmad.a.assaf", "skype"), "messaging"); 27 | assert.equal(plugins.getLinkType("@ahmadaassaf", "qq"), "messaging"); 28 | assert.equal(plugins.getLinkType("@ahmadaassaf", "twitter"), "social"); 29 | }); 30 | 31 | it('should return a correct link type considering the "service"', function(){ 32 | assert.equal(plugins.getLinkType("ahmad.a.assaf", "skype"), "messaging"); 33 | }); 34 | 35 | 36 | it('should return null if no valid arguments are passed', function(){ 37 | assert.equal(plugins.getLinkType(), null); 38 | assert.equal(plugins.getLinkType(true), null); 39 | assert.equal(plugins.getLinkType(null), null); 40 | assert.equal(plugins.getLinkType({}), null); 41 | assert.equal(plugins.getLinkType(undefined), null); 42 | }); 43 | 44 | }); 45 | -------------------------------------------------------------------------------- /test/plugins/getValueAtPath.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const assert = require('assert'); 5 | const { getValueAtPath } = require('../../ditto/plugins'); 6 | 7 | describe('getValueAtPath', function () { 8 | 9 | it('should return value at path specified in array or string', function() { 10 | assert.equal(getValueAtPath({a: {b: 1}}, ['a', 'b']), 1); 11 | assert.equal(getValueAtPath({a: {b: 1}}, 'a.b'), 1); 12 | }); 13 | 14 | it('should return undefined if no value at path', function() { 15 | assert.equal(getValueAtPath({a: {b: 1}}, ['a', 'b', 'c']), undefined); 16 | assert.equal(getValueAtPath({a: {b: 1}}, 'a.b.c'), undefined); 17 | }); 18 | 19 | }); 20 | -------------------------------------------------------------------------------- /test/plugins/minBy.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | const { minBy } = require('../../ditto/plugins'); 5 | 6 | describe('minBy', function () { 7 | 8 | it('should return min value from array if no path specified and items are numbers', function () { 9 | assert.equal(minBy([1, 3, 2]), 1); 10 | }); 11 | 12 | it('should return lowest value by code of first character if array values are strings', function() { 13 | assert.equal(minBy(['bbb', 'afdd', 'ffaaa']), 'afdd'); 14 | }); 15 | 16 | it('should, when given array of objects and path, return prop with lowest value at path', function() { 17 | const arr = [ { a: { a: 3} }, { a: { a: 2} }, { a: { a: 1} }]; 18 | assert.deepEqual(minBy(arr, 'a.a'), { a: { a: 1} }); 19 | }); 20 | 21 | it('should return undefined if given wrong or empty input in place of array argument', function() { 22 | assert.equal(minBy({a: 'a'}), undefined); 23 | assert.equal(minBy([]), undefined); 24 | assert.equal(minBy('string'), undefined); 25 | }); 26 | 27 | }); 28 | -------------------------------------------------------------------------------- /test/plugins/normalizeString.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('normalizeString plugin', function(){ 9 | 10 | it('should correctly clean a string and strip any un-needed characters', function(){ 11 | assert.equal(plugins.normalizeString("...///T<<><>"), "t"); 12 | assert.equal(plugins.normalizeString(".../// tEsT <<><>"), "test"); 13 | assert.equal(plugins.normalizeString(",,Test......"), "test."); 14 | assert.equal(plugins.normalizeString(".Test,,,,....."), "test."); 15 | assert.equal(plugins.normalizeString(" Test "), "test"); 16 | }); 17 | 18 | it('should return null if no valid string is found', function(){ 19 | assert.equal(plugins.normalizeString("...///<<><>"), null); 20 | }); 21 | 22 | it('should return null if no valid string is passed as a parameter', function(){ 23 | assert.equal(plugins.normalizeString(true), null); 24 | assert.equal(plugins.normalizeString({}), null); 25 | assert.equal(plugins.normalizeString(["test"]), null); 26 | assert.equal(plugins.normalizeString(undefined), null); 27 | }); 28 | 29 | }); 30 | -------------------------------------------------------------------------------- /test/plugins/parseDate.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const dittoInterface = require('../../ditto/ditto'); 7 | const plugins = require('../../ditto/plugins/index'); 8 | 9 | 10 | const isValidDate = (input) => !isNaN(Date.parse(input)); 11 | 12 | describe('parseDate plugin', function(){ 13 | 14 | it('should return null for not date strings', function(){ 15 | assert.equal(plugins.parseDate('notKnown'), null); 16 | }); 17 | 18 | it('should return a valid date for ISO date strings', function(){ 19 | assert.ok(isValidDate(plugins.parseDate('2017-01-30T11:04:04.277Z'))); 20 | }); 21 | 22 | it('should return a valid date for "full contact" date format', function(){ 23 | 24 | const result = plugins.parseDate('2014-10'); 25 | 26 | assert.ok(isValidDate(result)); 27 | assert.equal(result.getMonth() + 1, 10); 28 | assert.equal(result.getFullYear(), 2014); 29 | 30 | }); 31 | 32 | it('should parse the provided "month" argument', function(){ 33 | 34 | const result = plugins.parseDate(2016, 6); 35 | 36 | assert.ok(isValidDate(result)); 37 | assert.equal(result.getMonth() + 1, 6) 38 | }); 39 | 40 | it('should parse the provided "day" argument', function(){ 41 | 42 | const result = plugins.parseDate(2018, 10, 16); 43 | 44 | assert.ok(isValidDate(result)); 45 | assert.equal(result.getMonth() + 1, 10); 46 | assert.equal(result.getFullYear(), 2018); 47 | assert.equal(result.getDate(), 16); 48 | }); 49 | 50 | it('should default the month if not provided as an argument', function(){ 51 | 52 | const result = plugins.parseDate(2016); 53 | 54 | assert.ok(isValidDate(result)); 55 | assert.equal(result.getMonth() + 1, 1); 56 | }); 57 | 58 | it('should not set null or undefined values when no item present', function(){ 59 | 60 | const testMap = { 61 | "testDate": "@parseDate(data.testDate.year|data.testDate.month)" 62 | }; 63 | const dummySample = { 64 | data: { 65 | testDate: { 66 | year: 2016, 67 | month: 7 68 | } 69 | } 70 | }; 71 | 72 | return new dittoInterface().unify(dummySample, testMap).then((result) => { 73 | assert.equal(result.testDate.getFullYear(), 2016); 74 | assert.equal(result.testDate.getMonth()+1, 7); 75 | }); 76 | }); 77 | 78 | }); 79 | -------------------------------------------------------------------------------- /test/plugins/parseString.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('parseString plugin', function(){ 9 | 10 | it('should return a string from a non-string value', function(){ 11 | assert.equal(plugins.parseString(12311231.00), "12311231"); 12 | assert.equal(plugins.parseString(12311211), "12311211"); 13 | assert.equal(plugins.parseString(false), "false"); 14 | }); 15 | 16 | it('should return null if no valid arguments are passed', function(){ 17 | assert.equal(plugins.getLinkType(), null); 18 | assert.equal(plugins.getLinkType(true), null); 19 | assert.equal(plugins.getLinkType(null), null); 20 | assert.equal(plugins.getLinkType({}), null); 21 | assert.equal(plugins.getLinkType(undefined), null); 22 | }); 23 | 24 | }); 25 | -------------------------------------------------------------------------------- /test/plugins/splitList.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | 5 | const plugins = require('../../ditto/plugins/index'); 6 | 7 | describe('splitList plugin', function(){ 8 | 9 | it('should encode strings', function() { 10 | assert.deepEqual(plugins.splitList("leandro"), ["leandro"]); 11 | assert.deepEqual(plugins.splitList(null), null); 12 | assert.deepEqual(plugins.splitList(123), null); 13 | assert.deepEqual(plugins.splitList("a,b,c"), ["a", "b", "c"]); 14 | assert.deepEqual(plugins.splitList("a, b and c"), ["a", "b", "c"]); 15 | assert.deepEqual(plugins.splitList("a, b and c"), ["a", "b", "c"]); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /test/plugins/uniqueArray.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const plugins = require('../../ditto/plugins/index'); 7 | 8 | describe('uniqueArray plugin', function(){ 9 | 10 | 11 | it('should return a unique array if no key was not passed', function(){ 12 | assert.deepStrictEqual(plugins.uniqueArray([1,2,3,4,3]), [1,2,3,4]); 13 | assert.deepStrictEqual(plugins.uniqueArray([1,2,3,4,3], null), [1,2,3,4]); 14 | assert.deepStrictEqual(plugins.uniqueArray(["a", "b", "b"]), ["a", "b"]); 15 | assert.deepStrictEqual(plugins.uniqueArray([{key: "a"}, {key: "b"}, {key: "b"}]), [{key: "a"}, {key: "b"}, {key: "b"}]); 16 | }); 17 | 18 | it('should return a unqiue array based on a passed key', function(){ 19 | assert.deepStrictEqual(plugins.uniqueArray([{key: "a"}, {key: "b"}, {key: "b"}], "key"), [{key: "a"}, {key: "b"}]); 20 | }); 21 | 22 | it('should return an empty array if no valid arguments are passed', function(){ 23 | assert.deepStrictEqual(plugins.uniqueArray(null), []); 24 | assert.deepStrictEqual(plugins.uniqueArray(), []); 25 | assert.deepStrictEqual(plugins.uniqueArray([]), []); 26 | }); 27 | 28 | }); 29 | -------------------------------------------------------------------------------- /test/results/services/facebook.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "id": "0dfc3dc1-a304-4ccf-82cb-f5e173a55742", 5 | "firstName": "Test Ts", 6 | "lastName": "Ts", 7 | "fullName": "Test Ts Ts", 8 | "gender": "male", 9 | "links": { 10 | "values": { 11 | "38b70b4f2d5fb00c29f76c632d6a62c6": { 12 | "id": "38b70b4f2d5fb00c29f76c632d6a62c6", 13 | "type": "social", 14 | "value": "https://www.facebook.com/app_scoped_user_id/10153358626426860/", 15 | "service": "facebook", 16 | "username": "10153358626426860", 17 | "userId": "10153358626426860" 18 | } 19 | }, 20 | "keys": [ 21 | "38b70b4f2d5fb00c29f76c632d6a62c6" 22 | ] 23 | }, 24 | "emails": { 25 | "values": { 26 | "6b61c66d59f8fc6389cb717c88493993": { 27 | "id": "6b61c66d59f8fc6389cb717c88493993", 28 | "value": "test@test.edu", 29 | "type": "facebook" 30 | } 31 | }, 32 | "keys": [ 33 | "6b61c66d59f8fc6389cb717c88493993" 34 | ] 35 | }, 36 | "photos": { 37 | "values": { 38 | "b71b11623d78015c7484e8a4d6c45b58": { 39 | "id": "b71b11623d78015c7484e8a4d6c45b58", 40 | "type": "facebook", 41 | "value": "http://graph.facebook.com/10153358626426860/picture?type=large" 42 | } 43 | }, 44 | "keys": [ 45 | "b71b11623d78015c7484e8a4d6c45b58" 46 | ] 47 | }, 48 | "experience": { 49 | "values": { 50 | "73abd24ee6d284b8dcda4dfccfb4814d": { 51 | "id": "73abd24ee6d284b8dcda4dfccfb4814d", 52 | "organisationName": "Singapore Armed Forces" 53 | }, 54 | "a0596fa3a5bb2077ef3d358b524c7628": { 55 | "id": "a0596fa3a5bb2077ef3d358b524c7628", 56 | "organisationName": "APSC (Aquatic Performance Swim Club)" 57 | } 58 | }, 59 | "keys": [ 60 | "73abd24ee6d284b8dcda4dfccfb4814d", 61 | "a0596fa3a5bb2077ef3d358b524c7628" 62 | ] 63 | }, 64 | "education": { 65 | "values": { 66 | "28a3b3ea42a5b1dd96ca2d301548729f": { 67 | "id": "28a3b3ea42a5b1dd96ca2d301548729f", 68 | "organisationName": "Raffles Institution", 69 | "level": "High School", 70 | "degree": "High School" 71 | }, 72 | "ae9a6f89bba68684f282e98d68e3170a": { 73 | "id": "ae9a6f89bba68684f282e98d68e3170a", 74 | "organisationName": "Raffles Junior College", 75 | "level": "High School", 76 | "degree": "High School", 77 | "endDate": new Date("2009-01-01T00:00:00.000Z") 78 | }, 79 | "d57c572a03ade2066ef8142a6c7a7f50": { 80 | "id": "d57c572a03ade2066ef8142a6c7a7f50", 81 | "organisationName": "Duke University", 82 | "level": "College", 83 | "degree": "College" 84 | } 85 | }, 86 | "keys": [ 87 | "28a3b3ea42a5b1dd96ca2d301548729f", 88 | "ae9a6f89bba68684f282e98d68e3170a", 89 | "d57c572a03ade2066ef8142a6c7a7f50" 90 | ] 91 | }, 92 | "createdAt": "2017-02-07T11:43:37.784Z", 93 | "creationSource": { 94 | "value": "facebook" 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /test/results/services/facebook_raw.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "id": "ea51d22b-269c-4f62-b3c5-6ba66c13a9fd", 5 | "fullName": "Michael Paterson", 6 | "firstName": "Michael", 7 | "lastName": "Paterson", 8 | "gender": "male", 9 | "links": { 10 | "values": { 11 | "104aa9b93493b48451fd700e31693237": { 12 | "id": "104aa9b93493b48451fd700e31693237", 13 | "type": "social", 14 | "value": "https://www.facebook.com/app_scoped_user_id/522313161380912312/", 15 | "service": "facebook", 16 | "username": "522313161380912312", 17 | "userId": "522313161380912312" 18 | } 19 | }, 20 | "keys": [ 21 | "104aa9b93493b48451fd700e31693237" 22 | ] 23 | }, 24 | "emails": { 25 | "values": { 26 | "56b1c5f64491e3f92290d6d98ec363f3": { 27 | "id": "56b1c5f64491e3f92290d6d98ec363f3", 28 | "value": "mpaterson192@gmail.com", 29 | "type": "facebook" 30 | } 31 | }, 32 | "keys": [ 33 | "56b1c5f64491e3f92290d6d98ec363f3" 34 | ] 35 | }, 36 | "experience": { 37 | "values": { 38 | "691db995f25eba90936a791f1b4c589c": { 39 | "id": "691db995f25eba90936a791f1b4c589c", 40 | "startDate": new Date("2013-04-30T00:00:00.000Z"), 41 | "organisationName": "Seed", 42 | "role": "Founder and Head of Development" 43 | }, 44 | "f71caeef48e1b54825ca6a2b3074d6c5": { 45 | "id": "f71caeef48e1b54825ca6a2b3074d6c5", 46 | "startDate": new Date("2012-01-01T00:00:00.000Z"), 47 | "endDate": new Date("2013-04-01T00:00:00.000Z"), 48 | "location": "London, United Kingdom", 49 | "organisationName": "Morgan Stanley", 50 | "role": "Fixed Income Analyst" 51 | }, 52 | "306a00c9bcc0ba27d57d9c676f489e70": { 53 | "id": "306a00c9bcc0ba27d57d9c676f489e70", 54 | "startDate": new Date("2011-01-01T00:00:00.000Z"), 55 | "endDate": new Date("2012-01-01T00:00:00.000Z"), 56 | "organisationName": "Moving Mountains", 57 | "role": "Chalet Manager" 58 | } 59 | }, 60 | "keys": [ 61 | "691db995f25eba90936a791f1b4c589c", 62 | "f71caeef48e1b54825ca6a2b3074d6c5", 63 | "306a00c9bcc0ba27d57d9c676f489e70" 64 | ] 65 | }, 66 | "education": { 67 | "values": { 68 | "155fd84d89209c847a93500914e1e13c": { 69 | "id": "155fd84d89209c847a93500914e1e13c", 70 | "organisationName": "Trinity Academy", 71 | "endDate": new Date("2007-01-01T00:00:00.000Z"), 72 | "level": "High School", 73 | "degree": "High School" 74 | }, 75 | "4c0914a0b270ea72e5f376751b40d353": { 76 | "id": "4c0914a0b270ea72e5f376751b40d353", 77 | "organisationName": "University of Bristol", 78 | "endDate": new Date("2011-01-01T00:00:00.000Z"), 79 | "level": "College", 80 | "degree": "College" 81 | } 82 | }, 83 | "keys": [ 84 | "155fd84d89209c847a93500914e1e13c", 85 | "4c0914a0b270ea72e5f376751b40d353" 86 | ] 87 | }, 88 | "photos": { 89 | "values": { 90 | "8ab90b053af8da36b65b50addc13c819": { 91 | "id": "8ab90b053af8da36b65b50addc13c819", 92 | "value": "https://scontent.xx.fbcdn.net/v/t1.0-1/p320x320/10399620_10153397426446149_6697611637278028610_n.jpg?oh=911077657dcb5d8b84a27ce63cfdf0f1&oe=58556F74" 93 | } 94 | }, 95 | "keys": [ 96 | "8ab90b053af8da36b65b50addc13c819" 97 | ] 98 | }, 99 | "createdAt": "2016-10-20T16:03:26.424Z", 100 | "creationSource": { 101 | "value": "facebook_raw" 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /test/results/services/github.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "id": "b8184048-2b51-47ff-b12b-7878bcb24720", 5 | "fullName": "Test Tas", 6 | "firstName": "Test", 7 | "links": { 8 | "values": { 9 | "1e131b0e61bd23625143d710bd4fe612": { 10 | "id": "1e131b0e61bd23625143d710bd4fe612", 11 | "type": "social", 12 | "value": "https://github.com/testaaat", 13 | "service": "github", 14 | "username": "testaaat", 15 | "userId": "663460", 16 | "following": "1356", 17 | "followers": "150" 18 | }, 19 | "a17988f91399c9153f5b9aed14ad1323": { 20 | "id": "a17988f91399c9153f5b9aed14ad1323", 21 | "type": "website", 22 | "value": "http://testtas.name" 23 | } 24 | }, 25 | "keys": [ 26 | "1e131b0e61bd23625143d710bd4fe612", 27 | "a17988f91399c9153f5b9aed14ad1323" 28 | ] 29 | }, 30 | "experience": { 31 | "values": { 32 | "15206977c6880ac1e3407b84e5b750be": { 33 | "id": "15206977c6880ac1e3407b84e5b750be", 34 | "organisationName": "ODM Salas FP" 35 | } 36 | }, 37 | "keys": [ 38 | "15206977c6880ac1e3407b84e5b750be" 39 | ] 40 | }, 41 | "emails": { 42 | "values": { 43 | "cde26ea71f98503b6516da041b5e586e": { 44 | "id": "cde26ea71f98503b6516da041b5e586e", 45 | "value": "t.test@teseee.org", 46 | "type": "github" 47 | } 48 | }, 49 | "keys": [ 50 | "cde26ea71f98503b6516da041b5e586e" 51 | ] 52 | }, 53 | "location": { 54 | "address": "Valencia, Venezuela" 55 | }, 56 | "createdAt": "2016-10-20T16:03:26.435Z", 57 | "creationSource": { 58 | "value": "github" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /test/results/services/github_raw.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "id": "333f7fdd-2984-41a9-b375-6e4466d14f05", 5 | "fullName": "Yigang Wang", 6 | "firstName": "Yigang", 7 | "links": { 8 | "values": { 9 | "4e85aac0a520c1a54f5e8a790103c281": { 10 | "id": "4e85aac0a520c1a54f5e8a790103c281", 11 | "type": "social", 12 | "value": "https://github.com/testttgithub", 13 | "service": "github", 14 | "username": "testttgithub", 15 | "userId": "10148384", 16 | "following": 5, 17 | "followers": 1 18 | } 19 | }, 20 | "keys": [ 21 | "4e85aac0a520c1a54f5e8a790103c281" 22 | ] 23 | }, 24 | "experience": { 25 | "values": { 26 | "f54bdf2bcc94f91e9fa88c7070601c24": { 27 | "id": "f54bdf2bcc94f91e9fa88c7070601c24", 28 | "organisationName": "The New York Times" 29 | } 30 | }, 31 | "keys": [ 32 | "f54bdf2bcc94f91e9fa88c7070601c24" 33 | ] 34 | }, 35 | "emails": { 36 | "values": { 37 | "d1d9c1b84c6362307c0cbe2d4a8eaf9f": { 38 | "id": "d1d9c1b84c6362307c0cbe2d4a8eaf9f", 39 | "value": "test@tesmail.com", 40 | "type": "github" 41 | } 42 | }, 43 | "keys": [ 44 | "d1d9c1b84c6362307c0cbe2d4a8eaf9f" 45 | ] 46 | }, 47 | "photos": { 48 | "values": { 49 | "fd07e7b10c69f94984037f4c3e446b96": { 50 | "id": "fd07e7b10c69f94984037f4c3e446b96", 51 | "value": "https://avatars.githubusercontent.com/u/10148384?v=3" 52 | } 53 | }, 54 | "keys": [ 55 | "fd07e7b10c69f94984037f4c3e446b96" 56 | ] 57 | }, 58 | "location": { 59 | "address": "New York" 60 | }, 61 | "createdAt": "2016-10-20T16:03:26.449Z", 62 | "creationSource": { 63 | "value": "github_raw" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /test/results/services/google.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "id": "e3bc0a7a-b6ca-4675-96c4-d57eef15ee80", 5 | "fullName": "Michael Paterson", 6 | "firstName": "Michael", 7 | "lastName": "Paterson", 8 | "gender": "male", 9 | "links": { 10 | "values": { 11 | "c9f50bbc2d27b8934b4ac4ba67d5ab86": { 12 | "id": "c9f50bbc2d27b8934b4ac4ba67d5ab86", 13 | "type": "social", 14 | "username": "108066648140795390948", 15 | "userId": "108066648140795390948", 16 | "value": "https://plus.google.com/108066648140795390948", 17 | "service": "google" 18 | } 19 | }, 20 | "keys": [ 21 | "c9f50bbc2d27b8934b4ac4ba67d5ab86" 22 | ] 23 | }, 24 | "photos": { 25 | "values": { 26 | "c91add286ae6a4a11b79b9f9267b936e": { 27 | "id": "c91add286ae6a4a11b79b9f9267b936e", 28 | "value": "https://lh6.googleusercontent.com/-EbyaFlEmdIo/AAAAAAAAAAI/AAAAAAAAAfA/ZF7RWwJsobU/photo.jpg", 29 | "type": "google" 30 | } 31 | }, 32 | "keys": [ 33 | "c91add286ae6a4a11b79b9f9267b936e" 34 | ] 35 | }, 36 | "emails": { 37 | "values": { 38 | "56b1c5f64491e3f92290d6d98ec363f3": { 39 | "id": "56b1c5f64491e3f92290d6d98ec363f3", 40 | "value": "mpaterson192@gmail.com", 41 | "type": "google" 42 | } 43 | }, 44 | "keys": [ 45 | "56b1c5f64491e3f92290d6d98ec363f3" 46 | ] 47 | }, 48 | "createdAt": "2016-10-20T16:03:26.455Z", 49 | "creationSource": { 50 | "value": "google" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /test/results/services/linkedin.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "id": "c944e21b-5f93-47bf-a780-04a34e62eb68", 5 | "fullName": "Test Tase", 6 | "firstName": "Test", 7 | "lastName": "Tase", 8 | "links": { 9 | "values": { 10 | "7523d7416efdaca7dc1c8c0ee3b1d394": { 11 | "id": "7523d7416efdaca7dc1c8c0ee3b1d394", 12 | "type": "social", 13 | "value": "https://www.linkedin.com/in/testjetase", 14 | "service": "linkedin", 15 | "username": "UKJ0axFJ0n", 16 | "userId": "UKJ0axFJ0n", 17 | "bio": "Experienced product manager across financial products and services. Have lead teams from 2-20 across a range of leadership roles. Trusted Advisor on several start-ups and investor in to big data, cybersecuri, and general product development companies. Strong people management and influencing skills. Non-executive director positions held and executive coaching also supported. I've been lucky that over 15 years in a variety of businesses I've met and continue to met amazing people that have influenced, mentored, coached and partnered me me in business with. With such diversity and a strong ability to learn and evolve I bring an array of skills to any opportunity or engagement I have.\n\nCOO skill set comprising of:\nProduct Creation and Management\nBusiness Leadership\nP&L ownership\nBusiness Development and Client/Relationship Management\nOperations Leadership\nOrganisational design\nFinancial services across all asset classes and functions (front to back office)\n\n\nStarts ups in:\nRecruitment\nEnvironmental Capital Markets\nEducation\nBusiness Consultancy", 18 | "following": "452" 19 | } 20 | }, 21 | "keys": [ 22 | "7523d7416efdaca7dc1c8c0ee3b1d394" 23 | ] 24 | }, 25 | "emails": { 26 | "values": { 27 | "8b3d9116dbbcd197f0b386cafd714155": { 28 | "id": "8b3d9116dbbcd197f0b386cafd714155", 29 | "value": "t.test@gtest.com", 30 | "type": "linkedin" 31 | } 32 | }, 33 | "keys": [ 34 | "8b3d9116dbbcd197f0b386cafd714155" 35 | ] 36 | }, 37 | "location": { 38 | "address": "London, United Kingdom", 39 | "countryCode": "gb" 40 | }, 41 | "experience": { 42 | "values": { 43 | "ff57c6af866350e102b64b0765e91e46": { 44 | "id": "ff57c6af866350e102b64b0765e91e46", 45 | "startDate": new Date("2014-08-01T00:00:00.000Z"), 46 | "organisationName": "EC1 Capital Limited", 47 | "description": "EC1 Capital\nEC1 Capital is a web and mobile technology investment fund created in Feb 2012.\n\nWe back tenacious, aligned and focused founders addressing a significant pain point or disruption potential within a large addressable market.\n\nWe invest across the full early funding spectrum including Seed, Bridging and Series A participation and syndicate all our deals with angels who have domain experience as well as other VC firms to bring the greatest breadth of support.\n\nMy role:\nAdvisor for EC1 Capital, on early stage tech investment fund that invests in web technology businesses primarily located in London, Dublin and Edinburgh", 48 | "role": "Advisor", 49 | "current": "true" 50 | }, 51 | "8d5f73322f1c333263d213c9e41e9f10": { 52 | "id": "8d5f73322f1c333263d213c9e41e9f10", 53 | "startDate": new Date("2014-07-01T00:00:00.000Z"), 54 | "organisationName": "LearnerLane", 55 | "description": "Acting in the capacity of COO. Aiding go-live execution and operational readiness. Advisory on fund raising, marketing, corporate development and general start up management. Also assisted CEO with 1-2-1 coaching.", 56 | "role": "Senior Advisor", 57 | "current": "true" 58 | }, 59 | "cc85ce7cca623b5092356c62fe835907": { 60 | "id": "cc85ce7cca623b5092356c62fe835907", 61 | "startDate": new Date("2008-08-01T00:00:00.000Z"), 62 | "organisationName": "J.P. Morgan", 63 | "description": "Leading the Global Custody Product for EMEA as part of the senior leadership team. Primary responsibilities are product growth, business development, risk management, P&L governance, and client experience/ux", 64 | "role": "Executive Director-EMEA Head of Global Custody - Commercial Product Management", 65 | "current": "true" 66 | } 67 | }, 68 | "keys": [ 69 | "ff57c6af866350e102b64b0765e91e46", 70 | "8d5f73322f1c333263d213c9e41e9f10", 71 | "cc85ce7cca623b5092356c62fe835907" 72 | ] 73 | }, 74 | "photos": { 75 | "values": { 76 | "e4e335758c6fa7c195f24ed0df5b7512": { 77 | "id": "e4e335758c6fa7c195f24ed0df5b7512", 78 | "value": "https://media.licdn.com/mpr/mprx/0_D5pUSVZxoJtFnano_GseSMd0EY5Xnano7FH1231zETET_2BiJ22krLVzdB62QO5COEqb3Gs111jkja" 79 | } 80 | }, 81 | "keys": [ 82 | "e4e335758c6fa7c195f24ed0df5b7512" 83 | ] 84 | }, 85 | "education": { 86 | "values": {}, 87 | "keys": [] 88 | }, 89 | "languages": { 90 | "values": {}, 91 | "keys": [] 92 | }, 93 | "createdAt": "2016-10-20T16:03:26.466Z", 94 | "creationSource": { 95 | "value": "linkedin" 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /test/results/services/linkedin_raw.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "id": "b80bb7740288fda1f201890375a60c8f", 5 | "fullName": "Test Tu", 6 | "firstName": "Test", 7 | "lastName": "Tu", 8 | "links": { 9 | "values": { 10 | "05f842512b5f48315dbef03bda8aedcd": { 11 | "id": "05f842512b5f48315dbef03bda8aedcd", 12 | "type": "social", 13 | "value": "https://www.linkedin.com/in/teststu", 14 | "service": "linkedin", 15 | "username": "Qa3D3zIUQWkTh", 16 | "userId": "Qa3D3zIUQWkTh", 17 | "bio": "Experienced project manager, outstanding communication skills, organizer, solution-oriented, stickler for quality and precision, loves working with diverse and talented teams. \n\nQUALIFICATIONS\nExtensive experience in managing multiple and concurrent digital and print projects.\n\nDigital project work includes managing the development and delivery of: corporate website redesigns, career websites, branded blogs (custom-built and third-party hosted, event micro-sites, email marketing, site localization, social media strategy and campaign execution, user-generated video campaigns.\n\nProficient in MS Office and Project; Keynote; Adobe Photoshop, InDesign, Illustrator; JIRA; Pivotal Tracker; Basecamp; Smartsheet and Google Analytics.\n\nSolid understanding of UX/UI and web development technologies.\n\nExperience with agile methodologies, QA/UAT testing, content management systems and CRM systems. \n\nAn excellent communicator with a strong design sense and an eye for detail.", 18 | "following": 283 19 | } 20 | }, 21 | "keys": [ 22 | "05f842512b5f48315dbef03bda8aedcd" 23 | ] 24 | }, 25 | "emails": { 26 | "values": { 27 | "8ba96e0617d35c63e927bcb817bacd9c": { 28 | "id": "8ba96e0617d35c63e927bcb817bacd9c", 29 | "value": "test.js@test.com", 30 | "type": "linkedin" 31 | } 32 | }, 33 | "keys": [ 34 | "8ba96e0617d35c63e927bcb817bacd9c" 35 | ] 36 | }, 37 | "experience": { 38 | "values": { 39 | "e02961f48e4ac7e0a3aaaaedfcb9ae76": { 40 | "id": "e02961f48e4ac7e0a3aaaaedfcb9ae76", 41 | "startDate": new Date("2015-04-01T00:00:00.000Z"), 42 | "description": "• Implementing new web products and managing the development life \tcycle of career websites for clients at the world’s largest recruitment advertising firm;\n• Clients include VMware, Nike, Nestle, TDAmeritrade, Scripps Health, Cox Enterprises and Citizens Bank;\n• Ensuring all projects pass QA/UAT testing according to client requirements and web standard best practices;\n• Coordinating cross-functional teams of UX and visual designers, front- and back-end developers (including off-shore, product development team and QA engineers;\n• Proficiency in using JIRA, Smartsheet, Google Drive and Basecamp.", 43 | "role": "Digital Project Manager", 44 | "current": true, 45 | "organisationName": "TMP Worldwide" 46 | } 47 | }, 48 | "keys": [ 49 | "e02961f48e4ac7e0a3aaaaedfcb9ae76" 50 | ] 51 | }, 52 | "photos": { 53 | "values": { 54 | "20088adc0262acbdd7ac2b99caceeb23": { 55 | "id": "20088adc0262acbdd7ac2b99caceeb23", 56 | "value": "https://media.licdn.com/mpr/mprx/Qa3D3zIUQWkTh" 57 | } 58 | }, 59 | "keys": [ 60 | "20088adc0262acbdd7ac2b99caceeb23" 61 | ] 62 | }, 63 | "education": { 64 | "values": {}, 65 | "keys": [] 66 | }, 67 | "languages": { 68 | "values": {}, 69 | "keys": [] 70 | }, 71 | "createdAt": "2016-10-20T16:03:26.482Z", 72 | "creationSource": { 73 | "value": "linkedin_raw" 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /test/results/test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "name": "Ahmad", 5 | "default_name": "this_should_be_the_firstName", 6 | "nickname": "nickname_not_found", 7 | "isNickNameFound": false, 8 | "isDynamicDefault": "Ahmad", 9 | "fullName": "Ahmad Assaf", 10 | "fullNameDefault": "Ahmad Assaf", 11 | "fullNameDefaultHardcoded": "default", 12 | "completeName": "Ahmad Ahmad Assaf", 13 | "displayName": "Ahmad Assaf", 14 | "email": { 15 | "value": "ahmad.a.assaf@gmail.com" 16 | }, 17 | "links": [ 18 | "http://a.com", 19 | "http://b.com", 20 | "http://c.com" 21 | ], 22 | "messaging": [{ 23 | "service": "skype", 24 | "type": "messaging", 25 | "value": "ahmad.a.assaf" 26 | }], 27 | "social_links": [{ 28 | "value": "http://a.com", 29 | "type": "test", 30 | "order": 0, 31 | "social": true 32 | }, { 33 | "value": "http://b.com", 34 | "type": "test", 35 | "order": 1, 36 | "social": true 37 | }, { 38 | "value": "http://c.com", 39 | "type": "test", 40 | "order": 2, 41 | "social": true 42 | }, { 43 | "value": "http://github.com/ahmadassaf", 44 | "service": "github", 45 | "type": "social" 46 | }, { 47 | "value": "http://twitter.com/ahmadaassaf", 48 | "service": "twitter", 49 | "type": "social" 50 | }], 51 | "website_addresses_keyless": [{ 52 | "value": "https://gravatar.com/ahmadassaf", 53 | "type": "other" 54 | }, { 55 | "value": "http://klout.com/ahmadassaf", 56 | "type": "other" 57 | }, { 58 | "value": "http://ahmadassaf.com", 59 | "type": "other" 60 | }], 61 | "website_addresses": { 62 | "f5e32a6faaa7ead6ba201e8fa25733ee": { 63 | "value": "http://klout.com/ahmadassaf", 64 | "type": "other", 65 | "keys": [ 66 | "f5e32a6faaa7ead6ba201e8fa25733ee" 67 | ] 68 | } 69 | }, 70 | "social_media_addresses": [{ 71 | "value": "@ahmadaassaf" 72 | }], 73 | "social_links_objects": { 74 | "388495550d157e5c5d299b7ecfeb1c2d": { 75 | "value": "http://a.com" 76 | }, 77 | "715af3fd14408eda374ef3bcb23c10b6": { 78 | "value": "http://b.com" 79 | }, 80 | "e5bd27f12c3d3e03d1c4463bf0dfa035": { 81 | "value": "http://c.com" 82 | } 83 | }, 84 | "experience": [{ 85 | "name": "beamery", 86 | "role": "Data Scientist", 87 | "startDate": new Date("2015-08-28T00:00:00.000Z"), 88 | "current": true 89 | }, { 90 | "name": "SAP", 91 | "role": "Associate Researcher", 92 | "current": false 93 | }], 94 | "experience_primary": { 95 | "values": { 96 | "cd094dd92fe6ec408206c38421d0c835": { 97 | "id": "cd094dd92fe6ec408206c38421d0c835", 98 | "role": "Data Scientist", 99 | "organisationName": "Beamery" 100 | } 101 | } 102 | }, 103 | "primaryExperience": { 104 | "name": "beamery", 105 | "role": "Data Scientist", 106 | "startDate": new Date("2015-08-28T00:00:00.000Z"), 107 | "current": true 108 | }, 109 | "primaryRole": "Data Scientist", 110 | "experiences": [ 111 | "beamery", 112 | "SAP" 113 | ], 114 | "experience_object": { 115 | "values": { 116 | "894b6152a9dde92713a40590f6f4d5b8": { 117 | "id": "894b6152a9dde92713a40590f6f4d5b8", 118 | "name": "beamery", 119 | "role": "Data Scientist", 120 | "startDate": "2015-08-28", 121 | "current": true 122 | }, 123 | "31465d14a616920720fac760c503ada7": { 124 | "id": "31465d14a616920720fac760c503ada7", 125 | "name": "SAP", 126 | "role": "Associate Researcher", 127 | "current": false 128 | } 129 | } 130 | }, 131 | "education": [{ 132 | "universityName": "telecomParisTech", 133 | "degree": "phd", 134 | "location": "france" 135 | }, { 136 | "universityName": "st-andrews", 137 | "degree": "masters" 138 | }], 139 | "education_object": { 140 | "6ea8c4cf5c35861ba3b690f93828e44f": { 141 | "degree": "phd", 142 | "location": "france" 143 | }, 144 | "e4cf0b80ab478731917804503b2f3fc3": { 145 | "degree": "masters", 146 | "universityName": "University of St.Andrews" 147 | } 148 | }, 149 | "primaryPhoto": "http://photo.com/ahmadassaf" 150 | } 151 | -------------------------------------------------------------------------------- /test/samples/services/facebook.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "email" : "test@test.edu", 5 | "source" : "facebook", 6 | "updatedAt" : "2016-03-06T15:46:17.590+0000", 7 | "data" : { 8 | "user" : { 9 | "id" : "10153358626426860", 10 | "token" : "CAAJuuwb3kNMBAIA2lhUpmmngZCpyxz5RMflP6WLRUnYX3TQVpFicKDiFLKvqSPZCZC4dCHo3ZA3DOc16hOMXgcVSrHTUyKWxVELljZB2yIamjxfvQe8Tn5LXQJhlPG1vxcAYoXlSJf9Enh5Mo5H23YeRyDa9FTYViDH34BlUEHZBuGBl9ALesZB31VqcaLSJ5V5YvoUgSZBSkQZDZD", 11 | "name" : "Test Ts Ts", 12 | "email" : "test@test.edu" 13 | }, 14 | "profile" : { 15 | "id" : "10153358626426860", 16 | "displayName" : "Test Ts Ts", 17 | "name" : { 18 | "familyName" : "Ts", 19 | "givenName" : "Test Ts" 20 | }, 21 | "gender" : "male", 22 | "profileUrl" : "https://www.facebook.com/app_scoped_user_id/10153358626426860/", 23 | "emails" : [ 24 | { 25 | "value" : "test@test.edu" 26 | } 27 | ], 28 | "provider" : "facebook", 29 | "_raw" : "{\"id\":\"10153358626426860\",\"education\":[{\"school\":{\"id\":\"107950875905899\",\"name\":\"Raffles Institution\"},\"type\":\"High School\"},{\"school\":{\"id\":\"114908698520993\",\"name\":\"Raffles Junior College\"},\"type\":\"High School\",\"year\":{\"id\":\"136328419721520\",\"name\":\"2009\"}},{\"school\":{\"id\":\"21489041474\",\"name\":\"Duke University\"},\"type\":\"College\"}],\"email\":\"tn52\\u0040duke.edu\",\"first_name\":\"Test Ts\",\"gender\":\"male\",\"last_name\":\"Ng\",\"link\":\"https:\\/\\/www.facebook.com\\/app_scoped_user_id\\/10153358626426860\\/\",\"locale\":\"en_US\",\"name\":\"Test Ts Ts\",\"timezone\":-5,\"updated_time\":\"2015-06-06T00:07:28+0000\",\"verified\":true,\"work\":[{\"employer\":{\"id\":\"113799428632566\",\"name\":\"Singapore Armed Forces\"},\"projects\":[{\"id\":\"134626036607917\",\"name\":\"Platoon 2\",\"with\":[{\"name\":\"Pang Jun Xiang\",\"id\":\"10153443036978506\"}]}]},{\"employer\":{\"id\":\"162299510505768\",\"name\":\"APSC (Aquatic Performance Swim Club\"}}]}", 30 | "_json" : { 31 | "id" : "10153358626426860", 32 | "education" : [ 33 | { 34 | "school" : { 35 | "id" : "107950875905899", 36 | "name" : "Raffles Institution" 37 | }, 38 | "type" : "High School" 39 | }, 40 | { 41 | "school" : { 42 | "id" : "114908698520993", 43 | "name" : "Raffles Junior College" 44 | }, 45 | "type" : "High School", 46 | "year" : { 47 | "id" : "136328419721520", 48 | "name" : "2009" 49 | } 50 | }, 51 | { 52 | "school" : { 53 | "id" : "21489041474", 54 | "name" : "Duke University" 55 | }, 56 | "type" : "College" 57 | } 58 | ], 59 | "email" : "test@test.edu", 60 | "first_name" : "Test Ts", 61 | "gender" : "male", 62 | "last_name" : "Ts", 63 | "link" : "https://www.facebook.com/app_scoped_user_id/10153358626426860/", 64 | "locale" : "en_US", 65 | "name" : "Test Ts Ts", 66 | "timezone" : -5, 67 | "updated_time" : "2015-06-06T00:07:28+0000", 68 | "verified" : true, 69 | "work" : [ 70 | { 71 | "employer" : { 72 | "id" : "113799428632566", 73 | "name" : "Singapore Armed Forces" 74 | }, 75 | "projects" : [ 76 | { 77 | "id" : "134626036607917", 78 | "name" : "Platoon 2", 79 | "with" : [ 80 | { 81 | "name" : "Pang Jun Xiang", 82 | "id" : "10153443036978506" 83 | } 84 | ] 85 | } 86 | ] 87 | }, 88 | { 89 | "employer" : { 90 | "id" : "162299510505768", 91 | "name" : "APSC (Aquatic Performance Swim Club)" 92 | } 93 | } 94 | ] 95 | } 96 | } 97 | } 98 | }; 99 | -------------------------------------------------------------------------------- /test/samples/services/facebook_raw.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "email" : "mpaterson192@gmail.com", 5 | "source" : "facebook_raw", 6 | "data" : { 7 | "link" : "https://www.facebook.com/app_scoped_user_id/522313161380912312/", 8 | "first_name" : "Michael", 9 | "last_name" : "Paterson", 10 | "name" : "Michael Paterson", 11 | "email" : "mpaterson192@gmail.com", 12 | "education" : [ 13 | { 14 | "school" : { 15 | "id" : "107976902563320", 16 | "name" : "Trinity Academy" 17 | }, 18 | "type" : "High School", 19 | "year" : { 20 | "id" : "140617569303679", 21 | "name" : "2007" 22 | }, 23 | "id" : "10150092085701149" 24 | }, 25 | { 26 | "concentration" : [ 27 | { 28 | "id" : "109532545739630", 29 | "name" : "Engineering" 30 | } 31 | ], 32 | "school" : { 33 | "id" : "108242009204639", 34 | "name" : "University of Bristol" 35 | }, 36 | "type" : "College", 37 | "year" : { 38 | "id" : "144044875610606", 39 | "name" : "2011" 40 | }, 41 | "id" : "10150092085706149" 42 | } 43 | ], 44 | "work" : [ 45 | { 46 | "employer" : { 47 | "id" : "565207183512115", 48 | "name" : "Seed" 49 | }, 50 | "position" : { 51 | "id" : "213307258878570", 52 | "name" : "Founder and Head of Development" 53 | }, 54 | "start_date" : "2013-04-30", 55 | "id" : "10151436419181149" 56 | }, 57 | { 58 | "end_date" : "2013-04-01", 59 | "employer" : { 60 | "id" : "112316212117655", 61 | "name" : "Morgan Stanley" 62 | }, 63 | "location" : { 64 | "id" : "106078429431815", 65 | "name" : "London, United Kingdom" 66 | }, 67 | "position" : { 68 | "id" : "138253592863282", 69 | "name" : "Fixed Income Analyst" 70 | }, 71 | "start_date" : "2012-01-01", 72 | "id" : "10151436418711149" 73 | }, 74 | { 75 | "end_date" : "2012-01-01", 76 | "employer" : { 77 | "id" : "16082098580", 78 | "name" : "Moving Mountains" 79 | }, 80 | "position" : { 81 | "id" : "112591292131213", 82 | "name" : "Chalet Manager" 83 | }, 84 | "start_date" : "2011-01-01", 85 | "id" : "10150426110451149" 86 | } 87 | ], 88 | "picture" : { 89 | "data" : { 90 | "height" : 320, 91 | "is_silhouette" : false, 92 | "url" : "https://scontent.xx.fbcdn.net/v/t1.0-1/p320x320/10399620_10153397426446149_6697611637278028610_n.jpg?oh=911077657dcb5d8b84a27ce63cfdf0f1&oe=58556F74", 93 | "width" : 320 94 | } 95 | }, 96 | "gender" : "male", 97 | "id" : "522313161380912312" 98 | }, 99 | "updatedAt" : "2016-08-09T12:02:05.795+0000" 100 | } 101 | -------------------------------------------------------------------------------- /test/samples/services/github.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "email" : "t.test@teseee.org", 5 | "source" : "github", 6 | "data" : { 7 | "profile" : { 8 | "provider" : "github", 9 | "id" : "663460", 10 | "displayName" : "Test Tas", 11 | "username" : "testaaat", 12 | "profileUrl" : "https://github.com/testaaat", 13 | "emails" : [ 14 | { 15 | "value" : "t.test@teseee.org" 16 | } 17 | ], 18 | "_raw" : "{\"login\":\"testaaat\",\"id\":663460,\"avatar_url\":\"https://avatars.githubusercontent.com/u/663460?v=3\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/testaaat\",\"html_url\":\"https://github.com/testaaat\",\"followers_url\":\"https://api.github.com/users/testaaat/followers\",\"following_url\":\"https://api.github.com/users/testaaat/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/testaaat/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/testaaat/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/testaaat/subscriptions\",\"organizations_url\":\"https://api.github.com/users/testaaat/orgs\",\"repos_url\":\"https://api.github.com/users/testaaat/repos\",\"events_url\":\"https://api.github.com/users/testaaat/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/testaaat/received_events\",\"type\":\"User\",\"site_admin\":false,\"name\":\"Test Tas\",\"company\":\"ODM Salas FP\",\"blog\":\"http://testtas.name\",\"location\":\"Valencia, Venezuela\",\"email\":\"t.test@teseee.org\",\"hireable\":true,\"bio\":null,\"public_repos\":343,\"public_gists\":17,\"followers\":150,\"following\":1356,\"created_at\":\"2011-03-11T05:20:16Z\",\"updated_at\":\"2015-01-28T22:46:26Z\"}", 19 | "_json" : { 20 | "login" : "testaaat", 21 | "id" : "663460", 22 | "avatar_url" : "https://avatars.githubusercontent.com/u/663460?v=3", 23 | "gravatar_id" : "", 24 | "url" : "https://api.github.com/users/testaaat", 25 | "html_url" : "https://github.com/testaaat", 26 | "followers_url" : "https://api.github.com/users/testaaat/followers", 27 | "following_url" : "https://api.github.com/users/testaaat/following{/other_user}", 28 | "gists_url" : "https://api.github.com/users/testaaat/gists{/gist_id}", 29 | "starred_url" : "https://api.github.com/users/testaaat/starred{/owner}{/repo}", 30 | "subscriptions_url" : "https://api.github.com/users/testaaat/subscriptions", 31 | "organizations_url" : "https://api.github.com/users/testaaat/orgs", 32 | "repos_url" : "https://api.github.com/users/testaaat/repos", 33 | "events_url" : "https://api.github.com/users/testaaat/events{/privacy}", 34 | "received_events_url" : "https://api.github.com/users/testaaat/received_events", 35 | "type" : "User", 36 | "site_admin" : "false", 37 | "name" : "Test Tas", 38 | "company" : "ODM Salas FP", 39 | "blog" : "http://testtas.name", 40 | "location" : "Valencia, Venezuela", 41 | "email" : "t.test@teseee.org", 42 | "hireable" : "true", 43 | "public_repos" : "343", 44 | "public_gists" : "17", 45 | "followers" : "150", 46 | "following" : "1356", 47 | "created_at" : "2011-03-11T05:20:16Z", 48 | "updated_at" : "2015-01-28T22:46:26Z" 49 | } 50 | }, 51 | "user" : { 52 | "id" : "663460", 53 | "token" : "27f166b5259a859369d48b0b113c74a59de45585", 54 | "name" : "Test Tas", 55 | "email" : "t.test@teseee.org" 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /test/samples/services/github_raw.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "email" : "test@tesmail.com", 5 | "source" : "github_raw", 6 | "data" : { 7 | "login" : "testttgithub", 8 | "id" : 10148384, 9 | "avatar_url" : "https://avatars.githubusercontent.com/u/10148384?v=3", 10 | "gravatar_id" : "", 11 | "url" : "https://api.github.com/users/testttgithub", 12 | "html_url" : "https://github.com/testttgithub", 13 | "followers_url" : "https://api.github.com/users/testttgithub/followers", 14 | "following_url" : "https://api.github.com/users/testttgithub/following{/other_user}", 15 | "gists_url" : "https://api.github.com/users/testttgithub/gists{/gist_id}", 16 | "starred_url" : "https://api.github.com/users/testttgithub/starred{/owner}{/repo}", 17 | "subscriptions_url" : "https://api.github.com/users/testttgithub/subscriptions", 18 | "organizations_url" : "https://api.github.com/users/testttgithub/orgs", 19 | "repos_url" : "https://api.github.com/users/testttgithub/repos", 20 | "events_url" : "https://api.github.com/users/testttgithub/events{/privacy}", 21 | "received_events_url" : "https://api.github.com/users/testttgithub/received_events", 22 | "type" : "User", 23 | "site_admin" : false, 24 | "name" : "Yigang Wang", 25 | "company" : "The New York Times", 26 | "blog" : null, 27 | "location" : "New York", 28 | "email" : "test@tesmail.com", 29 | "hireable" : true, 30 | "bio" : "\"I drink and I know things.\" ----Tyrion Lannister", 31 | "public_repos" : 11, 32 | "public_gists" : 0, 33 | "followers" : 1, 34 | "following" : 5, 35 | "created_at" : "2014-12-11T00:19:47Z", 36 | "updated_at" : "2016-08-13T17:17:23Z" 37 | }, 38 | "updatedAt" : "2016-08-15T01:15:08.258+0000" 39 | } 40 | -------------------------------------------------------------------------------- /test/samples/services/google.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "email" : "mpaterson192@gmail.com", 5 | "source" : "google", 6 | "data" : { 7 | "profile" : { 8 | "provider" : "google", 9 | "id" : "108066648140795390948", 10 | "displayName" : "Michael Paterson", 11 | "name" : { 12 | "familyName" : "Paterson", 13 | "givenName" : "Michael" 14 | }, 15 | "emails" : [ 16 | { 17 | "value" : "mpaterson192@gmail.com" 18 | } 19 | ], 20 | "_raw" : "{\n \"id\": \"108066648140795390948\",\n \"email\": \"mpaterson192@gmail.com\",\n \"verified_email\": true,\n \"name\": \"Michael Paterson\",\n \"given_name\": \"Michael\",\n \"family_name\": \"Paterson\",\n \"link\": \"https://plus.google.com/108066648140795390948\",\n \"picture\": \"https://lh6.googleusercontent.com/-EbyaFlEmdIo/AAAAAAAAAAI/AAAAAAAAAfA/ZF7RWwJsobU/photo.jpg\",\n \"gender\": \"male\",\n \"locale\": \"en\"\n}\n", 21 | "_json" : { 22 | "id" : "108066648140795390948", 23 | "email" : "mpaterson192@gmail.com", 24 | "verified_email" : "true", 25 | "name" : "Michael Paterson", 26 | "given_name" : "Michael", 27 | "family_name" : "Paterson", 28 | "link" : "https://plus.google.com/108066648140795390948", 29 | "picture" : "https://lh6.googleusercontent.com/-EbyaFlEmdIo/AAAAAAAAAAI/AAAAAAAAAfA/ZF7RWwJsobU/photo.jpg", 30 | "gender" : "male", 31 | "locale" : "en" 32 | } 33 | }, 34 | "user" : { 35 | "id" : "108066648140795390948", 36 | "token" : "ya29.1gB0MvJmec9urWrPG0or1qKfyAmmhQqohtxK4XECkA3h4S012uspKp8i", 37 | "name" : "Michael Paterson", 38 | "email" : "mpaterson192@gmail.com" 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /test/samples/services/linkedin.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "email" : "t.test@gtest.com", 5 | "source" : "linkedin", 6 | "data" : { 7 | "profile" : { 8 | "provider" : "linkedin", 9 | "id" : "UKJ0axFJ0n", 10 | "displayName" : "Test Tase", 11 | "name" : { 12 | "familyName" : "Tase", 13 | "givenName" : "Test" 14 | }, 15 | "emails" : [ 16 | { 17 | "value" : "t.test@gtest.com" 18 | } 19 | ], 20 | "_json" : { 21 | "apiStandardProfileRequest" : { 22 | "headers" : { 23 | "_total" : "1", 24 | "values" : [ 25 | { 26 | "name" : "x-li-auth-token", 27 | "value" : "name:6pEX" 28 | } 29 | ] 30 | }, 31 | "url" : "https://api.linkedin.com/v1/people/UKJ0axFJ0n" 32 | }, 33 | "currentShare" : { 34 | "author" : { 35 | "firstName" : "Test", 36 | "id" : "UKJ0axFJ0n", 37 | "lastName" : "Tase" 38 | }, 39 | "comment" : "http://lnkd.in/p2MWBJ", 40 | "content" : { 41 | "description" : "Most advice on meetings focuses on the “how.” But the effort to improve meetings must start with the “what.”", 42 | "eyebrowUrl" : "https://www.linkedin.com/share?viewLink=&sid=s5775606237106298890&url=http%3A%2F%2Flnkd%2Ein%2Fp2MWBJ&urlhash=Ijtx", 43 | "resolvedUrl" : "http://www.linkedin.com/today/post/article/771239jmnq3121-36052017-cut-your-meeting-time-by-90", 44 | "shortenedUrl" : "http://lnkd.in/p2MWBJ", 45 | "submittedImageUrl" : "http://m.c.lnkd.licdn.com/mpr/mpr/p/2/005/009/1de/1a2ba39.jpg", 46 | "submittedUrl" : "http://www.linkedin.com/today/post/article/771239jmnq3121-36052017-cut-your-meeting-time-by-90?trk=api*a265395*s273144*", 47 | "thumbnailUrl" : "https://media.licdn.com/media-proxy/ext?w=80&h=100&hash=123218ijq11%2FDJ%2B9DbVr4YdejxA%3D&url=http%3A%2F%2Fm.c.lnkd.licdn.com%2Fmpr%2Fmpr%2Fp%2F2%2F005%2F009%2F1de%2F1a2ba39.jpg", 48 | "title" : "The Simplest Way to Cut Your Meeting Time by 90%" 49 | }, 50 | "id" : "s5775606237106298890", 51 | "source" : { 52 | "serviceProvider" : { 53 | "name" : "LINKEDIN" 54 | } 55 | }, 56 | "timestamp" : "1377011790000", 57 | "visibility" : { 58 | "code" : "anyone" 59 | } 60 | }, 61 | "distance" : "0", 62 | "emailAddress" : "t.test@gtest.com", 63 | "firstName" : "Test", 64 | "formattedName" : "Test Tase", 65 | "headline" : "Director level product manager in banking. VC advisory, angel investor, graduate mentor", 66 | "id" : "UKJ0axFJ0n", 67 | "industry" : "Investment Banking", 68 | "lastName" : "Tase", 69 | "location" : { 70 | "country" : { 71 | "code" : "gb" 72 | }, 73 | "name" : "London, United Kingdom" 74 | }, 75 | "numConnections" : "452", 76 | "numConnectionsCapped" : "false", 77 | "pictureUrl" : "https://media.licdn.com/mpr/mprx/0_D5pUSVZxoJtFnano_GseSMd0EY5Xnano7FH1231zETET_2BiJ22krLVzdB62QO5COEqb3Gs111jkja", 78 | "positions" : { 79 | "_total" : "3", 80 | "values" : [ 81 | { 82 | "company" : { 83 | "id" : "2452143", 84 | "name" : "EC1 Capital Limited" 85 | }, 86 | "id" : "577609241", 87 | "isCurrent" : "true", 88 | "startDate" : { 89 | "month" : "8", 90 | "year" : "2014" 91 | }, 92 | "summary" : "EC1 Capital\nEC1 Capital is a web and mobile technology investment fund created in Feb 2012.\n\nWe back tenacious, aligned and focused founders addressing a significant pain point or disruption potential within a large addressable market.\n\nWe invest across the full early funding spectrum including Seed, Bridging and Series A participation and syndicate all our deals with angels who have domain experience as well as other VC firms to bring the greatest breadth of support.\n\nMy role:\nAdvisor for EC1 Capital, on early stage tech investment fund that invests in web technology businesses primarily located in London, Dublin and Edinburgh", 93 | "title" : "Advisor" 94 | }, 95 | { 96 | "company" : { 97 | "name" : "LearnerLane" 98 | }, 99 | "id" : "556860238", 100 | "isCurrent" : "true", 101 | "startDate" : { 102 | "month" : "7", 103 | "year" : "2014" 104 | }, 105 | "summary" : "Acting in the capacity of COO. Aiding go-live execution and operational readiness. Advisory on fund raising, marketing, corporate development and general start up management. Also assisted CEO with 1-2-1 coaching.", 106 | "title" : "Senior Advisor" 107 | }, 108 | { 109 | "company" : { 110 | "id" : "1067", 111 | "name" : "J.P. Morgan" 112 | }, 113 | "id" : "107715419", 114 | "isCurrent" : "true", 115 | "startDate" : { 116 | "month" : "8", 117 | "year" : "2008" 118 | }, 119 | "summary" : "Leading the Global Custody Product for EMEA as part of the senior leadership team. Primary responsibilities are product growth, business development, risk management, P&L governance, and client experience/ux", 120 | "title" : "Executive Director-EMEA Head of Global Custody - Commercial Product Management" 121 | } 122 | ] 123 | }, 124 | "publicProfileUrl" : "https://www.linkedin.com/in/TestjeTase", 125 | "relationToViewer" : { 126 | "distance" : "0" 127 | }, 128 | "siteStandardProfileRequest" : { 129 | "url" : "https://www.linkedin.com/profile/view?id=16363999&authType=name&authToken=6pEX&trk=api*a265395*s273144*" 130 | }, 131 | "summary" : "Experienced product manager across financial products and services. Have lead teams from 2-20 across a range of leadership roles. Trusted Advisor on several start-ups and investor in to big data, cybersecuri, and general product development companies. Strong people management and influencing skills. Non-executive director positions held and executive coaching also supported. I've been lucky that over 15 years in a variety of businesses I've met and continue to met amazing people that have influenced, mentored, coached and partnered me me in business with. With such diversity and a strong ability to learn and evolve I bring an array of skills to any opportunity or engagement I have.\n\nCOO skill set comprising of:\nProduct Creation and Management\nBusiness Leadership\nP&L ownership\nBusiness Development and Client/Relationship Management\nOperations Leadership\nOrganisational design\nFinancial services across all asset classes and functions (front to back office)\n\n\nStarts ups in:\nRecruitment\nEnvironmental Capital Markets\nEducation\nBusiness Consultancy" 132 | } 133 | }, 134 | "user" : { 135 | "id" : "UKJ0axFJ0n", 136 | "token" : "AQWf0_1P27sYKh52P5Dh0tYsdbEWk_49q0zlPBcu4uYukrFhaDXi4k1QNdzSQWhvGxOLXz6Hmpqy-m5fQA_Dn89iYdDsqSWOETj2k9x4PBEl7rm7s48tR69Mc28HwBIBkj-eufCyHS7lq5BLXgpEhlo7IIkmggAq4nj9ZKQj2ZUewGr8BKE", 137 | "name" : "Test Tase", 138 | "email" : "t.test@gtest.com" 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /test/samples/services/linkedin_raw.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "email" : "test.js@test.com", 5 | "source" : "linkedin_raw", 6 | "data" : { 7 | "emailAddress" : "test.js@test.com", 8 | "firstName" : "Test", 9 | "formattedName" : "Test Tu", 10 | "headline" : "Digital Project Manager at TMP Worldwide", 11 | "id" : "Qa3D3zIUQWkTh", 12 | "industry" : "Architecture and Planning", 13 | "lastName" : "Tu", 14 | "numConnections" : 283, 15 | "pictureUrl" : "https://media.licdn.com/mpr/mprx/Qa3D3zIUQWkTh", 16 | "pictureUrls" : { 17 | "_total" : 1, 18 | "values" : [ 19 | "https://media.licdn.com/mpr/mprx/Qa3D3zIUQWkTh" 20 | ] 21 | }, 22 | "positions" : { 23 | "_total" : 1, 24 | "values" : [ 25 | { 26 | "company" : { 27 | "id" : 1843, 28 | "industry" : "Marketing & Advertising", 29 | "name" : "TMP Worldwide", 30 | "size" : "1001-5000", 31 | "type" : "Privately Held" 32 | }, 33 | "id" : 699221028, 34 | "isCurrent" : true, 35 | "startDate" : { 36 | "month" : 4, 37 | "year" : 2015 38 | }, 39 | "summary" : "• Implementing new web products and managing the development life \tcycle of career websites for clients at the world’s largest recruitment advertising firm;\n• Clients include VMware, Nike, Nestle, TDAmeritrade, Scripps Health, Cox Enterprises and Citizens Bank;\n• Ensuring all projects pass QA/UAT testing according to client requirements and web standard best practices;\n• Coordinating cross-functional teams of UX and visual designers, front- and back-end developers (including off-shore, product development team and QA engineers;\n• Proficiency in using JIRA, Smartsheet, Google Drive and Basecamp.", 40 | "title" : "Digital Project Manager" 41 | } 42 | ] 43 | }, 44 | "publicProfileUrl" : "https://www.linkedin.com/in/TestsTu", 45 | "summary" : "Experienced project manager, outstanding communication skills, organizer, solution-oriented, stickler for quality and precision, loves working with diverse and talented teams. \n\nQUALIFICATIONS\nExtensive experience in managing multiple and concurrent digital and print projects.\n\nDigital project work includes managing the development and delivery of: corporate website redesigns, career websites, branded blogs (custom-built and third-party hosted, event micro-sites, email marketing, site localization, social media strategy and campaign execution, user-generated video campaigns.\n\nProficient in MS Office and Project; Keynote; Adobe Photoshop, InDesign, Illustrator; JIRA; Pivotal Tracker; Basecamp; Smartsheet and Google Analytics.\n\nSolid understanding of UX/UI and web development technologies.\n\nExperience with agile methodologies, QA/UAT testing, content management systems and CRM systems. \n\nAn excellent communicator with a strong design sense and an eye for detail." 46 | }, 47 | "updatedAt" : "2016-06-23T19:05:55.290+0000" 48 | } 49 | -------------------------------------------------------------------------------- /test/samples/test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "firstName" : "Ahmad", 5 | "lastName": "Assaf", 6 | "email" : "ahmad.a.assaf@gmail.com", 7 | "title": "Data Scientist", 8 | "company": "Beamery", 9 | "links": [ 10 | "http://a.com", 11 | "http://b.com", 12 | "http://c.com" 13 | ], 14 | "linksv2" : { 15 | "values" : { 16 | "54b931981f03e4416a53c3096a32b134" : { 17 | "id" : "54b931981f03e4416a53c3096a32b134", 18 | "value" : "https://twitter.com/ahmadaassaf", 19 | "service" : "twitter", 20 | "keys": ["54b931981f03e4416a53c3096a32b134", "13efd8b0-4a7e-4201-8b6f-ceff2599117a"], 21 | "type" : "social" 22 | }, 23 | "54b931981f03e4416a53c3096a32b135" : { 24 | "id" : "54b931981f03e4416a53c3096a32b134", 25 | "value" : "https://twitter.com/ahmadaassaf", 26 | "service" : "twitter", 27 | "keys": ["54b931981f03e4416a53c3096a32b135", "13efd8b0-4a7e-4201-8b6f-ceff2599117a"], 28 | "type" : "social" 29 | }, 30 | "e9a5737987d042c6da01f3e740f3f7f7" : { 31 | "id" : "e9a5737987d042c6da01f3e740f3f7f7", 32 | "value" : "https://gravatar.com/ahmadassaf", 33 | "service" : "gravatar", 34 | "keys": ["e9a5737987d042c6da01f3e740f3f7f7", "edc1e66e-6b63-417f-9cdb-ba94bc3ff011"], 35 | "type" : "website" 36 | }, 37 | "f5e32a6faaa7ead6ba201e8fa25733ee" : { 38 | "id" : "f5e32a6faaa7ead6ba201e8fa25733ee", 39 | "value" : "http://klout.com/ahmadassaf", 40 | "service" : "klout", 41 | "keys": ["f5e32a6faaa7ead6ba201e8fa25733ee"], 42 | "type" : "website" 43 | }, 44 | "e960af6e-c99f-4981-81b3-0b09d99d7354" : { 45 | "id" : "1d87333c-5c60-4c5b-bd3f-611fc38db771", 46 | "value" : "ahmad.a.assaf", 47 | "service" : "skype", 48 | "keys": ["2e3a0ccf-1b27-4435-bfc8-1f751a7be6bd"] 49 | }, 50 | "ce621e6d5d201a8212c3684b2a40c813" : { 51 | "id" : "ce621e6d5d201a8212c3684b2a40c813", 52 | "value" : "http://ahmadassaf.com", 53 | "service" : "seed", 54 | "type" : "website", 55 | "deleted" : true 56 | } 57 | }, 58 | "keys" : [ 59 | "54b931981f03e4416a53c3096a32b134", 60 | "e9a5737987d042c6da01f3e740f3f7f7", 61 | "f5e32a6faaa7ead6ba201e8fa25733ee", 62 | "ce621e6d5d201a8212c3684b2a40c813" 63 | ] 64 | }, 65 | "social": [{ 66 | "value": "http://github.com/ahmadassaf", 67 | "service": "github" 68 | },{ 69 | "value": "http://twitter.com/ahmadaassaf", 70 | "service": "twitter" 71 | },{ 72 | "service": "google", 73 | "value": null 74 | }], 75 | "work": [ 76 | { 77 | "companyName": "beamery", 78 | "title": "Data Scientist", 79 | "startDate": "2015-08-28", 80 | "current": true 81 | }, 82 | { 83 | "companyName": "SAP", 84 | "title": "Associate Researcher", 85 | "current": false 86 | } 87 | ], 88 | "json": { 89 | "education" : { 90 | "telecomParisTech" : { 91 | "degree" : "phd", 92 | "location": "france" 93 | }, 94 | "st-andrews" : { 95 | "degree": "masters", 96 | "universityName": "University of St.Andrews" 97 | } 98 | } 99 | }, 100 | "photo" : "http://a photo .com" 101 | } 102 | -------------------------------------------------------------------------------- /test/socialServices.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const assert = require('assert'); 5 | 6 | const ditto = require('../ditto/ditto'); 7 | 8 | const propToExclude = ['createdAt', 'updatedAt']; 9 | 10 | /** 11 | * Assert a contact match, removing Date field that can't be asserted with exact match 12 | * @param result 13 | * @param fileImportJsonResult 14 | */ 15 | function assertContact(result, fileImportJsonResult) { 16 | assert.deepStrictEqual(_.omit(result, propToExclude), _.omit(fileImportJsonResult, propToExclude)); 17 | } 18 | 19 | describe('External Services Unification', function(){ 20 | 21 | const socialServices = ["facebook", "facebook_raw", "github", "github_raw", "google", "linkedin", "linkedin_raw"]; 22 | 23 | _.each(socialServices, function(service){ 24 | 25 | let contactJSONSample = require(`./samples/services/${service}`); 26 | let contactJSONResult = require(`./results/services/${service}`); 27 | let contactJSONMapping = require(`./mappings/services/${service}`); 28 | 29 | it(`should unify correctly ${service} JSON response`, function(){ 30 | return new ditto().unify(contactJSONSample, contactJSONMapping).then((result) => { 31 | assertContact(_.omit(result, "id"), _.omit(contactJSONResult, "id")); 32 | }); 33 | }); 34 | }); 35 | }); --------------------------------------------------------------------------------