├── .eslintrc.js ├── .gitignore ├── LICENSE ├── README.md ├── example.js ├── index.js └── package.json /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "es6": true, 4 | "node": true 5 | }, 6 | "extends": "eslint:recommended", 7 | "parserOptions": { 8 | "sourceType": "module" 9 | }, 10 | "rules": { 11 | "indent": [ 12 | "error", 13 | "tab" 14 | ], 15 | "linebreak-style": [ 16 | "error", 17 | "unix" 18 | ], 19 | "quotes": [ 20 | "error", 21 | "single" 22 | ], 23 | "semi": [ 24 | "error", 25 | "always" 26 | ] 27 | } 28 | }; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 sohamdodia 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DynamoDB-to-ElasticSearch 2 | 3 | A NPM module that will dump all DynamoDB data to AWS ElasticSearch indices. 4 | 5 | ## What this does 6 | 7 | There's no blueprint in AWS Lambda that allows to dump DynamoDB data into ElasticSearch. This module will facilitate exactly that. 8 | 9 | --- 10 | 11 | ### Installation 12 | 13 | ```sh 14 | $ npm install dynamodb-to-elasticsearch 15 | ``` 16 | 17 | ## [Guide to configure AWS to use this blueprint](https://aws.amazon.com/blogs/compute/indexing-amazon-dynamodb-content-with-amazon-elasticsearch-service-using-aws-lambda) 18 | 19 | ### Documentation 20 | 21 | `module.exec (table, region, es_endpoint, es_data = { id: 'sortKey', type: 'datatype', indiceName: ''})` 22 | 23 | | Parameter | Type | Description 24 | | ------ | ------ | ------ | 25 | | table | string | Table name of dynamoDB whose data you want to dump in elastic-search. 26 | | indiceName | string | indice name of elastic-search on which you can perform query. 27 | | region | string | dynamodb table region 28 | | es_endpoint | string | elastic-search endpoint 29 | | es_data | object | 30 | | es_data.id | object | The name of `primaryKey` field for DynamoDB table. It should be unique identifier for documents. By default it is set to `sortKey`. 31 | | es_data.type | object | Name of class of objects, which document represents. By default it is set to `datatype`. 32 | | es_data.indiceName | object | Index name of elastic-search on which you can perform query. 33 | 34 | ### Example 35 | 36 | ```javascript 37 | const d2es = require('dynamodb-to-elasticsearch'); 38 | 39 | const table = 'table', 40 | region = 'region', 41 | es_endpoint = 'es_endpoint_value', 42 | es_data = { id: 'sortKey', type: 'data', indiceName: 'candidates' }}; 43 | 44 | exports.handler = function(event, context, callback) { 45 | d2es.exec(table, region, es_endpoint, es_data, (err, success) => { 46 | if (err) { 47 | callback(err, null); 48 | } else { 49 | callback(null, success); 50 | } 51 | }); 52 | } 53 | 54 | ``` 55 | 56 | License 57 | ---- 58 | MIT 59 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | const d2es = require('dynamo-to-elasticsearch'), 2 | table = 'table', 3 | region = 'region', 4 | es_endpoint = 'es_endpoint_value', 5 | es_data = { id: 'url', type: 'datatype', indiceName: 'candidates' }; 6 | 7 | exports.handler = function (event, context, callback) { 8 | d2es.exec(table, region, es_endpoint, es_data, (err, success) => { 9 | if (err) { 10 | callback(err, null); 11 | } else { 12 | callback(null, success); 13 | } 14 | }); 15 | } 16 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // dependencies 2 | const AWS = require('aws-sdk'), 3 | _ = require('lodash'), 4 | elasticsearch = require('elasticsearch'), 5 | http_aws_es = require('http-aws-es'), 6 | when = require('when'); 7 | 8 | exports.exec = function (table, region, es_endpoint, es_data = { id: 'sortKey', type: 'datatype' }, callback) { 9 | if (!es_data.indiceName) throw new Error('You should provide es_data.indiceName') 10 | const docClient = new AWS.DynamoDB.DocumentClient({region: region}) 11 | 12 | // Promise - Describe the ES Domain in order to get the endpoint url 13 | when.promise(function (resolve) { 14 | resolve({ 15 | domain: { 16 | Endpoint: es_endpoint 17 | } 18 | }); 19 | }).then(function (result) { 20 | // Promise - Create the Index if needed - resolve if it is already there 21 | // or create and then resolve 22 | 23 | let promise = when.promise(function (resolve, reject) { 24 | let myCredentials = new AWS.EnvironmentCredentials('AWS'); 25 | 26 | let es = elasticsearch.Client({ 27 | hosts: result.domain.Endpoint, 28 | connectionClass: http_aws_es, 29 | amazonES: { 30 | region: region, 31 | credentials: myCredentials 32 | } 33 | }); 34 | 35 | es.indices.exists({ // check whether the indices exists or not 36 | index: es_data.indiceName 37 | }, function (err, response, status) { 38 | if (status == 200) { // if indices found 39 | resolve({es: es, domain: result.domain}) 40 | } else if (status == 404) { // if not found 41 | createIndex(es, es_data, function () { // create indices 42 | resolve({es: es, domain: result.domain}); 43 | }) 44 | } else { 45 | reject(err); 46 | } 47 | }); 48 | }); 49 | return promise 50 | }).then(function (result) { 51 | let scanningParameter = { 52 | TableName: table 53 | }; 54 | 55 | docClient.scan(scanningParameter, function (err, records) { // scanning database for getting all the records 56 | if (!err) { 57 | var recordsList = _.map(records.Items, function (record) { // for every record found 58 | return when.promise(function (resolve, reject) { 59 | recordExists(result.es, es_data, record).then(function (exists) { // check if record exists or not 60 | return putRecord(result.es, es_data, record, exists) 61 | }).then(function (record) { 62 | resolve(record) 63 | }, function (reason) { 64 | reject(reason) 65 | }); 66 | }); 67 | }); 68 | // return a promise array of all records 69 | } 70 | return when.all(recordsList); 71 | }) 72 | }).done(function (records) { 73 | callback(null, { 74 | status: true, 75 | message: 'Processed all records.' 76 | }); 77 | }, function (reason) { 78 | callback(reason, null); 79 | }); 80 | }; 81 | 82 | const createIndex = function (es, es_data, callback) { 83 | es.indices.create({ // create indices 84 | index: es_data.indiceName 85 | }, function (err) { 86 | if (err) { // if err return 87 | callback(err); 88 | } else { // index created 89 | callback(null); 90 | } 91 | }) 92 | }, 93 | recordExists = function (es, es_data, record) { // check if record exists in elasticsearch 94 | return when.promise(function (resolve, reject) { 95 | es.get({ 96 | index: es_data.indiceName, 97 | id: record[es_data.id], // this is the primaryKey of record in elasticsearch from which we can check weather the record is exists or not 98 | type: '_all' 99 | }, function (err, response, status) { 100 | if (status == 200) { // record exist 101 | resolve(true); 102 | } else if (status == 404) { // record not exist 103 | resolve(false); 104 | } else { // something went wrong 105 | reject(err); 106 | } 107 | }) 108 | }) 109 | }, 110 | putRecord = function (es, es_data, record, exists) { // insert record in elasticsearch 111 | return when.promise(function (resolve, reject) { 112 | var params = { 113 | index: es_data.indiceName, 114 | id: record[es_data.id], // this will be primaryKey of record in elasticsearch 115 | body: record, 116 | type: es_data.type 117 | }; 118 | 119 | const handler = function (err, response, status) { 120 | if (status == 200 || status == 201) { 121 | resolve(record) 122 | } else { 123 | reject(err) 124 | } 125 | } 126 | 127 | if (exists) { // if record exists in elasticsearch then update it 128 | params.body = { 129 | doc: record 130 | }; 131 | es.update(params, handler) 132 | } else { // if not exists then insert it into elasticsearch 133 | params.body = record 134 | es.create(params, handler) 135 | } 136 | }); 137 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamodb-to-elasticsearch", 3 | "version": "0.2.1", 4 | "description": "Module to dump DynamoDB data to ElasticSearch indices.", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/sohamdodia/dynamodb-to-elasticsearch" 8 | }, 9 | "main": "index.js", 10 | "scripts": { 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "keywords": [ 14 | "dynamodb-to-elastic-search", 15 | "dynamodb-to-elasticsearch", 16 | "aws", 17 | "dynamodb", 18 | "elasticsearch", 19 | "elastic-search" 20 | ], 21 | "author": "Soham Dodia", 22 | "license": "MIT", 23 | "dependencies": { 24 | "aws-sdk": "^2.122.0", 25 | "elasticsearch": "^13.3.1", 26 | "http-aws-es": "^3.1.0", 27 | "lodash": "^4.17.4", 28 | "when": "^3.7.8" 29 | } 30 | } 31 | --------------------------------------------------------------------------------