├── todo.md ├── README.md ├── public ├── js │ ├── app.js │ ├── index.js │ ├── controllers.js │ └── csv.min.js └── views │ └── index.html ├── test.csv ├── package.json ├── app └── helpers │ └── segment.js ├── .gitignore ├── server.js └── config └── routes.js /todo.md: -------------------------------------------------------------------------------- 1 | 1. all errors must propogate to client. 2 | 2. add error handling prior to server-side request being initiated. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Import 2 | 3 | You can now import a CSV directly into Segment for you to see what the data looks like. 4 | -------------------------------------------------------------------------------- /public/js/app.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // Declare app level module which depends on services, etc. ==================== 4 | var segImport = angular.module('segImport', []); -------------------------------------------------------------------------------- /test.csv: -------------------------------------------------------------------------------- 1 | type,userId,traits.company,traits.email,traits.favorite_band 2 | identify,a3810,segment,andy@segment.io,null 3 | identify,b1238,mixpanel,steve@mixpanel.com,nickelback 4 | identify,b0099,01,bob@01.com,0.1 5 | identify,b8877,01,bob@02.com,.1 -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "import", 3 | "description": "Import UI for Segment.", 4 | "version": "0.0.1", 5 | "author": "Andy Jiang", 6 | "main": "server.js", 7 | "repository": { 8 | "type": "git", 9 | "url": "https://github.com/lambtron/segment-import" 10 | }, 11 | "engines": { 12 | "node": "0.11.14", 13 | "npm": "1.1.x" 14 | }, 15 | "private": true, 16 | "dependencies": { 17 | "express": "^4.0.0", 18 | "http-assert": "^1.5.0", 19 | "request": "^2.88.2", 20 | "underscore": "latest" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /app/helpers/segment.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Segment helper library. 3 | */ 4 | 5 | (function() { 6 | var request = require('request'); 7 | var path = 'https://api.segment.io/v1/import/'; 8 | 9 | module.exports = { 10 | batchImport: function batchImport(writeKey, batch, fn) { 11 | var auth = { 12 | user: writeKey || '', 13 | pass: '' 14 | }; 15 | var opts = { 16 | uri: path, 17 | method: 'POST', 18 | timeout: 50000, 19 | followRedirect: true, 20 | maxRedirects: 10, 21 | auth: auth, 22 | body: batch, 23 | json: true 24 | }; 25 | request(opts, fn); 26 | } 27 | }; 28 | 29 | }()); -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Deployed apps should consider commenting this line out: 24 | # see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git 25 | node_modules 26 | 27 | ./config/config.js 28 | 29 | # Users Environment Variables 30 | .lock-wscript -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // Initialize server =========================================================== 4 | var express = require('express') 5 | , http = require('http') 6 | , app = express() 7 | , server = http.createServer(app) 8 | , port = process.env.PORT || 3000; 9 | 10 | // Configuration =============================================================== 11 | app.set('views', __dirname + 'public/views'); 12 | app.use(express.static(__dirname + '/public')); 13 | app.use(express.bodyParser()); 14 | 15 | // Routes ====================================================================== 16 | require('./config/routes.js')(app); 17 | 18 | // New relic. 19 | // require('newrelic'); 20 | 21 | // Listen (start app with node server.js) ====================================== 22 | server.listen(port, function() { 23 | console.log("App is now listening on port " + port); 24 | }); -------------------------------------------------------------------------------- /config/routes.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function() { 4 | 5 | /** 6 | * Import helpers ============================================================ 7 | */ 8 | var Segment = require('../app/helpers/segment.js'); 9 | var assert = require('http-assert'); 10 | var ok = require('assert'); 11 | 12 | // Public functions. ========================================================= 13 | module.exports = function(app) { 14 | // API routes ============================================================== 15 | app.post('/api/import', function(req, res) { 16 | // Send request to Segment. 17 | // Need writeKey 18 | var writeKey = req.body.writeKey; 19 | var batch = { 20 | batch: req.body.batch 21 | }; 22 | 23 | Segment.batchImport(writeKey, batch, function(err, http, body) { 24 | if (err) 25 | res.send(err, 400); 26 | 27 | console.log(body); 28 | 29 | res.send(body, 200); 30 | }); 31 | }); 32 | 33 | // Application routes ====================================================== 34 | app.get('/*', function(req, res) { 35 | res.sendfile('index.html', {'root': './public/views/'}); 36 | }); 37 | }; 38 | 39 | }()); -------------------------------------------------------------------------------- /public/js/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | window.onload = function() { 4 | var fileInput = document.getElementById('fileInput'); 5 | var fileDisplayArea = document.getElementById('fileDisplayArea'); 6 | var csvFile = {}; 7 | 8 | fileInput.addEventListener('change', function(e) { 9 | // Use DOM to get AngularJS root scope. 10 | var scope = angular.element(this).scope(); 11 | 12 | // Reset firebase and local data store. 13 | scope.$apply(function() { 14 | scope.csv.removeAll(); 15 | }); 16 | 17 | var file = fileInput.files[0]; 18 | var textType = /text.*/; 19 | 20 | if (file.type.match(textType)) { 21 | var reader = new FileReader(); 22 | 23 | reader.onload = function(e) { 24 | var csvString = reader.result; 25 | // Clean the headers. 26 | var firstLine = csvString.split('\n')[0]; 27 | var cleanedFirstLine = firstLine.replace(/\s+/g, '_'); 28 | csvString = csvString.replace(firstLine, cleanedFirstLine); 29 | 30 | // Parse csv. 31 | var csv = new CSV(csvString); 32 | scope.$apply(function() { 33 | scope.csv.addArray(csv); 34 | }); 35 | }; 36 | 37 | reader.readAsText(file); 38 | } else { 39 | fileDisplayArea.innerText = "File not supported!" 40 | }; 41 | }); 42 | } -------------------------------------------------------------------------------- /public/js/controllers.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | segImport.controller('mainController', ['$scope', '$http', 4 | function($scope, $http) 5 | { 6 | var csv = $scope.csv = {}; 7 | 8 | csv.array = []; // These are the rows. 9 | csv.JSON = []; // JSON object 10 | csv.JSONString = ''; 11 | csv.writeKey = ''; 12 | 13 | // Take csv string and turn it into array. 14 | csv.addArray = function addArray(csv) { 15 | csv.forEach(function(row) { 16 | this.array.push(row); 17 | }.bind(this)); 18 | this.arrayToJSON(); 19 | }; 20 | 21 | // Empty rows in csv.array. 22 | csv.removeAll = function removeAll() { 23 | this.array.length = 0; 24 | }; 25 | 26 | // Convert csv.array to csv.JSON. 27 | csv.arrayToJSON = function arrayToJSON() { 28 | 29 | var headers = this.array[0]; 30 | this.JSON.length = 0; 31 | 32 | for (var i = 1; i < this.array.length; i ++) { 33 | var obj = {}; 34 | var currentLine = this.array[i]; 35 | for (var j = 0; j < headers.length; j ++) { 36 | // Keep Nulls, ints and floats 37 | switch (true) { 38 | case currentLine[j] == 'null': 39 | currentLine[j] = null 40 | break 41 | case /^[0-9]+$/.test(currentLine[j]): 42 | currentLine[j] = parseInt(currentLine[j]) 43 | break 44 | case /^[0-9]*\.[0-9]+$/.test(currentLine[j]): 45 | currentLine[j] = parseFloat(currentLine[j]) 46 | break 47 | } 48 | if (headers[j].indexOf('.') > 0) { 49 | var prefix = headers[j].substring(0, headers[j].indexOf('.')); 50 | var suffix = headers[j].substring(headers[j].indexOf('.') + 1); 51 | if (!obj[prefix]) 52 | obj[prefix] = {}; 53 | obj[prefix][suffix] = currentLine[j] ; 54 | } else { 55 | obj[headers[j]] = currentLine[j]; 56 | } 57 | } 58 | this.JSON.push(obj); 59 | } 60 | this.JSONString = JSON.stringify(this.JSON, null, 2); 61 | }; 62 | 63 | // Post csv.JSON to end point. 64 | csv.importJSON = function importJSON() { 65 | console.log(this.JSON); 66 | $http.post('/api/import', {batch: this.JSON, writeKey: this.writeKey}) 67 | .success(function(err, data) { 68 | console.log(err); 69 | console.log(data); 70 | }) 71 | .error(function(err, data) { 72 | console.log(err); 73 | console.log(data); 74 | }); 75 | }; 76 | }]); -------------------------------------------------------------------------------- /public/views/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Import 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 |
34 |
35 |

36 |
37 |
38 | 39 | 40 |
41 |
42 | 43 | 44 |
45 | 46 |


47 | 48 |
49 |
50 | 51 | 52 | 53 | 54 | 57 | 58 | 59 | 60 | 61 | 64 | 65 | 66 |
55 | {{ header }} 56 |
62 | {{ item }} 63 |
67 |
68 |
69 |
70 |
71 | 72 |
73 |
74 | 
{{ csv.JSONString }}
75 |
76 |
77 | import 78 |
79 |
80 |
81 |
82 | 83 | 84 | -------------------------------------------------------------------------------- /public/js/csv.min.js: -------------------------------------------------------------------------------- 1 | !function(){"use strict";function t(t){var n=typeof t;return"function"===n||"object"===n&&!!t}function n(t){return"[object Array]"==toString.call(t)}function e(t){return"[object String]"===toString.call(t)}function i(t){return!isNaN(Number(t))}function r(t){return 0==t||1==t}function o(t){return null==t}function c(t){return null!=t}function u(t,n){return c(t)?t:n}function l(t,n){for(var e=0,i=t.length;i>e&&n(t[e],e)!==!1;)e++;return t}function a(t){return"attrs["+t+"]"}function s(t,n){return i(t)?"Number("+a(n)+")":r(t)?"Boolean("+a(n)+" == true)":"String("+a(n)+")"}function f(t,e,i){var r=[];return 2==arguments.length?(t?n(t)?l(e,function(n,e){r.push(t[e]+"("+a(e)+")")}):l(e,function(t,n){r.push(s(t,n))}):l(e,function(t,n){r.push(a(n))}),r="return ["+r.join(",")+"]"):(t?n(t)?l(e,function(n,e){r.push('"'+i[e]+'": '+t[e]+"("+a(e)+")")}):l(e,function(t,n){r.push('"'+i[n]+'": '+s(t,n))}):l(e,function(t,n){r.push('"'+i[n]+'": '+a(n))}),r="return {"+r.join(",")+"}"),new Function("attrs",r)}function h(t,n){var e,i=0;return l(n,function(n){var r,o=n;-1!=p.indexOf(n)&&(o="\\"+o),r=t.match(new RegExp(o,"g")),r&&r.length>i&&(i=r.length,e=n)}),e||n[0]}var p=["|","^"],d=[",",";"," ","|","^"],m=["\r\n","\r","\n"],g=function(){function i(t,i){if(i||(i={}),n(t))this.mode="encode";else{if(!e(t))throw new Error("Incompatible format!");this.mode="parse"}this.data=t,this.options={header:u(i.header,!1),cast:u(i.cast,!0)};var r=i.lineDelimiter||i.line,o=i.cellDelimiter||i.delimiter;this.isParser()?(this.options.lineDelimiter=r||h(this.data,m),this.options.cellDelimiter=o||h(this.data,d),this.data=c(this.data,this.options.lineDelimiter)):this.isEncoder()&&(this.options.lineDelimiter=r||"\r\n",this.options.cellDelimiter=o||",")}function r(t,n,e){t(new n(e))}function c(t,n){return t.slice(-n.length)!=n&&(t+=n),t}function a(i){return n(i)?"array":t(i)?"object":e(i)?"string":o(i)?"null":"primitive"}return i.prototype.set=function(t,n){return this.options[t]=n},i.prototype.isParser=function(){return"parse"==this.mode},i.prototype.isEncoder=function(){return"encode"==this.mode},i.prototype.parse=function(t){function e(){a={escaped:!1,quote:!1,cell:!0}}function i(){g.cell=""}function o(){g.line=[]}function c(t){g.line.push(a.escaped?t.slice(1,-1).replace(/""/g,'"'):t),i(),e()}function u(t){c(t.slice(0,1-d.lineDelimiter.length))}function l(){m?n(m)?(s=f(d.cast,g.line,m),(l=function(){r(t,s,g.line)})()):m=g.line:(s||(s=f(d.cast,g.line)),(l=function(){r(t,s,g.line)})())}if("parse"==this.mode){if(0===this.data.trim().length)return[];var a,s,h,p=this.data,d=this.options,m=d.header,g={cell:"",line:[]};t||(h=[],t=function(t){h.push(t)}),e(),1==d.lineDelimiter.length&&(u=c);var y,v,j,w=p.length,D=d.cellDelimiter.charCodeAt(0),b=d.lineDelimiter.charCodeAt(d.lineDelimiter.length-1);for(y=0,v=0;w>y;y++)j=p.charCodeAt(y),a.cell&&(a.cell=!1,34==j)?a.escaped=!0:a.escaped&&34==j?a.quote=!a.quote:(a.escaped&&a.quote||!a.escaped)&&(j==D?(c(g.cell+p.slice(v,y)),v=y+1):j==b&&(u(g.cell+p.slice(v,y)),v=y+1,l(),o()));return h?h:this}},i.prototype.serialize={object:function(t){var n=this,e=Object.keys(t),i=Array(e.length);return l(e,function(e,r){i[r]=n[a(t[e])](t[e])}),i},array:function(t){var n=this,e=Array(t.length);return l(t,function(t,i){e[i]=n[a(t)](t)}),e},string:function(t){return'"'+String(t).replace(/"/g,'""')+'"'},"null":function(){return""},primitive:function(t){return t}},i.prototype.encode=function(t){function e(t){return t.join(c.cellDelimiter)}if("encode"==this.mode){if(0==this.data.length)return"";var i,r,o=this.data,c=this.options,u=c.header,s=o[0],f=this.serialize,h=0;t||(r=Array(o.length),t=function(t,n){r[n+h]=t}),u&&(n(u)||(i=Object.keys(s),u=i),t(e(f.array(u)),0),h=1);var p,d=a(s);return"array"==d?(n(c.cast)?(p=Array(c.cast.length),l(c.cast,function(t,n){p[n]=t.toLowerCase()})):(p=Array(s.length),l(s,function(t,n){p[n]=a(t)})),l(o,function(n,i){var r=Array(p.length);l(n,function(t,n){r[n]=f[p[n]](t)}),t(e(r),i)})):"object"==d&&(i=Object.keys(s),n(c.cast)?(p=Array(c.cast.length),l(c.cast,function(t,n){p[n]=t.toLowerCase()})):(p=Array(i.length),l(i,function(t,n){p[n]=a(s[t])})),l(o,function(n,r){var o=Array(i.length);l(i,function(t,e){o[e]=f[p[e]](n[t])}),t(e(o),r)})),r?r.join(c.lineDelimiter):this}},i.prototype.forEach=function(t){return this[this.mode](t)},i}();g.parse=function(t,n){return new g(t,n).parse()},g.encode=function(t,n){return new g(t,n).encode()},g.forEach=function(t,n,e){return 2==arguments.length&&(e=n),new g(t,n).forEach(e)},"function"==typeof define&&define.amd?define("CSV",[],function(){return g}):"object"==typeof module&&module.exports?module.exports=g:window&&(window.CSV=g)}(); --------------------------------------------------------------------------------