├── .gitignore ├── CHANGELOG.md ├── package.json ├── README.md └── engage.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | 3 | *.txt 4 | *.csv 5 | *.json 6 | .env 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 0.2.1 2 | - Support for config in ~/.engagerc 3 | 4 | 0.2.0 5 | - First NPM release -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mixpanel-engage-query", 3 | "description": "Command-line tool to query People Data using Mixpanel Engage API.", 4 | "version": "0.2.1", 5 | "homepage": "https://github.com/stpe/mixpanel-engage-query", 6 | "author": "Stefan Pettersson", 7 | "repository": { 8 | "type": "git", 9 | "url": "git://github.com/stpe/mixpanel-engage-query.git" 10 | }, 11 | "bugs": { 12 | "url": "https://github.com/stpe/mixpanel-engage-query/issues" 13 | }, 14 | "scripts": { 15 | "start": "node engage.js" 16 | }, 17 | "dependencies": { 18 | "cli-color": "1.1.0", 19 | "crypto": "0.0.3", 20 | "dotenv": "1.2.0", 21 | "exit": "0.1.2", 22 | "needle": "0.11.0", 23 | "os-homedir": "1.0.1", 24 | "sugar-date": "1.5.1", 25 | "yargs": "3.30.0" 26 | }, 27 | "license": "MIT", 28 | "preferGlobal": true, 29 | "bin": { 30 | "engage": "engage.js" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Command-line tool to query the [Mixpanel Engage API](https://mixpanel.com/docs/api-documentation/data-export-api#engage-default) for People Data. With other words, export list of Mixpanel users with selected properties, optionally filtered by query on property values. 2 | 3 | This script is especially powerful in combination with [mixpanel-engage-post](https://github.com/stpe/mixpanel-engage-post), that allow you to batch post additions/updates/deletion of people profiles, and [jq](http://stedolan.github.io/jq) (a command-line JSON processor). 4 | 5 | ## Installation 6 | 7 | Install [Node.js](http://nodejs.org/). 8 | 9 | Type `npm install --global mixpanel-engage-query` 10 | 11 | That's it! Run it by typing `engage` in your terminal. 12 | 13 | ## Setup 14 | 15 | To run the script you must specify your Mixpanel API key and secret either as parameters, as environment variables `MIXPANEL_API_KEY` and `MIXPANEL_API_SECRET`, in a [.env](https://github.com/motdotla/dotenv) file located in the script's directory (typically useful if you check out the source from Github) or in a `.engagerc` file in your home directory. 16 | 17 | Example of `.env` and `~/.engagerc` file: 18 | ``` 19 | MIXPANEL_API_KEY=f49785f7a0yourkey2019c6ba15d71f5 20 | MIXPANEL_API_SECRET=a69ca325ayoursecret4f5ed45cafb66 21 | ``` 22 | 23 | ## Example Usage 24 | 25 | #### Get help 26 | 27 | ``` 28 | $ engage --help 29 | 30 | Usage: engage -k [string] -s [string] 31 | 32 | Options: 33 | -k, --key Mixpanel API key [string] 34 | -s, --secret Mixpanel API secret [string] 35 | -f, --format Output format, json or csv [string] [default: "json"] 36 | -t, --total Only return total count of results 37 | -q, --query A segmentation expression (see Mixpanel API doc) [string] 38 | -p, --properties Properties to output (e.g. '$email,$first_name'). Outputs 39 | all properties if none specified. 40 | -r, --required Skip entries where the required properties are not set (e.g. 41 | '$email $first_name'). 42 | --na, --noarray Output json as one object per row, instead of one array of 43 | objects. 44 | -u, --url Only return the URL of query without making the actual 45 | request. 46 | -h, --help Help [boolean] 47 | 48 | Examples: 49 | engage -q 'properties["$last_seen"] > Query using expression 50 | "2015-04-24T23:00:00"' 51 | engage -p '$email,$first_name' Limit output to only given list of 52 | comma delimited properties 53 | 54 | Note that Mixpanel API key/secret may also be set using environment variables. 55 | For more information, see https://github.com/stpe/mixpanel-engage-query 56 | ``` 57 | 58 | #### Get everything 59 | 60 | `$ engage` 61 | 62 | Example output: 63 | ``` 64 | [ 65 | { 66 | "$browser": "Chrome", 67 | "$city": "Kathmandu", 68 | "$country_code": "NP", 69 | "$initial_referrer": "$direct", 70 | "$initial_referring_domain": "$direct", 71 | "$os": "Windows", 72 | "$timezone": "Asia/Katmandu", 73 | "id": "279267", 74 | "nickname": "bamigasectorone", 75 | "$last_seen": "2015-04-15T13:07:30", 76 | "$distinct_id": "15b9cba739b75-03c7e24a3-459c0418-101270-13d9bfa739ca6" 77 | } 78 | ] 79 | ``` 80 | 81 | Default behaviour is to output the result as an array of entries. Using the `noarray` flag will instead output one entry per row. 82 | 83 | Note that `$distinct_id` is included as a property for convenience. 84 | 85 | #### Get just the number of results 86 | 87 | `engage -t` (assumes Mixpanel key/secret set as environment variables) 88 | 89 | Example output: 90 | ``` 91 | 1138 92 | ``` 93 | 94 | #### Only output specific fields 95 | 96 | `engage -p '$email,$first_name'` 97 | 98 | Example output: 99 | ``` 100 | [ 101 | { '$email': 'jocke@bigcompany.se', '$first_name': 'Joakim' }, 102 | { '$email': 'henke@gmail.com', '$first_name': 'Henrik' }, 103 | { '$email': 'theguy@gmail.com', '$first_name': 'Jonas' } 104 | ] 105 | ``` 106 | 107 | #### Output as CSV instead of JSON 108 | 109 | `engage -f csv` 110 | 111 | Example output: 112 | ``` 113 | jocke@bigcompany.se;Joakim 114 | henke@gmail.com;Henrik 115 | theguy@gmail.com;Jonas 116 | ``` 117 | 118 | Note: currently no special escaping or similar is implemented, so depending on values CSV may end up invalid. 119 | 120 | #### Query using expression 121 | 122 | This example returns people with $last_seen timestamp greater (later) than 24th of April (see the Mixpanel documentation for [segmentation expressions](https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions)). 123 | 124 | `engage -q 'properties["$last_seen"] > "2015-04-24T23:00:00"'` 125 | 126 | ##### Relative date parsing 127 | 128 | Often you need a query with a condition relative to today's date. In order to avoid having to generate the command-line parameters dynamically you can use a placeholder as `[[DATE:]]` which will be replaced by a correctly formatted date for the Mixpanel API. The `` may be formatted according to what [Sugar Dates](http://sugarjs.com/dates) supports. 129 | 130 | Examples: 131 | 132 | `engage -q 'properties["$last_seen"] > "[[DATE:yesterday]]"'` 133 | 134 | `engage -q 'properties["$last_seen"] > "[[DATE:the beginning of last month]]"'` 135 | 136 | 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /engage.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node --harmony 2 | 3 | /* jshint node: true */ 4 | 5 | "use strict"; 6 | 7 | var needle = require('needle'), 8 | crypto = require('crypto'), 9 | exit = require('exit'), 10 | dotenv = require("dotenv"), 11 | osHomedir = require('os-homedir'); // consider replacing with os.homedir() later 12 | 13 | // mixpanel 14 | var base_url = "http://mixpanel.com/api/2.0/"; 15 | 16 | require('sugar-date'); 17 | 18 | // add environment variables from .env if present, check (in prio order) 19 | // - .env in script directory 20 | // - .engagerc in home directory 21 | dotenv.load({ silent: true }) || dotenv.load({ silent: true, path: osHomedir() + '/.engagerc' }); 22 | 23 | // options 24 | var yargs = require('yargs') 25 | .usage('Usage: $0 -k [string] -s [string]') 26 | .options('k', { 27 | alias: 'key', 28 | describe: 'Mixpanel API key', 29 | nargs: 1, 30 | type: 'string' 31 | }) 32 | .options('s', { 33 | alias: 'secret', 34 | describe: 'Mixpanel API secret', 35 | nargs: 1, 36 | type: 'string' 37 | }) 38 | .options('f', { 39 | alias: 'format', 40 | default: 'json', 41 | describe: 'Output format, json or csv', 42 | type: 'string' 43 | }) 44 | .options('t', { 45 | alias: 'total', 46 | describe: 'Only return total count of results' 47 | }) 48 | .options('q', { 49 | alias: 'query', 50 | // https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions 51 | // example: 'properties["$last_seen"] > "2013-08-29T23:00:00"' 52 | describe: 'A segmentation expression (see Mixpanel API doc)', 53 | type: 'string' 54 | }) 55 | .example("$0 -q 'properties[\"$last_seen\"] > \"2015-04-24T23:00:00\"'", 'Query using expression') 56 | .options('p', { 57 | alias: 'properties', 58 | describe: "Properties to output (e.g. '$email,$first_name'). Outputs all properties if none specified." 59 | }) 60 | .example("$0 -p '$email $first_name'", 'Limit output to only given list of space delimited properties') 61 | .options('r', { 62 | alias: 'required', 63 | describe: "Skip entries where the required properties are not set (e.g. '$email $first_name')." 64 | }) 65 | .options('na', { 66 | alias: 'noarray', 67 | describe: 'Output json as one object per row, instead of one array of objects.' 68 | }) 69 | .options('u', { 70 | alias: 'url', 71 | describe: "Only return the URL of query without making the actual request." 72 | }) 73 | .help('h') 74 | .options('h', { 75 | alias: 'help', 76 | describe: 'Help' 77 | }) 78 | .epilogue('Note that Mixpanel API key/secret may also be set using environment variables. For more information, see https://github.com/stpe/mixpanel-engage-query'); 79 | 80 | if (!process.env.MIXPANEL_API_KEY) { 81 | yargs.demand(['k']); 82 | } 83 | 84 | if (!process.env.MIXPANEL_API_SECRET) { 85 | yargs.demand(['s']); 86 | } 87 | 88 | var argv = yargs.argv; 89 | 90 | var MIXPANEL_API_KEY = process.env.MIXPANEL_API_KEY || argv.key; 91 | var MIXPANEL_API_SECRET = process.env.MIXPANEL_API_SECRET || argv.secret; 92 | 93 | // get mp properties to output 94 | var properties = typeof argv.properties === "string" ? argv.properties.split(",") : []; 95 | 96 | // get required mp properties 97 | var required = typeof argv.required === "string" ? argv.required.split(",") : []; 98 | 99 | // parse special [[DATE:]] tags 100 | if (typeof argv.query === "string") { 101 | var matches; 102 | while (matches = argv.query.match(/\[\[DATE:(.*?)\]\]/)) { 103 | var tag = matches[0]; 104 | var date = matches[1]; 105 | 106 | try { 107 | var dateISOstring = Date.create(date).format('{yyyy}-{MM}-{dd}T{hh}:{mm}:{ss}'); 108 | argv.query = argv.query.replace(tag, dateISOstring); 109 | } catch(e) { 110 | console.log("Error parsing date '" + date + "': " + e.message); 111 | exit(1); 112 | } 113 | } 114 | } 115 | 116 | // do the stuff! 117 | queryEngageApi({ 118 | where: argv.query || "" 119 | }); 120 | 121 | // ------------------------------------------ 122 | 123 | function queryEngageApi(params) { 124 | var page_size, total; 125 | 126 | var doQuery = function(params) { 127 | var url = getUrl("engage", params); 128 | if (argv.url) { 129 | console.log(url); 130 | exit(0); 131 | } 132 | 133 | needle.get(url, {}, function(err, resp, data) { 134 | // request error 135 | if (err) { 136 | console.log(err.toString()); 137 | exit(1); 138 | } 139 | 140 | // Mixpanel API error 141 | if (data.error) { 142 | console.log('Mixpanel API error: ' + data.error); 143 | exit(1); 144 | } 145 | 146 | // return total count 147 | if (argv.total) { 148 | console.log(data.total); 149 | exit(0); 150 | } 151 | 152 | // note: properties page_size and total are only returned if no page parameter 153 | // is set in the request (not even page=0). Hence they are only available 154 | // in the first response. 155 | 156 | if (data.page == 0) { 157 | // remember total results and page_size 158 | total = data.total; 159 | page_size = data.page_size; 160 | 161 | // use session id from now on to speed up API response 162 | params.session_id = data.session_id; 163 | 164 | // beginning of json array 165 | if (argv.format == 'json' && !argv.noarray) { 166 | console.log('['); 167 | } 168 | } 169 | 170 | total -= data.results.length; 171 | var isLastQuery = total < 1; 172 | 173 | processResults(data, isLastQuery); 174 | 175 | // if not done, keep querying for additional pages 176 | if (!isLastQuery) { 177 | // get next page 178 | params.page = data.page + 1; 179 | 180 | doQuery(params); 181 | } else { 182 | // end of json array 183 | if (argv.format == 'json' && !argv.noarray) { 184 | console.log(']'); 185 | } 186 | 187 | exit(0); 188 | } 189 | }); 190 | }; 191 | 192 | doQuery(params); 193 | } 194 | 195 | function processResults(data, isLastQuery) { 196 | var i, csv, entry, len = data.results.length, output; 197 | 198 | for (i = 0; i < len; i++) { 199 | if (required.length > 0) { 200 | // skip if not required properties present 201 | if (!required.every(function(r) { 202 | return typeof data.results[i].$properties[r] !== 'undefined'; 203 | })) { 204 | continue; 205 | } 206 | } 207 | 208 | // include $distinct_id in property list for convenience 209 | if (data.results[i].$distinct_id) { 210 | data.results[i].$properties.$distinct_id = data.results[i].$distinct_id; 211 | } 212 | 213 | entry = {}; 214 | if (properties.length === 0) { 215 | // output all 216 | entry = data.results[i].$properties; 217 | } else { 218 | // only include given properties 219 | properties.forEach(function(p) { 220 | entry[p] = data.results[i].$properties[p] || ''; 221 | }); 222 | } 223 | 224 | // skip if object is empty 225 | if (Object.keys(entry).length === 0) { 226 | continue; 227 | } 228 | 229 | if (argv.format == "csv") { 230 | // csv 231 | csv = []; 232 | Object.keys(entry).forEach(function(k) { 233 | csv.push(entry[k]); 234 | }); 235 | output = csv.join(";"); 236 | } else { 237 | // json 238 | output = JSON.stringify(entry); 239 | 240 | // if not last result... 241 | if (!argv.noarray && (i < len - 1 || !isLastQuery)) { 242 | // ...append comma 243 | output += ","; 244 | } 245 | } 246 | 247 | console.log(output); 248 | } 249 | } 250 | 251 | function getUrl(endpoint, args) { 252 | // add api_key and expire in EXPIRE_IN_MINUTES 253 | var EXPIRE_IN_MINUTES = 10; 254 | args.api_key = MIXPANEL_API_KEY; 255 | args.expire = Math.round(Date.now() / 1000) + 60 * EXPIRE_IN_MINUTES; 256 | 257 | // see https://mixpanel.com/docs/api-documentation/data-export-api#auth-implementation 258 | var arg_keys = Object.keys(args), 259 | sorted_keys = arg_keys.sort(), 260 | concat_keys = "", 261 | params = []; 262 | 263 | for (var i = 0; i < sorted_keys.length; i++) { 264 | params.push(sorted_keys[i] + "=" + args[sorted_keys[i]]); 265 | concat_keys += params[params.length - 1]; 266 | } 267 | 268 | // sign 269 | var sig = crypto.createHash('md5').update(concat_keys + MIXPANEL_API_SECRET).digest("hex"); 270 | 271 | // return request url 272 | return base_url + endpoint + "/?" + params.join("&") + "&sig=" + sig; 273 | } 274 | --------------------------------------------------------------------------------