├── dist └── .gitkeep ├── .project.vim ├── coffee ├── csv │ ├── run.jade │ ├── start.jade │ └── csv_connector.coffee ├── forum │ ├── run.jade │ ├── start.jade │ └── forum_connector.coffee ├── sap_bo │ ├── run.jade │ ├── start.jade │ ├── configuration.jade │ ├── proxy_server.coffee │ └── sap_bo_connector.coffee ├── community │ ├── run.jade │ ├── start.jade │ └── community_connector.coffee ├── template │ ├── run.jade │ ├── start.jade │ └── connector.coffee ├── github_commits │ ├── run.jade │ ├── start.jade │ ├── source.jade │ ├── github_connector.coffee │ └── twdc-github-commits.coffee ├── json_connector │ ├── run.jade │ ├── start.jade │ ├── json_connector.coffee │ └── json_flattener.coffee ├── mongodb_connector │ ├── run.jade │ ├── json_flattener.coffee │ ├── mongodb_connector.coffee │ └── start.jade ├── twitter_connector │ ├── run.jade │ ├── twitter_keys.coffee.example │ ├── auth.jade │ ├── twitter_auth_server.coffee │ ├── start.jade │ └── twitter_connector.coffee ├── connector_base │ ├── tableau_helpers.coffee │ ├── underscore_starschema.coffee │ ├── state_machine.coffee │ └── starschema_wdc_base.coffee └── nasty_proxy │ └── nasty_proxy.coffee ├── .travis.yml ├── resources ├── bootstrap-3.3.5-dist │ ├── fonts │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.ttf │ │ ├── glyphicons-halflings-regular.woff │ │ └── glyphicons-halflings-regular.woff2 │ ├── js │ │ ├── npm.js │ │ └── bootstrap.min.js │ └── css │ │ ├── bootstrap-theme.min.css │ │ └── bootstrap-theme.css ├── forum.html ├── community.html ├── template.html ├── mongodb.html ├── twitter.html ├── github.html ├── json.html ├── csv.html ├── sapbo.html └── tableau-wdc-js │ └── tableauwdc-1.1.0.js ├── create_connector.sh ├── .gitignore ├── LICENSE ├── package.json ├── docs ├── make_wdc_better.markdown ├── github.md └── mongo.md └── README.md /dist/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.project.vim: -------------------------------------------------------------------------------- 1 | nmap :!./build-sap-bo.sh 2 | -------------------------------------------------------------------------------- /coffee/csv/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /coffee/forum/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /coffee/sap_bo/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /coffee/community/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /coffee/template/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | script: for i in build*.sh; do ./$i ; done 3 | node_js: 4 | - "0.12" 5 | -------------------------------------------------------------------------------- /coffee/github_commits/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /coffee/json_connector/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /coffee/mongodb_connector/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /coffee/twitter_connector/run.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h2 Loading data... 5 | -------------------------------------------------------------------------------- /resources/bootstrap-3.3.5-dist/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/starschema/tableau-web-table-connector/HEAD/resources/bootstrap-3.3.5-dist/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /resources/bootstrap-3.3.5-dist/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/starschema/tableau-web-table-connector/HEAD/resources/bootstrap-3.3.5-dist/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /resources/bootstrap-3.3.5-dist/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/starschema/tableau-web-table-connector/HEAD/resources/bootstrap-3.3.5-dist/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /resources/bootstrap-3.3.5-dist/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/starschema/tableau-web-table-connector/HEAD/resources/bootstrap-3.3.5-dist/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /coffee/twitter_connector/twitter_keys.coffee.example: -------------------------------------------------------------------------------- 1 | # RENAME ME TO twitter_keys.coffee after filling the keys 2 | 3 | module.exports = 4 | CONSUMER_KEY: "" 5 | CONSUMER_SECRET: "" 6 | ACCESS_TOKEN: "" 7 | ACCESS_TOKEN_SECRET: "" 8 | 9 | -------------------------------------------------------------------------------- /create_connector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -Ex 2 | 3 | cp -Rp coffee/template coffee/$1 4 | mv coffee/$1/connector.coffee coffee/$1/${1}_connector.coffee 5 | cp resources/template.html resources/$1.html 6 | 7 | sed -i'' "s/CONNECTORNAME/$1/" resources/$1.html 8 | 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Let's make it easy and add it. Maybe a git link to Tableau's repo would make more sense? 2 | # resources/tableau-wdc-js/* 3 | node_modules/**/* 4 | dist/**/* 5 | 6 | coffee/twitter_connector/twitter_keys.coffee 7 | 8 | # Editor files 9 | *tags 10 | *.swp 11 | *~ 12 | -------------------------------------------------------------------------------- /coffee/template/start.jade: -------------------------------------------------------------------------------- 1 | h3 Connect to Data 2 | 3 | .row 4 | .col-sm-12 5 | label(for="myinput") Myinput 6 | input.form-control(type="text" data-tableau-key="myinput") 7 | p.help-block 8 | | Add meaningful help with 9 | code some_code 10 | 11 | .row 12 | .col-sm-12 13 | a.btn.btn-default(href="#" data-state-to="run") Load Data 14 | -------------------------------------------------------------------------------- /coffee/forum/start.jade: -------------------------------------------------------------------------------- 1 | h3 Connect to Data 2 | 3 | .row 4 | .col-sm-12 5 | label(for="tags") Tags for search 6 | input.form-control(type="text" data-tableau-key="tags") 7 | p.help-block 8 | | Add coma separated list of tags like 9 | code wdc,api 10 | 11 | .row 12 | .col-sm-12 13 | a.btn.btn-default(href="#" data-state-to="run") Load Data 14 | -------------------------------------------------------------------------------- /coffee/community/start.jade: -------------------------------------------------------------------------------- 1 | h3 Connect to Data 2 | 3 | .row 4 | .col-sm-12 5 | label(for="after") Enter first date to check 6 | input.form-control(type="text" data-tableau-key="after") 7 | p.help-block 8 | | Search posts after a give date. Format is ISO date/time: 9 | code 2015-11-11T01:13:29.851+0200 or 2015-11-11 10 | 11 | .row 12 | .col-sm-12 13 | a.btn.btn-default(href="#" data-state-to="run") Load Data 14 | -------------------------------------------------------------------------------- /resources/bootstrap-3.3.5-dist/js/npm.js: -------------------------------------------------------------------------------- 1 | // This file is autogenerated via the `commonjs` Grunt task. You can require() this file in a CommonJS environment. 2 | require('../../js/transition.js') 3 | require('../../js/alert.js') 4 | require('../../js/button.js') 5 | require('../../js/carousel.js') 6 | require('../../js/collapse.js') 7 | require('../../js/dropdown.js') 8 | require('../../js/modal.js') 9 | require('../../js/tooltip.js') 10 | require('../../js/popover.js') 11 | require('../../js/scrollspy.js') 12 | require('../../js/tab.js') 13 | require('../../js/affix.js') -------------------------------------------------------------------------------- /coffee/sap_bo/start.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h1 5 | | SAP BO 6 | small 7 | WebI / QAAWS web data connector 8 | br 9 | | by  10 | 11 | a(href="https://starschema.net" target="_blank") StarSchema 12 | 13 | 14 | .col-sm-12 15 | pre.alert.alert-danger#error(style="display:none") 16 | .col-sm-12 17 | form.form-horizontal 18 | .form-group 19 | label(for="wsdl") WSDL Url 20 | input.form-control#wsdl(type="text", name="wsdl" data-tableau-key="wsdl") 21 | a.btn.btn-primary#submit-button(href="#" data-state-to="configuration") Next 22 | -------------------------------------------------------------------------------- /coffee/twitter_connector/auth.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | h1 Auth 4 | .col-sm-12 5 | a.btn.btn-primary(href="#" data-state-to="run") Run 6 | a.btn.btn-default(href="#" data-state-to=":back2") << Back 7 | 8 | 9 | .row 10 | .col-sm-12 11 | label(for="url") Consumer Key 12 | input.form-control(type="text" data-tableau-key="consumer_key" value="kWyxiJH90fDiHGFv372v6g1OZ") 13 | p.help-block 14 | | The Twitter Consumer Key. For example (not valid): 15 | code xvz1evFS4wEEPTGEFPHBog 16 | 17 | .col-sm-12 18 | label(for="url") Consumer Secret 19 | input.form-control(type="text" data-tableau-key="consumer_secret" value="yGxQNtqQL2KJXWmBYdJolG3y5I6YEmZOT3WXPbWNEzmlBBXPAf") 20 | p.help-block 21 | | The Twitter Consumer Secret. For example (not valid): 22 | code L8qq9PZyRg6ieKGEKhZolGC0vJWLw8iEJ88DRdyOg 23 | 24 | -------------------------------------------------------------------------------- /coffee/json_connector/start.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | h1 Raw JSON connect 4 | 5 | .row 6 | .col-sm-12 7 | .page-header 8 | h4 Connect to Raw JSON data source 9 | 10 | .col-sm-12 11 | 12 | form 13 | .row 14 | .col-sm-10 15 | label(for="url") JSON Url 16 | input.form-control(type="text" data-tableau-key="url" ) 17 | 18 | 19 | 20 | .col-sm-2 21 | label Connect 22 | a.btn.btn-default(href="#" data-state-to="run_json") Connect to JSON 23 | 24 | 25 | // 26 | .row 27 | .col-sm-12 28 | .page-header 29 | h4 Or connect to a raw JSON file 30 | 31 | .col-sm-12 32 | 33 | form 34 | .row 35 | .col-sm-10 36 | label(for="url") JSON Url 37 | input.form-control(type="text" data-tableau-key="url") 38 | 39 | .col-sm-2 40 | label Connect 41 | a.btn.btn-default(href="#" data-state-to="run") Get raw JSON 42 | 43 | -------------------------------------------------------------------------------- /coffee/sap_bo/configuration.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h1 5 | | SAP Business Objects 6 | small web data connector 7 | h6 8 | | Get live data with 9 | a(href="http://www.virtdb.com/webdataconnector") VirtDB 10 | 11 | .col-sm-12 12 | form.form-horizontal 13 | .form-group 14 | label(for="tablelist") Table 15 | select.form-control#tables(name="table" data-tableau-key="table" ) 16 | .form-group 17 | label(for="auth-username") Auth Username 18 | input.form-control#auth-username(type="text" data-tableau-key="auth_username" ) 19 | .form-group 20 | label(for="auth-username") Auth Password 21 | input.form-control#auth-password(type="password" data-tableau-key="auth_password" ) 22 | a.btn.btn-primary#submit-button(href="#" data-state-to="run") Load -------------------------------------------------------------------------------- /resources/forum.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /resources/community.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /resources/template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /resources/mongodb.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector -- Mongodb 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /resources/twitter.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector -- Twitter 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Starschema Limited 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /resources/github.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector -- Github 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /coffee/connector_base/tableau_helpers.coffee: -------------------------------------------------------------------------------- 1 | _ = require 'underscore' 2 | 3 | STRING = "string" 4 | INT = "int" 5 | FLOAT = "float" 6 | DATE = "date" 7 | # STRING = tableau.dataTypeEnum.string 8 | # INT = tableau.dataTypeEnum.int 9 | # FLOAT = tableau.dataTypeEnum.float 10 | # DATE = tableau.dataTypeEnum.date 11 | # We need to map the source column data type to tableau column 12 | # data type. This function tries to figure out the type based on 13 | # its value. 14 | guessDataType = (value)-> 15 | return STRING unless value 16 | switch 17 | when parseInt(value).toString() == value.toString() then INT 18 | when parseFloat(value).toString() == value.toString() then FLOAT 19 | when value == "true" || value == "false" || value == true || value == false then INT 20 | when _.isString(value) then STRING 21 | when isFinite(new Date(value).getTime()) then DATE 22 | else STRING 23 | 24 | getJsonType = (value)-> switch 25 | when _.isArray(value) then getJsonType(_.first(value)) 26 | when _.isObject(value) then guessDataType(value) 27 | else guessDataType(value) 28 | 29 | 30 | module.exports = 31 | guessDataType: guessDataType 32 | getJsonType: getJsonType 33 | INT: INT 34 | FLOAT: FLOAT 35 | DATE: DATE 36 | STRING: STRING 37 | -------------------------------------------------------------------------------- /resources/json.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector -- JSON 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /resources/csv.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector -- CSV 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /resources/sapbo.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Tableau Web Data Connector -- SAP Business Objects 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "starschema-wdc-91", 3 | "version": "2.0.1", 4 | "description": "", 5 | "main": "index.js", 6 | "dependencies": { 7 | "JSONSelect": "^0.4.0", 8 | "bobj-access": "^3.2.2", 9 | "bootstrap": "^3.3.4", 10 | "cors": "^2.7.1", 11 | "csv": "^0.4.1", 12 | "dateformat": "^2.0.0", 13 | "express": "^4.13.1", 14 | "jquery": "^2.1.3", 15 | "request": "^2.59.0", 16 | "twitter": "^1.2.5", 17 | "underscore": "^1.8.3" 18 | }, 19 | "devDependencies": { 20 | "browserify": "^9.0.7", 21 | "coffee-script": "^1.9.1", 22 | "coffeeify": "^1.0.0", 23 | "grunt": "^0.4.5", 24 | "grunt-browserify": "^3.6.0", 25 | "jade": "^1.9.2", 26 | "jasmine": "^2.2.1", 27 | "jadeify": "^4.1.0" 28 | }, 29 | "browserify": { 30 | "extensions": [ 31 | ".coffee", 32 | ".hbs" 33 | ], 34 | "transform": [ 35 | [ 36 | "coffeeify", 37 | { 38 | "bare": false 39 | } 40 | ], 41 | [ 42 | "jadeify", 43 | { 44 | "compileDebug": true, 45 | "pretty": true 46 | } 47 | ] 48 | ] 49 | }, 50 | "scripts": { 51 | "test": "echo \"Error: no test specified\" && exit 1" 52 | }, 53 | "author": "Tamas Foldi ", 54 | "license": "Eclipse" 55 | } 56 | -------------------------------------------------------------------------------- /coffee/github_commits/start.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h1 5 | | Github commits 6 | small web data connector 7 | 8 | .col-sm-12 9 | form.form-inline 10 | 11 | .form-group 12 | label(for="username") Username 13 | input.form-control#username(type="text", name="username" data-tableau-key="username" ) 14 | .form-group 15 | label(for="reponame") Repository name 16 | input.form-control#reponame(type="text", name="reponame" data-tableau-key="reponame" ) 17 | 18 | a.btn.btn-primary#submit-button(href="#" data-state-to="run") Load commits 19 | 20 | .col-sm-12 21 | .page-header 22 | h4 23 | | Authentication 24 | small Unatuhenticated API requests are limited to 60 req/hour. 25 | 26 | .col-sm-12 27 | form.form-inline 28 | .form-group 29 | label(for="auth-username") Auth Username 30 | input.form-control#auth-username(type="text" data-tableau-key="auth_username" ) 31 | 32 | .form-group 33 | label(for="auth-username") Auth Password 34 | input.form-control#auth-password(type="password" data-tableau-key="auth_password" ) 35 | 36 | .form-group 37 | label 38 | input#do-auth( type="checkbox" checked="checked" data-tableau-key="do_auth" ) 39 | | Do authentication? 40 | 41 | 42 | -------------------------------------------------------------------------------- /coffee/forum/forum_connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | 4 | 5 | tableauHelpers = require '../connector_base/tableau_helpers.coffee' 6 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 7 | 8 | # TODO: implement your HTTP logic 9 | load_url = (url, success_callback)-> 10 | $.ajax 11 | url: url 12 | dataType: "text" 13 | success: (res)-> 14 | res = res.substring(res.indexOf("\n") + 1) 15 | 16 | success_callback( JSON.parse(res) ) 17 | 18 | error: (xhr, ajaxOptions, thrownError)-> 19 | tableau.abortWithError "Error while trying to load '#{url}'. #{thrownError}" 20 | 21 | 22 | wdc_base.make_tableau_connector 23 | name: "Simple XXX connector" 24 | 25 | steps: 26 | start: 27 | template: require './start.jade' 28 | run: 29 | template: require './run.jade' 30 | 31 | 32 | transitions: 33 | "start > run": (data)-> 34 | _.extend data, wdc_base.fetch_inputs("#state-start") 35 | 36 | data.url = "http://localhost:1337/community.tableau.com/api/core/v3/contents?filter=tag(#{data.tags})&filter=type(document,discussion)" 37 | 38 | "enter run": (data)-> 39 | wdc_base.set_connection_data( data ) 40 | tableau.submit() 41 | 42 | rows: (connection_data, lastRecordToken)-> 43 | 44 | load_url connection_data.url, (data)-> 45 | 46 | # Call back tableau 47 | tableau.dataCallback data.list, "", false 48 | 49 | 50 | columns: (connection_data)-> 51 | 52 | # Call back tableau 53 | tableau.headersCallback ["subject", "viewCount", "published"], ["string", "int", "date"] 54 | -------------------------------------------------------------------------------- /coffee/template/connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | 4 | 5 | tableauHelpers = require '../connector_base/tableau_helpers.coffee' 6 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 7 | 8 | # TODO: implement your HTTP logic 9 | load_url = (url, success_callback)-> 10 | $.ajax 11 | url: url 12 | success: (res)-> 13 | success_callback(data) 14 | 15 | error: (xhr, ajaxOptions, thrownError)-> 16 | tableau.abortWithError "Error while trying to load '#{url}'. #{thrownError}" 17 | 18 | 19 | wdc_base.make_tableau_connector 20 | name: "Simple XXX connector" 21 | 22 | steps: 23 | start: 24 | template: require './start.jade' 25 | run: 26 | template: require './run.jade' 27 | 28 | 29 | transitions: 30 | "start > run": (data)-> 31 | _.extend data, wdc_base.fetch_inputs("#state-start") 32 | 33 | 34 | "enter run": (data)-> 35 | wdc_base.set_connection_data( data ) 36 | tableau.submit() 37 | 38 | rows: (connection_data, lastRecordToken)-> 39 | 40 | load_url connection_data.url, (data)-> 41 | 42 | # Call back tableau 43 | tableau.dataCallback data, "", false 44 | 45 | 46 | columns: (connection_data)-> 47 | 48 | load_url connection_data.url, (data)-> 49 | tableau.abortWithError("No rows available in data") if _.isEmpty(data) 50 | 51 | # get the first row 52 | first_row = _.first data 53 | 54 | # Guess the data types of the columns 55 | datatypes = _.mapObject first_row, (v,k,o)-> 56 | tableauHelpers.guessDataType(v) 57 | 58 | # Call back tableau 59 | tableau.headersCallback( _.keys(datatypes), _.values(datatypes)) 60 | -------------------------------------------------------------------------------- /coffee/twitter_connector/twitter_auth_server.coffee: -------------------------------------------------------------------------------- 1 | express = require 'express' 2 | Twitter = require('twitter') 3 | http_request = require 'request' 4 | 5 | app = express() 6 | 7 | console.log "startup" 8 | 9 | 10 | TWITTER_GET_OAUTH_TOKEN_URL = "https://api.twitter.com/oauth2/token" 11 | 12 | try 13 | keys = require './twitter_keys.coffee' 14 | 15 | client = new Twitter 16 | consumer_key: keys.CONSUMER_KEY, 17 | consumer_secret: keys.CONSUMER_SECRET, 18 | access_token_key: keys.ACCESS_TOKEN, 19 | access_token_secret: keys.ACCESS_TOKEN_SECRET 20 | 21 | app.get '/search', (req, res)-> 22 | console.log "--> #{req.path} -- #{JSON.stringify(req.query)}" 23 | #res.send("OK") 24 | #return 25 | client.get 'search/tweets', req.query, (error, tweets, response)-> 26 | if error 27 | console.log "<=== Error: #{error}" 28 | res.status(500).send("

Error: #{JSON.stringify(error)}

") 29 | 30 | tweet_json = JSON.stringify(tweets) 31 | console.log "<-- OK #{tweet_json.length} bytes" 32 | 33 | res.setHeader('Content-Type', 'application/json') 34 | res.send(tweet_json) 35 | catch ex 36 | console.log "Failed to initialize twitter server. Please make sure twitter_keys.coffee is filled with valid credentials.", ex 37 | process.exit 1 38 | 39 | app.get '/', (req, res)-> res.send("

Hello

") 40 | 41 | # serve the static files of the connector 42 | app.use(express.static('dist')) 43 | 44 | 45 | server = app.listen 3000, -> 46 | host = server.address().address 47 | port = server.address().port 48 | 49 | console.log('Twitter WDC Connection Server listening at http://%s:%s', host, port) 50 | -------------------------------------------------------------------------------- /coffee/github_commits/source.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | .page-header 4 | h1 5 | | Github commits 6 | small web data connector 7 | 8 | .col-sm-12 9 | form.form-inline 10 | 11 | .form-group 12 | label(for="username") Username 13 | input.form-control#username(type="text", name="username" value="gyulalaszlo") 14 | .form-group 15 | label(for="reponame") Repository name 16 | input.form-control#reponame(type="text", name="reponame" value="wave-d") 17 | 18 | input.btn.btn-primary#submit-button(type="submit" value="Load commits") 19 | 20 | .col-sm-12 21 | .page-header 22 | h4 23 | | Authentication 24 | small Unatuhenticated API requests are limited to 60 req/hour. 25 | 26 | .col-sm-12 27 | form.form-inline 28 | .form-group 29 | label(for="auth-username") Auth Username 30 | input.form-control#auth-username(type="text" value="gyulalaszlo") 31 | 32 | .form-group 33 | label(for="auth-username") Auth Password 34 | input.form-control#auth-password(type="password" value="") 35 | 36 | .form-group 37 | label 38 | input#do-auth( type="checkbox" checked="checked" ) 39 | | Do authentication? 40 | 41 | 42 | .col-sm-12 43 | .page-header 44 | h4 45 | | Nasty bits 46 | small Try to smuggle a web resource from a local network url 47 | 48 | .col-sm-12 49 | form.form-inline 50 | .form-group 51 | label(for="smuggle-url") Url to smuggle 52 | input.form-control#smuggle-url(type="text" value="http://192.168.86.250:9090/Simulator.html") 53 | 54 | .form-group 55 | label 56 | input#do-smuggle( type="checkbox" ) 57 | | Do smuggling? 58 | 59 | 60 | -------------------------------------------------------------------------------- /coffee/csv/start.jade: -------------------------------------------------------------------------------- 1 | h3 Connect to CSV Data 2 | 3 | .row 4 | .col-sm-12 5 | label(for="url") Url 6 | input.form-control(type="text" data-tableau-key="url") 7 | p.help-block 8 | | Use this URL to load a sample CSV file: 9 | code https://www.quandl.com/api/v1/datasets/MADDISON/GDPPC_CHN.csv 10 | 11 | .row 12 | 13 | .col-sm-4 14 | label(for="csv-quotes") Quotes 15 | select#csv-quotes.form-control(data-tableau-key="quote") 16 | option(value='"') " : Double quotes 17 | option(value="'") ' : Single quote 18 | option(value="`") ` : Backticks 19 | 20 | p.help-block 21 | | The character used to quote the fields. Defaults to ". Must 22 | | be a single character. 23 | 24 | .col-sm-4 25 | label(for="csv-delimieter") Delimiter 26 | select#csv-delimiter.form-control(data-tableau-source="csv" data-tableau-key="delimiter") 27 | option(value=",") , : Colon 28 | option(value=";") ; : Semicolon 29 | option(value="|") | : Pipe 30 | option(value="TAB") t : Tab 31 | 32 | p.help-block 33 | | The character used to separate the fields. Defaults to ,. 34 | | Must be a single character. 35 | 36 | .col-sm-4 37 | label(for="csv-charset") Charset 38 | select#csv-charset.form-control(data-tableau-key="charset") 39 | 40 | optgroup(label="Web Standards") 41 | option(value="UTF-8") UTF-8 42 | option(value="ISO8859-1") ISO8859-1 43 | 44 | optgroup(label="Central European") 45 | option(value="ISO8859-2") ISO8859-2 46 | 47 | p.help-block 48 | | EXPERIMENTAL: depending on the server hosting the data, setting this may or may not fix 49 | | character encoding problems. 50 | 51 | 52 | .row 53 | .col-sm-12 54 | a.btn.btn-default(href="#" data-state-to="run") Load CSV 55 | -------------------------------------------------------------------------------- /coffee/json_connector/json_connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | 4 | tableauHelpers = require '../connector_base/tableau_helpers.coffee' 5 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 6 | json_flattener = require './json_flattener.coffee' 7 | 8 | 9 | load_json = (url, success_callback)-> 10 | console.log("Getting URL", url) 11 | $.ajax 12 | url: url 13 | contentType: "application/json", 14 | success: (data, textStatus, request)-> 15 | flat = json_flattener.remap(data) 16 | success_callback(flat) 17 | 18 | error: (xhr, ajaxOptions, thrownError)-> 19 | console.error("Got error", thrownError) 20 | tableau.abortWithError "Error while trying to load '#{url}'. #{thrownError}" 21 | 22 | JSONP_CALLBACK_NAME = "jsondb_wdc_jsonp_callback" 23 | 24 | wdc_base.make_tableau_connector 25 | name: "Raw JSON connector" 26 | 27 | steps: 28 | start: 29 | template: require './start.jade' 30 | run_json: 31 | template: require './run.jade' 32 | 33 | 34 | transitions: 35 | "start > run_json": (data)-> 36 | _.extend data, wdc_base.fetch_inputs("#state-start") 37 | 38 | "enter run_json": (data)-> 39 | wdc_base.set_connection_data( data ) 40 | tableau.submit() 41 | 42 | rows: (data, table, doneCallback)-> 43 | load_json data.url, (rows)-> 44 | table.appendRows(rows.rows) 45 | doneCallback() 46 | 47 | 48 | columns: (connection_data, schemaCallback)-> 49 | 50 | load_json connection_data.url, (data)-> 51 | tableau.abortWithError("No rows available in data") if _.isEmpty(data) 52 | 53 | first_row = _.first(data.rows) 54 | dataTypes = _.map first_row, (v,k,o)-> 55 | { id: k, dataType: tableauHelpers.guessDataType(v)} 56 | 57 | tableInfo = { 58 | id : "json" 59 | alias: "JSON DATA" 60 | columns : dataTypes 61 | } 62 | 63 | # Call back tableau 64 | schemaCallback([tableInfo]) 65 | 66 | -------------------------------------------------------------------------------- /coffee/json_connector/json_flattener.coffee: -------------------------------------------------------------------------------- 1 | _ = require 'underscore' 2 | 3 | class Table 4 | constructor: (@rows=[])-> 5 | 6 | add_row: (row_data)-> 7 | @rows.push row_data 8 | 9 | 10 | 11 | merge_table_pair = (t1, t2)-> 12 | a = t1.rows 13 | b = t2.rows 14 | o = new Table 15 | 16 | for a_row in a 17 | for b_row in b 18 | o.add_row _.extend({}, a_row, b_row) 19 | 20 | o 21 | 22 | 23 | merge_tables = (t1, tables...)-> 24 | return t1 if _.isEmpty(tables) 25 | return merge_table_pair(t1, tables[0]) if tables.length == 1 26 | 27 | merge_table_pair(t1, merge_tables(tables...)) 28 | 29 | 30 | 31 | remap = (obj, base_key=null)-> 32 | base_row = {} 33 | 34 | # The function to retrieve the key 35 | key = if base_key then (k) -> "#{base_key}_#{k}" else (k) -> k 36 | 37 | # First add primitive values 38 | for k,v of obj 39 | if _.isString(v) or _.isNumber(v) 40 | base_row[key(k)] = v 41 | 42 | base_table = new Table([base_row]) 43 | children = [] 44 | choices = {} 45 | 46 | # For each attribute 47 | for k,v of obj 48 | switch 49 | 50 | # Add arrays as choices for the key 51 | # recursively 52 | when _.isArray(v) 53 | 54 | table_out = new Table 55 | 56 | for element in v 57 | for row in remap(element, key(k)).rows 58 | table_out.add_row(row) 59 | choices[k] = table_out 60 | 61 | # Add objects as extensions for their key 62 | # recursively 63 | when _.isObject(v) 64 | children.push remap(v, key(k)) 65 | 66 | # Merge the extensions to the base attributes after 67 | # all their children and choice tables are already extended 68 | base_out = merge_tables( base_table, children... ) 69 | 70 | # Then finally merge all the choices to the extended base table 71 | # ( so the attributes coming from the extensions are mapped across 72 | # all choices here) 73 | for k,v of choices 74 | base_out = merge_tables( base_out, v) 75 | 76 | base_out 77 | 78 | _.extend module.exports, 79 | remap: remap 80 | 81 | -------------------------------------------------------------------------------- /coffee/mongodb_connector/json_flattener.coffee: -------------------------------------------------------------------------------- 1 | _ = require 'underscore' 2 | 3 | class Table 4 | constructor: (@rows=[])-> 5 | 6 | add_row: (row_data)-> 7 | @rows.push row_data 8 | 9 | 10 | 11 | merge_table_pair = (t1, t2)-> 12 | a = t1.rows 13 | b = t2.rows 14 | o = new Table 15 | 16 | for a_row in a 17 | for b_row in b 18 | o.add_row _.extend({}, a_row, b_row) 19 | 20 | o 21 | 22 | 23 | merge_tables = (t1, tables...)-> 24 | return t1 if _.isEmpty(tables) 25 | return merge_table_pair(t1, tables[0]) if tables.length == 1 26 | 27 | merge_table_pair(t1, merge_tables(tables...)) 28 | 29 | 30 | 31 | remap = (obj, base_key=null)-> 32 | base_row = {} 33 | 34 | # The function to retrieve the key 35 | key = if base_key then (k) -> "#{base_key}.#{k}" else (k) -> k 36 | 37 | # First add primitive values 38 | for k,v of obj 39 | if _.isString(v) or _.isNumber(v) 40 | base_row[key(k)] = v 41 | 42 | base_table = new Table([base_row]) 43 | children = [] 44 | choices = {} 45 | 46 | # For each attribute 47 | for k,v of obj 48 | switch 49 | 50 | # Add arrays as choices for the key 51 | # recursively 52 | when _.isArray(v) 53 | 54 | table_out = new Table 55 | 56 | for element in v 57 | for row in remap(element, key(k)).rows 58 | table_out.add_row(row) 59 | choices[k] = table_out 60 | 61 | # Add objects as extensions for their key 62 | # recursively 63 | when _.isObject(v) 64 | children.push remap(v, key(k)) 65 | 66 | # Merge the extensions to the base attributes after 67 | # all their children and choice tables are already extended 68 | base_out = merge_tables( base_table, children... ) 69 | 70 | # Then finally merge all the choices to the extended base table 71 | # ( so the attributes coming from the extensions are mapped across 72 | # all choices here) 73 | for k,v of choices 74 | base_out = merge_tables( base_out, v) 75 | 76 | base_out 77 | 78 | _.extend module.exports, 79 | remap: remap 80 | 81 | -------------------------------------------------------------------------------- /coffee/connector_base/underscore_starschema.coffee: -------------------------------------------------------------------------------- 1 | _ = require 'underscore' 2 | 3 | toArr = (e)-> [e] 4 | 5 | byKey = (k)-> (e)->e[k] 6 | 7 | combineReduction = (memo,a)-> 8 | return memo if _.isEmpty(a) 9 | return _.map(a,toArr) if _.isEmpty(memo) 10 | o = [] 11 | for ae in a 12 | for row in memo 13 | o.push [row...,ae] 14 | o 15 | 16 | 17 | DEFAULT_FILTER_MAP_FN = (v,k,o)-> [k,v] 18 | 19 | _.mixin 20 | # Returns a function that steps through the elements of a collection 21 | # returning the next element on each successive call. 22 | iteratorFor: (coll)-> 23 | idx = 0 24 | ()-> if coll.length > idx then coll[idx++] else null 25 | 26 | 27 | # Takes an index or key and returns a function that tries to get 28 | # that key from the passed in collection or object 29 | nth: byKey 30 | byKey: byKey 31 | 32 | 33 | combineReduction: combineReduction 34 | 35 | # Returns the combination of any number of arrays 36 | combinationsOf: (colls)-> 37 | # Handle edge cases 38 | return [] if colls.length == 0 39 | return [colls[0]] if colls.length == 1 40 | 41 | 42 | 43 | _.reduce colls, combineReduction, [] 44 | 45 | # Allows fn to transform both the keys and values of the object. 46 | # 47 | # Like other underscore functions, fn takes a (value,key) as input and should 48 | # return an object whos keys will be added to the output object. 49 | # 50 | # Returns an object containing all the keys from each iteration of fn, 51 | # overwriting existing keys as the iteration progresses. 52 | remapObject: (obj,fn)-> 53 | # _.extend takes any number of arguments and the reducer 54 | # function gets called with more then 2 arguments, so it 55 | # whould merge them 56 | _.extend( {}, _.map(obj,fn)...) 57 | 58 | # Shortcut to create a brand new object from a key-value pair 59 | makePair: (key,value)-> o={};o[key]=value;o 60 | 61 | filterObject: (obj, filterFn, tranformFn=DEFAULT_FILTER_MAP_FN)-> 62 | o = {} 63 | for k, v of obj 64 | continue unless filterFn(v,k,obj) 65 | tmp = tranformFn(v,k,obj) 66 | o[tmp[0]] = tmp[1] 67 | 68 | o 69 | 70 | 71 | 72 | 73 | # Re-export undescore for easier importing 74 | module.exports = _ 75 | -------------------------------------------------------------------------------- /coffee/sap_bo/proxy_server.coffee: -------------------------------------------------------------------------------- 1 | express = require 'express' 2 | cors = require 'cors' 3 | http_request = require 'request' 4 | sap = require 'bobj-access' 5 | fs = require('fs') 6 | https = require('https') 7 | http = require('http') 8 | 9 | SERVER_CONFIG = 10 | ssl: false 11 | privateKey: 'cert.key' 12 | certificate: 'cert.crt' 13 | port:3000 14 | 15 | 16 | app = express() 17 | app.use(cors()) 18 | 19 | 20 | 21 | 22 | 23 | app.get '/', (req, res)-> res.send("

Hello

") 24 | 25 | app.get '/sap/tablelist', (req, res) -> 26 | sap.getTableList req.query.wsdl, (err, tableList) -> 27 | unless err? 28 | res.json tableList 29 | console.log "Table list response sent", tableList 30 | else 31 | console.log("ERROR:", err, err.stack) 32 | res.status(500).send("#{err}\n\n#{err.stack}") 33 | 34 | app.get '/sap/tabledefinitions', (req, res) -> 35 | sap.getFields req.query.wsdl, req.query.credentials, req.query.table, (err, tables) -> 36 | unless err? 37 | res.json tables 38 | console.log "Table definition response sent. table: ", req.query.table 39 | else 40 | console.log("ERROR:", err, err.stack) 41 | res.status(500).send("#{err}\n\n#{err.stack}") 42 | 43 | app.get '/sap/tablerows', (req, res) -> 44 | sap.getTableData req.query.wsdl, req.query.credentials, req.query.table, {}, (err, tables) -> 45 | unless err? 46 | res.json tables 47 | console.log "Data Response sent.table: ", req.query.table 48 | else 49 | console.log("ERROR:", err, err.stack) 50 | res.status(500).send("#{err}\n\n#{err.stack}") 51 | 52 | # serve the static files of the connector 53 | app.use(express.static('dist')) 54 | 55 | 56 | if SERVER_CONFIG.ssl 57 | fsr = fs.readFileSync 58 | key = fsr(SERVER_CONFIG.privateKey) 59 | cert = fsr(SERVER_CONFIG.certificate) 60 | https.createServer({ key, cert}, app).listen SERVER_CONFIG.port, ()-> 61 | console.log("SAP BO server listening on port https " + SERVER_CONFIG.port) 62 | 63 | 64 | else 65 | server = app.listen SERVER_CONFIG.port, -> 66 | host = server.address().address 67 | port = server.address().port 68 | 69 | console.log('SAP BO Connection Server listening at http://%s:%s', host, port) 70 | -------------------------------------------------------------------------------- /coffee/connector_base/state_machine.coffee: -------------------------------------------------------------------------------- 1 | _ = require 'underscore' 2 | $ = require 'jquery' 3 | 4 | 5 | # Generates a tab-handler function that acts like a state machine 6 | stateMachine = (startState, stateList, transitionHandlers={})-> 7 | # Set up the data store 8 | data = {} 9 | # Initialize the current tab 10 | history = [startState] 11 | 12 | runTransitions = (from, to, names, callback)-> 13 | for name in names 14 | transition = transitionHandlers[name] 15 | continue unless transition 16 | console.log("Running transition handler[#{from} -> #{to}]: #{name}") 17 | transition(data,from,to, transitionTo) 18 | 19 | # Shortcut for getting a transitions name 20 | nameOf = (from,to)-> "#{from} > #{to}" 21 | 22 | # The main transition function. 23 | # 24 | # Transition to a new state and use the history to figure out which 25 | # state we came from 26 | transitionTo = (to, withData={})-> 27 | return unless _.contains(stateList, to) 28 | from = _.last history 29 | return if to == from 30 | _.extend data, withData 31 | history.push(to) 32 | runTransitions(from, to, ["leave #{from}", nameOf(from,to),"*", "enter #{to}"]) 33 | 34 | # Go back in history 35 | goBack = (n)-> transitionTo(_.last( history,n)[0] ) 36 | 37 | return { 38 | # A function that can be called with the id of 39 | # the new tab and transitions 40 | to: transitionTo 41 | back: goBack 42 | data: -> data 43 | } 44 | 45 | # Provide a wizzard-like interface using the stateMachine. 46 | # 47 | # 48 | wizzard = (startState, steps, transitionHandlers={})-> 49 | $steps = (name)-> $(steps[name]) 50 | handlers = _.extend {}, transitionHandlers, 51 | "*": (data,from,to)-> 52 | $steps(from).fadeOut(100) 53 | $steps(to).removeClass('hide').fadeIn(100) 54 | 55 | # Get the state machine 56 | sm = stateMachine(startState, _.keys(steps), handlers) 57 | 58 | $ -> 59 | # Hook our transitioners 60 | $('body').on 'click', '*[data-state-to]', (e)-> 61 | e.preventDefault() 62 | transitions = $(this).data('state-to').split(/ +/) 63 | for to in transitions 64 | switch 65 | when to == ':back' then sm.back(1) 66 | when to == ':back2' then sm.back(2) 67 | else sm.to(to) 68 | 69 | sm 70 | 71 | 72 | _.extend module.exports, 73 | stateMachine: stateMachine 74 | wizzard: wizzard 75 | -------------------------------------------------------------------------------- /coffee/community/community_connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | 4 | 5 | tableauHelpers = require '../connector_base/tableau_helpers.coffee' 6 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 7 | 8 | load_url = (url, success_callback)-> 9 | $.ajax 10 | url: url 11 | dataType: "text" 12 | success: (res)-> 13 | 14 | res = res.substring(res.indexOf("\n") + 1) 15 | out = JSON.parse(res) 16 | 17 | list = for post in out.list 18 | updated: post.updated 19 | tags: JSON.stringify(post.tags) 20 | subject: post.subject 21 | viewCount: post.viewCount 22 | published: post.published 23 | categories: JSON.stringify(post.categories) 24 | resolved: post.resolved 25 | status: post.status 26 | place: post.parentPlace.name 27 | url: post.resources.html.ref 28 | 29 | out.list = list 30 | success_callback( out ) 31 | 32 | error: (xhr, ajaxOptions, thrownError)-> 33 | tableau.abortWithError "Error while trying to load '#{url}'. #{thrownError}" 34 | 35 | 36 | wdc_base.make_tableau_connector 37 | name: "Simple XXX connector" 38 | 39 | steps: 40 | start: 41 | template: require './start.jade' 42 | run: 43 | template: require './run.jade' 44 | 45 | 46 | transitions: 47 | "start > run": (data)-> 48 | _.extend data, wdc_base.fetch_inputs("#state-start") 49 | 50 | data.url = "http://community.tableau.com/api/core/v3/search/contents?filter=type(document,discussion)&filter=after(#{data.after})&filter=search(*a*)&sort=updatedDesc&fields=modified,published,tags,subject,categories,resolved,status,viewCount,updated" 51 | 52 | 53 | "enter run": (data)-> 54 | wdc_base.set_connection_data( data ) 55 | tableau.submit() 56 | 57 | rows: (connection_data, lastRecordToken)-> 58 | 59 | url = lastRecordToken || connection_data.url 60 | url = url.replace("http://", "http://localhost:1337/") 61 | 62 | load_url url, (data)-> 63 | 64 | # Call back tableau 65 | has_more = if data.links.next then true else false 66 | tableau.dataCallback data.list, data.links.next, has_more 67 | 68 | 69 | columns: (connection_data)-> 70 | 71 | # Call back tableau 72 | tableau.headersCallback( 73 | ["updated", "tags", "subject", "viewCount", "published", "categories", "resolved", "status", "url", "place"], 74 | ["string", "string", "string", "int", "string", "string", "string", "string", "string", "string" ] ) 75 | -------------------------------------------------------------------------------- /coffee/csv/csv_connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | csv = require 'csv' 4 | 5 | 6 | tableauHelpers = require '../connector_base/tableau_helpers.coffee' 7 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 8 | 9 | 10 | load_csv = (url, params, success_callback)-> 11 | console.log params, "000" 12 | 13 | opts = _.extend { 14 | quote: '"' 15 | delimiter: ',' 16 | columns: true 17 | auto_parse: true 18 | }, params 19 | 20 | # Handle tab as delimiter 21 | if opts.delimiter == 'TAB' 22 | opts.delimiter = "\t" 23 | 24 | $.ajax 25 | url: url 26 | # TODO: Setting this gives us some errors on some servers. 27 | #contentType: "text/html;charset=#{params.charset}" 28 | 29 | success: (res)-> 30 | csv.parse res, opts, (err,data)-> success_callback(data) 31 | 32 | error: (xhr, ajaxOptions, thrownError)-> 33 | tableau.abortWithError "Error while trying to load '#{url}'. #{thrownError}" 34 | 35 | aliasToId = (alias)-> alias.replace(/[^A-Za-z0-9]+/g, "_") 36 | 37 | wdc_base.make_tableau_connector 38 | name: "Simple CSV connector" 39 | 40 | steps: 41 | start: 42 | template: require './start.jade' 43 | run: 44 | template: require './run.jade' 45 | 46 | 47 | transitions: 48 | "start > run": (data)-> 49 | _.extend data, wdc_base.fetch_inputs("#state-start") 50 | 51 | "enter run": (data)-> 52 | wdc_base.set_connection_data( data ) 53 | tableau.submit() 54 | 55 | rows: (connection_data, table, doneCallback) -> 56 | console.log("Connection data", connection_data) 57 | 58 | load_csv connection_data.url, connection_data, (data)-> 59 | # Convert the columns aliases to ids 60 | table.appendRows(data.map( (row)-> 61 | o = {} 62 | Object.keys(row).forEach (k)-> 63 | o[aliasToId(k)] = row[k] 64 | return o 65 | )) 66 | 67 | ## Call back tableau 68 | doneCallback() 69 | 70 | 71 | columns: (connection_data, schemaCallback) -> 72 | 73 | load_csv connection_data.url, connection_data, (data)-> 74 | tableau.abortWithError("No rows available in data") if _.isEmpty(data) 75 | 76 | # get the first row 77 | first_row = _.first data 78 | 79 | # Call back tableau 80 | schemaCallback [ 81 | id: "CSV" 82 | # Guess the datatype for each column 83 | columns: Object.keys(first_row).map (key)-> 84 | { alias: key, id: aliasToId(key) , dataType: tableauHelpers.guessDataType( first_row[key] ) } 85 | ] 86 | 87 | -------------------------------------------------------------------------------- /coffee/mongodb_connector/mongodb_connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | 4 | tableauHelpers = require '../connector_base/tableau_helpers.coffee' 5 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 6 | json_flattener = require './json_flattener.coffee' 7 | 8 | 9 | load_jsonp = (url, success_callback)-> 10 | $.ajax 11 | url: url 12 | #async: false 13 | jsonpCallback: JSONP_CALLBACK_NAME 14 | contentType: "application/json", 15 | dataType: 'jsonp', 16 | success: (data, textStatus, request)-> 17 | success_callback(data) 18 | error: (xhr, ajaxOptions, thrownError)-> 19 | tableau.abortWithError "Error while trying to load '#{url}'. #{thrownError}" 20 | 21 | JSONP_CALLBACK_NAME = "mongodb_wdc_jsonp_callback" 22 | 23 | wdc_base.make_tableau_connector 24 | name: "MongoDB connector" 25 | 26 | steps: 27 | start: 28 | template: require './start.jade' 29 | run_mongo: 30 | template: require './run.jade' 31 | 32 | 33 | transitions: 34 | "start > run_mongo": (data)-> 35 | _.extend data, wdc_base.fetch_inputs("#state-start") 36 | url = "http://#{data.mongodb_host}:#{data.mongodb_port}/#{data.mongodb_db}/#{data.mongodb_collection}" 37 | 38 | # Add the jsonP stuff 39 | url = "#{url}/?jsonp=#{JSONP_CALLBACK_NAME}" 40 | 41 | # Add some default page size 42 | url = "#{url}&limit=#{data.page_size}" 43 | 44 | 45 | if data.mongodb_params && data.mongodb_params != "" 46 | url = "#{url}&#{data.mongodb_params}" 47 | 48 | data.url = url 49 | 50 | "enter run_mongo": (data)-> 51 | wdc_base.set_connection_data( data ) 52 | tableau.submit() 53 | 54 | rows: (connection_data, lastRecordToken)-> 55 | offset_str = if lastRecordToken == "" then 0 else lastRecordToken 56 | offseted_url = "#{connection_data.url}&skip=#{offset_str}" 57 | 58 | load_jsonp offseted_url, (data)-> 59 | 60 | {offset: offset, rows: rows, total_rows: total_rows} = data 61 | 62 | # when we reached the last page, it is signaled by returning 63 | # 0 rows 64 | has_more = (total_rows != 0) 65 | # the next page is the one after the current 66 | next_offset = offset + total_rows 67 | 68 | # Remap each row 69 | tableau_data = for row in rows 70 | json_flattener.remap(row, null).rows 71 | 72 | # Call back tableau 73 | tableau.dataCallback(_.flatten(tableau_data), next_offset.toString(), has_more) 74 | 75 | 76 | columns: (connection_data)-> 77 | 78 | load_jsonp connection_data.url, (data)-> 79 | tableau.abortWithError("No rows available in data") if _.isEmpty(data) 80 | 81 | # get the first row 82 | first_row = _.first( json_flattener.remap(_.first(data.rows)).rows ) 83 | 84 | # Guess the data types of the columns 85 | datatypes = _.mapObject first_row, (v,k,o)-> 86 | tableauHelpers.guessDataType(v) 87 | 88 | # Call back tableau 89 | tableau.headersCallback( _.keys(datatypes), _.values(datatypes)) 90 | 91 | -------------------------------------------------------------------------------- /coffee/mongodb_connector/start.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | h1 MongoDB / Raw JSON connect 4 | 5 | .row 6 | .col-sm-12 7 | .page-header 8 | h4 Connect to MongoDB 9 | 10 | .col-sm-12 11 | 12 | form 13 | .row 14 | .col-sm-3 15 | label(for="mongodb_host") MongoDB host 16 | input.form-control(type="text" data-tableau-key="mongodb_host" ) 17 | 18 | .col-sm-3 19 | label(for="mongodb_port") MongoDB port 20 | input.form-control(type="text" data-tableau-key="mongodb_port" value="28017") 21 | 22 | .col-sm-3 23 | label(for="mongodb_host") MongoDB Database 24 | input.form-control(type="text" data-tableau-key="mongodb_db" ) 25 | 26 | .col-sm-3 27 | label(for="mongodb_host") MongoDB Collection 28 | input.form-control(type="text" data-tableau-key="mongodb_collection" ) 29 | 30 | .row 31 | .col-sm-8 32 | label(for="mongodb_host") MongoDB additional Parameters 33 | input.form-control(type="text" data-tableau-key="mongodb_params") 34 | 35 | p 36 | | Query parameters can be found   37 | a(href="http://docs.mongodb.org/ecosystem/tools/http-interfaces/#simple-rest-api" target="_blank") in the MongoDB Simple REST API guide. 38 | 39 | .col-sm-2 40 | label Page size 41 | input.form-control(type="number" data-tableau-key="page_size" value="1000") 42 | 43 | 44 | .col-sm-2 45 | label Connect 46 | a.btn.btn-default(href="#" data-state-to="run_mongo") Connect to MongoDB 47 | 48 | .col-sm-12 49 | p 50 | strong 51 | | This connection requires   52 | a(href="http://docs.mongodb.org/ecosystem/tools/http-interfaces/#simple-rest-api" target="_blank") setting up MongoDB v1.4+: 53 | | This API is disabled by default, as it could provide unauthenticated access to data. Use --rest on the command line to enable, but be aware of security implications. 54 | 55 | 56 | .text-muted 57 | p Connecting to mongodb using the Simple REST API requires starting the mongod server with the --rest and --jsonp command line switches to enable the REST API and enable the connector to load data through JSONP. 58 | p For example on Ubuntu this may mean: 59 | code mongod --config /etc/mongodb.conf --rest --jsonp 60 | p To access the REST API from a different machine you also need to change the bind_ip configuration property in your mongodb configuration form 127.0.0.1 (the default that comes with most mongodb packages) to 0.0.0.0 61 | 62 | // 63 | .row 64 | .col-sm-12 65 | .page-header 66 | h4 Or connect to a raw JSON file 67 | 68 | .col-sm-12 69 | 70 | form 71 | .row 72 | .col-sm-10 73 | label(for="url") JSON Url 74 | input.form-control(type="text" data-tableau-key="url") 75 | 76 | .col-sm-2 77 | label Connect 78 | a.btn.btn-default(href="#" data-state-to="run") Get raw JSON 79 | 80 | -------------------------------------------------------------------------------- /docs/make_wdc_better.markdown: -------------------------------------------------------------------------------- 1 | 2 | # Security 3 | 4 | 5 | ### Scenario 1: Malicious connector on a remote server, well set-up target 6 | 7 | Since there is no whitelisting (at least I could not find any traces of 8 | it in the source code for the WDC API or the Simulator) HTTP resources 9 | on the internal network can be secured against smuggling out by setting 10 | up the Same Origin Policy on the target servers (by default servers 11 | should refuse. 12 | 13 | ### Scenario 2: Malicious connector th on a remote server, badly configured target 14 | 15 | Since the Same Origin Policy is applied after the request takes 16 | place (since its implemented on the server side), a server configured 17 | to share embeddable web resources may allow the attacker to both smuggle out 18 | the content and/or trigger actions (URLs having side-effects) on the target 19 | server (both in the Simulator and when using Tableau itself). 20 | 21 | 22 | ### Scenario 2: Malicious connector on the target server 23 | 24 | If the malicious party can trick the users to install the connector on 25 | the target server (like a Tomcat WAR package deployed to the companys 26 | internal workgroup server), its Game Over. 27 | 28 | The WDC framework cannot protect against compromise. 29 | 30 | 31 | ### Suggestions 32 | 33 | - Make all connectors declare their intentions in a non-programmatic 34 | way before use. For example: 35 | 36 | - A JSON file declaring the name of the connector, some signature data 37 | to verify its origin and untouched nature, and the url masks it will 38 | be using (which will be whitelisted for the connector if the user 39 | accepts them) 40 | 41 | - Before first use, show this data to the user and allow them to 42 | review the policies of the connector before any of its code is ran. 43 | (like mobile applications show the required permissions before 44 | install) 45 | 46 | - If the signature of the connector changes, notify the user and 47 | let them re-review and re-accept the policies / the connector 48 | 49 | 50 | # General 51 | 52 | ## Bugs(?) 53 | 54 | - When setting tableau.password or tableau.username before calling 55 | tableau.submit() the connector works fine in the simulator, but 56 | accessing tableau.username and tableau.password in the data callback 57 | they are blank in Tableau. 58 | 59 | We currently work around it by storing the username / password in the 60 | connectionData in plain text :( 61 | 62 | 63 | ## Things that could make WDC better: 64 | 65 | - better communication when using Tableua instead of the Simulator (error and 66 | log messages are swallowed by Tableau, and we cannot show a meaningful error 67 | message if the extraction fails after tableau.submit() is called ) 68 | 69 | - Perheaps a +tableau.encryptedData+ field where we can store 70 | arbitrary data that wont be stored plain-text in the Tableu file. 71 | Currently only tableau.password provides this capability. 72 | 73 | - The option to turn off showing of the password field in the simulator 74 | (during demos and more importanty debugging of failed connectors, 75 | private passwords used to access confidential sources may be leaked 76 | to the developer or anyone with a plain view of the screen) 77 | 78 | - Provide a callback to update some kind of progress meter on the 79 | Tableau interface during the data extract so the user can see 80 | the progress when importing a larger number of rows (for example: 81 | loading 4000 rows in 30 row pages on a throttled HTTP API) 82 | -------------------------------------------------------------------------------- /coffee/twitter_connector/start.jade: -------------------------------------------------------------------------------- 1 | .row 2 | .col-sm-12 3 | h1 Start 4 | 5 | .col-sm-12 6 | a.btn.btn-default(href="#" data-state-to="run") Run query 7 | 8 | .row 9 | .col-sm-8 10 | form 11 | .row 12 | .col-sm-12 13 | label(for="url") What to search for? 14 | input.form-control(type="text" data-tableau-key="q") 15 | 16 | .row 17 | .col-sm-12 18 | h4 Query operators 19 | p The query can have operators that modify its behavior, the available operators are: 20 | 21 | table.table 22 | tr 23 | th Operator 24 | th Finds tweets… 25 | 26 | tr 27 | th watching now 28 | td containing both “watching” and “now”. This is the default operator. 29 | 30 | tr 31 | th “happy hour” 32 | td containing the exact phrase “happy hour”. 33 | 34 | tr 35 | th love OR hate 36 | td containing either “love” or “hate” (or both). 37 | 38 | tr 39 | th beer -root 40 | td containing “beer” but not “root”. 41 | 42 | tr 43 | th #haiku 44 | td containing the hashtag “haiku”. 45 | 46 | tr 47 | th from:alexiskold 48 | td sent from person “alexiskold”. 49 | 50 | tr 51 | th to:techcrunch 52 | td sent to person “techcrunch”. 53 | 54 | tr 55 | th @mashable 56 | td referencing person “mashable”. 57 | 58 | tr 59 | th superhero since:2015-07-19 60 | td containing “superhero” and sent since date “2015-07-19” (year-month-day). 61 | 62 | tr 63 | th ftw until:2015-07-19 64 | td containing “ftw” and sent before the date “2015-07-19”. 65 | 66 | tr 67 | th movie -scary :) 68 | td containing “movie”, but not “scary”, and with a positive attitude. 69 | 70 | tr 71 | th flight :( 72 | td containing “flight” and with a negative attitude. 73 | 74 | tr 75 | th traffic ? 76 | td containing “traffic” and asking a question. 77 | 78 | tr 79 | th hilarious filter:links 80 | td containing “hilarious” and linking to URL. 81 | 82 | tr 83 | th news source:twitterfeed 84 | td containing “news” and entered via TwitterFeed 85 | 86 | 87 | .col-sm-4 88 | .row 89 | .col-sm-12 90 | h4 Extra data to include 91 | 92 | .row 93 | .col-sm-12 94 | form 95 | .form-group 96 | label(for="url") Number of pages to load 97 | input.form-control(type="number" data-tableau-key="page_count" value="4") 98 | 99 | 100 | .checkbox.form-group 101 | label 102 | input(type="checkbox" data-tableau-key="include_geolocation") 103 | | Include GeoLocation? 104 | 105 | .checkbox.form-group 106 | label 107 | input(type="checkbox" data-tableau-key="include_userdata") 108 | | Include User Data? 109 | 110 | .checkbox.form-group 111 | label 112 | input(type="checkbox" data-tableau-key="expand_hashtags") 113 | | Generate a row for each hashtag? 114 | 115 | .checkbox.form-group 116 | label 117 | input(type="checkbox" data-tableau-key="expand_mentions") 118 | | Generate a row for each @mention? 119 | -------------------------------------------------------------------------------- /coffee/nasty_proxy/nasty_proxy.coffee: -------------------------------------------------------------------------------- 1 | # Lets require/import the HTTP module 2 | http = require 'http' 3 | url = require 'url' 4 | querystring= require 'querystring' 5 | http_request = require 'request' 6 | 7 | _ = require 'underscore' 8 | 9 | PORT=8080 10 | 11 | 12 | #SKIP_RESPONSE_HEADERS = ["content-encoding"] 13 | SKIP_RESPONSE_HEADERS = ["transfer-encoding", "content-encoding"] 14 | 15 | SKIP_REQUEST_HEADERS = ["host", "if-modified-since", "if-none-match"] 16 | 17 | should_skip_header = (name, coll)-> (name.toLowerCase() in coll) 18 | 19 | 20 | # Helpers to copy headers from one request to another 21 | copy_response_headers = (req_from, req_to)-> 22 | return unless req_from.headers 23 | 24 | for k,v of req_from.headers 25 | if should_skip_header(k, SKIP_RESPONSE_HEADERS) 26 | console.log " - Skipping header: '#{k}' : '#{v}'" 27 | continue 28 | 29 | req_to.setHeader k, v 30 | console.log " + Copying header '#{k}' : '#{v}'" 31 | 32 | copy_request_headers = (src_headers)-> 33 | h = {} 34 | 35 | for k,v of src_headers 36 | if should_skip_header(k, SKIP_REQUEST_HEADERS) 37 | console.log " - Skipping REQ header: '#{k}' : '#{v}'" 38 | continue 39 | 40 | h[k] = v 41 | console.log " + Copying REQ header '#{k}' : '#{v}'" 42 | 43 | 44 | h 45 | #_.pick( src_headers, (v,k,o)-> k not in ["host"]) 46 | 47 | # handler to proxy a request to a remote target 48 | proxy_req = (proxied_req, callback)-> 49 | return callback() unless proxied_req.url 50 | 51 | req_str = "#{proxied_req.method} #{proxied_req.url}" 52 | 53 | new_req = 54 | # make sure we allow bad certs so GitHub works 55 | rejectUnauthorized: false 56 | url: proxied_req.url 57 | gzip: true 58 | headers: copy_request_headers(proxied_req.headers) 59 | 60 | if proxied_req.auth 61 | new_req.headers['Authorization'] = proxied_req.auth 62 | 63 | console.log "--> Request: #{req_str} #{JSON.stringify(proxied_req, null, " ")}" 64 | console.log " Headers: #{JSON.stringify(new_req, null, " ")}" 65 | 66 | req = http_request new_req, (error, response, body)-> 67 | 68 | if error 69 | console.log " Error!: #{error}" 70 | return callback( error, null, null) 71 | 72 | console.log "<-- Response #{response.statusCode} #{req_str} #{body.length} bytes" 73 | callback( error, response, body ) 74 | 75 | req.on 'data', (data)-> 76 | console.log "Got data:", data 77 | 78 | 79 | # The main webapp handler 80 | handleRequest = (request, response)-> 81 | query = url.parse( request.url, true ).query 82 | console.log "-------> Incoming request: #{request.url}" 83 | 84 | query_url = query.url 85 | query_method = query.method ? "GET" 86 | 87 | unless query_url 88 | console.log " 404..." 89 | response.setHeader 'Server', 'got_yout_bitch_ass_chumped v0.9' 90 | response.end("Not found", 404) 91 | return 92 | 93 | proxied_req = 94 | url: query.url 95 | method: query.method ? "GET" 96 | params: JSON.parse( query.params ? "{}" ) 97 | auth: query.auth 98 | headers: request.headers 99 | 100 | # store the credentials passed 101 | stored_credentials = JSON.parse( query.store ? "{}") 102 | console.log "====== STORING: #{JSON.stringify(stored_credentials)} =====" 103 | console.log request.auth 104 | 105 | proxy_req proxied_req, (err, proxied_resp, body)-> 106 | if err 107 | return response.end("Request errror", 502) 108 | 109 | copy_response_headers( proxied_resp, response ) 110 | #response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept") 111 | response.setHeader("Access-Control-Allow-Credentials", "true") 112 | response.setHeader("Access-Control-Allow-Origin", "*") 113 | response.end( body ) 114 | 115 | # Create a server 116 | server = http.createServer(handleRequest) 117 | 118 | # Lets start our server 119 | server.listen PORT, -> 120 | # Callback triggered when server is successfully listening. Hurray! 121 | console.log("Server listening on: http:# localhost:%s", PORT); 122 | -------------------------------------------------------------------------------- /coffee/connector_base/starschema_wdc_base.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | 4 | state_machine = require '../connector_base/state_machine' 5 | 6 | 7 | make_tableau_connector = (connector_data)-> 8 | $ -> 9 | $('#twdc-ui').html( build_html( connector_data )) 10 | sm = build_state_machine( connector_data ) 11 | build_tableau_connector(connector_data) 12 | 13 | 14 | 15 | # Build the HTML from the states templates 16 | build_html = (description, start_state="start")-> 17 | states = for name, desc of description.steps 18 | style = [] 19 | style.push("display:none;") unless name == start_state 20 | ["
", desc.template(), "
"].join("") 21 | 22 | states.join("") 23 | 24 | 25 | build_state_machine = (description)-> 26 | state_ids = _.mapObject( description.steps, (v,k,o)-> "#state-#{k}") 27 | #for k,v of description.steps 28 | #state_ids[k] = "#state-#{k}" 29 | transitionHandlers = description.transitions 30 | 31 | handlers = _.extend {}, transitionHandlers, 32 | "*": (data,from,to)-> 33 | $steps(from).fadeOut(100) 34 | $steps(to).removeClass('hide').fadeIn(100) 35 | 36 | sm = state_machine.wizzard( "start", state_ids, transitionHandlers) 37 | sm 38 | 39 | 40 | 41 | # HTML Helpers 42 | # ------------ 43 | 44 | # Gets the value of an HTML input, and works on checkboxes too 45 | $.fn.realVal = ()-> 46 | $obj = $(this) 47 | val = $obj.val() 48 | type = $obj.attr('type') 49 | return val unless (type && type == 'checkbox') 50 | $obj.prop('checked') ? val : ($obj.attr('data-unchecked') ? '') 51 | 52 | 53 | # Generates a reducer function to consturct an object from the *dataField* data 54 | # attribute of an HTML input or select element and the value of that element. 55 | dataKeyValueReducer = (dataField)-> 56 | (memo,e)-> 57 | $e = $(e) 58 | memo[$e.data(dataField)] = $e.realVal() 59 | memo 60 | 61 | fetchInputs = (wrap_selector)-> 62 | inputs = $("#{wrap_selector} [data-tableau-key]") 63 | console.log "GOT INPUTS:", inputs 64 | # Collect the input values into an object 65 | formData = _.reduce inputs, dataKeyValueReducer('tableau-key'), {} 66 | 67 | 68 | # Add the transition event handlers 69 | add_event_handlers = (description)-> 70 | $('body').on 'click', '*[data-state-to]', (e)-> 71 | e.preventDefault() 72 | transitions = $(@).data('state-to').split(/ +/) 73 | 74 | # Tableau Connector related 75 | # ------------------------- 76 | 77 | get_connection_data = -> JSON.parse( tableau.connectionData ) 78 | set_connection_data = (cd)-> tableau.connectionData = JSON.stringify( cd ) 79 | 80 | 81 | build_tableau_connector = (description)-> 82 | connector = tableau.makeConnector() 83 | 84 | #connector.getColumnHeaders = -> 85 | connector.getSchema = (schemaCallback)-> 86 | description.columns(get_connection_data(), schemaCallback) 87 | 88 | connector.getData = (table, doneCallback)-> 89 | description.rows( get_connection_data(), table, doneCallback) 90 | 91 | tableau.registerConnector(connector) 92 | 93 | 94 | extract_column = (data, col_name)-> 95 | col_name_parts = col_name.split('.') 96 | obj = data 97 | o = {} 98 | o[col_name] = null 99 | for col_name_part in col_name_parts 100 | if /^[0-9]+$/.test(col_name_part) 101 | col_name_part = parseInt( col_name_part ) 102 | 103 | return o if obj[col_name_part] == null 104 | obj = obj[col_name_part] 105 | 106 | o[col_name] = obj 107 | o 108 | 109 | 110 | 111 | make_columns = (data)-> 112 | cols = {names: [], types: [], extractors:[]} 113 | 114 | for col in data 115 | c = col.split(':') 116 | col_name = c[0] 117 | #sanitized_col_name = sanitize_col_name(col_name) 118 | cols.names.push col_name 119 | cols.types.push c[1] 120 | 121 | 122 | 123 | cols 124 | 125 | _.extend module.exports, 126 | make_tableau_connector: make_tableau_connector 127 | fetch_inputs: fetchInputs 128 | set_connection_data: set_connection_data 129 | get_connection_data: get_connection_data 130 | make_columns: make_columns 131 | extract_column: extract_column 132 | -------------------------------------------------------------------------------- /coffee/github_commits/github_connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | 4 | tableauHelpers = require '../connector_base/tableau_helpers.coffee' 5 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 6 | 7 | 8 | # get the commits url for a repo 9 | github_commits_url = (user,repo)-> "https://api.github.com/repos/#{user}/#{repo}/commits" 10 | 11 | GITHUB_COLUMNS = 12 | names: ['author_name', 'committer_name', 'author', 'authored_at', 'committer', 'committed_at'] 13 | types: ['text', 'text','text', 'date', 'text', 'date'] 14 | 15 | # THe regexp to parse the Link header for pagination 16 | LINK_REGEXP = /<([^>]+)>; rel="(next|last)"/g 17 | 18 | # Parses the value of the Link header returned by GitHub 19 | parse_link_header = (link_header)-> 20 | return {} unless link_header 21 | o = {} 22 | match = LINK_REGEXP.exec link_header 23 | while match != null 24 | console.log match 25 | o[match[2]] = match[1] 26 | match = LINK_REGEXP.exec link_header 27 | 28 | o 29 | 30 | # AUTH STUFF 31 | # ---------- 32 | 33 | 34 | make_base_auth = (user, password)-> "Basic #{btoa("#{user}:#{password}")}" 35 | 36 | apply_auth = (params, username, password)-> 37 | _.extend {}, params, 38 | beforeSend: (xhr)-> 39 | xhr.setRequestHeader('Authorization', make_base_auth(username, password)) 40 | 41 | 42 | 43 | 44 | wdc_base.make_tableau_connector 45 | steps: 46 | start: 47 | template: require './start.jade' 48 | run: 49 | template: require './run.jade' 50 | 51 | transitions: 52 | "start > run" : (data)-> 53 | _.extend data, wdc_base.fetch_inputs("#state-start") 54 | 55 | "enter run": (data)-> 56 | # save the password 57 | tableau.password = JSON.stringify 58 | user: data.auth_username 59 | password: data.auth_password 60 | 61 | # Remove the sensitive data from the connection_data 62 | delete data.auth_username 63 | delete data.auth_password 64 | 65 | wdc_base.set_connection_data(data) 66 | 67 | tableau.submit() 68 | 69 | rows: ( connection_data, lastRecordToken)-> 70 | {user: auth_user, password: auth_password} = JSON.parse(tableau.password) 71 | 72 | # the URL of the first page 73 | connectionUrl = github_commits_url(connection_data.username, connection_data.reponame) 74 | 75 | # if we are in a pagination loop, use the last record token to load the next page 76 | if lastRecordToken.length > 0 77 | connectionUrl = lastRecordToken 78 | 79 | tableau.log "Connecting to #{connectionUrl}" 80 | 81 | xhr_params = 82 | url: connectionUrl, 83 | dataType: 'json', 84 | success: (data, textStatus, request)-> 85 | link_headers = parse_link_header( request.getResponseHeader('Link') ) 86 | tableau.log "Got response - links: #{JSON.stringify(link_headers)}" 87 | 88 | # Stop if no commits present 89 | unless _.isArray(data) 90 | tableau.abortWithError "GitHub returned an invalid response." 91 | 92 | out = for commit_data in data 93 | # shorten names 94 | {committer: cc, author: ca} = commit_data.commit 95 | # return the data 96 | { 97 | author_name: ca.name 98 | committer_name: cc.name 99 | 100 | author: ca.email 101 | committer: cc.email 102 | 103 | authored_at: ca.date 104 | committed_at: cc.date 105 | } 106 | 107 | has_more = if link_headers.next then true else false 108 | tableau.dataCallback( out, link_headers.next, has_more) 109 | 110 | error: (xhr, ajaxOptions, thrownError)-> 111 | # Add something to the log and return an empty set if there 112 | # was problem with the connection 113 | err = "Connection error: #{xhr.responseText} -- #{thrownError}" 114 | tableau.log err 115 | tableau.abortWithError "Cannot connect to the specified GitHub repository. -- #{err}" 116 | 117 | # Add the auth handler if necessary 118 | if connection_data.do_auth 119 | xhr_params = apply_auth(xhr_params, auth_user, auth_password) 120 | 121 | 122 | $.ajax xhr_params 123 | 124 | 125 | columns: (connection_data)-> 126 | tableau.headersCallback( GITHUB_COLUMNS.names, GITHUB_COLUMNS.types ) 127 | 128 | -------------------------------------------------------------------------------- /coffee/twitter_connector/twitter_connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | 4 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 5 | 6 | # Generate the list of column names based on the filtering flags 7 | get_twitter_columns = (filter_flags)-> 8 | o = [ 9 | 'lang:text' 10 | 'created_at:date' 11 | 'text:text' 12 | 'user.name:text' 13 | 'retweet_count:int' 14 | 'favorite_count:int' 15 | ] 16 | if filter_flags.include_geolocation 17 | o.push( 'geo.coordinates.0:text' 18 | 'geo.coordinates.1:text' 19 | ) 20 | 21 | if filter_flags.include_userdata 22 | o.push( 23 | 'user.screen_name:text' 24 | 'user.location:text' 25 | 'user.followers_count:int' 26 | ) 27 | 28 | if filter_flags.expand_hashtags 29 | o.push "hashtag:text" 30 | 31 | if filter_flags.expand_mentions 32 | o.push "mention_screen_name:text" 33 | o.push "mention_username:text" 34 | 35 | # Generate the column data 36 | wdc_base.make_columns(o) 37 | 38 | 39 | # generate new rows from data based on the filter/expand flags 40 | expand_twitter_rows = (row_data, column_names, filter_flags)-> 41 | 42 | # Extract data from a single row 43 | extract_row = (row)-> 44 | _.extend( {}, _.map(column_names, (c)-> wdc_base.extract_column(row,c) )...) 45 | 46 | 47 | unless filter_flags.expand_hashtags or filter_flags.expand_mentions 48 | return extract_row(row_data) 49 | 50 | out_rows = [] 51 | 52 | # the base for the row 53 | row_out = extract_row(row_data) 54 | 55 | # Add each hashtag 56 | if filter_flags.expand_hashtags 57 | if _.isEmpty( row_data.entities.hashtags ) 58 | out_rows.push _.clone( row_out ) 59 | else 60 | for hashtag in row_data.entities.hashtags 61 | out_rows.push _.extend( {}, row_out, {hashtag: hashtag.text} ) 62 | 63 | # Add each hashtag 64 | if filter_flags.expand_mentions 65 | if _.isEmpty( row_data.entities.user_mentions ) 66 | out_rows.push _.clone( row_out ) 67 | else 68 | for mention in row_data.entities.user_mentions 69 | out_rows.push _.extend( {}, row_out, {mention_screen_name: mention.screen_name, mention_username: mention.name} ) 70 | 71 | out_rows 72 | 73 | 74 | # The twitter connector itself 75 | connector_data = 76 | name: "Twitter Connector" 77 | 78 | steps: 79 | 80 | # The start where we ask for credentials and what to search for 81 | start: 82 | template: require './start.jade' 83 | auth: 84 | template: require './auth.jade' 85 | 86 | run: 87 | template: require './run.jade' 88 | 89 | transitions: 90 | 91 | "start > run": (data, from, to)-> 92 | _.extend data, wdc_base.fetch_inputs("#state-start") 93 | #inputs = 94 | #data.q = inputs.q 95 | 96 | #"auth > run": (data)-> 97 | #_.extend data, fetchInputs("#state-auth") 98 | 99 | "enter run": (data)-> 100 | wdc_base.set_connection_data( data ) 101 | tableau.submit() 102 | 103 | rows: (connection_data, lastRecordTokenBase64)-> 104 | # De-base64 the last record token to get around a Tableau Desktop issue 105 | # with quotes in the lastRecordToken 106 | lastRecordToken = atob(lastRecordTokenBase64) 107 | 108 | tableau.log "Starting to fetch a batch -- lastRecordToken:'#{lastRecordToken}'" 109 | tableau.abortWithError("No search terms provided") unless connection_data.q 110 | 111 | pagination = { 112 | url: "/search?q=#{encodeURIComponent(connection_data.q)}" 113 | pages: parseInt(connection_data.page_count) 114 | } 115 | 116 | if lastRecordToken and lastRecordToken != "" 117 | pagination = JSON.parse(lastRecordToken) 118 | 119 | $.ajax 120 | url: pagination.url 121 | datatype: "json" 122 | success: (data, textStatus, request)-> 123 | 124 | extract_column_desc = get_twitter_columns(connection_data) 125 | 126 | tableau_data = [] 127 | for row in data.statuses 128 | tableau_data = tableau_data.concat expand_twitter_rows( row, extract_column_desc.names, connection_data) 129 | 130 | 131 | next_page = { 132 | url: "/search#{data.search_metadata.next_results}" 133 | pages: pagination.pages - 1 134 | } 135 | 136 | have_more = (next_page.pages > 0) 137 | 138 | # Re-encode the new lastRecordToken with base64 to get around 139 | # the Tableau Desktop lastRecordToken issue. 140 | reencodedLastRecordToken = btoa(JSON.stringify(next_page)) 141 | 142 | tableau.dataCallback(tableau_data, reencodedLastRecordToken, have_more ) 143 | 144 | error: (xhr, ajaxOptions, thrownError)-> 145 | console.error("Error during search request", thrownError) 146 | tableau.abortWithError "Error while trying to load the tweets. #{thrownError}" 147 | 148 | columns: (connection_data)-> 149 | cols = get_twitter_columns(connection_data) 150 | tableau.headersCallback( cols.names, cols.types ) 151 | 152 | 153 | wdc_base.make_tableau_connector( connector_data ) 154 | #$ -> 155 | #$('#twdc-ui').html( build_html( connector_data )) 156 | #sm = build_state_machine( connector_data ) 157 | #build_tableau_connector(connector_data) 158 | ##add_event_handlers( connector_data ) 159 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Starschema Tableau Web Table Connector (WDC) Toolkit / ConnectorBase [![Build Status](https://travis-ci.org/starschema/tableau-web-table-connector.svg)](https://travis-ci.org/starschema/tableau-web-table-connector) 2 | 3 | Starschema's Tableau Web Table Connector Toolkit enables you to develop Web Data Connectors in CoffeeScript more easily, elegantly. It comes with sample connectors for **SAP BusinessObjects. Twitter, MongoDB, Github and generic CSV**. 4 | 5 | The framework (or as we call _ConnectorBase_) has a few concepts that may feel different to the usual WDC development process: 6 | 7 | - A connector is built from a number of states/steps (like setup, authentication, running the connector), where each state/step has a separate representation in the UI (like a setup page, an authentication page, etc.) 8 | 9 | - The form inputs used by the connector are declared in the template for each state (instead of declaring them in the connector code itself), so you can keep your connector code DRY 10 | 11 | - The connector source code defines the JavaScript to be ran during transitions from one state to another. (like get the data from all the inputs and call tableau.submit() when transitioning from the start state to the run state.) 12 | 13 | 14 | 15 | # Business Objects connector 16 | 17 | ![](http://www.virtdb.com/images/bo-wdc-tableau.gif) 18 | 19 | ## Building the connector 20 | 21 | You should never trust any tableau connector on the internet. Check the source codes before you deploy or run anything on your environment. Therefore we don't provide any "prebuilt" connector, you should do it for yourselves. 22 | 23 | ### The necessary steps to compile 24 | 25 | - download node.js and install it (https://nodejs.org/en/) 26 | - Update NPM by typing "npm install npm -g" 27 | - Install Coffee "npm install -g coffee-script" (CoffeeScript is a little language that compiles into JavaScript.) 28 | - Download the tableau-web-table-connector zip, and extract into a directory 29 | - Open a command shell as Administrator 30 | - run "npm install" from the administrator command shell while in the extracted directory 31 | - run "build-sap-bo.sh" from the administrator command shell while in the extracted directory 32 | - run "coffee coffee/sap_bo/proxy_server.coffee" from the administrator command shell while in the extracted directory 33 | 34 | 35 | this copies the resources and builds the client using browserify. For more information on how to use SAP BusinessObjects connection [check out this article](http://databoss.starschema.net/accessing-sap-businessobjects-from-tableau-using-web-data-connector/). 36 | 37 | ### Testing 38 | 39 | [Serve](https://www.npmjs.com/package/serve) your preferred WDC on HTTP. 40 | 41 | Get the WDC SDK, follow setup instructions, open the simulator in your browser, enter the WDC URL. See: [https://tableau.github.io/webdataconnector/](https://tableau.github.io/webdataconnector/) 42 | 43 | 44 | ### Adding the WDC to Tableau 45 | 46 | See [http://tableau.github.io/webdataconnector/docs/wdc_use_in_tableau](http://tableau.github.io/webdataconnector/docs/wdc_use_in_tableau) 47 | 48 | 49 | ## Running the BO proxy web service 50 | 51 | ``` 52 | coffee coffee/sap_bo/proxy_server.coffee 53 | ``` 54 | 55 | Then navigate to ```http://localhost:3000/sapbo.html``` in the 56 | Simulator / Tableau. 57 | 58 | # Twitter Client 59 | 60 | ## Building the twitter client 61 | 62 | ``` 63 | npm install 64 | ./build-twitter.sh 65 | ``` 66 | 67 | this copies the resources and builds the client using browserify. 68 | 69 | ## Running the twitter client / server 70 | 71 | ``` 72 | coffee coffee/twitter_connector/twitter_auth_server.coffee 73 | ``` 74 | 75 | Then navigate to ```http://localhost:3000/twitter.html``` in the 76 | Simulator / Tableau. 77 | 78 | # Building the raw JSON client 79 | 80 | ``` 81 | npm install 82 | ./build-json.sh 83 | ``` 84 | this copies the resources and builds the client using browserify. 85 | 86 | 87 | # Building the MongoDB REST JSON client 88 | 89 | Connecting to mongodb using the Simple REST API requires starting the 90 | mongod server with the ```--rest``` and ```--jsonp``` command line 91 | switches to enable the REST API and enable the connector to load data 92 | through JSONP. 93 | 94 | For example on Ubuntu this may mean: 95 | 96 | ``` 97 | mongod --config /etc/mongodb.conf --rest --jsonp 98 | ``` 99 | 100 | To access the REST API from a different machine you also need to change 101 | the ```bind_ip``` configuration property in your mongodb configuration 102 | form 127.0.0.1 (the default that comes with most mongodb packages) to 103 | 0.0.0.0 104 | 105 | After changing the bind address, starting mongod will issue a warning: 106 | 107 | ``` 108 | warning: bind_ip of 0.0.0.0 is unnecessary; listens on all ips by default 109 | ``` 110 | 111 | but this warning is a lie when it comes to the web interface used for 112 | the REST API. 113 | 114 | 115 | Then build the mongodb connector: 116 | 117 | ``` 118 | ./build-mongodb.sh 119 | ``` 120 | 121 | this copies the resources and builds the client using browserify and 122 | places the results in the ```dist``` folder. 123 | 124 | 125 | Then fire up a web server in the dist directory and the mongodb 126 | connector should be accessible with ```mongodb.html```. 127 | 128 | 129 | # CSV Connector 130 | 131 | ## Building the CSV connector 132 | 133 | ``` 134 | # Linux 135 | ./build-csv.sh 136 | 137 | # Windows 138 | build-csv.bat 139 | ``` 140 | 141 | ## Using the CSV Connector 142 | 143 | Copy the built files in the `dist` folder to the root of your webserver, then open `csv.html` in your browser. 144 | 145 | 146 | -------------------------------------------------------------------------------- /coffee/sap_bo/sap_bo_connector.coffee: -------------------------------------------------------------------------------- 1 | $ = require 'jquery' 2 | _ = require 'underscore' 3 | dateFormat = require('dateformat') 4 | helpers = require '../connector_base/tableau_helpers' 5 | wdc_base = require '../connector_base/starschema_wdc_base.coffee' 6 | 7 | PROXY_SERVER_CONFIG = 8 | protocol: 'http' 9 | 10 | rowConverter = {} 11 | 12 | # Transforms type to tableau type 13 | transformType = (type) -> 14 | switch type 15 | when 'STRING' then tableau.dataTypeEnum.string 16 | when 'DOUBLE', 'FLOAT' then tableau.dataTypeEnum.float 17 | when 'INT32', 'INT64', 'UINT32', 'UINT64' then tableau.dataTypeEnum.int 18 | when 'DATE' then tableau.dataTypeEnum.date 19 | when 'DATETIME' then tableau.dataTypeEnum.datetime 20 | else tableau.dataTypeEnum.string 21 | 22 | 23 | # Attempts to convert a list of fields to a table schema compatible 24 | # with tableau 25 | toTableauSchema = (fields)-> 26 | fields.map (field)-> {id: sanitizeId(field.name), dataType: transformType(field.type) } 27 | 28 | # Creates a converter from the given schema 29 | makeRowConverter = (schema) -> 30 | converters = {} 31 | Object.keys(schema).map (k)-> 32 | converters[schema[k].id] = switch schema[k].dataType 33 | when tableau.dataTypeEnum.datetime then (x) -> convertToDateString(x, "yyyy-mm-dd HH:MM:ss") 34 | when tableau.dataTypeEnum.date then (x) -> convertToDateString(x, "yyyy-mm-dd") 35 | else (x)-> x 36 | (row) -> 37 | Object.keys(row).map (k)-> 38 | converters[sanitizeId k](row[k]) 39 | 40 | # Replaces any non-id characters with an underscore 41 | sanitizeId = (name)-> 42 | name.replace(/[^a-zA-Z0-9_]/g, '_') 43 | 44 | # Parse date to a given format 45 | convertToDateString = (dateString, format) -> 46 | date = new Date(dateString) 47 | return dateFormat(date, format) if not isNaN(date) and isFinite(date.getTime()) 48 | dateString 49 | 50 | wdc_base.make_tableau_connector 51 | steps: 52 | start: 53 | template: require './start.jade' 54 | configuration: 55 | template: require './configuration.jade' 56 | run: 57 | template: require './run.jade' 58 | 59 | transitions: 60 | "enter start": (data)-> 61 | if data.error 62 | $('#error').show().text(data.error) 63 | else 64 | $('#error').hide().text() 65 | 66 | "start > configuration": (data) -> 67 | _.extend data, wdc_base.fetch_inputs("#state-start") 68 | 69 | "configuration > run": (data) -> 70 | _.extend data, wdc_base.fetch_inputs("#state-configuration") 71 | 72 | "enter configuration": (data, from,to, transitionTo) -> 73 | url = "#{PROXY_SERVER_CONFIG.protocol}://#{window.location.host}/sap/tablelist" 74 | $.ajax 75 | url: url 76 | dataType: 'json' 77 | data: 78 | "wsdl": data.wsdl 79 | success: (data, textStatus, request) -> 80 | for table in data 81 | $("