├── .idea ├── .name ├── misc.xml ├── scopes │ └── scope_settings.xml ├── encodings.xml ├── vcs.xml ├── modules.xml ├── jsLibraryMappings.xml ├── SqlProject.iml ├── libraries │ └── SqlProject_node_modules.xml ├── runConfigurations │ └── bin_www.xml └── dbnavigator.xml ├── README.md ├── views ├── error.ejs └── index.ejs ├── public └── stylesheets │ └── style.css ├── routes ├── index.js └── users.js ├── package.json ├── bin └── www └── app.js /.idea/.name: -------------------------------------------------------------------------------- 1 | SqlProject -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Database Management for Treebook 2 | -------------------------------------------------------------------------------- /views/error.ejs: -------------------------------------------------------------------------------- 1 |

<%= message %>

2 |

<%= error.status %>

3 |

<%= error.stack %>

4 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /public/stylesheets/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding: 50px; 3 | font: 14px "Lucida Grande", Helvetica, Arial, sans-serif; 4 | } 5 | 6 | a { 7 | color: #00B7FF; 8 | } -------------------------------------------------------------------------------- /.idea/scopes/scope_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /routes/index.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | 4 | /* GET home page. */ 5 | router.get('/', function(req, res, next) { 6 | res.render('index', { title: 'Express' }); 7 | }); 8 | 9 | module.exports = router; 10 | -------------------------------------------------------------------------------- /routes/users.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | 4 | /* GET users listing. */ 5 | router.get('/', function(req, res, next) { 6 | res.send('respond with a resource'); 7 | }); 8 | 9 | module.exports = router; 10 | -------------------------------------------------------------------------------- /views/index.ejs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | <%= title %> 5 | 6 | 7 | 8 |

<%= title %>

9 |

Welcome to <%= title %>

10 | 11 | 12 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/jsLibraryMappings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/SqlProject.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /.idea/libraries/SqlProject_node_modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SqlProject", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "start": "node ./bin/www" 7 | }, 8 | "dependencies": { 9 | "async": "^0.9.0", 10 | "body-parser": "~1.12.0", 11 | "cookie-parser": "~1.3.4", 12 | "debug": "~2.1.1", 13 | "ejs": "~2.3.1", 14 | "express": "~4.12.0", 15 | "morgan": "~1.5.1", 16 | "pg": "^4.3.0", 17 | "pg-hstore": "^2.3.1", 18 | "sequelize": "^2.0.4", 19 | "serve-favicon": "~2.2.0" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /.idea/runConfigurations/bin_www.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /bin/www: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /** 4 | * Module dependencies. 5 | */ 6 | 7 | var app = require('../app'); 8 | var debug = require('debug')('SqlProject:server'); 9 | var http = require('http'); 10 | 11 | /** 12 | * Get port from environment and store in Express. 13 | */ 14 | 15 | var port = normalizePort(process.env.PORT || '3000'); 16 | app.set('port', port); 17 | 18 | /** 19 | * Create HTTP server. 20 | */ 21 | 22 | var server = http.createServer(app); 23 | 24 | /** 25 | * Listen on provided port, on all network interfaces. 26 | */ 27 | 28 | server.listen(port); 29 | server.on('error', onError); 30 | server.on('listening', onListening); 31 | 32 | /** 33 | * Normalize a port into a number, string, or false. 34 | */ 35 | 36 | function normalizePort(val) { 37 | var port = parseInt(val, 10); 38 | 39 | if (isNaN(port)) { 40 | // named pipe 41 | return val; 42 | } 43 | 44 | if (port >= 0) { 45 | // port number 46 | return port; 47 | } 48 | 49 | return false; 50 | } 51 | 52 | /** 53 | * Event listener for HTTP server "error" event. 54 | */ 55 | 56 | function onError(error) { 57 | if (error.syscall !== 'listen') { 58 | throw error; 59 | } 60 | 61 | var bind = typeof port === 'string' 62 | ? 'Pipe ' + port 63 | : 'Port ' + port; 64 | 65 | // handle specific listen errors with friendly messages 66 | switch (error.code) { 67 | case 'EACCES': 68 | console.error(bind + ' requires elevated privileges'); 69 | process.exit(1); 70 | break; 71 | case 'EADDRINUSE': 72 | console.error(bind + ' is already in use'); 73 | process.exit(1); 74 | break; 75 | default: 76 | throw error; 77 | } 78 | } 79 | 80 | /** 81 | * Event listener for HTTP server "listening" event. 82 | */ 83 | 84 | function onListening() { 85 | var addr = server.address(); 86 | var bind = typeof addr === 'string' 87 | ? 'pipe ' + addr 88 | : 'port ' + addr.port; 89 | debug('Listening on ' + bind); 90 | } 91 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var path = require('path'); 3 | var favicon = require('serve-favicon'); 4 | var logger = require('morgan'); 5 | var cookieParser = require('cookie-parser'); 6 | var bodyParser = require('body-parser'); 7 | var request = require('request'); 8 | var routes = require('./routes/index'); 9 | var users = require('./routes/users'); 10 | var pg = require('pg'); 11 | var async = require('async'); 12 | var names = require('./names.js'); 13 | 14 | var app = express(); 15 | 16 | // view engine setup 17 | app.set('views', path.join(__dirname, 'views')); 18 | app.set('view engine', 'ejs'); 19 | 20 | // uncomment after placing your favicon in /public 21 | //app.use(favicon(__dirname + '/public/favicon.ico')); 22 | app.use(logger('dev')); 23 | app.use(bodyParser.json()); 24 | app.use(bodyParser.urlencoded({extended: false})); 25 | app.use(cookieParser()); 26 | app.use(express.static(path.join(__dirname, 'public'))); 27 | 28 | 29 | var conString = info; 30 | //pg.connect(conString, function(err, client, done) { 31 | // console.log(err); 32 | // var locationQuery = "INSERT INTO location (xcoord, ycoord , latitude, longitude) select $1, $2, $3, $4 WHERE NOT EXISTS (SELECT xcoord FROM location WHERE xcoord = $1 and ycoord = $2);"; 33 | // var treeQuery = "INSERT INTO tree (name, qspeciesid, siteorder, qsiteinfo, qcaretaker, plantdate, dbh, plotsize, permitnotes, treeid, locationid) SELECT $1, (select distinct qspeciesid from qspecies where qspecies = $2 limit 1), $3, $4, $5, $6, $7, $8, $9, $10, (select distinct locationid from location where xcoord = $11 limit 1) WHERE NOT EXISTS (SELECT treeid FROM tree WHERE treeid = $10);"; 34 | // var qspeciesQuery = "INSERT INTO qspecies (qspecies) SELECT $1 WHERE NOT EXISTS (SELECT qspecies FROM qspecies WHERE qspecies = $1);"; 35 | // client.query(locationQuery, [xcoord, ycoord, latitude, longitude]); 36 | // function(error, results) { 37 | // //console.log("Finished location inserts!", error, results); 38 | // done(); 39 | // } 40 | // client.query(qspeciesQuery, [name, qspecies, siteorder, qsiteinfo, qcaretaker, plantdate, dbh, plotsize, permitnotes, treeid, xcoord], function(error, results) { 41 | // //console.log("Finished tree inserts!", error, results); 42 | // done(); 43 | // }); 44 | // client.query(treeQuery, [tree.qspecies], function(error, results) { 45 | // //console.log("Finished tree inserts!", error, results); 46 | // done(); 47 | // }); 48 | //}); 49 | 50 | var treeInsert = function(error, response, body, callback) { 51 | //console.log("inside request"); 52 | //console.log(error, "error"); 53 | //console.log(response); 54 | //console.log(body); 55 | var trees = JSON.parse(body); 56 | //console.log(trees[0].qspecies); 57 | pg.connect(conString, function(err, client, done) { 58 | console.log(err); 59 | var locationQuery = "INSERT INTO location (xcoord, ycoord , latitude, longitude) select $1, $2, $3, $4 WHERE NOT EXISTS (SELECT xcoord FROM location WHERE xcoord = $1 and ycoord = $2);"; 60 | var treeQuery = "INSERT INTO tree (name, qspeciesid, siteorder, qsiteinfo, qcaretaker, plantdate, dbh, plotsize, permitnotes, treeid, locationid) SELECT $1, (select distinct qspeciesid from qspecies where qspecies = $2 limit 1), $3, $4, $5, $6, $7, $8, $9, $10, (select distinct locationid from location where xcoord = $11 limit 1) WHERE NOT EXISTS (SELECT treeid FROM tree WHERE treeid = $10);"; 61 | var qspeciesQuery = "INSERT INTO qspecies (qspecies) SELECT $1 WHERE NOT EXISTS (SELECT qspecies FROM qspecies WHERE qspecies = $1);"; 62 | async.map(trees, function(tree, cb) { 63 | var longitude, 64 | latitude, 65 | xcoord, 66 | ycoord; 67 | if (tree.location) { 68 | longitude = tree.location.longitude; 69 | latitude = tree.location.latitude; 70 | xcoord = tree.xcoord; 71 | ycoord = tree.ycoord; 72 | } 73 | else { 74 | longitude = "9999"; 75 | latitude = "9999"; 76 | xcoord = "9999"; 77 | ycoord = "9999"; 78 | } 79 | client.query(locationQuery, [xcoord, ycoord, latitude, longitude]); 80 | }, function(error, results) { 81 | //console.log("Finished location inserts!", error, results); 82 | done(); 83 | }); 84 | async.map(trees, function(tree, cb) { 85 | client.query(qspeciesQuery, [tree.qspecies]); 86 | }, function(error, results) { 87 | //console.log("Finished qspecies inserts!", error, results); 88 | done(); 89 | }); 90 | async.map(trees, function(tree, cb) { 91 | var xcoord = tree.xcoord || 999, 92 | treeid = tree.treeid, 93 | qspecies = tree.qspecies, 94 | siteorder = tree.siteorder || 9999, 95 | qsiteinfo = tree.qsiteinfo || 'unknown', 96 | qcaretaker = tree.qcaretaker || 'unknown', 97 | plantdate = tree.plantdate || new Date(0), 98 | dbh = tree.dbh || 999, 99 | plotsize = tree.plotsize || "unknown", 100 | permitnotes = tree.permitnotes || "unknown"; 101 | var name = names.pop()[0]; 102 | client.query(treeQuery, [name, qspecies, siteorder, qsiteinfo, qcaretaker, plantdate, dbh, plotsize, permitnotes, treeid, xcoord]); 103 | }, function(error, results) { 104 | //console.log("Finished tree inserts!", error, results); 105 | done(); 106 | }); 107 | }); 108 | }; 109 | 110 | var imageInsert = function() { 111 | pg.connect(conString, function(err, client, done) { 112 | console.log(err); 113 | var selectQuery = "select * from qspecies"; 114 | client.query(selectQuery, function(err, results) { 115 | console.log(err, ' inside select for image'); 116 | var queryResults = results.rows; 117 | var insertFunction = function(qspecies) { 118 | //console.log(qspecies); 119 | //console.log(qspecies.qspecies); 120 | //console.log(qspecies.qspeciesid); 121 | var qspeciesid = (qspecies.qspeciesid); 122 | var qspeciesname = (qspecies.qspecies); 123 | var qspeciesname = qspeciesname; 124 | var uri = "https://api.datamarket.azure.com/Bing/Search/v1/Image?Query='" + encodeURIComponent(qspeciesname) + "'&$top=1&$format=json"; 125 | //console.log(uri); 126 | var options = { 127 | uri: uri, 128 | auth: { 129 | user: 'hGDwtBW/yO/Dv0DAefKYF45CYLvgutQMwqUe2PSNL+w', 130 | pass: 'hGDwtBW/yO/Dv0DAefKYF45CYLvgutQMwqUe2PSNL+w' 131 | } 132 | }; 133 | 134 | var selectQuery = "SELECT * FROM image WHERE qspeciesid = $1;"; 135 | client.query(selectQuery, [qspeciesid], function(err, results) { 136 | if (results.rows === undefined || results.rows[0] === undefined) { 137 | console.log('database results', results); 138 | console.log('database rows', results.rows); 139 | request(options, function(err, response, results, body) { 140 | console.log(err, ' error inside picture request'); 141 | var results = JSON.parse(results); 142 | console.log(results); 143 | console.log(results['d']); 144 | console.log(results.d.results[0]); 145 | var mediaurl = results.d.results[0] ? results.d.results[0].MediaUrl : "unknown"; 146 | var mediaw = results.d.results[0] ? results.d.results[0].Width : 9999; 147 | var mediah = results.d.results[0] ? results.d.results[0].Height : 9999; 148 | var mediat = results.d.results[0] ? results.d.results[0].ContentType : "unknown"; 149 | var thumbnailurl = results.d.results[0] ? results.d.results[0].Thumbnail.MediaUrl : "unknown"; 150 | var thumbnailt = results.d.results[0] ? results.d.results[0].Thumbnail.ContentType : "unknown"; 151 | var thumbnailw = results.d.results[0] ? results.d.results[0].Thumbnail.Width : 9999; 152 | var thumbnailh = results.d.results[0] ? results.d.results[0].Thumbnail.Height : 9999; 153 | var insertQuery = "INSERT INTO image (imageurl, imagewidth, imageheight, imagetype, qspeciesid) values ($1, $2, $3, $4, $5);"; 154 | client.query(insertQuery, [mediaurl, mediaw, mediah, mediat, qspeciesid], function(err, results) { 155 | console.log(err, " error inside insert query"); 156 | console.log(results); 157 | }); 158 | var insertQuery2 = "INSERT INTO thumbnail (contenttype, url, width, height, qspeciesid) values ($1, $2, $3, $4, $5);"; 159 | client.query(insertQuery2, [thumbnailt, thumbnailurl, thumbnailw, thumbnailh, qspeciesid], function(err, results) { 160 | console.log(err, " error inside insert query"); 161 | console.log(results); 162 | }); 163 | }); 164 | } 165 | }); 166 | }; 167 | async.map(queryResults, insertFunction, function(err, results) { 168 | console.log(results); 169 | console.log(err); 170 | }); 171 | }); 172 | }); 173 | }; 174 | 175 | var options = { 176 | url: 'https://data.sfgov.org/resource/tkzw-k3nq.json?$limit=50000' 177 | }; 178 | var options1 = { 179 | url: 'https://data.sfgov.org/resource/tkzw-k3nq.json?$limit=50000&$offset=50000' 180 | }; 181 | 182 | async.series([ 183 | function(callback) { 184 | request(options, treeInsert); 185 | callback(null) 186 | }, 187 | function(callback) { 188 | request(options1, treeInsert); 189 | callback(null) 190 | }, 191 | function(callback) { 192 | imageInsert(); 193 | callback(null); 194 | } 195 | ]); 196 | 197 | 198 | module.exports = app; 199 | -------------------------------------------------------------------------------- /.idea/dbnavigator.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | --------------------------------------------------------------------------------