├── .bowerrc ├── .gitignore ├── .npmrc ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── COPYING.txt ├── Gruntfile.js ├── README.md ├── TERMS.md ├── Vagrantfile ├── bower.json ├── dev └── user.clj ├── doc ├── .gitignore ├── config.edn ├── manifests │ └── vagrant.pp └── vagrant.md ├── npm-shrinkwrap.json ├── package.json ├── project.clj ├── resources ├── codox │ ├── css │ │ └── codox-md.css │ ├── index_template.html │ └── ns_template.html ├── datasets │ ├── census │ │ ├── SC-EST2011-ALLDATA6.pdf │ │ ├── SC-EST2011-alldata6-AL_ID.csv │ │ ├── SC-EST2011-alldata6-IL_MO.csv │ │ ├── SC-EST2011-alldata6-MT_PA.csv │ │ ├── SC-EST2011-alldata6-RI_WY.csv │ │ └── definition.json │ ├── county_taxes │ │ ├── .gitignore │ │ ├── Drakefile │ │ ├── definition.json │ │ └── states.csv │ └── integration_test │ │ ├── definition.json │ │ ├── states.csv │ │ └── test_data.csv ├── mime.types └── static │ ├── .gitignore │ ├── background.png │ ├── cfpb-minicon │ ├── css │ │ ├── custom.css │ │ ├── custom.min.css │ │ ├── icons-ie7.css │ │ ├── icons-ie7.min.css │ │ ├── icons.css │ │ └── icons.min.css │ └── fonts │ │ ├── CFPB_minicons.eot │ │ ├── CFPB_minicons.otf │ │ ├── CFPB_minicons.svg │ │ ├── CFPB_minicons.ttf │ │ └── CFPB_minicons.woff │ ├── css │ └── data-api.less │ ├── favicon.ico │ └── js │ ├── data-api.js │ └── html5shiv.js ├── src ├── monger │ └── key_compression.clj └── qu │ ├── app.clj │ ├── app │ ├── mongo.clj │ ├── options.clj │ └── webserver.clj │ ├── cache.clj │ ├── data.clj │ ├── data │ ├── aggregation.clj │ ├── compression.clj │ ├── definition.clj │ ├── result.clj │ └── source.clj │ ├── env.clj │ ├── etag.clj │ ├── generate │ ├── static.clj │ └── templates.clj │ ├── loader.clj │ ├── logging.clj │ ├── main.clj │ ├── metrics.clj │ ├── middleware │ ├── keyword_params.clj │ ├── stacktrace.clj │ └── uri_rewrite.clj │ ├── query.clj │ ├── query │ ├── mongo.clj │ ├── parser.clj │ ├── select.clj │ ├── validation.clj │ └── where.clj │ ├── resources.clj │ ├── routes.clj │ ├── swagger.clj │ ├── templates │ ├── 404.mustache │ ├── 500.mustache │ ├── concept.mustache │ ├── dataset.mustache │ ├── index.mustache │ ├── layout.mustache │ ├── pagination.mustache │ ├── slice-metadata.mustache │ └── slice.mustache │ ├── urls.clj │ ├── util.clj │ ├── views.clj │ └── writer.clj └── test ├── integration └── test │ ├── aggregation.clj │ ├── cache.clj │ ├── loader.clj │ ├── main.clj │ ├── mongo_query.clj │ ├── slice.clj │ └── writer.clj └── qu ├── test ├── cache.clj ├── data.clj ├── main.clj ├── metrics.clj ├── query.clj └── query │ ├── mongo.clj │ ├── parser.clj │ ├── validation.clj │ └── where.clj └── test_util.clj /.bowerrc: -------------------------------------------------------------------------------- 1 | { 2 | "directory": "resources/components" 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | report.xml 3 | *.jar 4 | *.class 5 | .DS_Store 6 | .lein-deps-sum 7 | .lein-failures 8 | .lein-plugins 9 | .lein-env 10 | *.iml 11 | .lein-repl-history 12 | sandbox/ 13 | *.iml 14 | *.log 15 | .drake 16 | .sly 17 | .nrepl-port 18 | .vagrant 19 | build.xml 20 | test-results/ 21 | /docs 22 | /logs 23 | /target 24 | /lib 25 | /classes 26 | /checkouts 27 | /out 28 | /node_modules/ 29 | /resources/components 30 | /resources/datasets/hmda 31 | /src/qu/project.clj 32 | 33 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | save=true 2 | save-exact=true 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | sudo: false 3 | lein: lein 4 | script: lein inttest 5 | jdk: 6 | - oraclejdk7 7 | - oraclejdk8 8 | - openjdk7 9 | 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v1.1.13 - 2016-09-22 4 | 5 | - Frontend improvements to pagination and limits for HTML format. 6 | - Fixes SSL bug when retrieving dependencies. 7 | 8 | ## v1.1.12 - 2016-04-08 9 | 10 | - Lock down front-end dependencies using `npm shrinkwrap` (c962a27) 11 | 12 | ## v1.1.11 - 2016-02-08 13 | 14 | - Validate `$page` argument (a44a85b) 15 | 16 | ## v1.1.10 - 2015-12-29 17 | 18 | - Disables pagination links in HTML view when they should not be clickable (43516f9) 19 | - No longer sends statsd metrics for responses with 4XX status codes (37e6d8a) 20 | 21 | ## v1.1.7 - 2015-09-18 22 | 23 | ### Added 24 | 25 | - Do not escape html in dataset description text when browsing the html api 26 | 27 | ## v1.1.6 - 2015-07-27 28 | 29 | ### Added 30 | 31 | - Logging to print out raw result of aggregation 32 | - For data load, add assertion that .csv file exists before processing 33 | 34 | ## v1.1.5 - 2014-09-19 35 | 36 | ### Added 37 | 38 | - Logging to print out aggregation id if use query matches pre-built aggregation. 39 | 40 | ## v1.1.4 - 2014-08-26 41 | 42 | ### Removed 43 | 44 | - Disabled the use of large offsets, which are extremely inefficient in mongo 45 | - Offset must be less than 10,000 in HTML requests (7b8d9a9) 46 | - Remove the "last" button in the website pagination (ae91e39) 47 | 48 | ### Fixed 49 | 50 | - Fix loading of derived slices, which had incorrect field titles (aedd275) 51 | - Drakefile for [cfpb/api](https://github.com/cfpb/api) no longer fails in Vagrant environment due to unzip/7z issues (9ecdbe6) 52 | - Fixed mistake in download instructions (c55236c) 53 | 54 | ## v1.1.3 - 2014-07-24 55 | 56 | ### Added 57 | 58 | - All incrementing metrics, such as cache hit, now end in `.count` (0e366c4) 59 | - This will require any graphite dashboards to be updated. 60 | 61 | ### Fixed 62 | 63 | - Data loader uses `allowDiskUse` for derived slices. Without this, errors ensue. (80ebb01) 64 | - Fix 404 error for lein in vagrant provisioning (0c22dd7) 65 | 66 | ## v1.1.2 - 2014-06-13 67 | 68 | ### Added 69 | - MongoDB 2.6 now required. Aggregations now use the Mongo Aggregation Framework and no longer use MapReduce (ca9940c) 70 | - Existing cached aggregations will need to be removed and recreated 71 | - Liberator tracing to dev environment (efbbf5a) 72 | - More integration tests (6c7b5b8) 73 | - Support for new style configuration file (fc06091) 74 | - Metrics around open/close/cancel stream channels (92d2840) 75 | 76 | 77 | ### Removed 78 | - Support for MongoDB 2.4. Aggregations now require MongoDB 2.6 (ca9940c) 79 | - JDK 6 no longer a testing target in TravisCI (94de20b) 80 | _ "_id" is no longer included in aggregation results (661120e) 81 | 82 | ### Fixed 83 | 84 | - Sorting on aggregations (1f04252) 85 | - StatsD port can be passed from environment variable without validation error (92d2840) 86 | - One-off error in HTML view for query (df51ed3) 87 | 88 | ### Security 89 | 90 | - DB authentication details no longer printed in logs if environment is not "DEV" (7688434) 91 | 92 | 93 | ## v0.9.2 94 | 95 | ### Major additions 96 | 97 | * Added a query cache to cache all aggregations 98 | * Added 2012 HMDA data 99 | * Added multiple-column joins for concept loading 100 | 101 | ### Minor changes 102 | 103 | * Added `ez-load-` functions to `cfpb.qu.loader` to break loading from a monolithic task to smaller ones 104 | * Changed public domain statement from Unlicense to Creative Commons 0. 105 | * Added route-generation through `route-one` library 106 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Public domain 2 | 3 | The project is in the public domain within the United States, and 4 | copyright and related rights in the work worldwide are waived through 5 | the [CC0 1.0 Universal public domain dedication][CC0]. 6 | 7 | All contributions to this project will be released under the CC0 8 | dedication. By submitting a pull request, you are agreeing to comply 9 | with this waiver of copyright interest. 10 | 11 | [CC0]: http://creativecommons.org/publicdomain/zero/1.0/ 12 | -------------------------------------------------------------------------------- /COPYING.txt: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | module.exports = function(grunt) { 2 | 3 | // Project configuration. 4 | grunt.initConfig({ 5 | pkg: grunt.file.readJSON('package.json'), 6 | uglify: { 7 | options: { 8 | banner: '/*! <%= pkg.name %> <%= grunt.template.today("yyyy-mm-dd") %> */\n' 9 | }, 10 | build: { 11 | src: [ 12 | 'resources/components/jquery/jquery.js', 13 | 'resources/components/underscore/underscore.js', 14 | 'resources/components/bootstrap/docs/assets/js/bootstrap.js', 15 | 'resources/components/bootstrap/docs/assets/js/bootstrap-typeahead.js', 16 | 'resources/components/bootstrap/docs/assets/js/bootstrap-tooltip.js', 17 | 'resources/components/bootstrap/docs/assets/js/bootstrap-popover.js', 18 | 'resources/components/jquery-textrange/jquery-textrange.js', 19 | 'resources/components/localforage/dist/localforage.js', 20 | 'resources/static/js/<%= pkg.name %>.js' 21 | ], 22 | dest: 'resources/static/js/<%= pkg.name %>.min.js' 23 | } 24 | }, 25 | recess: { 26 | dist: { 27 | src: ['resources/components/bootstrap/less/bootstrap.less', 28 | 'resources/static/css/data-api.less'], 29 | dest: 'resources/static/css/data-api.min.css', 30 | options: { 31 | compile: true, 32 | compress: true 33 | } 34 | } 35 | }, 36 | watch: { 37 | files: ['Gruntfile.js', 'resources/components/**/*', 'resources/static/**/*'], 38 | tasks: ['default'] 39 | } 40 | }); 41 | 42 | grunt.loadNpmTasks('grunt-recess'); 43 | grunt.loadNpmTasks('grunt-contrib-uglify'); 44 | grunt.loadNpmTasks('grunt-contrib-watch'); 45 | 46 | // Default task. 47 | grunt.registerTask('default', ['recess', 'uglify']); 48 | }; 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # qu 2 | 3 | :warning: **This project was archived on September 25th, 2020 and is no longer maintained** :warning: 4 | 5 | 6 | --- 7 | --- 8 | --- 9 | 10 | 11 | 12 | 13 | 14 | _qu_ is a data platform created to serve our public data sets. You can use it to serve your data sets, as well. 15 | 16 | The goals of this platform are to: 17 | * Import data in our 18 | [Google-Dataset-inspired format][dataset-inspired] 19 | * Query data using our 20 | [Socrata-Open-Data-API][soda]-inspired API 21 | * Export data in JSON or CSV format 22 | 23 | [CFPB]: http://www.consumerfinance.gov/ 24 | [dataset-inspired]: https://github.com/cfpb/qu/wiki/Dataset-publishing-format 25 | [soda]: http://dev.socrata.com/consumers/getting-started/ 26 | 27 | ## Developing with Vagrant 28 | 29 | If you are using Vagrant, life is clean and easy for you! Go to [our Vagrant documentation](doc/vagrant.md) to get started. 30 | 31 | ## Getting started without Vagrant 32 | 33 | ### Prerequisites 34 | 35 | In order to work on _qu_, you need the following languages and tools 36 | installed: 37 | 38 | * [Node.js][] 39 | * [Grunt][] 40 | * [Bower][] 41 | * [Java][] 42 | * [Leiningen][] 43 | * [MongoDB][] 44 | 45 | [Java]: http://www.java.com/en/ 46 | [Node.js]: http://nodejs.org/ 47 | [Leiningen]: http://leiningen.org/ 48 | [Grunt]: http://gruntjs.com/ 49 | [Bower]: http://bower.io/ 50 | [MongoDB]: http://www.mongodb.org/ 51 | 52 | ### Setup 53 | 54 | #### Front-end assets 55 | 56 | Once you have the prerequisites installed and the code downloaded and 57 | expanded into a directory (which we will call "qu"), run the following 58 | commands: 59 | 60 | ```sh 61 | cd qu 62 | lein deps 63 | npm install -g grunt-cli bower 64 | npm install && bower install 65 | grunt 66 | ``` 67 | 68 | If editing the JavaScript or CSS, run the following to watch the JS 69 | and CSS and make sure your changes are compiled: 70 | 71 | ```sh 72 | grunt watch 73 | ``` 74 | 75 | You can run `grunt` to compile the files once. 76 | 77 | #### Vagrant 78 | 79 | Start a VM by running `vagrant up`. Provisioning will take a few minutes. 80 | 81 | After a VM is started, you should be able to run `vagrant ssh` to SSH to the VM. Then run: 82 | 83 | ``` 84 | cd /vagrant 85 | ``` 86 | 87 | to change the working directory to the Qu codebase. 88 | 89 | #### Clojure 90 | 91 | To start a Clojure REPL to work with the software, run: 92 | 93 | ```sh 94 | lein repl 95 | ``` 96 | 97 | In order to run the API as a web server, run: 98 | 99 | ```sh 100 | lein run 101 | ``` 102 | 103 | Go to http://localhost:3000 (or http://localhost:3333 if using Vagrant) and you should see the app running. 104 | 105 | Before starting the API, you will want to start MongoDB and load some 106 | data into it. 107 | 108 | ### Configuration 109 | 110 | All the settings below are shown via environment variables, but they 111 | can also be set via Java properties. See 112 | [the documentation for environ][https://github.com/weavejester/environ/blob/master/README.md] 113 | for more information on how to use Java properties if you prefer. 114 | 115 | #### Configuration file 116 | 117 | Besides using environment variables, you can also use a configuration 118 | file. This file must contain a Clojure map with your configuration set 119 | in it. Unlike with environment variables, where each setting is 120 | uppercased and SNAKE_CASED, these settings must be lowercase keywords 121 | with dashes, like so: 122 | 123 | ```clojure 124 | { :http-port 8080 125 | :mongo-host "127.0.0.1" } 126 | ``` 127 | 128 | In order to use a configuration file, set `QU_CONFIG` to the file's 129 | location, like so: 130 | 131 | ```sh 132 | QU_CONFIG=/etc/qu-conf.clj 133 | ``` 134 | 135 | Note that the configuration file overrides environment variables. 136 | 137 | #### HTTP server 138 | 139 | By default, the server will come up on port 3000 and 4 threads will be 140 | allocated to handle requests. The server will be bound to 141 | localhost. You can change these settings via environment variables: 142 | 143 | ```sh 144 | HTTP_IP=0.0.0.0 145 | HTTP_PORT=3000 146 | HTTP_THREADS=4 147 | ``` 148 | 149 | You can also do this in the `QU_CONFIG` config file: 150 | 151 | ```clojure 152 | { :http-ip "0.0.0.0" 153 | :http-port 3000 154 | :http-threads 50 } 155 | ``` 156 | 157 | #### MongoDB 158 | 159 | In development mode, the application will connect to your local MongoDB server. In production, or if you want to connect to a different Mongo server in dev, you will have to specify the Mongo host and port. 160 | 161 | You can do this via setting environment variables: 162 | 163 | ```sh 164 | MONGO_HOST=192.168.21.98 165 | MONGO_PORT=27017 166 | ``` 167 | 168 | You can also do this in the `QU_CONFIG` config file: 169 | 170 | ```clojure 171 | { :mongo-host "192.168.21.98" 172 | :mongo-port 27017 } 173 | ``` 174 | 175 | If you prefer to connect via a URI, use `MONGO_URI`. 176 | 177 | If you need to connect to several servers to read from multiple replica sets, set specific Mongo options, or authenticate, you will have to set your configuration in a file as specified under `QU_CONFIG`. Your configuration should look like the following: 178 | 179 | ```clojure 180 | { 181 | ;; General settings 182 | :http-ip "0.0.0.0" 183 | :http-port 3000 184 | :http-threads 50 185 | 186 | ;; Set a vector of vectors, each made up of the IP address and port. 187 | :mongo-hosts [["127.0.0.1" 27017] ["192.168.1.1" 27017]] 188 | 189 | ;; Mongo options should be in a map. 190 | :mongo-options {:connections-per-host 20 191 | :connect-timeout 60} 192 | 193 | ;; Authentication should be a map of database names to vectors containing username and password. 194 | ;; If you have a user on the admin database with the roles "readWriteAnyDatabase", that user should 195 | ;; work for running the entire API. To load data, that user needs the roles "clusterAdmin" and 196 | ;; "dbAdminAnyDatabase" as well. 197 | ;; If you choose not to have a user on the admin database, you will need a user for every dataset 198 | ;; and for the "metadata" database. 199 | :mongo-auth { 200 | :admin ["admin-user" "s3cr3t"] 201 | :slicename ["admin-user" "s3cr3t"] 202 | :metadata ["admin-user" "s3cr3t"] 203 | :query_cache ["admin-user" "s3cr3t"]} 204 | } 205 | ``` 206 | 207 | See [the Monger documentation for all available Mongo connection options](http://clojuremongodb.info/articles/connecting.html#connecting_to_mongodb_using_connection_options). 208 | 209 | #### StatsD 210 | 211 | The application can generate metrics related to its execution and send them to statsd. 212 | 213 | However by default metrics publishing is disabled. To enable it you need to provide statsd hostname in the configuration file: 214 | 215 | ```clojure 216 | { 217 | :statsd-host "localhost" 218 | ;; Standard statsd port 219 | :statsd-port 8125 220 | } 221 | ``` 222 | 223 | #### App URL 224 | 225 | To control the HREF of the links that are created for data slices, you can set the APP_URL environment variable. 226 | 227 | For example, given a slice at `/data/a_resource/a_slice`, setting the APP_URL variable like so 228 | 229 | ```sh 230 | APP_URL=https://my.data.platform/data-api 231 | ``` 232 | 233 | will create links such as 234 | 235 | ```sh 236 | _links":[{"rel":"self","href":"https://my.data.platform/data-api/data/a_resource/a_slice.json?...."}] 237 | ``` 238 | 239 | when emitted in JSON, JSONP, XML, and so on. 240 | 241 | If the variable is not set, then relative HREFs such as `/data/a_resource/a_slice.json` are used. This variable is most useful in production hosting situations where an application server is behind a proxy, and you wish to granularly control the HREFs that are created independent of how the application server sees the request URI. 242 | 243 | #### API Name 244 | 245 | In order for your API to show a custom name (such as "Spiffy Lube 246 | API"), set the `API_NAME` environment variable. This is probably best 247 | set in an external config file. 248 | 249 | ### Loading data 250 | 251 | Make sure you have MongoDB started. To load some sample data, run 252 | `lein repl` and enter the following: 253 | 254 | ```clojure 255 | (go) 256 | (load-dataset "census") ; Takes quite a while to run; can skip. 257 | (stop) 258 | ``` 259 | 260 | ### Testing 261 | 262 | To execute the project's tests, run: 263 | 264 | ```sh 265 | lein test 266 | ``` 267 | 268 | We also have integration tests that run tests against a Mongo database. 269 | To run these tests: 270 | 271 | ```sh 272 | lein with-profile integration embongo test 273 | ``` 274 | 275 | or, even more easily: 276 | 277 | ```sh 278 | lein inttest 279 | ``` 280 | 281 | ### Nginx 282 | 283 | We recommend serving Qu behind a proxy. Nginx works well for this, and 284 | there is a [sample configuration file](doc/nginx.conf) available. 285 | -------------------------------------------------------------------------------- /TERMS.md: -------------------------------------------------------------------------------- 1 | As a work of the United States Government, this package is in the 2 | public domain within the United States. Additionally, we waive 3 | copyright and related rights in the work worldwide through the CC0 1.0 4 | Universal public domain dedication. 5 | 6 | ## CC0 1.0 Universal Summary 7 | 8 | This is a human-readable summary of the 9 | [Legal Code (read the full text)](http://creativecommons.org/publicdomain/zero/1.0/legalcode). 10 | 11 | ### No Copyright 12 | 13 | The person who associated a work with this deed has dedicated the work to 14 | the public domain by waiving all of his or her rights to the work worldwide 15 | under copyright law, including all related and neighboring rights, to the 16 | extent allowed by law. 17 | 18 | You can copy, modify, distribute and perform the work, even for commercial 19 | purposes, all without asking permission. See Other Information below. 20 | 21 | ### Other Information 22 | 23 | In no way are the patent or trademark rights of any person affected by CC0, 24 | nor are the rights that other persons may have in the work or in how the 25 | work is used, such as publicity or privacy rights. 26 | 27 | Unless expressly stated otherwise, the person who associated a work with 28 | this deed makes no warranties about the work, and disclaims liability for 29 | all uses of the work, to the fullest extent permitted by applicable law. 30 | When using or citing the work, you should not imply endorsement by the 31 | author or the affirmer. 32 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | # Every Vagrant virtual environment requires a box to build off of. 6 | config.vm.box = "centos-6.4-x86_64" 7 | config.vm.box_url = "http://developer.nrel.gov/downloads/vagrant-boxes/CentOS-6.4-x86_64-v20130731.box" 8 | 9 | config.vm.network "forwarded_port", guest: 3000, host: 3333 # Web Server 10 | config.vm.network "forwarded_port", guest: 5678, host: 5678 # Leiningen 11 | config.vm.network "forwarded_port", guest: 27017, host: 27017 # MongoDB 12 | config.vm.network "forwarded_port", guest: 28017, host: 28017 # MongoDB status 13 | 14 | config.vm.provision :puppet do |puppet| 15 | puppet.manifests_path = "doc/manifests" 16 | puppet.manifest_file = "vagrant.pp" 17 | end 18 | 19 | # Optional - expand memory limit for the virtual machine to improve performance with larger datasets 20 | # config.vm.provider "virtualbox" do |v| 21 | # v.memory = 2048 22 | # end 23 | end 24 | -------------------------------------------------------------------------------- /bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "data-api", 3 | "version": "1.1.0", 4 | "main": [ 5 | "./resources/css/app.css", 6 | "./resources/js/app.js" 7 | ], 8 | "dependencies": { 9 | "jquery": "1.10.2", 10 | "underscore": "1.5.0", 11 | "bootstrap": "2.3.2", 12 | "jquery-textrange": "~1.1.0" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /dev/user.clj: -------------------------------------------------------------------------------- 1 | (ns user) 2 | 3 | (def namespaces (atom {})) 4 | 5 | (alter-var-root (var clojure.core/load-lib) 6 | (fn [f] 7 | (fn [prefix lib & options] 8 | (let [start (. java.lang.System (nanoTime)) 9 | return (apply f prefix lib options) 10 | end (. java.lang.System (nanoTime)) 11 | ms (/ (double (- end start)) 1000000.0) 12 | lib-name (if prefix 13 | (keyword (str prefix "." lib)) 14 | (keyword lib))] 15 | (swap! namespaces 16 | update-in [lib-name] (fnil max ms) ms) 17 | return)))) 18 | 19 | (defn ns-times 20 | [] 21 | (reverse (sort-by second @namespaces))) 22 | 23 | (require '[alembic.still :refer [distill]] 24 | '[clojure.string :as str] 25 | '[clojure.pprint :refer [pprint]] 26 | '[clojure.repl :refer :all] 27 | '[clojure.tools.namespace.repl :refer [refresh refresh-all set-refresh-dirs]] 28 | '[com.stuartsierra.component :as component] 29 | '[qu.main :as main] 30 | '[qu.app :refer [new-qu-system]] 31 | '[qu.app.options :refer [inflate-options]] 32 | '[qu.util :refer :all] 33 | '[qu.loader :as loader :refer [load-dataset]]) 34 | 35 | (set-refresh-dirs "src/" "dev/") 36 | 37 | (def system (atom nil)) 38 | 39 | (defn init 40 | ([] (init {})) 41 | ([options] 42 | (reset! system (new-qu-system (combine (main/default-options) options))))) 43 | 44 | (defn start 45 | [] 46 | (swap! system component/start) 47 | (swap! system assoc :running true)) 48 | 49 | (defn stop 50 | [] 51 | (swap! system component/stop) 52 | (swap! system assoc :running false)) 53 | 54 | (defn go 55 | ([] (go {})) 56 | ([options] 57 | (init options) 58 | (start))) 59 | 60 | (defn reset 61 | [] 62 | (stop) 63 | (refresh :after 'user/go)) 64 | 65 | (defn load-sample-data 66 | [] 67 | (if (:running @system) 68 | (load-dataset "county_taxes") 69 | (println "System not running"))) 70 | -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | codox/ 2 | -------------------------------------------------------------------------------- /doc/config.edn: -------------------------------------------------------------------------------- 1 | {:dev false 2 | :log {:file "qu.log" 3 | :level "info"} 4 | :mongo {:conn {:uri uri 5 | :hosts hosts 6 | :host host 7 | :port port} 8 | :options {} 9 | :auth {}} 10 | :http {:ip ip 11 | :port port 12 | :thread number-of-threads 13 | :queue-size queue-size 14 | :view {:api_name (:api-name env) 15 | :qu_version (:version @project) 16 | :build_number (:build-number @project) 17 | :build_url (:build-url @project) 18 | :base_url (:app-url @project) 19 | :dev_mode (:dev env)}}} 20 | -------------------------------------------------------------------------------- /doc/manifests/vagrant.pp: -------------------------------------------------------------------------------- 1 | yumrepo { "mongodb": 2 | name => 'mongodb', 3 | baseurl => 'http://downloads-distro.mongodb.org/repo/redhat/os/$basearch/', 4 | gpgcheck => 0, 5 | enabled => 1, 6 | } 7 | 8 | yumrepo { 'epel': 9 | name => 'EPEL', 10 | mirrorlist => 'http://mirrors.fedoraproject.org/mirrorlist?repo=epel-6&arch=$basearch', 11 | gpgcheck => 0, 12 | enabled => 1, 13 | } 14 | 15 | package { "mongodb-org": 16 | ensure => present, 17 | require => Yumrepo["mongodb"], 18 | } 19 | 20 | service { "mongod": 21 | ensure => running, 22 | require => Package["mongodb-org"], 23 | } 24 | 25 | package { "nodejs": 26 | ensure => present, 27 | require => Yumrepo["epel"] 28 | } 29 | 30 | package { "npm": 31 | ensure => present, 32 | require => [Yumrepo["epel"], Package["nodejs"]], 33 | } 34 | 35 | package { "unzip": 36 | ensure => present, 37 | } 38 | 39 | package { "p7zip": 40 | ensure => present, 41 | require => Yumrepo["epel"], 42 | } 43 | 44 | file { "/usr/bin/7z": 45 | ensure => link, 46 | target => "/usr/bin/7za", 47 | mode => "0755", 48 | owner => "root", 49 | require => Package["p7zip"], 50 | } 51 | 52 | package { "git": 53 | ensure => present, 54 | } 55 | 56 | package { "java-1.8.0-openjdk": 57 | ensure => present, 58 | } 59 | 60 | exec { 'install leiningen': 61 | command => "/usr/bin/curl -sL https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein > /usr/bin/lein", 62 | creates => "/usr/bin/lein", 63 | } 64 | 65 | file { "/usr/bin/lein": 66 | ensure => present, 67 | mode => "0755", 68 | owner => "root", 69 | require => Exec["install leiningen"], 70 | } 71 | 72 | package{"nss": 73 | ensure => "latest", 74 | } 75 | 76 | exec { 'install bower': 77 | timeout => 1800, 78 | command => "/usr/bin/npm install -g bower", 79 | creates => "/usr/bin/bower", 80 | require => Package["npm"], 81 | } 82 | 83 | exec { 'install grunt': 84 | command => "/usr/bin/npm install -g grunt-cli", 85 | creates => "/usr/bin/grunt", 86 | require => Package["npm"], 87 | } 88 | 89 | service { "iptables": 90 | ensure => "stopped", 91 | enable => false, 92 | } 93 | -------------------------------------------------------------------------------- /doc/vagrant.md: -------------------------------------------------------------------------------- 1 | # Developing with Vagrant 2 | 3 | With Vagrant, everything you need for Qu should be isolated on a virtual machine that you can develop on. 4 | 5 | ## Getting started 6 | 7 | After downloading the code via git like so: 8 | 9 | ```sh 10 | git clone https://github.com/cfpb/qu.git 11 | cd qu 12 | ``` 13 | 14 | you should be able to run the following command to build your virtual machine: 15 | 16 | ```sh 17 | vagrant up --provision 18 | ``` 19 | 20 | **NOTE:** This will take a very long time the first time. Plan a coffee break. 21 | 22 | Once this is built, you should be able to run: 23 | 24 | ```sh 25 | vagrant ssh 26 | cd /vagrant 27 | ``` 28 | 29 | which will put you into a shell on the virtual machine in the Qu working directory. All other commands in this documentation are run on the virtual machine in this directory. 30 | 31 | 32 | ## Front-end assets 33 | 34 | Front-end assets for Qu are managed by Bower and Grunt. The first time you start working with Qu, you will need to install the front-end dependencies like so: 35 | 36 | ```sh 37 | npm install && bower install 38 | grunt 39 | ``` 40 | 41 | From then on, if editing the JavaScript or CSS, run the following to watch the JS 42 | and CSS and make sure your changes are compiled: 43 | 44 | ```sh 45 | grunt watch 46 | ``` 47 | 48 | You can run `grunt` again to compile the files once. 49 | 50 | ## Loading data 51 | 52 | You will need to load some data to get started working with Qu. At the shell, run: 53 | 54 | ``` 55 | lein repl 56 | ``` 57 | 58 | This will start the Clojure REPL, a shell, that you can run Clojure commands in. Run the following: 59 | 60 | ```clojure 61 | (go) 62 | (load-dataset "county_taxes") 63 | (stop) 64 | ``` 65 | 66 | After that, you can type `Ctrl+D` to leave the Clojure REPL. You are ready to work. 67 | 68 | ## Running the API 69 | 70 | In order to run the API as a web server, run: 71 | 72 | ```sh 73 | lein run 74 | ``` 75 | 76 | To start a Clojure REPL to work with the software, run: 77 | 78 | ```sh 79 | lein repl 80 | ``` 81 | 82 | Inside the REPL, you can run the following commands to start and stop the app: 83 | 84 | ```clojure 85 | (go) ;; starts the app 86 | (stop) ;; stops the app 87 | (reset) ;; resets a running app, reloading all the code 88 | ``` 89 | 90 | 91 | Go to http://localhost:3333 on your machine and you should see the app running. This is running on port 3000 on the virtual machine if you need to check it from there. 92 | 93 | ## Testing 94 | 95 | To execute the project's tests, run: 96 | 97 | ```sh 98 | lein test 99 | ``` 100 | 101 | We also have integration tests that run tests against a Mongo database. 102 | To run these tests: 103 | 104 | ```sh 105 | lein inttest 106 | ``` 107 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "data-api", 3 | "version": "1.1.1", 4 | "repository": { 5 | "type": "git", 6 | "url": "https://github.com/cfpb/qu" 7 | }, 8 | "description": "A data platform created to serve public data sets.", 9 | "license": "SEE LICENSE IN TERMS.md", 10 | "devDependencies": { 11 | "bower": "1.2.8", 12 | "grunt": "1.0.1", 13 | "grunt-contrib-jshint": "0.1.1", 14 | "grunt-contrib-nodeunit": "0.1.2", 15 | "grunt-contrib-requirejs": "0.4.0", 16 | "grunt-contrib-uglify": "3.0.0", 17 | "grunt-contrib-watch": "1.0.0", 18 | "grunt-recess": "1.0.1" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (def build-number (or (System/getenv "BUILD_NUMBER") "handbuilt")) 2 | (def build-url (System/getenv "BUILD_URL")) 3 | (def git-commit (or (System/getenv "GIT_COMMIT") 4 | (System/getenv "TRAVIS_COMMIT"))) 5 | 6 | (defproject qu/qu-core "1.1.14" 7 | :description "qu is a data platform created by the CFPB to serve their public data sets." 8 | :license {:name "Public Domain"} 9 | :scm {:name "git" 10 | :url "https://github.com/cfpb/qu.git"} 11 | :build-number ~build-number 12 | :build-url ~build-url 13 | :git-commit ~git-commit 14 | :url "https://github.com/cfpb/qu" 15 | :min-lein-version "2.0.0" 16 | :source-paths ["src"] 17 | :main ^:skip-aot qu.main 18 | :repl-options {:init-ns user 19 | :timeout 600000} 20 | :plugins [[lein-environ "0.4.0"] 21 | [lein-embongo "0.2.2"] 22 | [lein-cloverage "1.0.2"] 23 | [test2junit "1.0.1"] 24 | [slothcfg "1.0.1"] 25 | [codox "0.6.4"]] 26 | :dependencies [[org.clojure/clojure "1.6.0"] 27 | [caribou/antlers "0.6.1"] 28 | [cheshire "5.3.1"] 29 | [clj-statsd "0.3.10"] 30 | [clj-time "0.7.0"] 31 | [clojurewerkz/route-one "1.1.0"] 32 | [clojurewerkz/urly "2.0.0-alpha5" :exclusions [com.google.guava/guava]] 33 | [com.novemberain/monger "1.7.0"] 34 | [com.stuartsierra/component "0.2.1"] 35 | [com.stuartsierra/dependency "0.1.1"] 36 | [com.taoensso/timbre "3.1.6" :exclusions [expectations]] 37 | [compojure "1.1.6" :exclusions [ring/ring-core]] 38 | [digest "1.4.4"] 39 | [environ "0.5.0"] 40 | [halresource "0.1.1-20130809.164342-1"] 41 | [http-kit "2.1.18"] 42 | [liberator "0.11.0"] 43 | [lonocloud/synthread "1.0.5"] 44 | [me.raynes/fs "1.4.5"] 45 | [org.clojure/core.cache "0.6.3"] 46 | [org.clojure/data.csv "0.1.2"] 47 | [org.clojure/data.json "0.2.4"] 48 | [org.codehaus.jsr166-mirror/jsr166y "1.7.0"] 49 | [parse-ez "0.3.6"] 50 | [prismatic/schema "0.2.1"] 51 | [ring "1.2.2"] 52 | [ring.middleware.mime-extensions "0.2.0"] 53 | [ring-middleware-format "0.3.2"] 54 | [scriptjure "0.1.24"] 55 | ] 56 | :aliases {"inttest" ["with-profile" "integration" "embongo" "test"] 57 | "jenkins" ["with-profile" "integration" "embongo" "test2junit"] 58 | "coverage" ["with-profile" "integration" "embongo" "cloverage"] 59 | "vagrant-repl" ["repl" ":start" ":host" "0.0.0.0" ":port" "5678"]} 60 | :jar-exclusions [#"(^|/)\." #"datasets/.*" ] 61 | :uberjar-exclusions [#"(^|/)\." #"datasets/.*" 62 | #"META-INF/.*\.SF" #"META-INF/.*\.[RD]SA"] 63 | :slothcfg {:namespace qu.project 64 | :config-source-path "src"} 65 | :test-selectors {:default (fn [t] (not (:integration t))) 66 | :all (constantly true)} 67 | :test2junit-output-dir "test-results" 68 | :profiles {:uberjar {:aot [#"qu\.(?!loader)" monger.key-compression] 69 | :env {:dev false}} 70 | :test {:injections [(taoensso.timbre/set-level! :error)]} 71 | :dev {:source-paths ["dev"] 72 | :env {:dev true} 73 | :embongo {:version "3.0.6"} 74 | :codox {:output-dir "doc/codox" 75 | :src-dir-uri "https://github.com/cfpb/qu/blob/master" 76 | :src-linenum-anchor-prefix "L" 77 | :writer codox-md.writer/write-docs} 78 | :dependencies [[alembic "0.2.1"] 79 | [clj-http "0.9.1"] 80 | [factual/drake "0.1.5-SNAPSHOT"] 81 | [org.clojure/tools.namespace "0.2.4"] 82 | [org.clojure/java.classpath "0.2.2"] 83 | [ring-mock "0.1.5"] 84 | [codox-md "0.2.0"] 85 | ]} 86 | :integration [:default 87 | {:test-selectors {:default (constantly true)} 88 | :env {:mongo-port 37017 89 | :integration true} 90 | :embongo {:port 37017}}]}) 91 | 92 | -------------------------------------------------------------------------------- /resources/codox/css/codox-md.css: -------------------------------------------------------------------------------- 1 | .doc { 2 | clear: both; 3 | } 4 | 5 | h4.fn, h4.var { 6 | margin: 0; 7 | float: left; 8 | } 9 | 10 | h4.fn, h4.var { 11 | font-variant: small-caps; 12 | font-size: 13px; 13 | font-weight: bold; 14 | color: #717171; 15 | margin-top: 3px; 16 | margin-left: 10px; 17 | } 18 | 19 | div.usage { 20 | font-family: Monaco, "DejaVu Sans Mono", Consolas, monospace; 21 | } 22 | -------------------------------------------------------------------------------- /resources/codox/index_template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | project title 5 | 6 | 7 | 8 | 9 | 10 | 11 |
12 | 15 |
16 | 17 | 23 | 24 |
25 |
26 |

Project Title

27 |
project description
28 |
29 |
30 |

namespace name

31 |
namespace doc
32 |
33 |

Public variables and functions:

34 |
    35 |
  • var
    • 36 |
37 |
38 |
39 |
40 |
41 |
42 | 43 | 44 | -------------------------------------------------------------------------------- /resources/codox/ns_template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | namespace title 5 | 6 | 7 | 8 | 9 | 10 | 11 |
12 | 15 |
16 | 17 | 23 | 29 | 30 |
31 |
32 |

Namespace Title

33 |
namespace doc
34 |
35 |
36 |

var name

37 |

38 |
39 | arglist 40 |
41 |
42 |

Added: version

43 |
44 |
45 |

Deprecated: version

46 |
47 |
docs
48 |
49 |
50 |
51 |
52 | 53 | 54 | -------------------------------------------------------------------------------- /resources/datasets/census/SC-EST2011-ALLDATA6.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/qu/f460d9ab2f05ac22f6d68a98a9641daf0f7c7ba4/resources/datasets/census/SC-EST2011-ALLDATA6.pdf -------------------------------------------------------------------------------- /resources/datasets/census/definition.json: -------------------------------------------------------------------------------- 1 | { "info": { 2 | "name": "2011 State Populations", 3 | "description": "State population by age, sex, race, and Hispanic origin.", 4 | "url": "http://www.census.gov/popest/data/datasets.html" 5 | }, 6 | "slices": { 7 | "population_raw": { 8 | "type": "table", 9 | "table": "population", 10 | "info": { 11 | "name": "Population", 12 | "description": "Raw data for 2010 Census population counts by state." 13 | }, 14 | "dimensions": [ 15 | "region", "division", "state", 16 | "sex", "origin", "race", "age" 17 | ], 18 | "metrics": [ 19 | "population_2010", 20 | "population_estimate_2010", 21 | "population_estimate_2011" 22 | ] 23 | }, 24 | "population_by_state": { 25 | "type": "derived", 26 | "slice": "population_raw", 27 | "info": { 28 | "name": "Population by State", 29 | "description": "Rolled-up populations for each combination of race, origin, and sex by state." 30 | }, 31 | "dimensions": [ 32 | "state", "race", "origin", "sex" 33 | ], 34 | "metrics": [ 35 | "population_2010", 36 | "population_estimate_2010", 37 | "population_estimate_2011" 38 | ], 39 | "aggregations": { 40 | "population_2010": ["sum", "population_2010"], 41 | "population_estimate_2010": ["sum", "population_estimate_2010"], 42 | "population_estimate_2011": ["sum", "population_estimate_2011"] 43 | }, 44 | "where": "race != 'Total' AND sex != 'Total' AND origin != 'Total'" 45 | } 46 | }, 47 | "tables": { 48 | "population": { 49 | "sources": [ 50 | "SC-EST2011-alldata6-AL_ID.csv", 51 | "SC-EST2011-alldata6-IL_MO.csv", 52 | "SC-EST2011-alldata6-MT_PA.csv", 53 | "SC-EST2011-alldata6-RI_WY.csv" 54 | ], 55 | "columns": { 56 | "SUMLEV": { 57 | "skip": true, 58 | "type": "string" 59 | }, 60 | "REGION": { 61 | "name": "region", 62 | "type": "lookup", 63 | "lookup": { 64 | "1": "Northeast", 65 | "2": "Midwest", 66 | "3": "South", 67 | "4": "West" 68 | } 69 | }, 70 | "DIVISION": { 71 | "name": "division", 72 | "type": "lookup", 73 | "lookup": { 74 | "1": "New England", 75 | "2": "Middle Atlantic", 76 | "3": "East North Central", 77 | "4": "West North Central", 78 | "5": "South Atlantic", 79 | "6": "East South Central", 80 | "7": "West South Central", 81 | "8": "Mountain", 82 | "9": "Pacific" 83 | } 84 | }, 85 | "STATE": { 86 | "name": "state", 87 | "type": "lookup", 88 | "lookup": { 89 | "01": "Alabama", 90 | "02": "Alaska", 91 | "04": "Arizona", 92 | "05": "Arkansas", 93 | "06": "California", 94 | "08": "Colorado", 95 | "09": "Connecticut", 96 | "10": "Delaware", 97 | "11": "District of Columbia", 98 | "12": "Florida", 99 | "13": "Georgia", 100 | "15": "Hawaii", 101 | "16": "Idaho", 102 | "17": "Illinois", 103 | "18": "Indiana", 104 | "19": "Iowa", 105 | "20": "Kansas", 106 | "21": "Kentucky", 107 | "22": "Louisiana", 108 | "23": "Maine", 109 | "24": "Maryland", 110 | "25": "Massachusetts", 111 | "26": "Michigan", 112 | "27": "Minnesota", 113 | "28": "Mississippi", 114 | "29": "Missouri", 115 | "30": "Montana", 116 | "31": "Nebraska", 117 | "32": "Nevada", 118 | "33": "New Hampshire", 119 | "34": "New Jersey", 120 | "35": "New Mexico", 121 | "36": "New York", 122 | "37": "North Carolina", 123 | "38": "North Dakota", 124 | "39": "Ohio", 125 | "40": "Oklahoma", 126 | "41": "Oregon", 127 | "42": "Pennsylvania", 128 | "44": "Rhode Island", 129 | "45": "South Carolina", 130 | "46": "South Dakota", 131 | "47": "Tennessee", 132 | "48": "Texas", 133 | "49": "Utah", 134 | "50": "Vermont", 135 | "51": "Virginia", 136 | "53": "Washington", 137 | "54": "West Virginia", 138 | "55": "Wisconsin", 139 | "56": "Wyoming", 140 | "72": "Puerto Rico" 141 | } 142 | }, 143 | "SEX": { 144 | "name": "sex", 145 | "type": "lookup", 146 | "lookup": { 147 | "0": "Total", 148 | "1": "Male", 149 | "2": "Female" 150 | } 151 | }, 152 | "ORIGIN": { 153 | "name": "origin", 154 | "type": "lookup", 155 | "lookup": { 156 | "0": "Total", 157 | "1": "Not Hispanic", 158 | "2": "Hispanic" 159 | } 160 | }, 161 | "RACE": { 162 | "name": "race", 163 | "type": "lookup", 164 | "lookup": { 165 | "1": "White", 166 | "2": "Black or African American", 167 | "3": "American Indian or Alaska Native", 168 | "4": "Asian", 169 | "5": "Native Hawaiian or Pacific Islander", 170 | "6": "Two or More Races" 171 | } 172 | }, 173 | "AGE": { 174 | "name": "age", 175 | "type": "integer" 176 | }, 177 | "CENSUS2010POP": { 178 | "name": "population_2010", 179 | "type": "integer" 180 | }, 181 | "ESTIMATESBASE2010": { 182 | "skip": true, 183 | "type": "integer" 184 | }, 185 | "POPESTIMATE2010": { 186 | "name": "population_estimate_2010", 187 | "type": "integer" 188 | }, 189 | "POPESTIMATE2011": { 190 | "name": "population_estimate_2011", 191 | "type": "integer" 192 | } 193 | } 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /resources/datasets/county_taxes/.gitignore: -------------------------------------------------------------------------------- 1 | *.csv* 2 | !states.csv 3 | -------------------------------------------------------------------------------- /resources/datasets/county_taxes/Drakefile: -------------------------------------------------------------------------------- 1 | countyincome07.csv <- [-timecheck] 2 | curl -O http://www.irs.gov/pub/irs-soi/countyincome07.csv 3 | 4 | county_incomes.csv.header <- [-timecheck] 5 | echo "state_code,county_code,state,county,tax_returns,exemptions,adjusted_gross_income,wages_and_salaries_income,dividend_income,interest_income" > $OUTPUT 6 | 7 | county_incomes.csv <- county_incomes.csv.header, countyincome07.csv 8 | cat $INPUT0 > $OUTPUT 9 | tail -n +2 $INPUT1 | grep -v '^$' | grep -v '"000"' >> $OUTPUT 10 | -------------------------------------------------------------------------------- /resources/datasets/county_taxes/definition.json: -------------------------------------------------------------------------------- 1 | { "info": { 2 | "name": "Tax Year 2007 County Income Data", 3 | "description": "Contains selected individual income tax return data items classified by state and county.", 4 | "url": "https://explore.data.gov/Population/Tax-Year-2007-County-Income-Data/wvps-imhx" 5 | }, 6 | "concepts": { 7 | "state": { 8 | "name": "State Abbreviation", 9 | "type": "string", 10 | "table": "states", 11 | "properties": { 12 | "name": { 13 | "name": "Name", 14 | "description": "The name of the state", 15 | "type": "string" 16 | } 17 | } 18 | }, 19 | "county": { 20 | "name": "County" 21 | }, 22 | "tax_returns": { 23 | "name": "Total Number of Tax Returns" 24 | }, 25 | "adjusted_gross_income": { 26 | "name": "Adjusted Gross Income (In Thousands)" 27 | }, 28 | "wages_and_salaries_income": { 29 | "name": "Wages and Salaries (In Thousands)" 30 | }, 31 | "dividend_income": { 32 | "name": "Dividend Income (In Thousands)" 33 | }, 34 | "interest_income": { 35 | "name": "Interest Income (In Thousands)" 36 | } 37 | }, 38 | "slices": { 39 | "incomes": { 40 | "type": "table", 41 | "table": "incomes", 42 | "dimensions": [ 43 | "state", "county" 44 | ], 45 | "metrics": [ 46 | "tax_returns", 47 | "adjusted_gross_income", 48 | "wages_and_salaries_income", 49 | "dividend_income", 50 | "interest_income" 51 | ], 52 | "references": { 53 | "state_name": { 54 | "column": "state", 55 | "concept": "state", 56 | "value": "name" 57 | } 58 | } 59 | }, 60 | "incomes_by_state": { 61 | "type": "derived", 62 | "slice": "incomes", 63 | "dimensions": [ 64 | "state" 65 | ], 66 | "metrics": [ 67 | "tax_returns", 68 | "adjusted_gross_income", 69 | "wages_and_salaries_income", 70 | "dividend_income", 71 | "interest_income" 72 | ], 73 | "aggregations": { 74 | "tax_returns": ["sum", "tax_returns"], 75 | "adjusted_gross_income": ["sum", "adjusted_gross_income"], 76 | "wages_and_salaries_income": ["sum", "wages_and_salaries_income"], 77 | "dividend_income": ["sum", "dividend_income"], 78 | "interest_income": ["sum", "interest_income"] 79 | } 80 | } 81 | }, 82 | "tables": { 83 | "states": { 84 | "sources": [ 85 | "states.csv" 86 | ], 87 | "columns": { 88 | "state": { 89 | "name": "_id", 90 | "type": "string" 91 | }, 92 | "name": { 93 | "type": "string" 94 | } 95 | } 96 | }, 97 | "incomes": { 98 | "sources": [ 99 | "county_incomes.csv" 100 | ], 101 | "columns": { 102 | "state_code": { 103 | "type": "integer" 104 | }, 105 | "county_code": { 106 | "type": "integer" 107 | }, 108 | "state": { 109 | "type": "string" 110 | }, 111 | "county": { 112 | "type": "string" 113 | }, 114 | "tax_returns": { 115 | "type": "integer" 116 | }, 117 | "exemptions": { 118 | "type": "integer" 119 | }, 120 | "adjusted_gross_income": { 121 | "type": "integer" 122 | }, 123 | "wages_and_salaries_income": { 124 | "type": "integer" 125 | }, 126 | "dividend_income": { 127 | "type": "integer" 128 | }, 129 | "interest_income": { 130 | "type": "integer" 131 | } 132 | } 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /resources/datasets/county_taxes/states.csv: -------------------------------------------------------------------------------- 1 | name,state 2 | "Alabama","AL" 3 | "Alaska","AK" 4 | "Arizona","AZ" 5 | "Arkansas","AR" 6 | "California","CA" 7 | "Colorado","CO" 8 | "Connecticut","CT" 9 | "Delaware","DE" 10 | "District of Columbia","DC" 11 | "Florida","FL" 12 | "Georgia","GA" 13 | "Hawaii","HI" 14 | "Idaho","ID" 15 | "Illinois","IL" 16 | "Indiana","IN" 17 | "Iowa","IA" 18 | "Kansas","KS" 19 | "Kentucky","KY" 20 | "Louisiana","LA" 21 | "Maine","ME" 22 | "Montana","MT" 23 | "Nebraska","NE" 24 | "Nevada","NV" 25 | "New Hampshire","NH" 26 | "New Jersey","NJ" 27 | "New Mexico","NM" 28 | "New York","NY" 29 | "North Carolina","NC" 30 | "North Dakota","ND" 31 | "Ohio","OH" 32 | "Oklahoma","OK" 33 | "Oregon","OR" 34 | "Maryland","MD" 35 | "Massachusetts","MA" 36 | "Michigan","MI" 37 | "Minnesota","MN" 38 | "Mississippi","MS" 39 | "Missouri","MO" 40 | "Pennsylvania","PA" 41 | "Rhode Island","RI" 42 | "South Carolina","SC" 43 | "South Dakota","SD" 44 | "Tennessee","TN" 45 | "Texas","TX" 46 | "Utah","UT" 47 | "Vermont","VT" 48 | "Virginia","VA" 49 | "Washington","WA" 50 | "West Virginia","WV" 51 | "Wisconsin","WI" 52 | "Wyoming","WY" 53 | -------------------------------------------------------------------------------- /resources/datasets/integration_test/definition.json: -------------------------------------------------------------------------------- 1 | { "info": { 2 | "name": "Integration Test Data", 3 | "description": "Data for integration tests.", 4 | "copyright": "This dataset is a public domain work of the US Government." 5 | }, 6 | "concepts": { 7 | "state_abbr": { 8 | "description": "State Abbreviation", 9 | "type": "string", 10 | "table": "states", 11 | "properties": { 12 | "name": { 13 | "name": "Name", 14 | "description": "The name of the state", 15 | "type": "string" 16 | } 17 | } 18 | }, 19 | "county": { 20 | "description": "County" 21 | }, 22 | "tax_returns": { 23 | "description": "Total Number of Tax Returns" 24 | }, 25 | "adjusted_gross_income": { 26 | "description": "Adjusted Gross Income (In Thousands)" 27 | } 28 | }, 29 | "slices": { 30 | "incomes": { 31 | "type": "table", 32 | "table": "incomes", 33 | "dimensions": [ 34 | "state_abbr", "state_name", "county" 35 | ], 36 | "indexes": [ 37 | ["state_abbr", "state_name"], 38 | ["county", "state_name"], 39 | ["state_abbr", "county"], 40 | "state_abbr", 41 | "county" 42 | ], 43 | "metrics": [ 44 | "tax_returns", 45 | "adjusted_gross_income", 46 | "date_observed" 47 | ], 48 | "references": { 49 | "state_name": { 50 | "column": "state_abbr", 51 | "concept": "state_abbr", 52 | "value": "name" 53 | } 54 | } 55 | }, 56 | "incomes_by_state": { 57 | "type": "derived", 58 | "slice": "incomes", 59 | "dimensions": [ 60 | "state_abbr" 61 | ], 62 | "metrics": [ 63 | "counties", 64 | "tax_returns", 65 | "median_tax_return" 66 | ], 67 | "aggregations": { 68 | "counties": ["count"], 69 | "tax_returns": ["sum", "tax_returns"], 70 | "avg_tax_return": ["avg", "tax_returns"] 71 | } 72 | } 73 | }, 74 | "tables": { 75 | "states": { 76 | "sources": [ 77 | "states.csv" 78 | ], 79 | "columns": { 80 | "state_abbr": { 81 | "name": "_id", 82 | "type": "string" 83 | }, 84 | "name": { 85 | "type": "string" 86 | } 87 | } 88 | }, 89 | "incomes": { 90 | "sources": [ 91 | "test_data.csv" 92 | ], 93 | "columns": { 94 | "County Code": { 95 | "skip": true 96 | }, 97 | "State Abbreviation": { 98 | "name": "state_abbr", 99 | "type": "string" 100 | }, 101 | "County Name": { 102 | "name": "county", 103 | "type": "string" 104 | }, 105 | "Total Number of Tax Returns": { 106 | "name": "tax_returns", 107 | "type": "integer" 108 | }, 109 | "Adjusted Gross Income (In Thousands)": { 110 | "name": "adjusted_gross_income", 111 | "type": "dollars" 112 | }, 113 | "Date Observed": { 114 | "name": "date_observed", 115 | "type": "date", 116 | "format": "YYYY/MM/dd" 117 | } 118 | } 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /resources/datasets/integration_test/states.csv: -------------------------------------------------------------------------------- 1 | name,state_abbr 2 | "Alabama","AL" 3 | "Alaska","AK" 4 | "Arizona","AZ" 5 | "Arkansas","AR" 6 | "California","CA" 7 | "Colorado","CO" 8 | "Connecticut","CT" 9 | "Delaware","DE" 10 | "District of Columbia","DC" 11 | "Florida","FL" 12 | "Georgia","GA" 13 | "Hawaii","HI" 14 | "Idaho","ID" 15 | "Illinois","IL" 16 | "Indiana","IN" 17 | "Iowa","IA" 18 | "Kansas","KS" 19 | "Kentucky","KY" 20 | "Louisiana","LA" 21 | "Maine","ME" 22 | "Montana","MT" 23 | "Nebraska","NE" 24 | "Nevada","NV" 25 | "New Hampshire","NH" 26 | "New Jersey","NJ" 27 | "New Mexico","NM" 28 | "New York","NY" 29 | "North Carolina","NC" 30 | "North Dakota","ND" 31 | "Ohio","OH" 32 | "Oklahoma","OK" 33 | "Oregon","OR" 34 | "Maryland","MD" 35 | "Massachusetts","MA" 36 | "Michigan","MI" 37 | "Minnesota","MN" 38 | "Mississippi","MS" 39 | "Missouri","MO" 40 | "Pennsylvania","PA" 41 | "Rhode Island","RI" 42 | "South Carolina","SC" 43 | "South Dakota","SD" 44 | "Tennessee","TN" 45 | "Texas","TX" 46 | "Utah","UT" 47 | "Vermont","VT" 48 | "Virginia","VA" 49 | "Washington","WA" 50 | "West Virginia","WV" 51 | "Wisconsin","WI" 52 | "Wyoming","WY" 53 | -------------------------------------------------------------------------------- /resources/datasets/integration_test/test_data.csv: -------------------------------------------------------------------------------- 1 | County Code,State Abbreviation,County Name,Total Number of Tax Returns,Adjusted Gross Income (In Thousands),Date Observed 2 | 1,NC,County 1,1,100,2013/01/01 3 | 2,NC,County 2,2,200,2013/01/02 4 | 3,NC,County 3,3,300,2013/01/03 5 | 4,NC,County 4,4,400,2013/01/04 6 | 5,NC,County 5,5,500,2013/01/05 7 | 6,PA,County 6,6,600,2013/01/06 8 | 7,PA,County 7,7,700,2013/01/07 9 | 8,PA,County 8,8,800,2013/01/08 10 | 9,PA,County 9,9,900,2013/01/09 11 | 10,PA,County 10,10,1000,2013/01/10 12 | 11,NY,County 11,11,1100,2013/01/11 13 | 12,NY,County 12,12,1200,2013/01/12 14 | 13,NY,County 13,13,1300,2013/01/13 15 | 14,NY,County 14,14,1400,2013/01/14 16 | 15,NY,County 15,15,1500,2013/01/15 17 | 16,DC,County 16,16,1600,2013/01/16 18 | 17,DC,County 17,17,1700,2013/01/17 19 | -------------------------------------------------------------------------------- /resources/mime.types: -------------------------------------------------------------------------------- 1 | application/atom+xml atom 2 | application/docbook+xml dbk docbook 3 | application/epub+zip epub 4 | application/java-archive jar 5 | application/java-vm class 6 | application/json json 7 | application/jsonml+json jsonml 8 | application/mathml+xml mathml 9 | application/msword doc dot 10 | application/pdf pdf 11 | application/pgp-encrypted pgp 12 | application/postscript ps 13 | application/rdf+xml rdf 14 | application/rsd+xml rsd 15 | application/rss+xml rss 16 | application/rss+xml rss 17 | application/rtf rtf 18 | application/vnd.amazon.ebook azw 19 | application/vnd.android.package-archive apk 20 | application/vnd.apple.installer+xml mpkg 21 | application/vnd.google-earth.kml+xml kml 22 | application/vnd.ms-excel xls xlm xla xlc xlt xlw 23 | application/vnd.ms-powerpoint ppt pps pot 24 | application/wsdl+xml wsdl 25 | application/x-7z-compressed 7z 26 | application/x-apple-diskimage dmg 27 | application/x-bittorrent torrent 28 | application/x-bzip bz 29 | application/x-bzip2 bz2 boz 30 | application/x-cpio cpio 31 | application/x-dvi dvi 32 | application/x-font-ttf ttf ttc 33 | application/x-font-ttf ttf 34 | application/x-font-woff woff 35 | application/x-font-woff woff 36 | application/x-gtar gtar 37 | application/x-gzip gz 38 | application/x-latex latex 39 | application/x-sql sql 40 | application/xml xml xsl 41 | application/xml xml xsl 42 | application/xml-dtd dtd 43 | application/xml-dtd dtd 44 | application/xop+xml xop 45 | application/zip zip 46 | application/zip zip 47 | image/bmp bmp 48 | image/cgm cgm 49 | image/gif gif 50 | image/gif gif 51 | image/jpeg jpeg jpg jpe 52 | image/jpeg jpeg jpg 53 | image/png png 54 | image/svg+xml svg svgz 55 | image/svg+xml svg 56 | image/tiff tiff tif 57 | image/tiff tiff tif 58 | image/vnd.adobe.photoshop psd 59 | image/vnd.microsoft.icon ico 60 | image/webp webp 61 | text/cache-manifest appcache 62 | text/calendar ics 63 | text/css css 64 | text/csv csv 65 | text/html html htm 66 | text/javascript js jsonp 67 | text/plain txt text conf def list log in 68 | text/plain txt md markdown mdown log text conf def list in 69 | text/tab-separated-values tsv 70 | text/troff t tr roff man me ms 71 | text/vcard vcard 72 | text/x-asm s asm 73 | text/x-c c cc cxx cpp h hh dic 74 | text/x-clojure clj cljs 75 | text/x-fortran f for f77 f90 76 | text/x-java-source java 77 | text/x-nfo nfo 78 | text/x-opml opml 79 | text/x-pascal p pas 80 | text/x-setext etx 81 | text/x-sfv sfv 82 | text/x-uuencode uu 83 | text/x-vcalendar vcs 84 | text/x-vcard vcf 85 | -------------------------------------------------------------------------------- /resources/static/.gitignore: -------------------------------------------------------------------------------- 1 | css/data-api.min.css 2 | js/data-api.min.js 3 | -------------------------------------------------------------------------------- /resources/static/background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/qu/f460d9ab2f05ac22f6d68a98a9641daf0f7c7ba4/resources/static/background.png -------------------------------------------------------------------------------- /resources/static/cfpb-minicon/css/custom.css: -------------------------------------------------------------------------------- 1 | body { 2 | color: #101820; 3 | font-family: "Avenir Next", Arial, sans-serif; 4 | margin: 0; 5 | padding: 0; 6 | } 7 | 8 | .wrap { 9 | width: 860px; 10 | padding: 40px; 11 | margin: 0 auto; 12 | } 13 | 14 | h1 { 15 | font-size: 2.125em; 16 | line-height: 2; 17 | text-align: center; 18 | margin: 0; 19 | } 20 | 21 | h2 { 22 | font-size: 1.625em; 23 | margin-top: 3em; 24 | color: #101820; 25 | text-transform: none; 26 | letter-spacing: 0; 27 | padding-bottom: 0.5em; 28 | border-bottom: 3px solid #2CB34A; 29 | } 30 | 31 | h3 { 32 | font-size: 1.375em; 33 | font-family: "Avenir Next", Arial, sans-serif; 34 | font-weight: 600; 35 | } 36 | 37 | header { 38 | width: 100%; 39 | position: fixed; 40 | top: 0; 41 | z-index: 2; 42 | background: #E7E7E6; 43 | box-shadow: 0 1px 4px 2px rgba(255, 255, 255, 1); 44 | } 45 | 46 | ul, 47 | li { 48 | padding: 0; 49 | margin: 0; 50 | } 51 | 52 | li { 53 | list-style: none; 54 | display: inline-block; 55 | /*font-size: 22px;*/ 56 | line-height: 1.5; 57 | margin-bottom: 16px; 58 | width: 24%; 59 | text-transform: lowercase; 60 | } 61 | -------------------------------------------------------------------------------- /resources/static/cfpb-minicon/css/custom.min.css: -------------------------------------------------------------------------------- 1 | body{color:#101820;font-family:"Avenir Next",Arial,sans-serif;margin:0;padding:0}.wrap{width:860px;padding:40px;margin:0 auto}h1{font-size:2.125em;line-height:2;text-align:center;margin:0}h2{font-size:1.625em;margin-top:3em;color:#101820;text-transform:none;letter-spacing:0;padding-bottom:.5em;border-bottom:3px solid #2CB34A}h3{font-size:1.375em;font-family:"Avenir Next",Arial,sans-serif;font-weight:600}header{width:100%;position:fixed;top:0;z-index:2;background:#E7E7E6;box-shadow:0 1px 4px 2px rgba(255,255,255,1)}li,ul{padding:0;margin:0}li{list-style:none;display:inline-block;line-height:1.5;margin-bottom:16px;width:24%;text-transform:lowercase} -------------------------------------------------------------------------------- /resources/static/cfpb-minicon/css/icons.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * CFPB Web Icons 3 | * CSS based on Font Awesome 4 | * the iconic font designed for Bootstrap 5 | * by Dave Gandy 6 | * ------------------------------------------------------------------------------ 7 | * The full suite of pictographic icons, examples, and documentation can be 8 | * found at http://fontawesome.io. Stay up to date on Twitter at 9 | * http://twitter.com/fontawesome. 10 | * 11 | * Font Awesome License 12 | * ------------------------------------------------------------------------------ 13 | * - Font Awesome CSS, LESS, and SASS files are licensed under MIT License - 14 | * http://opensource.org/licenses/mit-license.html 15 | */@font-face{font-family:"CFPB Icons";src:url(../fonts/CFPB_minicons.eot);src:url(../fonts/CFPB_minicons.eot?#iefix) format('embedded-opentype'),url(../fonts/CFPB_minicons.svg) format('svg'),url(../fonts/CFPB_minicons.woff) format('woff'),url(../fonts/CFPB_minicons.ttf) format('truetype');font-weight:400;font-style:normal}[class*=" icon-"],[class^=icon-]{font-family:"CFPB Icons";font-weight:400;font-style:normal;text-decoration:inherit;-webkit-font-smoothing:antialiased;*margin-right:.3em}[class*=" icon-"]:before,[class^=icon-]:before{text-decoration:inherit;display:inline-block;speak:none}.icon-large:before{vertical-align:-10%;font-size:1.3333333333333333em}a [class*=" icon-"],a [class^=icon-]{display:inline}[class*=" icon-"].icon-fixed-width,[class^=icon-].icon-fixed-width{display:inline-block;width:1.1428571428571428em;text-align:right;padding-right:.2857142857142857em}[class*=" icon-"].icon-fixed-width.icon-large,[class^=icon-].icon-fixed-width.icon-large{width:1.4285714285714286em}.icons-ul{margin-left:2.142857142857143em;list-style-type:none}.icons-ul>li{position:relative}.icons-ul .icon-li{position:absolute;left:-2.142857142857143em;width:2.142857142857143em;text-align:center;line-height:inherit}[class*=" icon-"].hide,[class^=icon-].hide{display:none}.icon-muted{color:#eee}.icon-light{color:#fff}.icon-dark{color:#333}.icon-border{border:solid 1px #eee;padding:.2em .25em .15em;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.icon-2x{font-size:2em}.icon-2x.icon-border{border-width:2px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.icon-3x{font-size:3em}.icon-3x.icon-border{border-width:3px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.icon-4x{font-size:4em}.icon-4x.icon-border{border-width:4px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.icon-5x{font-size:5em}.icon-5x.icon-border{border-width:5px;-webkit-border-radius:7px;-moz-border-radius:7px;border-radius:7px}.pull-right{float:right}.pull-left{float:left}[class*=" icon-"].pull-left,[class^=icon-].pull-left{margin-right:.3em}[class*=" icon-"].pull-right,[class^=icon-].pull-right{margin-left:.3em}.icon-left:before{content:"\e000"}.icon-left-alt:before{content:"\e001"}.icon-right:before{content:"\e002"}.icon-right-alt:before{content:"\e003"}.icon-up:before{content:"\e004"}.icon-up-alt:before{content:"\e005"}.icon-down:before{content:"\e006"}.icon-down-alt:before{content:"\e007"}.icon-arrow-left:before{content:"\e008"}.icon-arrow-left-alt:before{content:"\e009"}.icon-arrow-right:before{content:"\e010"}.icon-arrow-right-alt:before{content:"\e011"}.icon-arrow-up:before{content:"\e012"}.icon-arrow-up-alt:before{content:"\e013"}.icon-arrow-down:before{content:"\e014"}.icon-arrow-down-alt:before{content:"\e015"}.icon-approved:before{content:"\e100"}.icon-approved-alt:before{content:"\e101"}.icon-error:before{content:"\e102"}.icon-error-alt:before{content:"\e103"}.icon-help:before{content:"\e104"}.icon-help-alt:before{content:"\e105"}.icon-delete:before{content:"\e106"}.icon-delete-alt:before{content:"\e107"}.icon-plus:before{content:"\e108"}.icon-plus-alt:before{content:"\e109"}.icon-minus:before{content:"\e110"}.icon-minus-alt:before{content:"\e111"}.icon-update:before{content:"\e112"}.icon-update-alt:before{content:"\e113"}.icon-youtube:before{content:"\e200"}.icon-youtube-alt:before{content:"\e201"}.icon-linkedin:before{content:"\e202"}.icon-linkedin-alt:before{content:"\e203"}.icon-facebook:before{content:"\e204"}.icon-facebook-alt:before{content:"\e205"}.icon-flickr:before{content:"\e206"}.icon-flickr-alt:before{content:"\e207"}.icon-twitter:before{content:"\e208"}.icon-twitter-alt:before{content:"\e209"}.icon-github:before{content:"\e210"}.icon-github-alt:before{content:"\e211"}.icon-email-social:before{content:"\e212"}.icon-email-social-alt:before{content:"\e213"}.icon-web:before{content:"\e300"}.icon-web-alt:before{content:"\e301"}.icon-email:before{content:"\e302"}.icon-email-alt:before{content:"\e303"}.icon-mail:before{content:"\e304"}.icon-mail-alt:before{content:"\e305"}.icon-phone:before{content:"\e306"}.icon-phone-alt:before{content:"\e307"}.icon-technology:before{content:"\e308"}.icon-technology-alt:before{content:"\e309"}.icon-fax:before{content:"\e310"}.icon-fax-alt:before{content:"\e311"}.icon-document:before{content:"\e400"}.icon-document-alt:before{content:"\e401"}.icon-pdf:before{content:"\e402"}.icon-pdf-alt:before{content:"\e403"}.icon-upload:before{content:"\e404"}.icon-upload-alt:before{content:"\e405"}.icon-download:before{content:"\e406"}.icon-download-alt:before{content:"\e407"}.icon-copy:before{content:"\e408"}.icon-copy-alt:before{content:"\e409"}.icon-edit:before{content:"\e410"}.icon-edit-alt:before{content:"\e411"}.icon-attach:before{content:"\e412"}.icon-attach-alt:before{content:"\e413"}.icon-print:before{content:"\e414"}.icon-print-alt:before{content:"\e415"}.icon-save:before{content:"\e416"}.icon-save-alt:before{content:"\e417"}.icon-appendix:before{content:"\e418"}.icon-appendix-alt:before{content:"\e419"}.icon-supplement:before{content:"\e420"}.icon-supplement-alt:before{content:"\e421"}.icon-bank-account:before{content:"\e500"}.icon-bank-account-alt:before{content:"\e501"}.icon-credit-card:before{content:"\e502"}.icon-credit-card-alt:before{content:"\e503"}.icon-loan:before{content:"\e504"}.icon-loan-alt:before{content:"\e505"}.icon-money-transfer:before{content:"\e506"}.icon-money-transfer-alt:before{content:"\e507"}.icon-mortgage:before{content:"\e508"}.icon-mortgage-alt:before{content:"\e509"}.icon-debt-collection:before{content:"\e510"}.icon-debt-collection-alt:before{content:"\e511"}.icon-credit-report:before{content:"\e512"}.icon-credit-report-alt:before{content:"\e513"}.icon-money:before{content:"\e514"}.icon-money-alt:before{content:"\e515"}.icon-quick-cash:before{content:"\e516"}.icon-quick-cash-alt:before{content:"\e517"}.icon-contract:before{content:"\e518"}.icon-contract-alt:before{content:"\e519"}.icon-complaint:before{content:"\e520"}.icon-complaint-alt:before{content:"\e521"}.icon-getting-credit-card:before{content:"\e522"}.icon-getting-credit-card-alt:before{content:"\e523"}.icon-buying-car:before{content:"\e524"}.icon-buying-car-alt:before{content:"\e525"}.icon-paying-college:before{content:"\e526"}.icon-paying-college-alt:before{content:"\e527"}.icon-owning-home:before{content:"\e528"}.icon-owning-home-alt:before{content:"\e529"}.icon-debt:before{content:"\e530"}.icon-debt-alt:before{content:"\e531"}.icon-building-credit:before{content:"\e532"}.icon-building-credit-alt:before{content:"\e533"}.icon-prepaid-cards:before{content:"\e534"}.icon-prepaid-cards-alt:before{content:"\e535"}.icon-payday-loan:before{content:"\e536"}.icon-payday-loan-alt:before{content:"\e537"}.icon-retirement:before{content:"\e538"}.icon-retirement-alt:before{content:"\e539"}.icon-user:before{content:"\e600"}.icon-user-alt:before{content:"\e601"}.icon-wifi:before{content:"\e602"}.icon-wifi-alt:before{content:"\e603"}.icon-search:before{content:"\e604"}.icon-search-alt:before{content:"\e605"}.icon-share:before{content:"\e606"}.icon-share-alt:before{content:"\e607"}.icon-link:before{content:"\e608"}.icon-link-alt:before{content:"\e609"}.icon-external-link:before{content:"\e610"}.icon-external-link-alt:before{content:"\e611"}.icon-audio-mute:before{content:"\e612"}.icon-audio-mute-alt:before{content:"\e616"}.icon-audio-low:before{content:"\e613"}.icon-audio-low-alt:before{content:"\e617"}.icon-audio-medium:before{content:"\e614"}.icon-audio-medium-alt:before{content:"\e618"}.icon-audio-max:before{content:"\e615"}.icon-audio-max-alt:before{content:"\e619"}.icon-favorite:before{content:"\e620"}.icon-favorite-alt:before{content:"\e621"}.icon-unfavorite:before{content:"\e622"}.icon-unfavorite-alt:before{content:"\e623"}.icon-bookmark:before{content:"\e624"}.icon-bookmark-alt:before{content:"\e625"}.icon-unbookmark:before{content:"\e626"}.icon-unbookmark-alt:before{content:"\e627"}.icon-settings:before{content:"\e628"}.icon-settings-alt:before{content:"\e629"}.icon-menu:before{content:"\e630"}.icon-menu-alt:before{content:"\e631"}.icon-lock:before{content:"\e632"}.icon-lock-alt:before{content:"\e633"}.icon-unlock:before{content:"\e634"}.icon-unlock-alt:before{content:"\e635"}.icon-clock:before{content:"\e636"}.icon-clock-alt:before{content:"\e637"}.icon-chart:before{content:"\e638"}.icon-chart-alt:before{content:"\e639"}.icon-play:before{content:"\e640"}.icon-play-alt:before{content:"\e641"} -------------------------------------------------------------------------------- /resources/static/cfpb-minicon/fonts/CFPB_minicons.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/qu/f460d9ab2f05ac22f6d68a98a9641daf0f7c7ba4/resources/static/cfpb-minicon/fonts/CFPB_minicons.eot -------------------------------------------------------------------------------- /resources/static/cfpb-minicon/fonts/CFPB_minicons.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/qu/f460d9ab2f05ac22f6d68a98a9641daf0f7c7ba4/resources/static/cfpb-minicon/fonts/CFPB_minicons.otf -------------------------------------------------------------------------------- /resources/static/cfpb-minicon/fonts/CFPB_minicons.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/qu/f460d9ab2f05ac22f6d68a98a9641daf0f7c7ba4/resources/static/cfpb-minicon/fonts/CFPB_minicons.ttf -------------------------------------------------------------------------------- /resources/static/cfpb-minicon/fonts/CFPB_minicons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/qu/f460d9ab2f05ac22f6d68a98a9641daf0f7c7ba4/resources/static/cfpb-minicon/fonts/CFPB_minicons.woff -------------------------------------------------------------------------------- /resources/static/css/data-api.less: -------------------------------------------------------------------------------- 1 | .container { 2 | padding: 0 1em; 3 | background-color: white; 4 | } 5 | 6 | 7 | #footer { 8 | border-top: 1px solid #666; 9 | padding-top: 20px; 10 | font-size: 90%; 11 | width: 100%; 12 | 13 | .container { 14 | background-color: transparent; 15 | box-shadow: none; 16 | } 17 | } 18 | 19 | .dimension-fields { 20 | max-height: 600px; 21 | overflow: auto; 22 | } 23 | 24 | 25 | .properties { 26 | DT { 27 | margin-top: 1em; 28 | } 29 | DD { 30 | margin-left: 0; 31 | } 32 | } 33 | 34 | 35 | .bottom-pad { 36 | padding-bottom: 1em; 37 | } 38 | 39 | 40 | .icon-help-alt:hover, .icon-help-alt:focus { 41 | text-decoration: none; 42 | } 43 | 44 | 45 | #save-query .icon-bookmark { 46 | margin-top: .25em; 47 | } 48 | 49 | .save-options .icon-help-alt { 50 | margin-top: .25em; 51 | margin-left: .5em; 52 | } 53 | 54 | #query-url { 55 | width: 92%; 56 | margin: 0 auto 1em auto; 57 | } 58 | -------------------------------------------------------------------------------- /resources/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/qu/f460d9ab2f05ac22f6d68a98a9641daf0f7c7ba4/resources/static/favicon.ico -------------------------------------------------------------------------------- /resources/static/js/data-api.js: -------------------------------------------------------------------------------- 1 | (function ($) { 2 | var $form = $("#query-form"); 3 | 4 | 5 | var formVals = function(){ 6 | return _($form.serializeArray()) 7 | .chain() 8 | .reject(function (field) { 9 | return $.trim(field.value) === ""; 10 | }) 11 | .reduce(function (memo, field) { 12 | memo[field.name] = field.value; 13 | return memo; 14 | }, {}) 15 | .value(); 16 | }; 17 | 18 | 19 | var setFormOptions = function(){ 20 | // set options based on selected format 21 | var fv = formVals(); 22 | var format = fv["$format"] || "html"; 23 | 24 | if (format === 'html') { 25 | $('#field-limit').attr('disabled', 'disabled'); 26 | $('#field-limit').val(100); 27 | } else { 28 | $('#field-limit').removeAttr('disabled'); 29 | } 30 | 31 | if ($('#field-callback').length > 0) { 32 | var callback_container = $form.find("#field-callback").closest('.control-group'); 33 | if (format === 'jsonp') { 34 | callback_container.removeClass('hide'); 35 | $("#field-callback").prop('disabled', ''); 36 | } else { 37 | callback_container.addClass('hide'); 38 | $("#field-callback").val('').prop('disabled', 'disabled'); 39 | } 40 | } 41 | }; 42 | 43 | 44 | var buildQueryUrl = function () { 45 | var href = $form.data('href'); 46 | var fv = formVals(); 47 | var format = fv["$format"] || "html"; 48 | delete fv["$format"]; 49 | var action = href + "." + format; 50 | 51 | var formString = _(fv) 52 | .chain() 53 | .pairs() 54 | .map(function (pair) { 55 | return pair[0] + "=" + 56 | encodeURIComponent(pair[1]).replace(/%20/g,'+'); 57 | }) 58 | .value() 59 | .join("&"); 60 | 61 | $form.attr("action", action); 62 | 63 | $("#query-url").html((formString === "") ? action : action + "?" + formString) 64 | }; 65 | 66 | 67 | var rebuildQuery = function(){ 68 | if ($form.length == 0) return; 69 | 70 | setFormOptions(); 71 | buildQueryUrl(); 72 | }; 73 | 74 | 75 | $(document).ready(function () { 76 | rebuildQuery(); 77 | $form.on("change", "input[type=text]", rebuildQuery); 78 | $form.on("click", "input[type=radio]", rebuildQuery); 79 | 80 | $form.find('#field-select, #field-group, #field-where, #field-orderBy').typeahead({ 81 | source: (jQuery('#typeahead-candidates').val() || '').split(','), 82 | matcher: function (item) { 83 | var term; 84 | 85 | // To allow typeahead within an aggregation function, strip away the function and its parens 86 | term = this.query.split(',').pop().toLowerCase(); 87 | term = term.replace(/[a-zA-Z]+\(/, ''); 88 | term = term.replace(')', ''); 89 | term = term.trim(); 90 | 91 | if (!term) { 92 | return false; 93 | } 94 | 95 | return ~item.toLowerCase().indexOf(term) 96 | }, 97 | updater: function(item) { 98 | var field, parens_index, query_tail; 99 | field = this.$element; 100 | item = item.trim(); 101 | query_tail = this.query.split(',').pop(); 102 | 103 | 104 | // If the field value ends with an empty aggregation 105 | // function, place the item inside it. Otherwise, just 106 | // append. 107 | if (query_tail.match(/\([a-z]+\)$/)) { 108 | item = field.val().replace(query_tail, query_tail.replace(/\(.*/, '(') + item + ')'); 109 | } else { 110 | item = field.val().replace(/[^,]*$/,'') + ' ' + item; 111 | item = item.trim(); 112 | } 113 | 114 | // If there is an empty aggregation function, place the cursor inside it. 115 | parens_index = item.lastIndexOf('()'); 116 | 117 | if (parens_index > 0) { 118 | setTimeout(function () { 119 | field.textrange('setcursor', parens_index + 1); 120 | }); 121 | } 122 | 123 | return item; 124 | }, 125 | minLength: 1, 126 | items: 5 127 | }); 128 | 129 | // tooltips via boostrap-popover.js 130 | $('.icon-help-alt').popover({ 131 | trigger: 'click', 132 | placement: 'bottom' 133 | }).on('click', function (e) { e.preventDefault(); e.stopPropagation() }); 134 | }); 135 | 136 | })(jQuery); 137 | -------------------------------------------------------------------------------- /resources/static/js/html5shiv.js: -------------------------------------------------------------------------------- 1 | /* 2 | HTML5 Shiv v3.7.0 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed 3 | */ 4 | (function(l,f){function m(){var a=e.elements;return"string"==typeof a?a.split(" "):a}function i(a){var b=n[a[o]];b||(b={},h++,a[o]=h,n[h]=b);return b}function p(a,b,c){b||(b=f);if(g)return b.createElement(a);c||(c=i(b));b=c.cache[a]?c.cache[a].cloneNode():r.test(a)?(c.cache[a]=c.createElem(a)).cloneNode():c.createElem(a);return b.canHaveChildren&&!s.test(a)?c.frag.appendChild(b):b}function t(a,b){if(!b.cache)b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag(); 5 | a.createElement=function(c){return!e.shivMethods?b.createElem(c):p(c,a,b)};a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+m().join().replace(/[\w\-]+/g,function(a){b.createElem(a);b.frag.createElement(a);return'c("'+a+'")'})+");return n}")(e,b.frag)}function q(a){a||(a=f);var b=i(a);if(e.shivCSS&&!j&&!b.hasCSS){var c,d=a;c=d.createElement("p");d=d.getElementsByTagName("head")[0]||d.documentElement;c.innerHTML="x"; 6 | c=d.insertBefore(c.lastChild,d.firstChild);b.hasCSS=!!c}g||t(a,b);return a}var k=l.html5||{},s=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,r=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,j,o="_html5shiv",h=0,n={},g;(function(){try{var a=f.createElement("a");a.innerHTML="";j="hidden"in a;var b;if(!(b=1==a.childNodes.length)){f.createElement("a");var c=f.createDocumentFragment();b="undefined"==typeof c.cloneNode|| 7 | "undefined"==typeof c.createDocumentFragment||"undefined"==typeof c.createElement}g=b}catch(d){g=j=!0}})();var e={elements:k.elements||"abbr article aside audio bdi canvas data datalist details dialog figcaption figure footer header hgroup main mark meter nav output progress section summary template time video",version:"3.7.0",shivCSS:!1!==k.shivCSS,supportsUnknownElements:g,shivMethods:!1!==k.shivMethods,type:"default",shivDocument:q,createElement:p,createDocumentFragment:function(a,b){a||(a=f); 8 | if(g)return a.createDocumentFragment();for(var b=b||i(a),c=b.frag.cloneNode(),d=0,e=m(),h=e.length;dLog options)) 23 | 24 | (defrecord CacheWorker [] 25 | component/Lifecycle 26 | 27 | (start [component] 28 | (let [cache (qc/create-query-cache) 29 | worker (qc/create-worker cache)] 30 | (assoc component 31 | :worker worker 32 | :worker-agent (qc/start-worker worker)))) 33 | 34 | (stop [component] 35 | (let [worker-agent (:worker-agent component)] 36 | (qc/stop-worker worker-agent) 37 | component))) 38 | 39 | (defrecord Metrics [provider host port] 40 | component/Lifecycle 41 | 42 | (start [component] 43 | (when provider 44 | (metrics/setup host port)) 45 | component) 46 | 47 | (stop [component] 48 | component)) 49 | 50 | (defn new-metrics [options] 51 | (map->Metrics options)) 52 | 53 | (def components [:log :db :api :cache-worker :metrics]) 54 | 55 | (defrecord QuSystem [options api db log cache-worker] 56 | component/Lifecycle 57 | 58 | (start [system] 59 | (let [system (component/start-system system components)] 60 | (log/info "Started with settings" (str (update-in options [:mongo :auth] (fn [_] str "*****")))) 61 | system)) 62 | 63 | (stop [system] 64 | (component/stop-system system components))) 65 | 66 | (defn new-qu-system [options] 67 | (let [{:keys [http dev log mongo metrics] :as options} (inflate-options options)] 68 | (map->QuSystem {:options options 69 | :db (new-mongo mongo) 70 | :log (new-log log) 71 | :api (new-webserver http dev) 72 | :cache-worker (->CacheWorker) 73 | :metrics (new-metrics metrics) 74 | }))) 75 | -------------------------------------------------------------------------------- /src/qu/app/mongo.clj: -------------------------------------------------------------------------------- 1 | (ns qu.app.mongo 2 | (:require [com.stuartsierra.component :as component] 3 | [monger.core :as mongo] 4 | [qu.util :refer :all] 5 | [taoensso.timbre :as log])) 6 | 7 | (defn authenticate-mongo 8 | [auth] 9 | (doseq [[db [username password]] auth] 10 | (mongo/authenticate (mongo/get-db (name db)) 11 | username 12 | (.toCharArray password)))) 13 | 14 | (defn connect-mongo 15 | [{:keys [uri hosts host port] :as conn} options auth] 16 | (let [options (apply-kw mongo/mongo-options options) 17 | connection 18 | (cond 19 | uri (try (mongo/connect-via-uri! uri) 20 | (catch Exception e 21 | (log/error "The Mongo URI specified is invalid."))) 22 | hosts (let [addresses (map #(apply mongo/server-address %) hosts)] 23 | (mongo/connect! addresses options)) 24 | :else (mongo/connect! (mongo/server-address host port) options))] 25 | (if (map? auth) 26 | (authenticate-mongo auth)) 27 | connection)) 28 | 29 | (defn disconnect-mongo 30 | [] 31 | (when (bound? #'mongo/*mongodb-connection*) 32 | (mongo/disconnect!))) 33 | 34 | (defrecord Mongo [conn options auth] 35 | component/Lifecycle 36 | 37 | (start [component] 38 | (connect-mongo conn options auth) 39 | component) 40 | 41 | (stop [component] 42 | (disconnect-mongo) 43 | component)) 44 | 45 | (defn new-mongo [options] 46 | (map->Mongo options)) 47 | -------------------------------------------------------------------------------- /src/qu/app/options.clj: -------------------------------------------------------------------------------- 1 | (ns qu.app.options 2 | (:require [qu.util :refer :all] 3 | [schema.core :as s])) 4 | 5 | (def HttpOptionsS 6 | {:view {:api_name s/Str 7 | :base_url s/Str 8 | :dev_mode s/Bool 9 | (s/optional-key :build_number) s/Str 10 | (s/optional-key :build_url) s/Str 11 | (s/optional-key :qu_version) s/Str} 12 | :ip s/Str 13 | :port s/Int 14 | :threads s/Int 15 | :queue-size s/Int}) 16 | 17 | (def MongoOptionsS 18 | (let [database (s/either s/Str s/Keyword) 19 | conn-uri-s {:uri s/Str} 20 | conn-hosts-s {:hosts [[(s/one s/Str "ip") 21 | (s/one s/Int "port")]]} 22 | conn-host-s {:host s/Str :port s/Int}] 23 | {:conn (s/either conn-uri-s 24 | conn-hosts-s 25 | conn-host-s) 26 | :options {s/Any s/Any} 27 | (s/optional-key :auth) (s/maybe {database 28 | [(s/one s/Str "username") 29 | (s/one s/Str "password")]})})) 30 | 31 | (def MetricsOptionsS 32 | (s/either 33 | {:provider s/Keyword 34 | :host s/Str 35 | :port s/Int} 36 | {})) 37 | 38 | (def OptionsS 39 | {(s/optional-key :dev) s/Bool 40 | :log {(s/optional-key :file) s/Str 41 | :level s/Keyword} 42 | :mongo MongoOptionsS 43 | :http HttpOptionsS 44 | :metrics MetricsOptionsS}) 45 | 46 | (defn inflate-options 47 | [options] 48 | 49 | (let [default {:dev false 50 | :http {:ip "0.0.0.0" 51 | :port 3000 52 | :threads 4 53 | :queue-size 20480 54 | :view {:base_url "" 55 | :api_name "Data API"}} 56 | :mongo {:conn {:host "127.0.0.1" 57 | :port 27017} 58 | :options {:connect-timeout 2000}} 59 | :log {:level :info} 60 | :metrics {}} 61 | set-dev-mode (fn [opts] 62 | (assoc-in opts 63 | [:http :view :dev_mode] 64 | (:dev opts)))] 65 | (->> options 66 | (remove-nil-vals) 67 | (combine default) 68 | (set-dev-mode) 69 | (s/validate OptionsS)))) 70 | -------------------------------------------------------------------------------- /src/qu/app/webserver.clj: -------------------------------------------------------------------------------- 1 | (ns qu.app.webserver 2 | (:require [com.stuartsierra.component :as component] 3 | [liberator.dev :refer [wrap-trace]] 4 | [org.httpkit.server :refer [run-server]] 5 | [qu.etag :refer [wrap-etag]] 6 | [qu.logging :refer [wrap-with-logging]] 7 | [qu.middleware.keyword-params :refer [wrap-keyword-params]] 8 | [qu.middleware.stacktrace :as prod-stacktrace] 9 | [qu.middleware.uri-rewrite :refer [wrap-ignore-trailing-slash]] 10 | [qu.routes :refer [create-app-routes]] 11 | [ring.middleware.mime-extensions :refer [wrap-convert-extension-to-accept-header]] 12 | [ring.middleware.nested-params :refer [wrap-nested-params]] 13 | [ring.middleware.params :refer [wrap-params]] 14 | [ring.middleware.reload :as reload] 15 | [ring.middleware.stacktrace :as dev-stacktrace] 16 | [ring.util.response :as response] 17 | [taoensso.timbre :as log])) 18 | 19 | (defn- wrap-cors 20 | [handler] 21 | (fn [req] 22 | (let [resp (handler req)] 23 | (response/header resp "access-control-allow-origin" "*")))) 24 | 25 | (defn get-handler 26 | "Create the entry point into the web API. We look for URI suffixes 27 | and strip them to set the Accept header before handing off the 28 | request to Compojure." 29 | [webserver] 30 | (let [handler (-> (create-app-routes webserver) 31 | wrap-ignore-trailing-slash 32 | wrap-keyword-params 33 | wrap-nested-params 34 | wrap-params 35 | wrap-with-logging 36 | wrap-etag 37 | wrap-convert-extension-to-accept-header 38 | wrap-cors)] 39 | (if (:dev webserver) 40 | (-> handler 41 | reload/wrap-reload 42 | (wrap-trace :header :ui) 43 | dev-stacktrace/wrap-stacktrace-web) 44 | (-> handler 45 | prod-stacktrace/wrap-stacktrace-web)))) 46 | 47 | (defrecord WebServer [ip port threads queue-size dev view] 48 | component/Lifecycle 49 | 50 | (start [component] 51 | (let [options {:ip ip 52 | :port port 53 | :thread threads 54 | :queue-size queue-size} 55 | handler (get-handler component)] 56 | (log/info "Starting web server on" (str ip ":" port)) 57 | (assoc component :server (run-server handler options)))) 58 | 59 | (stop [component] 60 | (log/info "Stopping web server") 61 | (let [stop-server (:server component)] 62 | (stop-server :timeout 100)) 63 | (dissoc component :server))) 64 | 65 | (defn new-webserver [options dev] 66 | (map->WebServer (merge {:dev dev} options))) 67 | -------------------------------------------------------------------------------- /src/qu/data.clj: -------------------------------------------------------------------------------- 1 | (ns qu.data 2 | "This namespace contains all our functions for retrieving data from 3 | MongoDB, including creating queries and light manipulation of the data 4 | after retrieval." 5 | (:require [taoensso.timbre :as log] 6 | [clojure.string :as str] 7 | [clojure.walk :refer [postwalk]] 8 | [clojure.core.cache :as cache] 9 | [qu.util :refer :all] 10 | [qu.logging :refer [log-with-time]] 11 | [qu.cache :as qc :refer [create-query-cache add-to-cache]] 12 | [qu.data.result :refer [->DataResult map->DataResult]] 13 | [qu.data.compression :as compression] 14 | [qu.data.definition :as definition] 15 | [qu.metrics :as metrics] 16 | [cheshire.core :as json] 17 | [cheshire.generate :refer [add-encoder encode-str]] 18 | [monger 19 | [core :as mongo :refer [with-db get-db]] 20 | [query :as q] 21 | [collection :as coll] 22 | [conversion :as conv] 23 | joda-time 24 | json])) 25 | 26 | ;; Prevent encoding regexes in JSON in the logs from throwing an error. 27 | (add-encoder java.util.regex.Pattern encode-str) 28 | 29 | (defn get-datasets 30 | "Get metadata for all datasets. Information about the datasets is 31 | stored in a Mongo database called 'metadata'." 32 | [] 33 | (with-db (get-db "metadata") 34 | (coll/find-maps "datasets" {}))) 35 | 36 | (defn get-dataset-names 37 | "List all datasets." 38 | [] 39 | (map :name (get-datasets))) 40 | 41 | (defn get-metadata 42 | "Get metadata for one dataset." 43 | [dataset] 44 | (with-db (get-db "metadata") 45 | (coll/find-one-as-map "datasets" {:name dataset}))) 46 | 47 | (defn slice-columns 48 | "Slices are made up of dimensions, columns that can be queried, and 49 | metrics, which are columns, usually numeric, connected to a set of 50 | those dimensions. This function retrieves the names of all the 51 | columns, both dimensions and metrics." 52 | [slicedef] 53 | (concat (:dimensions slicedef) (:metrics slicedef))) 54 | 55 | (defn concept-collection [concept] 56 | (str "concept__" (name concept))) 57 | 58 | (defn concept-data 59 | "Get the data table for a concept." 60 | [dataset concept] 61 | (with-db (get-db dataset) 62 | (coll/find-maps (concept-collection concept)))) 63 | 64 | (defn field-zip-fn 65 | "Given a dataset and a slice, return a function that will compress 66 | field names." 67 | [dataset slice] 68 | (let [metadata (get-metadata dataset) 69 | slicedef (get-in metadata [:slices (keyword slice)])] 70 | (compression/field-zip-fn slicedef))) 71 | 72 | (defn field-unzip-fn 73 | "Given a dataset and a slice, return a function that will decompress 74 | field names." 75 | [dataset slice] 76 | (let [metadata (get-metadata dataset) 77 | slicedef (get-in metadata [:slices (keyword slice)])] 78 | (compression/field-unzip-fn slicedef))) 79 | 80 | (defn- text? [text] 81 | (or (string? text) 82 | (symbol? text))) 83 | 84 | (defn get-find 85 | "Given a collection and a Mongo find map, return a Result of the form: 86 | :total - Total number of documents for the input query irrespective of skip or limit 87 | :size - Number of documents for the input query after skip and limit are applied 88 | :data - Seq of maps with the IDs stripped out" 89 | [database collection find-map] 90 | (metrics/with-timing "queries.find" 91 | (let [zipfn (field-zip-fn database collection) 92 | find-map (compression/compress-find find-map zipfn) 93 | unzipfn (field-unzip-fn database collection)] 94 | (log-with-time 95 | :info 96 | (str/join " " ["Mongo query" 97 | (str database "/" (name collection)) 98 | (json/generate-string find-map)]) 99 | (with-db (get-db database) 100 | (with-open [cursor (doto (coll/find collection (:query find-map) (:fields find-map)) 101 | (.limit (:limit find-map 0)) 102 | (.skip (:skip find-map 0)) 103 | (.sort (conv/to-db-object (:sort find-map))))] 104 | (->DataResult 105 | (.count cursor) 106 | (.size cursor) 107 | (->> cursor 108 | (map (fn [x] (-> x 109 | (conv/from-db-object false) 110 | (convert-keys unzipfn) 111 | (dissoc :_id)))))))))))) 112 | 113 | (defn get-aggregation 114 | "Given a collection and a Mongo aggregation, return a Result of the form: 115 | :total - Total number of results returned 116 | :size - Same as :total 117 | :data - Seq of maps with the IDs stripped out 118 | 119 | After adding the compression processing, $match MUST come before $group." 120 | [database collection {:keys [query] :as aggmap}] 121 | (metrics/with-timing "queries.aggregation" 122 | (let [cache (create-query-cache) 123 | cache-record (if (cache/has? cache query) 124 | (do (cache/hit cache query) nil) 125 | (qc/add-to-queue cache aggmap))] 126 | (cache/lookup cache query 127 | (map->DataResult {:computing (dissoc cache-record :aggmap)}))))) 128 | 129 | (defn get-data-table 130 | "Given retrieved data (a seq of maps) and the columns you want from 131 | that data, return a seq of seqs representing the data in columnar 132 | format." 133 | [data columns] 134 | (let [columns (map keyword columns)] 135 | (map (fn [row] 136 | (map (fn [column] (column row)) columns)) data))) 137 | -------------------------------------------------------------------------------- /src/qu/data/aggregation.clj: -------------------------------------------------------------------------------- 1 | (ns qu.data.aggregation 2 | "This namespace contains functions for generating aggregations 3 | within Mongo." 4 | (:require [lonocloud.synthread :as ->] 5 | [qu.data.compression :refer [compress-where field-unzip-fn 6 | field-zip-fn]] 7 | [qu.util :refer :all])) 8 | 9 | (defn- select-to-agg 10 | "Convert a aggregation map into the Mongo equivalent for the 11 | $group filter of the aggregation framework. Used in the group 12 | function. Non-public." 13 | [field-zip-fn] 14 | (fn [[alias [agg field]]] 15 | (if (= agg "count") 16 | {alias {"$sum" 1}} 17 | {alias {(str "$" agg) (str "$" (name (field-zip-fn field)))}}))) 18 | 19 | (defn aggregation-group-args 20 | "Build the arguments for the group section of the aggregation framework." 21 | [group aggregations field-zip-fn] 22 | (let [id (into {} (map (fn [field] 23 | (vector field (str "$" (name (field-zip-fn field))))) 24 | group)) 25 | aggregations (map (select-to-agg field-zip-fn) aggregations) 26 | group (apply merge {:_id id} aggregations)] 27 | group)) 28 | 29 | (defn aggregation-project-args 30 | [group aggregations] 31 | (let [project-map {:_id 0} 32 | project-map (reduce (fn [project-map field] 33 | (assoc project-map field (str "$_id." (name field)))) 34 | project-map group) 35 | project-map (reduce (fn [project-map field] 36 | (assoc project-map field 1)) 37 | project-map (keys aggregations))] 38 | project-map)) 39 | 40 | (defn generate-agg-query 41 | [{:keys [to group aggregations filter sort slicedef] :as aggmap}] 42 | (let [field-zip-fn (if slicedef 43 | (field-zip-fn slicedef) 44 | identity) 45 | field-unzip-fn (if slicedef 46 | (field-unzip-fn slicedef) 47 | identity) 48 | match (if filter 49 | (-> filter 50 | (compress-where field-zip-fn) 51 | (convert-keys name))) 52 | group-args (aggregation-group-args group aggregations field-zip-fn) 53 | project-args (aggregation-project-args group aggregations) 54 | ] 55 | (-> [] 56 | (->/when match 57 | (conj {"$match" match})) 58 | (conj {"$group" group-args}) 59 | (conj {"$project" project-args}) 60 | (->/when sort 61 | (conj {"$sort" sort})) 62 | (conj {"$out" to})))) 63 | -------------------------------------------------------------------------------- /src/qu/data/compression.clj: -------------------------------------------------------------------------------- 1 | (ns qu.data.compression 2 | "Functions for compressing and uncompressing data going into and 3 | coming out of Mongo." 4 | (:require [clojure.walk :refer [postwalk]] 5 | [monger.key-compression :as mzip] 6 | [qu.metrics :as metrics] 7 | [qu.util :refer :all])) 8 | 9 | (defn- slice-columns 10 | [slicedef] 11 | (concat (:dimensions slicedef) (:metrics slicedef))) 12 | 13 | (defn field-zip-fn 14 | "Given a slice definition, return a function that will compress 15 | field names." 16 | [slicedef] 17 | (let [fields (slice-columns slicedef)] 18 | (metrics/with-timing "queries.fields.zip" 19 | (mzip/compression-fn fields)))) 20 | 21 | (defn field-unzip-fn 22 | "Given a slice definition, return a function that will decompress 23 | field names." 24 | [slicedef] 25 | (let [fields (slice-columns slicedef)] 26 | (metrics/with-timing "queries.fields.unzip" 27 | (mzip/decompression-fn fields)))) 28 | 29 | (defn compress-fields 30 | [fields zipfn] 31 | (convert-keys fields zipfn)) 32 | 33 | (defn compress-where 34 | [where zipfn] 35 | (let [f (fn [[k v]] (if (keyword? k) [(zipfn k) v] [k v]))] 36 | ;; only apply to maps 37 | (postwalk (fn [x] (if (map? x) (into {} (map f x)) x)) where))) 38 | 39 | (defn compress-find 40 | [find-map zipfn] 41 | (-> find-map 42 | (update-in [:query] compress-where zipfn) 43 | (update-in [:fields] convert-keys zipfn) 44 | (update-in [:sort] convert-keys zipfn))) 45 | -------------------------------------------------------------------------------- /src/qu/data/definition.clj: -------------------------------------------------------------------------------- 1 | (ns qu.data.definition 2 | "Functions for reading and altering dataset definitions. Includes 3 | schema for validation." 4 | (:require [cheshire.core :as json] 5 | [cheshire.factory :as factory] 6 | [schema.core :as s])) 7 | 8 | (def InfoS {(s/required-key :name) s/Str 9 | (s/optional-key :description) s/Str 10 | (s/optional-key :url) s/Str 11 | s/Keyword s/Any 12 | }) 13 | 14 | (def TypeS (s/enum "string" "integer" "date" "dollars" "number" "boolean" "lookup")) 15 | 16 | (def IndexS (s/either s/Str [s/Str])) 17 | 18 | (defn ref-col-count-eq? 19 | "Make sure that if you have multiple columns in a reference that you 20 | have the same number of columns in the id." 21 | [ref] 22 | (if (coll? (:column ref)) 23 | (= (count (:column ref)) (count (:id ref))) 24 | (not (coll? (:id ref))))) 25 | 26 | (def ReferenceS (s/both (s/pred ref-col-count-eq?) 27 | {(s/required-key :column) (s/either s/Str [s/Str]) 28 | (s/required-key :concept) s/Str 29 | (s/optional-key :id) (s/either s/Str [s/Str]) 30 | (s/required-key :value) s/Str 31 | })) 32 | 33 | (def TableSliceS {(s/optional-key :info) {s/Keyword s/Str} 34 | (s/required-key :type) (s/eq "table") 35 | (s/required-key :table) s/Str 36 | (s/required-key :dimensions) [s/Str] 37 | (s/required-key :metrics) [s/Str] 38 | (s/optional-key :indexes) [IndexS] 39 | (s/optional-key :references) {s/Keyword ReferenceS} 40 | (s/optional-key :max-group-fields) Integer 41 | }) 42 | 43 | (def DerivedSliceS {(s/optional-key :info) {s/Keyword s/Str} 44 | (s/required-key :type) (s/eq "derived") 45 | (s/required-key :slice) s/Str 46 | (s/required-key :dimensions) [s/Str] 47 | (s/required-key :metrics) [s/Str] ;; TODO remove metrics from here 48 | (s/optional-key :indexes) [IndexS] 49 | (s/required-key :aggregations) {s/Keyword [s/Str]} 50 | (s/optional-key :where) s/Str 51 | }) 52 | 53 | (def SliceS (s/either TableSliceS DerivedSliceS)) 54 | 55 | (def SimpleConceptS {(s/optional-key :name) s/Str 56 | (s/optional-key :description) s/Str 57 | (s/optional-key :type) TypeS}) 58 | (def TableConceptS (merge SimpleConceptS 59 | {(s/required-key :table) s/Str 60 | (s/required-key :properties) {s/Keyword 61 | {(s/required-key :type) TypeS 62 | s/Keyword s/Str}}})) 63 | (def ConceptS (s/either SimpleConceptS TableConceptS)) 64 | 65 | (def ColumnS {(s/optional-key :name) s/Str 66 | (s/optional-key :skip) s/Bool 67 | (s/optional-key :type) TypeS 68 | (s/optional-key :lookup) {s/Any s/Any} 69 | (s/optional-key :format) s/Str}) 70 | 71 | (def TableS {:sources [s/Str] 72 | :columns {s/Keyword ColumnS}}) 73 | 74 | (def DataDefinitionS {(s/required-key :info) InfoS 75 | (s/required-key :slices) {s/Keyword SliceS} 76 | (s/optional-key :concepts) {s/Keyword ConceptS} 77 | (s/required-key :tables) {s/Keyword TableS} 78 | }) 79 | 80 | (defn read-definition 81 | "Read the definition of a dataset from disk or URL." 82 | [f] 83 | (binding [factory/*json-factory* (factory/make-json-factory {:allow-comments true})] 84 | (s/validate DataDefinitionS 85 | (-> (slurp f) 86 | (json/parse-string true))))) 87 | 88 | (defn dimensions 89 | "Get the list of dimensions for a slice." 90 | [definition slice] 91 | (map keyword (get-in definition [:slices slice :dimensions]))) 92 | 93 | (defn metrics 94 | "Get the list of metrics for a slice." 95 | [definition slice] 96 | (let [slicedef (get-in definition [:slices slice])] 97 | (if (= (:type slicedef) "table") 98 | (map keyword (:metrics slicedef)) 99 | (keys (:aggregations slicedef))))) 100 | 101 | (defn fields 102 | "Get the list of fields for a slice." 103 | [definition slice] 104 | (apply concat ((juxt dimensions metrics) definition slice))) 105 | 106 | (defn indexes 107 | "Get the list of indexed fields for a slice." 108 | [definition slice] 109 | (let [slicedef (get-in definition [:slices slice])] 110 | (or (:indexes slicedef) 111 | (:index_only slicedef) ; deprecated 112 | (:dimensions slicedef)))) 113 | 114 | ;; (read-definition (io/resource "datasets/integration_test/definition.json")) 115 | ;; (read-definition (io/resource "datasets/hmda/definition.json")) 116 | ;; (s/explain DataDefinitionS) 117 | -------------------------------------------------------------------------------- /src/qu/data/result.clj: -------------------------------------------------------------------------------- 1 | (ns qu.data.result) 2 | 3 | (defrecord DataResult [total size data]) 4 | -------------------------------------------------------------------------------- /src/qu/data/source.clj: -------------------------------------------------------------------------------- 1 | (ns qu.data.source) 2 | 3 | (defprotocol DataSource 4 | (get-datasets [source]) ;; returns a list of all datasets 5 | (get-metadata [source dataset]) ;; returns the metadata for a dataset 6 | (get-concept-data [source dataset concept]) ;; returns the data table for a concept 7 | (get-results [source query]) 8 | (load-dataset [source definition options])) 9 | -------------------------------------------------------------------------------- /src/qu/env.clj: -------------------------------------------------------------------------------- 1 | (ns qu.env 2 | (:require [environ.core :as environ])) 3 | 4 | (def ^{:doc "A map of environment variables."} 5 | env 6 | (let [config-file (:qu-config environ/env)] 7 | (if config-file 8 | (binding [*read-eval* false] 9 | (read-string (slurp config-file))) 10 | environ/env))) 11 | -------------------------------------------------------------------------------- /src/qu/etag.clj: -------------------------------------------------------------------------------- 1 | (ns qu.etag 2 | (:require [digest :refer [md5]]) 3 | (:import (java.io File))) 4 | 5 | (defmulti calculate-etag class) 6 | (defmethod calculate-etag String [s] (md5 s)) 7 | (defmethod calculate-etag File 8 | [f] 9 | (str (.lastModified f) "-" (.length f))) 10 | (defmethod calculate-etag :default 11 | [_] 12 | nil) 13 | 14 | (defn- not-modified-response [etag] 15 | {:status 304 :body "" :headers {"etag" etag}}) 16 | 17 | (defn wrap-etag [handler] 18 | "Generates an etag header by hashing response body (currently only 19 | supported for string bodies). If the request includes a matching 20 | 'if-none-match' header then return a 304." 21 | (fn [req] 22 | (let [{body :body 23 | status :status 24 | {etag "ETag"} :headers 25 | :as resp} (handler req) 26 | if-none-match (get-in req [:headers "if-none-match"])] 27 | (if (and etag (not= status 304)) 28 | (if (= etag if-none-match) 29 | (not-modified-response etag) 30 | resp) 31 | (if (and (or (string? body) (instance? File body)) 32 | (= status 200)) 33 | (let [etag (calculate-etag body)] 34 | (if (= etag if-none-match) 35 | (not-modified-response etag) 36 | (assoc-in resp [:headers "ETag"] etag))) 37 | resp))))) 38 | -------------------------------------------------------------------------------- /src/qu/generate/static.clj: -------------------------------------------------------------------------------- 1 | (ns qu.generate.static 2 | (:require [clojure.java.io :as io] 3 | [me.raynes.fs :as fs])) 4 | 5 | (defn -main 6 | [& args] 7 | (let [resource-dir (fs/file (io/resource "static/")) 8 | out-dir (if-let [path (first args)] 9 | (fs/file path) 10 | (fs/file "resources/static"))] 11 | 12 | (when (fs/exists? out-dir) 13 | (println "Output dir exists. Delete first. Cancelling.") 14 | (System/exit 1)) 15 | 16 | (fs/copy-dir resource-dir out-dir) 17 | (println "Qu static files copied into" (str out-dir) "."))) 18 | -------------------------------------------------------------------------------- /src/qu/generate/templates.clj: -------------------------------------------------------------------------------- 1 | (ns qu.generate.templates 2 | (:require [clojure.java.io :as io] 3 | [me.raynes.fs :as fs])) 4 | 5 | (defn -main 6 | [& args] 7 | (let [resource-dir (fs/file (io/resource "qu/templates/")) 8 | out-dir (if-let [path (first args)] 9 | (fs/file path) 10 | (fs/file "resources/qu/templates"))] 11 | 12 | (when (fs/exists? out-dir) 13 | (println "Output dir exists. Delete first. Cancelling.") 14 | (System/exit 1)) 15 | 16 | (fs/copy-dir resource-dir out-dir) 17 | (println "Qu template files copied into" (str out-dir) "."))) 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/qu/logging.clj: -------------------------------------------------------------------------------- 1 | (ns qu.logging 2 | (:require [clojure.string :as str] 3 | [qu.metrics :as metrics] 4 | [taoensso.timbre :as log :refer [error info]])) 5 | 6 | (def ^:dynamic *log-id* "------") 7 | 8 | (defn make-log-id 9 | [] 10 | (subs (str (java.util.UUID/randomUUID)) 0 6)) 11 | 12 | (defn format-output-fn 13 | [{:keys [level throwable message timestamp]} 14 | ;; Any extra appender-specific opts: 15 | & [{:keys [nofonts?] :as appender-fmt-output-opts}]] 16 | (format "%s %s [%s] - %s%s" 17 | timestamp 18 | (-> level name str/upper-case) 19 | *log-id* 20 | (or message "") 21 | (or (log/stacktrace throwable "\n" (when nofonts? {})) ""))) 22 | 23 | (defn config 24 | [level file] 25 | (log/set-level! level) 26 | (log/set-config! [:timestamp-pattern] "yyyy-MM-dd'T'HH:mm:ssZZ") 27 | (log/set-config! [:fmt-output-fn] format-output-fn) 28 | (when file 29 | (println "Sending log output to" file) 30 | (log/set-config! [:appenders :spit :enabled?] true) 31 | (log/set-config! [:shared-appender-config :spit-filename] file) 32 | (log/set-config! [:appenders :standard-out :enabled?] false))) 33 | 34 | (defn- log-request-msg 35 | [verb {:keys [request-method uri remote-addr query-string params] :as req}] 36 | (let [uri (if query-string (str uri "?" query-string) uri)] 37 | (str/join " " [verb (str/upper-case (name request-method)) uri]))) 38 | 39 | (defn- log-request 40 | [{:keys [params] :as req}] 41 | (info (log-request-msg "Started" req))) 42 | 43 | (defn- log-response 44 | [req {:keys [status] :as resp} total] 45 | (let [msg (log-request-msg "Finished" req) 46 | ms (str total "ms")] 47 | (if (and (number? status) 48 | (>= status 500)) 49 | (error msg ms status) 50 | (info msg ms status)))) 51 | 52 | (defn- log-exception 53 | [req ex total] 54 | (metrics/increment "request.exception.count") 55 | (error (log-request-msg "Exception" req) (str total "ms")) 56 | (error ex) 57 | (error "--- END STACKTRACE ---")) 58 | 59 | (defmacro log-with-time 60 | [level msg & body] 61 | `(let [start# (System/currentTimeMillis) 62 | result# (do ~@body) 63 | finish# (System/currentTimeMillis) 64 | ms# (- finish# start#)] 65 | (log/log ~level ~msg (str ms# "ms")) 66 | result#)) 67 | 68 | 69 | 70 | 71 | 72 | (defn wrap-with-logging 73 | [handler] 74 | (fn [request] 75 | (binding [*log-id* (make-log-id)] 76 | (let [start (System/currentTimeMillis)] 77 | (try 78 | (log-request request) 79 | (let [response (handler request) 80 | finish (System/currentTimeMillis) 81 | total (- finish start)] 82 | (log-response request response total) 83 | (metrics/time-request request response total) 84 | response) 85 | (catch Throwable ex 86 | (let [finish (System/currentTimeMillis) 87 | total (- finish start)] 88 | (log-exception request ex total)) 89 | (throw ex))))))) 90 | -------------------------------------------------------------------------------- /src/qu/main.clj: -------------------------------------------------------------------------------- 1 | (ns qu.main 2 | (:require [com.stuartsierra.component :as component] 3 | [qu.app :refer [new-qu-system]] 4 | [qu.env :refer [env]] 5 | [qu.project :refer [project]] 6 | [qu.util :refer [->bool ->int]] 7 | [taoensso.timbre :as log]) 8 | (:gen-class :main true)) 9 | 10 | (defn- print-live-threads 11 | [] 12 | (let [mx-bean (java.lang.management.ManagementFactory/getThreadMXBean) 13 | stack (seq (.dumpAllThreads mx-bean true true))] 14 | (fn [] 15 | (doseq [thread stack] 16 | (println thread))))) 17 | 18 | (defn add-shutdown-hook 19 | "Add a shutdown hook that prints all live threads" 20 | [] 21 | (.addShutdownHook (Runtime/getRuntime) (Thread. (print-live-threads)))) 22 | 23 | (defn default-log-options 24 | [] 25 | (let [log-file (:log-file env) 26 | log-level (:log-level env)] 27 | {:file log-file 28 | :level log-level})) 29 | 30 | (defn default-mongo-options 31 | [] 32 | (let [uri (env :mongo-uri) 33 | hosts (env :mongo-hosts) 34 | host (env :mongo-host) 35 | port (->int (env :mongo-port)) 36 | options (env :mongo-options {}) 37 | auth (env :mongo-auth)] 38 | {:conn {:uri uri 39 | :hosts hosts 40 | :host host 41 | :port port} 42 | :options options 43 | :auth auth})) 44 | 45 | (defn default-view-data 46 | [] 47 | {:qu_version (:version @project) 48 | :build_number (:build-number @project) 49 | :build_url (:build-url @project) 50 | :base_url (:app-url env) 51 | :api_name (:api-name env) 52 | :dev_mode (:dev env)}) 53 | 54 | (defn default-http-options 55 | [] 56 | {:ip (:http-ip env) 57 | :port (->int (:http-port env)) 58 | :threads (->int (:http-threads env)) 59 | :queue-size (->int (:http-queue-size env)) 60 | :view (default-view-data)}) 61 | 62 | (defn default-metrics-options 63 | [] 64 | (if (:statsd-host env) 65 | {:provider :statsd 66 | :host (:statsd-host env) 67 | :port (->int (:statsd-port env))} 68 | {})) 69 | 70 | (defn default-options 71 | [] 72 | {:dev (->bool (:dev env)) 73 | :http (default-http-options) 74 | :log (default-log-options) 75 | :mongo (default-mongo-options) 76 | :metrics (default-metrics-options)}) 77 | 78 | (defn load-config 79 | "Load configuration from an outside file." 80 | [config-file] 81 | (binding [*read-eval* false] 82 | (read-string (slurp config-file)))) 83 | 84 | (defn -main 85 | [& args] 86 | (let [config (if (= (count args) 1) 87 | (load-config (first args)) 88 | (default-options))] 89 | (add-shutdown-hook) 90 | (component/start (new-qu-system config)) 91 | (when (:dev env) 92 | (log/info "Dev mode enabled")))) 93 | 94 | -------------------------------------------------------------------------------- /src/qu/metrics.clj: -------------------------------------------------------------------------------- 1 | (ns qu.metrics 2 | "Captures application metrics. Currently uses clj-statsd to pipe metrics through StatsD" 3 | (:require [clj-statsd :as sd] 4 | [clojure.string :as str] 5 | [taoensso.timbre :as log])) 6 | 7 | (def prefix (delay (str "qu." (.getHostName (java.net.InetAddress/getLocalHost)) "."))) 8 | 9 | (defn setup 10 | [host port] 11 | (log/info "Starting statsd metrics at" (str host ":" port)) 12 | (sd/setup host port)) 13 | 14 | (defn prefix-metric 15 | "Ensure consistent prefix for all metrics" 16 | [key] 17 | (str @prefix key)) 18 | 19 | (defn increment 20 | [key & args] 21 | (apply sd/increment (prefix-metric key) args)) 22 | 23 | (defn decrement 24 | [key & args] 25 | (apply sd/decrement (prefix-metric key) args)) 26 | 27 | (defn gauge 28 | [key & args] 29 | (apply sd/gauge (prefix-metric key) args)) 30 | 31 | (defn unique 32 | [key & args] 33 | (apply sd/unique (prefix-metric key) args)) 34 | 35 | (defmacro with-sampled-timing 36 | [key rate & body] 37 | `(sd/with-sampled-timing (prefix-metric ~key) ~rate ~@body)) 38 | 39 | (defmacro with-timing 40 | [key & body] 41 | `(sd/with-timing (prefix-metric ~key) ~@body)) 42 | 43 | (defn metrics-path 44 | [uri] 45 | (let [parts (str/split uri #"\.")] 46 | (cond 47 | (str/blank? (first parts)) "/data.html" 48 | (= 1 (count parts)) (str uri ".html") 49 | :else uri))) 50 | 51 | (defn- is-4xx? 52 | [response] 53 | (let [status (:status response)] 54 | (= (quot status 100) 4))) 55 | 56 | (defn time-request 57 | [request response total] 58 | (let [key (str "request.url." (metrics-path (:uri request)) ".time")] 59 | (when (not (is-4xx? response)) 60 | (sd/timing (prefix-metric key) total)))) -------------------------------------------------------------------------------- /src/qu/middleware/keyword_params.clj: -------------------------------------------------------------------------------- 1 | (ns qu.middleware.keyword-params 2 | "Convert param keys to keywords. This replaces 3 | ring.middleware.keyword-params as that does not turn all potential 4 | keys to keywords.") 5 | 6 | (defn- keyword-syntax? [s] 7 | (re-matches #"[A-Za-z\.*+!-?$_%&=][A-Za-z0-9\.*+!-?$_%&=#]*" s)) 8 | 9 | (defn- keyify-params [target] 10 | (cond 11 | (map? target) 12 | (into {} 13 | (for [[k v] target] 14 | [(if (and (string? k) (keyword-syntax? k)) 15 | (keyword k) 16 | k) 17 | (keyify-params v)])) 18 | (vector? target) 19 | (vec (map keyify-params target)) 20 | :else 21 | target)) 22 | 23 | (defn keyword-params-request 24 | "Converts string keys in :params map to keywords." 25 | [req] 26 | (update-in req [:params] keyify-params)) 27 | 28 | (defn wrap-keyword-params 29 | "Middleware that converts the string-keyed :params map to one with keyword 30 | keys before forwarding the request to the given handler. 31 | Does not alter the maps under :*-params keys; these are left with strings." 32 | [handler] 33 | (fn [req] 34 | (-> req 35 | keyword-params-request 36 | handler))) 37 | -------------------------------------------------------------------------------- /src/qu/middleware/stacktrace.clj: -------------------------------------------------------------------------------- 1 | (ns qu.middleware.stacktrace 2 | "Catch exceptions and render web and log stacktraces for debugging." 3 | (:require [qu.views :as views])) 4 | 5 | (defn- ex-response 6 | [req ex] 7 | (if-let [accept (get-in req [:headers "accept"])] 8 | (cond 9 | (re-find #"^json" accept) (views/json-error 500) 10 | :else (views/error-html)) 11 | (views/error-html))) 12 | 13 | (defn wrap-stacktrace-web 14 | "Wrap a handler such that exceptions are caught and no information 15 | is leaked to the user." 16 | [handler] 17 | (fn [request] 18 | (try 19 | (handler request) 20 | (catch Exception ex 21 | (ex-response request ex))))) 22 | -------------------------------------------------------------------------------- /src/qu/middleware/uri_rewrite.clj: -------------------------------------------------------------------------------- 1 | (ns qu.middleware.uri-rewrite 2 | (:require [clojure.string :as str])) 3 | 4 | (defn with-uri-rewrite 5 | "Rewrites a request uri with the result of calling f with the 6 | request's original uri. If f returns nil the handler is not called." 7 | [handler f] 8 | (fn [request] 9 | (let [uri (:uri request) 10 | rewrite (f uri)] 11 | (if rewrite 12 | (handler (assoc request :uri rewrite)) 13 | nil)))) 14 | 15 | (defn- uri-snip-slash 16 | "Removes a trailing slash from all uris except \"/\"." 17 | [uri] 18 | (if (= "/" uri) 19 | uri 20 | (str/replace uri #"/$" ""))) 21 | 22 | (defn wrap-ignore-trailing-slash 23 | "Makes routes match regardless of whether or not a uri ends in a slash." 24 | [handler] 25 | (with-uri-rewrite handler uri-snip-slash)) 26 | -------------------------------------------------------------------------------- /src/qu/query.clj: -------------------------------------------------------------------------------- 1 | (ns qu.query 2 | "Functions to build and execute queries." 3 | (:require [clojure.string :as str] 4 | [monger.query :as q] 5 | [qu.cache :refer [query-to-key]] 6 | [qu.data :as data] 7 | [qu.metrics :as metrics] 8 | [qu.query.mongo :as mongo] 9 | [qu.query.select :as select] 10 | [qu.query.validation :as validation] 11 | [qu.util :refer [->int]] 12 | [taoensso.timbre :as log])) 13 | 14 | (defrecord ^{:doc "This record contains all the information about a 15 | query. Much of this comes from requests to the system. The rest 16 | is accreted throughout the query parsing and verification process. 17 | This record uses camelCase for orderBy, even though that is 18 | non-idiomatic for Clojure, to highlight the parallel to the 19 | orderBy GET parameter, which is part of the established API."} 20 | Query 21 | [select group where orderBy limit offset callback 22 | mongo errors 23 | dataset slice metadata slicedef]) 24 | 25 | (def default-limit 100) 26 | (def default-offset 0) 27 | (def allowed-clauses 28 | #{:$select :$where :$orderBy :$group :$limit :$offset :$callback :$page :$perPage}) 29 | 30 | (defn valid? 31 | [query] 32 | (validation/valid? query)) 33 | 34 | (defn parse-params 35 | "Given a slice definition and the request parameters, convert those 36 | parameters into something we can use. Specifically, pull out the clauses." 37 | [params] 38 | (into {} (filter (fn [[key value]] 39 | (and 40 | (not= value "") 41 | (allowed-clauses key))) params))) 42 | 43 | (defn mongo-find 44 | "Create a Mongo find map from the query." 45 | [query] 46 | (let [mongo (q/partial-query 47 | (q/find (get-in query [:mongo :match])) 48 | (q/limit (->int (:limit query) default-limit)) 49 | (q/skip (->int (:offset query) default-offset)) 50 | (q/sort (get-in query [:mongo :sort] {})) 51 | (q/fields (or (get-in query [:mongo :project]) {})))] 52 | mongo)) 53 | 54 | (defn mongo-aggregation 55 | "Create a Mongo map-reduce aggregation map from the query." 56 | [{:keys [dataset slice mongo] :as query}] 57 | (let [filter (:match mongo {}) 58 | fields (get-in mongo [:project :fields]) 59 | aggregations (get-in mongo [:project :aggregations]) 60 | a-query (dissoc query :metadata :slicedef) 61 | to-collection (query-to-key query)] 62 | {:query a-query 63 | :dataset dataset 64 | :from slice 65 | :to to-collection 66 | :group (:group mongo) 67 | :aggregations aggregations 68 | :filter filter 69 | :fields fields 70 | :sort (:sort mongo) 71 | :limit (->int (:limit query)) 72 | :offset (->int (:offset query)) 73 | :slicedef (:slicedef query)})) 74 | 75 | (defn is-aggregation? [query] 76 | (:group query false)) 77 | 78 | (defn- resolve-limit-and-offset [{:keys [limit offset page perPage] :as query}] 79 | (let [limit (or (->int limit nil) 80 | (->int perPage nil) 81 | default-limit) 82 | offset (->int offset nil) 83 | page (->int page (if (zero? limit) 84 | nil 85 | (when (and offset 86 | (zero? (mod offset limit))) 87 | (inc (/ offset limit)))))] 88 | (cond 89 | page (merge query {:offset (-> page 90 | dec 91 | (* limit)) 92 | :limit limit 93 | :page page}) 94 | offset (merge query {:offset offset 95 | :limit limit 96 | :page page}) 97 | :default (merge query {:offset default-offset 98 | :limit limit 99 | :page 1})))) 100 | 101 | (defn prepare 102 | "Prepare the query for execution." 103 | [query] 104 | (-> query 105 | validation/validate 106 | resolve-limit-and-offset 107 | mongo/process 108 | (assoc :prepared? true))) 109 | 110 | (defn execute 111 | "Execute the query against the provided collection. This function 112 | does not follow the same API as the rest of the functions in this 113 | namespace: that is, take a query + params, return a query. Instead 114 | we return the query results. The results are kept out of the query 115 | record so that they can be garbage-collected as we iterate through 116 | them." 117 | [{:keys [dataset slice] :as query}] 118 | 119 | (metrics/with-timing "queries.execute" 120 | (let [_ (log/info "Execute query" (str (into {} (dissoc query :metadata :slicedef)))) 121 | query (if (:prepared? query) query (prepare query))] 122 | (cond 123 | (not (valid? query)) 124 | (do 125 | (metrics/increment "queries.invalid.count") 126 | []) 127 | 128 | (is-aggregation? query) 129 | (let [agg (mongo-aggregation query)] 130 | (data/get-aggregation dataset slice agg)) 131 | 132 | :default 133 | (data/get-find dataset slice (mongo-find query)))))) 134 | 135 | (defn params->Query 136 | "Convert params from a web request plus a dataset definition and a 137 | slice name into a Query record." 138 | [params metadata slice] 139 | (let [slicedef (get-in metadata [:slices (keyword slice)]) 140 | dataset (:name metadata) 141 | clauses (parse-params params) 142 | {select :$select 143 | group :$group 144 | orderBy :$orderBy 145 | where :$where 146 | limit :$limit 147 | offset :$offset 148 | page :$page 149 | perPage :$perPage 150 | callback :$callback} clauses] 151 | (map->Query {:select select 152 | :group group 153 | :where where 154 | :limit limit 155 | :offset offset 156 | :page page 157 | :perPage perPage 158 | :orderBy orderBy 159 | :callback callback 160 | :metadata metadata 161 | :dataset dataset 162 | :slice slice 163 | :slicedef slicedef}))) 164 | 165 | (defn columns 166 | "Return list of columns to be used in results. Assumes a prepared query." 167 | [{:keys [select slicedef] :as query}] 168 | (if (or (str/blank? select) 169 | (seq (:errors query))) 170 | (data/slice-columns slicedef) 171 | (map (comp name :select) (select/parse select)))) 172 | 173 | (defn make-query 174 | "Convenience function to quickly make a query for testing at the 175 | REPL." 176 | [{:keys [dataset slice] :as q}] 177 | {:pre [(every? #(not (nil? %)) [dataset slice])]} 178 | (let [metadata (data/get-metadata dataset) 179 | slicedef (get-in metadata [:slices (keyword slice)])] 180 | (-> q 181 | (assoc :metadata metadata :slicedef slicedef) 182 | (map->Query)))) 183 | -------------------------------------------------------------------------------- /src/qu/query/mongo.clj: -------------------------------------------------------------------------------- 1 | (ns qu.query.mongo 2 | "This namespace populates a query with a Mongo translation of that 3 | query. `process` is the main entry point, although the individual 4 | functions have been left public, mainly for testing. 5 | 6 | All public functions in this namespace should adhere to the 7 | following contract: 8 | 9 | * They take a qu.query/Query. 10 | * They return a qu.query/Query. 11 | * If any errors are found, they populate :errors on that Query and abort. 12 | * If not, they populate :mongo on that Query." 13 | (:require [clojure.set :as set] 14 | [clojure.string :as str] 15 | [clojure.walk :as walk] 16 | [protoflex.parse :refer [parse]] 17 | [qu.query.parser :as parser] 18 | [qu.query.select :as select] 19 | [qu.query.validation :refer [add-error valid? 20 | validate-field]] 21 | [qu.query.where :as where]) 22 | (:refer-clojure :exclude [sort])) 23 | 24 | (declare match project group sort post-validate) 25 | 26 | (defn process 27 | "Process the original query through the various filters used to 28 | create the Mongo representation of the query. Main entry point into 29 | this namespace." 30 | [query] 31 | (if (valid? query) 32 | (-> query 33 | match 34 | project 35 | group 36 | sort 37 | post-validate) 38 | query)) 39 | 40 | (defn match 41 | "Add the :match provision of the Mongo query. Assemble the match 42 | from the :where of the origin query." 43 | [query] 44 | (if (:where query) 45 | (let [parse #(where/mongo-eval (where/parse %)) 46 | match (parse (str (:where query)))] 47 | (assoc-in query [:mongo :match] match)) 48 | query)) 49 | 50 | (defn project 51 | "Add the :project provision of the Mongo query from the :select of 52 | the origin query." 53 | [query] 54 | (if-let [select (str (:select query))] 55 | (let [project (select/mongo-eval (select/parse select) 56 | :aggregation (:group query false))] 57 | (assoc-in query [:mongo :project] project)) 58 | query)) 59 | 60 | (defn group 61 | "Add the :group provision of the Mongo query, using both the :select 62 | and :group provisions of the original query." 63 | [query] 64 | (if-let [group (str (:group query))] 65 | (let [columns (parse parser/group-expr group)] 66 | (assoc-in query [:mongo :group] columns)) 67 | query)) 68 | 69 | (defn sort 70 | "Add the :sort provision of the Mongo query." 71 | [query] 72 | (let [order (str (:orderBy query))] 73 | (if-not (str/blank? order) 74 | (let [sort (->> order 75 | (parse parser/order-by-expr) 76 | (map (fn [[field dir]] 77 | (if (= dir :ASC) 78 | [field 1] 79 | [field -1]))) 80 | flatten 81 | (apply array-map))] 82 | (assoc-in query [:mongo :sort] sort)) 83 | query))) 84 | 85 | (defn- match-fields [match] 86 | (->> match 87 | (walk/prewalk (fn [element] 88 | (if (map? element) 89 | (vec element) 90 | element))) 91 | flatten 92 | (filter #(and (keyword? %) (not (= \$ (first (name %)))))))) 93 | 94 | (defn- validate-match-fields [query metadata slice] 95 | (let [fields (match-fields (get-in query [:mongo :match]))] 96 | (reduce #(validate-field %1 :where %2) query fields))) 97 | 98 | (defn- validate-order-fields-aggregation [query order-fields] 99 | (let [available-fields (set (get-in query [:mongo :project :fields])) 100 | order-fields (set (map keyword order-fields)) 101 | invalid-fields (set/difference order-fields available-fields)] 102 | (reduce #(add-error %1 :orderBy 103 | (str "\"" (name %2) 104 | "\" is not an available field for sorting.")) 105 | query invalid-fields))) 106 | 107 | (defn- validate-order-fields [query metadata slice] 108 | (let [order-fields (keys (get-in query [:mongo :sort])) 109 | group (get-in query [:mongo :group])] 110 | (if (str/blank? (:group query)) 111 | (reduce #(validate-field %1 :orderBy %2) query order-fields) 112 | (validate-order-fields-aggregation query order-fields)))) 113 | 114 | (defn post-validate [query] 115 | (let [metadata (:metadata query) 116 | slice (:slice query)] 117 | (-> query 118 | (validate-match-fields metadata slice) 119 | (validate-order-fields metadata slice)))) 120 | 121 | -------------------------------------------------------------------------------- /src/qu/query/parser.clj: -------------------------------------------------------------------------------- 1 | (ns qu.query.parser 2 | "Parse functions for queries." 3 | (:require [clj-time.core :as time] 4 | [clojure.string :as str] 5 | [protoflex.parse :refer [any attempt chr chr-in dq-str 6 | multi* number parens regex sep-by* 7 | series sq-str starts-with? 8 | string-in]])) 9 | 10 | (def identifier-regex #"[A-Za-z][A-Za-z0-9\-_]*") 11 | 12 | (defn- ci-string 13 | "Match case insensitive strings." 14 | [string] 15 | (regex (re-pattern (str "(?i)\\Q" string "\\E")))) 16 | 17 | (defn- string-literal [] 18 | (any dq-str sq-str)) 19 | 20 | (defn- date-literal [] 21 | (let [year (regex #"\d{4}") 22 | _ (chr-in [\- \/]) 23 | month (regex #"\d{1,2}") 24 | _ (chr-in [\- \/]) 25 | day (regex #"\d{1,2}") 26 | ymd (map #(Integer/parseInt %) [year month day])] 27 | (apply time/date-time ymd))) 28 | 29 | (defn- boolean-literal 30 | "Match the boolean literals true and false. Case-insensitive. We 31 | have to return a map instead of the boolean value because returning 32 | false will make the parser think it's failed to match." 33 | [] 34 | (let [lit (any #(ci-string "true") 35 | #(ci-string "false"))] 36 | {:bool (= (str/lower-case lit) "true")})) 37 | 38 | (defn value 39 | "Parse expression for values in WHERE queries. Valid values are numbers, 40 | numeric expressions, strings, and booleans." 41 | [] 42 | (any date-literal number string-literal boolean-literal)) 43 | 44 | (defn list-of-values 45 | [] 46 | (let [_ (chr \() 47 | values (sep-by* value #(chr \,) #(chr \)))] 48 | values)) 49 | 50 | (defn- comparison-operator [] 51 | (let [op (string-in [">" ">=" "=" "!=" "<" "<=" "LIKE" "ILIKE"])] 52 | (keyword op))) 53 | 54 | (defn identifier 55 | "Parse function for identifiers in WHERE queries. Valid identifiers 56 | begin with a letter and are made up of letters, numbers, dashes, and 57 | underscores." 58 | [] 59 | (let [ident (regex identifier-regex)] 60 | (keyword ident))) 61 | 62 | (defn- comparison-normal [] 63 | (let [[identifier op value] 64 | (series identifier comparison-operator value)] 65 | {:comparison [identifier op value]})) 66 | 67 | (defn- comparison-null [] 68 | (let [identifier (identifier) 69 | is-null (ci-string "IS NULL")] 70 | {:comparison [identifier := nil]})) 71 | 72 | (defn- comparison-not-null [] 73 | (let [identifier (identifier) 74 | is-null (ci-string "IS NOT NULL")] 75 | {:comparison [identifier :!= nil]})) 76 | 77 | (defn- comparison-in [] 78 | (let [identifier (identifier) 79 | _ (ci-string "IN") 80 | values (list-of-values)] 81 | {:comparison [identifier :IN values]})) 82 | 83 | (defn comparison 84 | "Parse function for comparisons in WHERE queries. Comparisons are 85 | made up of an identifier and then either a comparison operator and a 86 | value or the phrases 'IS NULL' or 'IS NOT NULL'." 87 | [] 88 | (any comparison-normal 89 | comparison-in 90 | comparison-null 91 | comparison-not-null)) 92 | 93 | (defn- and-or-operator [] 94 | (let [op (any #(ci-string "AND") 95 | #(ci-string "OR"))] 96 | (keyword (str/upper-case op)))) 97 | 98 | (defn- not-operator [] 99 | (let [op (ci-string "NOT")] 100 | (keyword (str/upper-case op)))) 101 | 102 | (declare where-expr) 103 | 104 | (defn- paren-where-expr [] 105 | (chr \() 106 | (let [expr (where-expr)] 107 | (chr \)) 108 | expr)) 109 | 110 | (defn- boolean-factor [] 111 | (let [not-operator (attempt not-operator) 112 | factor (if (starts-with? "(") 113 | (paren-where-expr) 114 | (comparison))] 115 | (if not-operator 116 | {:not factor} 117 | factor))) 118 | 119 | (defn- build-boolean-tree 120 | "Take a vector of boolean factors separated by boolean operators and 121 | turn it into a tree built in proper precedence order." 122 | [nodes] 123 | (let [nc (count nodes)] 124 | (assert (and 125 | (>= nc 3) 126 | (odd? nc))) 127 | (if (= nc 3) 128 | {:left (nth nodes 0) :op (nth nodes 1) :right (nth nodes 2)} 129 | {:left (build-boolean-tree (take (- nc 2) nodes)) 130 | :op (nth nodes (- nc 2)) 131 | :right (nth nodes (- nc 1))}))) 132 | 133 | (defn where-expr 134 | "The parse function for valid WHERE expressions." 135 | [] 136 | (if-let [left (attempt boolean-factor)] 137 | (if-let [rhs (multi* #(series and-or-operator boolean-factor))] 138 | (build-boolean-tree 139 | (into [left] (apply concat rhs))) 140 | left))) 141 | 142 | (defn- comma [] 143 | (chr \,)) 144 | 145 | (defn- simple-select 146 | [] 147 | (let [column (identifier)] 148 | {:select column})) 149 | 150 | (defn- aggregation [] 151 | (let [agg (any #(ci-string "SUM") 152 | #(ci-string "COUNT") 153 | #(ci-string "MAX") 154 | #(ci-string "MIN") 155 | #(ci-string "AVG"))] 156 | (keyword (str/upper-case agg)))) 157 | 158 | (defn- count-select [] 159 | (let [_ (ci-string "COUNT()")] 160 | {:aggregation [:COUNT :_id] 161 | :select :count})) 162 | 163 | (defn- aggregation-select 164 | [] 165 | (let [aggregation (aggregation) 166 | column (parens identifier) 167 | alias (keyword (str (str/lower-case (name aggregation)) 168 | "_" 169 | (str/join "_" (map name (flatten (vector column))))))] 170 | {:aggregation [aggregation column] 171 | :select alias})) 172 | 173 | (defn- select 174 | [] 175 | (any aggregation-select 176 | count-select 177 | simple-select)) 178 | 179 | (defn select-expr 180 | "The parse function for valid SELECT expressions. 181 | 182 | - state 183 | - state, county 184 | - state, SUM(population) 185 | - state, SUM(population)" 186 | [] 187 | (if-let [fst (attempt select)] 188 | (if-let [rst (multi* #(series comma select))] 189 | (concat (vector fst) (map second rst)) 190 | (vector fst)))) 191 | 192 | (defn group-expr 193 | "The parse function for valid GROUP expressions." 194 | [] 195 | (if-let [fst (attempt identifier)] 196 | (if-let [rst (multi* #(series comma identifier))] 197 | (concat (vector fst) (map second rst)) 198 | (vector fst)))) 199 | 200 | (defn- order-by 201 | [] 202 | (let [mod-expr #(regex #"(?i)ASC|DESC") 203 | column (identifier) 204 | modifier (attempt mod-expr)] 205 | [column (keyword (str/upper-case (or modifier "ASC")))])) 206 | 207 | (defn order-by-expr 208 | "The parse function for valid ORDER BY expressions. 209 | 210 | - state, 211 | - state, county 212 | - state ASC 213 | - state DESC 214 | - state DESC, county 215 | 216 | ASC is the default." 217 | [] 218 | (if-let [fst (attempt order-by)] 219 | (if-let [rst (multi* #(series comma order-by))] 220 | (concat (vector fst) (map second rst)) 221 | (vector fst)))) 222 | -------------------------------------------------------------------------------- /src/qu/query/select.clj: -------------------------------------------------------------------------------- 1 | (ns qu.query.select 2 | "This namespace parses SELECT clauses into an AST." 3 | (:require [clojure.string :as str] 4 | [protoflex.parse :as p] 5 | [qu.query.parser :refer [select-expr]])) 6 | 7 | (defn parse [select] 8 | (p/parse select-expr select)) 9 | 10 | (defn- is-aggregation? [ast] 11 | (some :aggregation ast)) 12 | 13 | (defn mongo-eval-aggregation [ast] 14 | (let [ast (vec ast) 15 | aggregations (->> (filter :aggregation ast) 16 | (map (fn [{:keys [select aggregation]}] 17 | (let [aggregation (map (comp str/lower-case name) aggregation)] 18 | (if (= (first aggregation) "count") 19 | {select ["count" 1]} 20 | {select aggregation})))) 21 | (apply merge)) 22 | fields (map :select ast)] 23 | {:fields fields 24 | :aggregations aggregations})) 25 | 26 | (defn mongo-eval [ast & {:keys [aggregation]}] 27 | (if (or aggregation (is-aggregation? ast)) 28 | (mongo-eval-aggregation ast) 29 | (->> ast 30 | (map :select) 31 | (map #(hash-map % 1)) 32 | (apply merge)))) 33 | 34 | -------------------------------------------------------------------------------- /src/qu/query/validation.clj: -------------------------------------------------------------------------------- 1 | (ns qu.query.validation 2 | (:require [clojure.set :as set] 3 | [protoflex.parse :refer [parse]] 4 | [qu.data :refer [slice-columns]] 5 | [qu.query.parser :as parser] 6 | [qu.query.select :as select] 7 | [qu.query.where :as where] 8 | [qu.util :refer [->int]])) 9 | 10 | (defn valid? [query] 11 | (or (not (:errors query)) 12 | (zero? (reduce + (map count (:errors query)))))) 13 | 14 | (defn valid-field? [{:keys [metadata slice]} field] 15 | (let [fields (->> (slice-columns (get-in metadata [:slices (keyword slice)])) 16 | (map name) 17 | set) 18 | fields (conj fields "_id")] 19 | (contains? fields field))) 20 | 21 | (defn add-error 22 | [query field message] 23 | (update-in query [:errors field] 24 | (fnil #(conj % message) (vector)))) 25 | 26 | (defn validate-field 27 | [query clause field] 28 | (let [field (name field)] 29 | (if (valid-field? query field) 30 | query 31 | (add-error query clause (str "\"" field "\" is not a valid field."))))) 32 | 33 | 34 | (defn- validate-select-fields 35 | [query select] 36 | (let [fields (map (comp name #(if (:aggregation %) 37 | (second (:aggregation %)) 38 | (:select %))) 39 | select)] 40 | (reduce (fn [query field] (validate-field query :select field)) query fields))) 41 | 42 | (defn- validate-select-no-aggregations-without-group 43 | [query select] 44 | (if (:group query) 45 | query 46 | (if (some :aggregation select) 47 | (add-error query :select 48 | (str "You cannot use aggregation operators " 49 | "without specifying a group clause.")) 50 | query))) 51 | 52 | (defn- validate-select-no-non-aggregated-fields 53 | [query select] 54 | (if (:group query) 55 | (let [group-fields (set (map name (parse parser/group-expr (:group query)))) 56 | non-aggregated-fields (set (map (comp name :select) (remove :aggregation select))) 57 | invalid-fields (set/difference non-aggregated-fields group-fields)] 58 | (reduce #(add-error %1 :select 59 | (str "\"" (name %2) 60 | "\" must either be aggregated or be in the group clause.")) 61 | query invalid-fields)) 62 | query)) 63 | 64 | (defn- validate-select 65 | [query] 66 | (try 67 | (let [select (select/parse (str (:select query)))] 68 | (-> query 69 | (validate-select-fields select) 70 | (validate-select-no-aggregations-without-group select) 71 | (validate-select-no-non-aggregated-fields select))) 72 | (catch Exception e 73 | (add-error query :select "Could not parse this clause.")))) 74 | 75 | (defn- validate-group-requires-select 76 | [query] 77 | (if (:select query) 78 | query 79 | (add-error query :group "You must have a select clause to use grouping."))) 80 | 81 | (defn- validate-group-fields 82 | [query group] 83 | (reduce #(validate-field %1 :group %2) query group)) 84 | 85 | (defn- validate-group-fields-max-count 86 | [{:keys [slicedef] :as query} group] 87 | (let [max-field-count (:max-group-fields slicedef 5)] 88 | (if (> (count group) max-field-count) 89 | (add-error query :group (str "Number of group fields exceeds maximum allowed (" max-field-count ").")) 90 | query))) 91 | 92 | (defn- validate-group-only-group-dimensions 93 | [{:keys [slicedef] :as query} group] 94 | (let [dimensions (set (:dimensions slicedef))] 95 | (reduce 96 | (fn [query field] 97 | (let [field (name field)] 98 | (if (contains? dimensions field) 99 | query 100 | (add-error query :group (str "\"" field "\" is not a dimension."))))) 101 | query group))) 102 | 103 | (defn- validate-group 104 | [query] 105 | (if-let [group (:group query)] 106 | (try 107 | (let [group (parse parser/group-expr group)] 108 | (-> query 109 | validate-group-requires-select 110 | (validate-group-fields-max-count group) 111 | (validate-group-fields group) 112 | (validate-group-only-group-dimensions group))) 113 | (catch Exception e 114 | (add-error query :group "Could not parse this clause."))) 115 | query)) 116 | 117 | (defn- validate-where 118 | [query] 119 | (try 120 | (let [_ (where/parse (str (:where query)))] 121 | query) 122 | (catch Exception e 123 | (add-error query :where "Could not parse this clause.")))) 124 | 125 | (defn- validate-order-by 126 | [query] 127 | (try 128 | (let [_ (parse parser/order-by-expr (str (:orderBy query)))] 129 | query) 130 | (catch Exception e 131 | (add-error query :orderBy "Could not parse this clause.")))) 132 | 133 | (defn- validate-integer 134 | [query clause] 135 | (let [val (clause query)] 136 | (if (integer? val) 137 | query 138 | (try 139 | (let [_ (cond 140 | (integer? val) val 141 | (nil? val) 0 142 | :default (Integer/parseInt val))] 143 | query) 144 | (catch NumberFormatException e 145 | (add-error query clause "Please use an integer.")))))) 146 | 147 | (defn- validate-positive-integer 148 | [query clause] 149 | (let [query (validate-integer query clause) 150 | val (->int (clause query) 1)] 151 | (if (>= 0 val) 152 | (add-error query clause "Should be positive") 153 | query))) 154 | 155 | (defn- validate-non-negative-integer 156 | [query clause] 157 | (let [query (validate-integer query clause) 158 | val (->int (clause query) 0)] 159 | (if (> 0 val) 160 | (add-error query clause "Should be non-negative") 161 | query))) 162 | 163 | 164 | (defn- validate-max-offset 165 | [query] 166 | (let [offset (->int (:offset query) 0)] 167 | (if (> offset 10000) 168 | (add-error query :offset (str "The maximum offset is 10,000.")) 169 | query))) 170 | 171 | (defn validate-max-limit 172 | [query max] 173 | (let [limit (->int (:limit query) 0)] 174 | (if (> limit max) 175 | (add-error query :limit (str "The maximum HTML limit is " 176 | max 177 | ". The entire result set is available in CSV, JSON and XML formats.")) 178 | query))) 179 | 180 | (defn- validate-limit 181 | [query] 182 | (validate-non-negative-integer query :limit)) 183 | 184 | 185 | (defn- validate-offset 186 | [query] 187 | (validate-integer query :offset)) 188 | 189 | (defn- validate-page 190 | [query] 191 | (validate-positive-integer query :page)) 192 | 193 | 194 | (defn validate 195 | "Check the query for any errors." 196 | [query] 197 | (if-let [metadata (:metadata query)] 198 | (-> query 199 | (assoc :errors {}) 200 | validate-select 201 | validate-group 202 | validate-where 203 | validate-order-by 204 | validate-limit 205 | validate-offset 206 | validate-page 207 | validate-max-offset) 208 | query)) 209 | -------------------------------------------------------------------------------- /src/qu/query/where.clj: -------------------------------------------------------------------------------- 1 | (ns qu.query.where 2 | "This namespace parses WHERE clauses into an AST and turns that AST 3 | into a Monger query." 4 | (:require [clojure.string :as str] 5 | [clojure.walk :refer [postwalk]] 6 | [protoflex.parse :as p] 7 | [taoensso.timbre :as log] 8 | [qu.query.parser :refer [where-expr]]) 9 | (:import (java.util.regex Pattern))) 10 | 11 | (defn parse 12 | "Parse a valid WHERE expression and return an abstract syntax tree 13 | for use in constructing Mongo queries." 14 | [clause] 15 | (p/parse where-expr clause)) 16 | 17 | (def mongo-operators 18 | {:AND :$and 19 | :OR :$or 20 | :IN :$in 21 | :< :$lt 22 | :<= :$lte 23 | :> :$gt 24 | :>= :$gte 25 | :!= :$ne}) 26 | 27 | (defn mongo-not [comparison] 28 | (let [ident (first (keys comparison)) 29 | operation (first (vals comparison))] 30 | 31 | (cond 32 | (map? operation) 33 | (let [operator (first (keys operation)) 34 | value (first (vals operation))] 35 | (if (= operator :$ne) 36 | {ident value} 37 | {ident {:$not operation}})) 38 | 39 | (= (type operation) Pattern) 40 | {ident {:$not operation}} 41 | 42 | :default 43 | {ident {:$ne operation}}))) 44 | 45 | (declare mongo-eval-not) 46 | 47 | (defn sql-pattern-to-regex-str 48 | "Converts a SQL search string, such as 'foo%', into a regular expression string" 49 | [value] 50 | (str "^" 51 | (str/replace value 52 | #"[%_]|[^%_]+" 53 | (fn [match] 54 | (case match 55 | "%" ".*" 56 | "_" "." 57 | (Pattern/quote match)))) 58 | "$")) 59 | 60 | (defn like-to-regex 61 | "Converts a SQL LIKE value into a regular expression." 62 | [like] 63 | (re-pattern (sql-pattern-to-regex-str like))) 64 | 65 | (defn ilike-to-regex 66 | "Converts a SQL ILIKE value into a regular expression." 67 | [ilike] 68 | (re-pattern 69 | (str "(?i)" 70 | (sql-pattern-to-regex-str ilike)))) 71 | 72 | (defn- has-key? 73 | "Returns true if map? is a map and has the key key." 74 | [map key] 75 | (and (map? map) 76 | (contains? map key))) 77 | 78 | (defn- fix-mongo-maps 79 | "In Mongo queries, equality or comparison is done with one-element 80 | maps. mapcat turns these into 2-element vectors, so this fixes them." 81 | [evaled-operands] 82 | (postwalk #(if (and (vector? %) 83 | (= 2 (count %)) 84 | (keyword? (first %))) 85 | (apply hash-map %) 86 | %) 87 | evaled-operands)) 88 | 89 | (defn mongo-eval 90 | "Take an abstract syntax tree generated by `parse` and turn it into 91 | a valid Monger query." 92 | [ast] 93 | (cond 94 | (has-key? ast :not) 95 | (mongo-eval-not (:not ast)) 96 | 97 | ;; TODO explain this nonsense 98 | (has-key? ast :op) 99 | (let [{:keys [op left right]} ast 100 | operand-eval (fn operand-eval [operand] 101 | (if (= op (:op operand)) 102 | (mapcat operand-eval ((juxt :left :right) operand)) 103 | (mongo-eval operand)))] 104 | {(op mongo-operators) 105 | (fix-mongo-maps (mapcat operand-eval [left right]))}) 106 | 107 | (has-key? ast :comparison) 108 | (let [[ident op value] (:comparison ast) 109 | value (mongo-eval value)] 110 | (case op 111 | := {ident value} 112 | :LIKE {ident (like-to-regex value)} 113 | :ILIKE {ident (ilike-to-regex value)} 114 | {ident {(op mongo-operators) value}})) 115 | 116 | (has-key? ast :bool) 117 | (:bool ast) 118 | 119 | :default 120 | ast)) 121 | 122 | (defn- mongo-eval-not [ast] 123 | (cond 124 | (has-key? ast :not) 125 | (mongo-eval (:not ast)) 126 | 127 | (has-key? ast :op) 128 | (let [{:keys [op left right]} ast 129 | mongo-op (case op 130 | :OR :$nor 131 | :AND :$or) 132 | next-eval (case op 133 | :OR mongo-eval 134 | :AND mongo-eval-not) 135 | operand-eval (fn operand-eval [operand] 136 | (if (= op (:op operand)) 137 | (mapcat operand-eval ((juxt :left :right) operand)) 138 | (next-eval operand)))] 139 | {mongo-op 140 | (fix-mongo-maps (mapcat operand-eval [left right]))}) 141 | 142 | (has-key? ast :comparison) 143 | (mongo-not (mongo-eval ast)) 144 | 145 | :default 146 | (throw (Exception. (str "Cannot evaluate " ast " in a negative context for Mongo."))))) 147 | 148 | 149 | -------------------------------------------------------------------------------- /src/qu/routes.clj: -------------------------------------------------------------------------------- 1 | (ns qu.routes 2 | (:require [compojure.core :refer [GET routes]] 3 | [compojure.route :as route] 4 | [qu.resources :as resources] 5 | [qu.swagger :as swagger] 6 | [qu.urls :refer :all] 7 | [ring.util.response :refer [redirect]])) 8 | 9 | (defn create-app-routes 10 | "Create the app routes. Provides GET-only access to the list of 11 | datasets, individual datasets, and slices. Static files are served 12 | through Jetty, not through another web server." 13 | [webserver] 14 | (routes 15 | (GET "/" [] (redirect datasets-template)) 16 | (GET "/data.:extension" [] (resources/index webserver)) 17 | (GET datasets-template [] (resources/index webserver)) 18 | (GET "/data/:dataset.:extension" [dataset] (resources/dataset webserver)) 19 | (GET dataset-template [dataset] (resources/dataset webserver)) 20 | (GET "/data/:dataset/concept/:concept.:extension" [dataset concept] (resources/concept webserver)) 21 | (GET concept-template [dataset concept] (resources/concept webserver)) 22 | (GET "/data/:dataset/slice/:slice/metadata.:extension" [dataset slice] (resources/slice-metadata webserver)) 23 | (GET slice-metadata-template [dataset slice] (resources/slice-metadata webserver)) 24 | (GET "/data/:dataset/slice/:slice.:extension" [dataset slice] (resources/slice-query webserver)) 25 | (GET slice-query-template [dataset slice] (resources/slice-query webserver)) 26 | (GET swagger-resource-listing-template [:as req] (swagger/resource-listing-json req)) 27 | (GET swagger-api-declaration-template [api :as req] (swagger/api-declaration-json api req)) 28 | (route/resources "/static" {:root "static"}) 29 | (route/not-found (resources/not-found)))) 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/qu/templates/404.mustache: -------------------------------------------------------------------------------- 1 | {{< qu/templates/layout}} 2 | 3 | {{%body}} 4 |

Not Found

5 | 6 |
7 | {{message}} 8 |
9 | {{/body}} 10 | -------------------------------------------------------------------------------- /src/qu/templates/500.mustache: -------------------------------------------------------------------------------- 1 | {{< qu/templates/layout}} 2 | 3 | {{%body}} 4 |

Server Error

5 | 6 |
7 | The server has had an unexpected error. The site administrators have been alerted. 8 |
9 | {{/body}} 10 | -------------------------------------------------------------------------------- /src/qu/templates/concept.mustache: -------------------------------------------------------------------------------- 1 | {{< qu/templates/layout}} 2 | 3 | {{%body}} 4 |

Concept: 5 | /data/{{dataset}}/concepts/{{concept}} 6 |

7 | 8 |

Formats

9 | 14 | 15 | {{#resource.properties}} 16 |

Properties

17 |
18 | {{#description}} 19 |
Description
20 |
{{description}}
21 | {{/description}} 22 | {{#type}} 23 |
Type
24 |
{{type}}
25 | {{/type}} 26 |
27 | {{/resource.properties}} 28 | 29 | {{#has-table?}} 30 |

Table

31 | 32 | 33 | 34 | {{#columns}} 35 | 36 | {{/columns}} 37 | 38 | 39 | 40 | 41 | {{#table}} 42 | 43 | {{#.}} 44 | 45 | {{/.}} 46 | 47 | {{/table}} 48 | 49 |
{{.}}
{{.}}
50 | {{/has-table?}} 51 | {{/body}} 52 | -------------------------------------------------------------------------------- /src/qu/templates/dataset.mustache: -------------------------------------------------------------------------------- 1 | {{< qu/templates/layout}} 2 | 3 | {{%body}} 4 |

Dataset: 5 | /data/{{dataset}} 6 |

7 | 8 |

Formats

9 | 14 | 15 | {{#resource.properties}} 16 |

Properties

17 |
18 | {{#name}} 19 |
Name
20 |
{{name}}
21 | {{/name}} 22 | {{#description}} 23 |
Description
24 |
{{{description}}}
25 | {{/description}} 26 | {{#url}} 27 |
URL
28 |
29 | {{url}} 30 |
31 | {{/url}} 32 | {{#copyright}} 33 |
Copyright
34 |
{{copyright}}
35 | {{/copyright}} 36 |
37 | {{/resource.properties}} 38 |

Slices

39 | 47 |

Concepts

48 | 55 | {{/body}} 56 | -------------------------------------------------------------------------------- /src/qu/templates/index.mustache: -------------------------------------------------------------------------------- 1 | {{< qu/templates/layout}} 2 | 3 | {{%body}} 4 |

{{api_name}}

5 | 6 |

Welcome to this data API. To get started, click on one of the datasets below.

7 | 8 |

Datasets

9 | 10 | 18 | {{/body}} 19 | -------------------------------------------------------------------------------- /src/qu/templates/layout.mustache: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | {{api_name}} 9 | 10 | 11 | {{#resource}} 12 | {{#href}} 13 | 14 | {{/href}} 15 | {{#links}} 16 | 17 | {{/links}} 18 | {{/resource}} 19 | 22 | 23 | 24 | 25 |
26 |
27 | {{%body}}{{/body}} 28 |
29 |
30 | {{#dev_mode}} 31 | 51 | {{/dev_mode}} 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /src/qu/templates/pagination.mustache: -------------------------------------------------------------------------------- 1 | 13 | -------------------------------------------------------------------------------- /src/qu/templates/slice-metadata.mustache: -------------------------------------------------------------------------------- 1 | {{< qu/templates/layout}} 2 | 3 | {{%body}} 4 |

Slice: 5 | /data/{{dataset}}/slice/{{slice}}/metadata 6 |

7 | 8 |

Formats

9 | 14 | 15 | {{#resource.properties.info}} 16 |

Properties

17 |
18 | {{#name}} 19 |
Name
20 |
{{name}}
21 | {{/name}} 22 | {{#description}} 23 |
Description
24 |
{{description}}
25 | {{/description}} 26 |
27 | {{/resource.properties.info}} 28 | 29 |

Dimensions

30 | 35 | 36 |

Metrics

37 | 42 | 43 |

Types

44 |
45 | {{#types}} 46 |
{{column}}
47 |
{{type}}
48 | {{/types}} 49 |
50 | 51 | {{#has-references?}} 52 |

References

53 | 72 | {{/has-references?}} 73 | {{/body}} 74 | -------------------------------------------------------------------------------- /src/qu/templates/slice.mustache: -------------------------------------------------------------------------------- 1 | {{< qu/templates/layout}} 2 | 3 | {{%body}} 4 |
5 |

6 | Slice: /data/{{dataset}}/slice/{{slice}} 7 |

8 | 9 | {{#metadata}} 10 |

Properties

11 |
12 | {{#name}} 13 |
Name
14 |
{{name}}
15 | {{/name}} 16 | 17 | {{#description}} 18 |
Description
19 |
{{description}}
20 | {{/description}} 21 | 22 | {{#dimensions}} 23 |
Dimensions
24 |
{{dimensions}}
25 | {{/dimensions}} 26 | 27 | {{#metrics}} 28 |
Metrics
29 |
{{metrics}}
30 | {{/metrics}} 31 |
32 | 33 |

34 | View full metadata 35 |

36 | {{/metadata}} 37 |
38 | 39 |
40 |

Query This Slice

41 | 42 |

43 | Qu's query language is documented at GitHub . 44 |

45 | 46 | 47 |
48 | 49 | 50 |
51 | 52 | {{#clauses}} 53 |
54 | 58 | 59 | {{#errors}} 60 | {{.}} 61 | {{/errors}} 62 |
63 | {{/clauses}} 64 | 65 |
66 | 67 | 68 | 72 | 76 | 80 | 84 | 88 |
89 | 90 |
91 | 95 | 96 |
97 | 98 |
99 | 100 | 101 |
102 | 103 |
104 |
105 |
106 | 107 | 108 |
109 |

Query URL

110 |
111 |
112 |
113 |
114 | 115 |
116 | {{#computing?}} 117 |
118 |

Data computing

119 |

Your aggregation is currently computing. Feel free to refresh this page to see if it is available. We will refresh this page every 10 seconds until it is.

120 | {{#computing}} 121 |
122 |
Status
{{status}}
123 |
Created
{{created}}
124 |
Started
{{started}}
125 |
126 | {{/computing}} 127 | 130 |
131 | {{/computing?}} 132 |
133 | 134 | {{#has-data?}} 135 |
136 |

Query Results ({{start}}-{{end}} of {{total}})

137 | 138 | 139 | {{> qu/templates/pagination}} 140 | 141 | 142 | 143 | 144 | {{#columns}} 145 | 146 | {{/columns}} 147 | 148 | 149 | 150 | 151 | {{#data}} 152 | 153 | {{#.}} 154 | 155 | {{/.}} 156 | 157 | {{/data}} 158 | 159 |
{{.}}
{{.}}
160 | 161 | {{> qu/templates/pagination}} 162 |
163 | {{/has-data?}} 164 | {{/body}} 165 | -------------------------------------------------------------------------------- /src/qu/urls.clj: -------------------------------------------------------------------------------- 1 | (ns qu.urls 2 | (:require [clojurewerkz.route-one.core :refer [defroute]])) 3 | 4 | (defroute datasets "/data") 5 | (defroute dataset "/data/:dataset") 6 | (defroute concept "/data/:dataset/concept/:concept") 7 | (defroute slice-query "/data/:dataset/slice/:slice") 8 | (defroute slice-metadata "/data/:dataset/slice/:slice/metadata") 9 | 10 | (defroute swagger-resource-listing "/api-docs") 11 | (defroute swagger-api-declaration "/api-docs/:api") 12 | -------------------------------------------------------------------------------- /src/qu/util.clj: -------------------------------------------------------------------------------- 1 | (ns qu.util 2 | "Utility functions for use throughout Qu." 3 | (:require [cheshire.core :as json] 4 | [clojure.string :as str] 5 | [clojure.walk :refer [postwalk]] 6 | [ring.util.response :refer [content-type]])) 7 | 8 | (defn json-response 9 | "Wraps the response in the json content type and generates JSON from the content" 10 | [content] 11 | (content-type {:body (json/generate-string content)} 12 | "application/json; charset=utf-8")) 13 | 14 | (defn request-protocol 15 | [request] 16 | (if-let [proto (get-in request [:headers "x-forwarded-proto"])] 17 | proto 18 | (name (:scheme request)))) 19 | 20 | (defn base-url 21 | [req] 22 | (str (request-protocol req) "://" (get-in req [:headers "host"]))) 23 | 24 | (defn str+ 25 | [& args] 26 | (str/join (map name args))) 27 | 28 | (defn apply-kw 29 | "Like apply, but f takes keyword arguments and the last argument is 30 | not a seq but a map with the arguments for f." 31 | [f & args] 32 | {:pre [(map? (last args))]} 33 | (apply f (apply concat 34 | (butlast args) (last args)))) 35 | 36 | (defn is-int? [^Double num] 37 | (and (not (or (Double/isNaN num) 38 | (Double/isInfinite num))) 39 | (= num (Math/rint num)))) 40 | 41 | (defn ->int 42 | "Convert strings and integers to integers. A blank string or 43 | anything but a string or integer will return the default value, which 44 | is nil unless specified." 45 | ([val] (->int val nil)) 46 | ([val default] 47 | (cond 48 | (integer? val) val 49 | (and (string? val) (not (str/blank? val))) 50 | (try (Integer/parseInt val) 51 | (catch NumberFormatException e default)) 52 | :default default))) 53 | 54 | (defn ->num 55 | "Convert strings and numbers to numbers. A blank string or 56 | anything but a string or number will return the default value, which 57 | is nil unless specified." 58 | ([val] (->num val nil)) 59 | ([val default] 60 | (cond 61 | (number? val) val 62 | (and (string? val) (not (str/blank? val))) 63 | (try (Float/parseFloat val) 64 | (catch NumberFormatException e default)) 65 | :default default))) 66 | 67 | (defn ->bool 68 | "Convert anything to a boolean." 69 | [val] 70 | (not (not val))) 71 | 72 | (defn first-or-identity 73 | "If the argument is a collection, return the first element in the 74 | collection, else return the argument." 75 | [thing] 76 | (if (coll? thing) 77 | (first thing) 78 | thing)) 79 | 80 | (defn convert-keys 81 | "Recursively transforms all map keys using the provided function." 82 | [m fun] 83 | (let [f (fn [[k v]] [(fun k) v])] 84 | ;; only apply to maps 85 | (postwalk (fn [x] (if (map? x) (into {} (map f x)) x)) m))) 86 | 87 | (defn convert-keys1 88 | "Transform all map keys at the top level, no recursion." 89 | [m fun] 90 | (->> m 91 | (map (fn [[k v]] [(fun k) v])) 92 | (into {}))) 93 | 94 | (defn coll-wrap 95 | "Given an argument, return it if coll? is true, else wrap it in a list." 96 | [thing] 97 | (if (coll? thing) 98 | thing 99 | (list thing))) 100 | 101 | (defn ->print 102 | [x] 103 | (println x) 104 | x) 105 | 106 | (defn combine 107 | "Recursively merges maps. If vals are not maps, the last non-nil value wins." 108 | [& vals] 109 | (if (every? map? vals) 110 | (apply merge-with combine vals) 111 | (last (remove nil? vals)))) 112 | 113 | (defn remove-nil-vals 114 | "Remove all nil values from a map." 115 | [a-map] 116 | (postwalk (fn [x] 117 | (if (map? x) 118 | (into {} (remove (comp nil? second) x)) 119 | x)) a-map)) 120 | -------------------------------------------------------------------------------- /src/qu/writer.clj: -------------------------------------------------------------------------------- 1 | (ns qu.writer 2 | (:require [clojure.data.csv :as csv] 3 | [clojure.java.io :as io] 4 | [qu.data :as data] 5 | [qu.query :as query])) 6 | 7 | (defn gzip-writer 8 | [filename] 9 | (-> filename 10 | io/output-stream 11 | java.util.zip.GZIPOutputStream. 12 | io/writer)) 13 | 14 | (defn query->csv 15 | [query writer] 16 | (let [query (query/prepare query) 17 | results (query/execute query) 18 | columns (query/columns query) 19 | data (:data results) 20 | rows (data/get-data-table data columns)] 21 | (if (not (:computing results)) 22 | (with-open [w writer] 23 | (csv/write-csv w (vector columns)) 24 | (csv/write-csv w rows))))) 25 | -------------------------------------------------------------------------------- /test/integration/test/aggregation.clj: -------------------------------------------------------------------------------- 1 | (ns ^:integration integration.test.aggregation 2 | (:require [clojure.test :refer :all] 3 | [qu.data.aggregation :refer :all] 4 | [qu.test-util :refer :all] 5 | [qu.data :as data] 6 | [qu.data.compression :refer [compress-where field-zip-fn field-unzip-fn]] 7 | [qu.loader :as loader])) 8 | 9 | (use-fixtures :once (mongo-setup-fn "integration_test")) 10 | 11 | (deftest test-agg-query 12 | (testing "it generates the appropriate agg-query" 13 | (let [metadata (data/get-metadata "integration_test") 14 | slicedef (get-in metadata [:slices :incomes])] 15 | (let [agg-query (generate-agg-query {:dataset "integration_test" 16 | :from "incomes" 17 | :to "test1" 18 | :group [:state_abbr] 19 | :aggregations {:max_tax_returns ["max" "tax_returns"]} 20 | :slicedef slicedef}) 21 | z (comp name (field-zip-fn slicedef))] 22 | (is (= agg-query 23 | [{"$group" {:_id {:state_abbr (str "$" (z "state_abbr"))} 24 | :max_tax_returns {"$max" (str "$" (z "tax_returns"))}}} 25 | {"$project" {:_id 0 26 | :state_abbr "$_id.state_abbr" 27 | :max_tax_returns 1}} 28 | {"$out" "test1"}])))))) 29 | 30 | ;; (run-tests) 31 | -------------------------------------------------------------------------------- /test/integration/test/cache.clj: -------------------------------------------------------------------------------- 1 | (ns ^:integration integration.test.cache 2 | (:require [clojure.test :refer :all] 3 | [clojure.core.cache :as cache] 4 | [qu.test-util :refer :all] 5 | [qu.data :as data] 6 | [qu.query :as q] 7 | [qu.cache :as c] 8 | [qu.loader :as loader] 9 | [monger.core :as mongo] 10 | [monger.collection :as coll] 11 | [monger.conversion :as conv] 12 | [monger.db :as db] 13 | [qu.main :as main] 14 | [qu.app :as app] 15 | [qu.app.mongo :refer [new-mongo]] 16 | [com.stuartsierra.component :as component])) 17 | 18 | (def db "integration_test") 19 | (def coll "incomes") 20 | (def qmap {:dataset db 21 | :slice coll 22 | :select "state_abbr, COUNT()" 23 | :group "state_abbr"}) 24 | (def ^:dynamic cache nil) 25 | (def ^:dynamic worker nil) 26 | (def ^:dynamic query nil) 27 | (def ^:dynamic agg nil) 28 | (def ^:dynamic dbo nil) 29 | (def worker-agent (atom nil)) 30 | 31 | (defn run-all-jobs [worker] 32 | (reset! worker-agent (c/start-worker worker)) 33 | (await @worker-agent) 34 | (swap! worker-agent c/stop-worker)) 35 | 36 | (defn mongo-setup 37 | [test] 38 | (let [mongo (new-mongo (main/default-mongo-options))] 39 | (component/start mongo) 40 | (loader/load-dataset db) 41 | (binding [cache (c/create-query-cache) 42 | dbo (mongo/get-db db)] 43 | (db/drop-db (:database cache)) 44 | (binding [query (q/prepare (q/make-query qmap))] 45 | (binding [worker (c/create-worker cache) 46 | agg (q/mongo-aggregation query)] 47 | (test)))) 48 | (component/stop mongo))) 49 | 50 | (use-fixtures :once mongo-setup) 51 | 52 | (deftest ^:integration test-cache 53 | (testing "the default cache uses the query-cache db" 54 | (does= (str (:database cache)) "query_cache")) 55 | 56 | (testing "you can use other databases" 57 | (does= (str (:database (c/create-query-cache "cashhhh"))) 58 | "cashhhh")) 59 | 60 | (testing "it can be wiped" 61 | (data/get-aggregation db coll agg) 62 | (run-all-jobs worker) 63 | (is (coll/exists? dbo (:to agg))) 64 | 65 | (c/wipe-cache cache) 66 | (is (not (coll/exists? dbo (:to agg)))))) 67 | 68 | (deftest ^:integration test-cleaning-cache 69 | (testing "it can be cleaned" 70 | (data/get-aggregation db coll agg) 71 | (run-all-jobs worker) 72 | (is (coll/exists? dbo (:to agg))) 73 | 74 | (c/clean-cache cache (constantly [(:to agg)])) 75 | (is (not (coll/exists? dbo (:to agg))))) 76 | 77 | (testing "by default, it cleans nothing" 78 | (data/get-aggregation db coll agg) 79 | (run-all-jobs worker) 80 | (is (coll/exists? dbo (:to agg))) 81 | 82 | (c/clean-cache cache) 83 | (is (coll/exists? dbo (:to agg)))) 84 | 85 | (testing "it runs cleaning operations as part of the worker cycle" 86 | (let [cleanups (atom 0) 87 | cache (c/create-query-cache "query_cache" (fn [_] (swap! cleanups inc) [])) 88 | worker (c/create-worker cache)] 89 | (run-all-jobs worker) 90 | (is (>= @cleanups 1))))) 91 | 92 | (deftest ^:integration test-add-to-queue 93 | (testing "it adds a document to jobs" 94 | (c/clean-cache cache (constantly [(:to agg)])) 95 | (does-contain (conv/from-db-object (c/add-to-queue cache agg) true) 96 | {:_id (:to agg) :status "unprocessed"}))) 97 | 98 | (deftest ^:integration test-add-to-cache 99 | (testing "it puts the aggregation results into the cache" 100 | (c/add-to-cache cache agg) 101 | (is (coll/exists? dbo (:to agg))))) 102 | 103 | (deftest ^:integration test-lookup 104 | (testing "it returns an empty result if not in the cache" 105 | (c/clean-cache cache (constantly [(:to agg)])) 106 | (is (nil? (cache/lookup cache query)))) 107 | 108 | (testing "it returns the cached results if they exist" 109 | (c/add-to-cache cache agg) 110 | (let [result (cache/lookup cache query)] 111 | (is (not (nil? result))) 112 | (is (= 4 (:total result)))))) 113 | 114 | (deftest ^:integration test-worker 115 | (testing "it will process jobs" 116 | (c/clean-cache cache (constantly [(:to agg)])) 117 | (is (:computing (data/get-aggregation db coll agg))) 118 | 119 | (run-all-jobs worker) 120 | (is (not (:computing (data/get-aggregation db coll agg)))))) 121 | 122 | ;; (run-tests) 123 | -------------------------------------------------------------------------------- /test/integration/test/loader.clj: -------------------------------------------------------------------------------- 1 | (ns ^:integration integration.test.loader 2 | "The 'integration_test' dataset is designed to simplify testing. 3 | Its data values are simple and map to the row number. 4 | 5 | The tests below are to ensure data is loaded correctly." 6 | (:require [clojure.test :refer :all] 7 | [qu.test-util :refer :all] 8 | [qu.data :as data] 9 | [qu.loader :as loader])) 10 | 11 | (def db "integration_test") 12 | (def coll "incomes") 13 | 14 | (use-fixtures :once (mongo-setup-fn db)) 15 | 16 | (deftest ^:integration test-dates 17 | (testing "stored as dates in MongoDB" 18 | (let [query {:query {} :fields {} :limit 100 :skip 0 :sort {}} 19 | result (data/get-find db coll query)] 20 | (doseq [datum (:data result)] 21 | (does= (class (:date_observed datum)) 22 | org.joda.time.DateTime))))) 23 | 24 | ;; (run-tests) 25 | -------------------------------------------------------------------------------- /test/integration/test/main.clj: -------------------------------------------------------------------------------- 1 | (ns ^:integration integration.test.main 2 | (:require [clojure.test :refer :all] 3 | [qu.test-util :refer :all])) 4 | 5 | (use-fixtures :once (mongo-setup-fn "integration_test")) 6 | 7 | (deftest ^:integration test-index-url 8 | (testing "it redirects to /data" 9 | (does-contain (GET "/") {:status 302}) 10 | (does-contain (:headers (GET "/")) 11 | {"Location" "/data"}))) 12 | 13 | (deftest ^:integration test-data-url 14 | (testing "it returns successfully" 15 | (let [resp (GET "/data")] 16 | (does= (:status resp) 200) 17 | (does-contain (:headers resp) 18 | {"Content-Type" "text/html;charset=UTF-8"})) 19 | 20 | (let [resp (GET "/data.xml")] 21 | (does= (:status resp) 200) 22 | (does-contain (:headers resp) 23 | {"Content-Type" "application/xml;charset=UTF-8"})))) 24 | 25 | (deftest ^:integration test-dataset-url 26 | (testing "it returns successfully" 27 | (let [resp (GET "/data/integration_test")] 28 | (does= (:status resp) 200) 29 | (does-contain (:headers resp) 30 | {"Content-Type" "text/html;charset=UTF-8"})) 31 | 32 | (let [resp (GET "/data/integration_test.xml")] 33 | (does= (:status resp) 200) 34 | (does-contain (:headers resp) 35 | {"Content-Type" "application/xml;charset=UTF-8"})))) 36 | 37 | (deftest ^:integration test-dataset-url-does-not-exist 38 | (testing "it returns a 404" 39 | (let [resp (GET "/data/bad_dataset")] 40 | (does= (:status resp) 404) 41 | (does-contain (:headers resp) 42 | {"Content-Type" "text/html"})) 43 | 44 | (let [resp (GET "/data/bad_dataset.xml")] 45 | (does= (:status resp) 404) 46 | (does-contain (:headers resp) 47 | {"Content-Type" "application/xml;charset=UTF-8"})))) 48 | 49 | ;; (run-tests) 50 | -------------------------------------------------------------------------------- /test/integration/test/slice.clj: -------------------------------------------------------------------------------- 1 | (ns ^:integration integration.test.slice 2 | (:require [clojure.test :refer :all] 3 | [qu.loader :as loader] 4 | [qu.data :as data] 5 | [qu.test-util :refer :all])) 6 | 7 | (use-fixtures :once (mongo-setup-fn "integration_test")) 8 | 9 | (deftest ^:integration test-query-slice-with-no-params 10 | (testing "it returns successfully as text/html" 11 | (let [resp (GET "/data/integration_test/slice/incomes")] 12 | (does= (:status resp) 200) 13 | (does-contain (:headers resp) 14 | {"Content-Type" "text/html;charset=UTF-8" "Vary" "Accept"})))) 15 | 16 | (deftest ^:integration test-query-slice-does-not-exist 17 | (testing "it returns a 404" 18 | (let [resp (GET "/data/bad-dataset/slice/bad-slice")] 19 | (does= (:status resp) 404)) 20 | 21 | (let [resp (GET "/data/bad-dataset/slice/bad-slice.xml")] 22 | (does= (:status resp) 404) 23 | (does-contain (:headers resp) 24 | {"Content-Type" "application/xml;charset=UTF-8" "Vary" "Accept"})))) 25 | 26 | (deftest ^:integration test-json 27 | (testing "it returns a content-type of application/json" 28 | (let [resp (GET "/data/integration_test/slice/incomes.json")] 29 | (does= (:status resp) 200) 30 | (does-contain (:headers resp) 31 | {"Content-Type" "application/json;charset=UTF-8"})))) 32 | 33 | (deftest ^:integration test-jsonp 34 | (testing "it uses the callback we supply" 35 | (let [resp (GET "/data/integration_test/slice/incomes.jsonp?$callback=foo")] 36 | (does= (:status resp) 200) 37 | (does-contain (:headers resp) 38 | {"Content-Type" "text/javascript;charset=UTF-8"}) 39 | (does-re-find (:body resp) #"^foo\("))) 40 | 41 | (testing "it uses 'callback' by default" 42 | (let [resp (GET "/data/integration_test/slice/incomes.jsonp")] 43 | (does= (:status resp) 200) 44 | (does-contain (:headers resp) 45 | {"Content-Type" "text/javascript;charset=UTF-8"}) 46 | (does-re-find (:body resp) #"^callback\(")))) 47 | 48 | (deftest ^:integration test-xml 49 | (testing "it returns a content-type of application/xml" 50 | (let [resp (GET "/data/integration_test/slice/incomes.xml")] 51 | (does= (:status resp) 200) 52 | (does-contain (:headers resp) 53 | {"Content-Type" "application/xml;charset=UTF-8"})))) 54 | 55 | (deftest ^:integration test-query-with-error 56 | (testing "it returns the status code for bad request" 57 | (let [resp (GET "/data/integration_test/slice/incomes?$where=peanut%20butter")] 58 | (does= (:status resp) 400)))) 59 | 60 | ;; (run-tests) 61 | -------------------------------------------------------------------------------- /test/integration/test/writer.clj: -------------------------------------------------------------------------------- 1 | (ns ^:integration integration.test.writer 2 | (:require [clojure.test :refer :all] 3 | [qu.writer :refer :all] 4 | [qu.test-util :refer :all] 5 | [qu.query :as query] 6 | [qu.data :as data] 7 | [clojure.java.io :as io] 8 | [clojure.data.csv :as csv])) 9 | 10 | (def db "integration_test") 11 | (def coll "incomes") 12 | (defn tempfile 13 | [] 14 | (java.io.File/createTempFile "query" ".csv")) 15 | 16 | (use-fixtures :once (mongo-setup-fn db)) 17 | 18 | (deftest ^:integration test-write-csv 19 | (let [outfile (tempfile) 20 | writer (io/writer outfile) 21 | query (query/make-query {:dataset db :slice coll :limit 0})] 22 | (query->csv query writer) 23 | 24 | (with-open [in (io/reader outfile)] 25 | (let [csv-data (csv/read-csv in) 26 | columns (query/columns query)] 27 | (does= (first csv-data) columns 28 | (count (rest csv-data)) 29 | (count (:data (query/execute query)))))))) 30 | -------------------------------------------------------------------------------- /test/qu/test/cache.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.cache 2 | (:refer-clojure :exclude [sort]) 3 | (:require [clojure.test :refer :all] 4 | [qu.query :as q] 5 | [qu.cache :refer :all])) 6 | 7 | (deftest test-query-to-key 8 | (let [query1 (q/map->Query {:select "state_id, county_id, MAX(tax_returns)" :group "state_id, county_id" :metadata {:database "test"} :slice "test"}) 9 | query2 (q/map->Query {:select "state_id,county_id,MAX(tax_returns)" :group "state_id,county_id" :metadata {:database "test"} :slice "test"})] 10 | 11 | (testing "it eliminates space in the SELECT and GROUP BY fields" 12 | (is (= (query-to-key query1) (query-to-key query2)))) 13 | 14 | (testing "different WHERE queries make different keys" 15 | (is (not (= (query-to-key (assoc query1 :where "state_id = 1")) 16 | (query-to-key (assoc query1 :where "state_id = 2")))))) 17 | 18 | (testing "different ORDER BY queries make the same key" 19 | (is (= (query-to-key (assoc query1 :orderBy "state_id")) 20 | (query-to-key (assoc query1 :orderBy "max_tax_returns"))))) 21 | 22 | (testing "different LIMIT and OFFSET queries make the same key" 23 | (is (= (query-to-key (assoc query1 :limit 10)) 24 | (query-to-key (assoc query1 :limit 20 :offset 10))))))) 25 | 26 | ;; (run-tests) 27 | -------------------------------------------------------------------------------- /test/qu/test/data.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.data 2 | (:require [clojure.test :refer :all] 3 | [qu.data :refer :all] 4 | [qu.query :refer [is-aggregation? params->Query]])) 5 | 6 | (deftest test-get-data-table 7 | (testing "it converts maps to seqs" 8 | (let [raw-data [{:name "Pete" :age 36 :city "York"} 9 | {:name "Sarah" :age 34 :city "Wallingford"} 10 | {:name "Shawn" :age 29 :city "Portland"}] 11 | data-table [["Pete" 36] 12 | ["Sarah" 34] 13 | ["Shawn" 29]]] 14 | (is (= (get-data-table raw-data [:name :age]) 15 | data-table))))) 16 | 17 | ;; (run-tests) 18 | -------------------------------------------------------------------------------- /test/qu/test/main.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.main 2 | (:require [clojure.test :refer :all] 3 | [ring.mock.request :refer :all] 4 | [qu.test-util :refer [GET]])) 5 | 6 | (deftest index-url 7 | (testing "it redirects to /data" 8 | (let [resp (GET "/")] 9 | (is (= (:status resp) 10 | 302)) 11 | (is (= (get-in resp [:headers "Location"])) 12 | "/data")))) 13 | 14 | (deftest data-url 15 | (testing "it returns successfully" 16 | (with-redefs [qu.data/get-datasets (constantly [])] 17 | (let [resp (GET "/data")] 18 | (is (= (:status resp) 19 | 200)) 20 | (is (= (get-in resp [:headers "Content-Type"]) 21 | "text/html;charset=UTF-8"))) 22 | 23 | (let [resp (GET "/data.xml")] 24 | (is (= (:status resp) 25 | 200)) 26 | (is (= (get-in resp [:headers "Content-Type"]) 27 | "application/xml;charset=UTF-8")))))) 28 | 29 | (deftest dataset-url 30 | (testing "it returns successfully when the dataset exists" 31 | (with-redefs [qu.data/get-metadata (constantly {})] 32 | (let [resp (GET "/data/good-dataset")] 33 | (is (= (:status resp) 34 | 200)) 35 | (is (= (get-in resp [:headers "Content-Type"]) 36 | "text/html;charset=UTF-8"))) 37 | 38 | (let [resp (GET "/data/good-dataset.xml")] 39 | (is (= (:status resp) 40 | 200)) 41 | (is (= (get-in resp [:headers "Content-Type"]) 42 | "application/xml;charset=UTF-8")))))) 43 | 44 | (deftest slice-url 45 | (testing "it returns successfully when the dataset and slice exist" 46 | (with-redefs [qu.data/get-metadata (constantly {:slices {:whoa {}}}) 47 | qu.query/execute (constantly {:total 0 :size 0 :data []})] 48 | (let [resp (GET "/data/good-dataset/slice/whoa")] 49 | (is (= (:status resp) 50 | 200)) 51 | (is (= (get-in resp [:headers "Content-Type"]) 52 | "text/html;charset=UTF-8"))) 53 | 54 | (let [resp (GET "/data/good-dataset/slice/whoa.xml")] 55 | (is (= (:status resp) 56 | 200)) 57 | (is (= (get-in resp [:headers "Content-Type"]) 58 | "application/xml;charset=UTF-8"))))) 59 | 60 | (testing "it returns a 404 when the dataset does not exist" 61 | (with-redefs [qu.data/get-metadata (constantly nil)] 62 | (let [resp (GET "/data/bad-dataset/what")] 63 | (is (= (:status resp) 404))))) 64 | 65 | (testing "it returns a 404 when the slice does not exist" 66 | (with-redefs [qu.data/get-metadata (constantly {:slices {}})] 67 | (let [resp (GET "/data/bad-dataset/what")] 68 | (is (= (:status resp) 404)))))) 69 | 70 | ;; (run-tests) 71 | -------------------------------------------------------------------------------- /test/qu/test/metrics.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.metrics 2 | (:require [clojure.test :refer :all] 3 | [qu.metrics :as metrics])) 4 | 5 | (deftest metrics-path 6 | (testing "it returns data.html when empty" 7 | (is (= (metrics/metrics-path "") "/data.html"))) 8 | 9 | (testing "it adds html extension when no extension is present" 10 | (is (= (metrics/metrics-path "/honeybadger") "/honeybadger.html"))) 11 | 12 | (testing "it uses extension when present" 13 | (are [x y] (= x y) 14 | (metrics/metrics-path "/honeybadger.json") "/honeybadger.json" 15 | (metrics/metrics-path "/honeybadger.csv") "/honeybadger.csv" 16 | (metrics/metrics-path "/honeybadger.xml") "/honeybadger.xml" 17 | (metrics/metrics-path "/honeybadger/dont/take/no.xml") "/honeybadger/dont/take/no.xml" 18 | (metrics/metrics-path "/honeybadger/dont/take/no") "/honeybadger/dont/take/no.html"))) 19 | 20 | ;; (run-tests) 21 | -------------------------------------------------------------------------------- /test/qu/test/query.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.query 2 | (:require [clojure.test :refer :all] 3 | [qu.test-util :refer :all] 4 | [qu.query :refer :all] 5 | [qu.query.mongo :as mongo])) 6 | 7 | (def metadata 8 | {:slices {:county_taxes {:dimensions ["state" "county"] 9 | :metrics ["tax_returns" "population"]}}}) 10 | 11 | (defn make-test-query 12 | [q] 13 | (merge {:metadata metadata 14 | :slicedef (get-in metadata [:slices :county_taxes]) 15 | :limit "0" :offset "0"} q)) 16 | 17 | (def query (make-test-query {})) 18 | 19 | (deftest test-parse-params 20 | (testing "it pulls out clauses" 21 | (let [params {:$select "age,race" :$foo "1"}] 22 | (is (= (parse-params params) {:$select "age,race"}))))) 23 | 24 | (deftest test-is-aggregation? 25 | (testing "returns true if we have a group key" 26 | (is (is-aggregation? {:group "state"}))) 27 | 28 | (testing "returns false if we don't have a group key" 29 | (is (not (is-aggregation? {}))))) 30 | 31 | (deftest test-mongo-find 32 | (testing "it populates fields if :select exists" 33 | (does-contain (mongo-find (mongo/process (make-test-query {:select "county, state"}))) 34 | {:fields {:county 1 :state 1}})) 35 | 36 | (testing "it returns empty fields if :select does not exist" 37 | (does-contain (mongo-find query) {:fields {}})) 38 | 39 | (testing "it returns empty sort if :sort does not exist" 40 | (does-contain (mongo-find query) {:sort {}}))) 41 | 42 | (deftest test-mongo-aggregation 43 | (testing "it creates a map-reduce query for Mongo" 44 | (let [query (mongo/process (make-test-query {:select "state, SUM(population)" 45 | :limit 100 46 | :offset 0 47 | :where "land_area > 1000000" 48 | :orderBy "state" 49 | :group "state"}))] 50 | (does-contain (mongo-aggregation query) 51 | {:group [:state] 52 | :aggregations {:sum_population ["sum" "population"]} 53 | :sort {:state 1} 54 | :limit 100 55 | :offset 0})))) 56 | 57 | ;; (run-tests) 58 | -------------------------------------------------------------------------------- /test/qu/test/query/mongo.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.query.mongo 2 | (:refer-clojure :exclude [sort]) 3 | (:require [clojure.test :refer :all] 4 | [qu.test-util :refer :all] 5 | [qu.query.mongo :refer :all])) 6 | 7 | (deftest test-sort 8 | (testing "it transforms :orderBy into a sorted-map" 9 | (does-contain (:mongo (sort {:orderBy "name"})) 10 | {:sort (sorted-map :name 1)}) 11 | 12 | (does-contain (:mongo (sort {:orderBy "state, name"})) 13 | {:sort (sorted-map :state 1 :name 1)}) 14 | 15 | (does-contain (:mongo (sort {:orderBy "state DESC, name"})) 16 | {:sort (sorted-map :state -1 :name 1)}))) 17 | 18 | (deftest test-match 19 | (testing "it transforms :where into :match" 20 | (does-contain (:mongo (match {:where "a > 2"})) 21 | {:match {:a {:$gt 2}}}))) 22 | 23 | (deftest test-project 24 | (testing "it transforms :select into :project" 25 | (does-contain (:mongo (project {:select "name, city"})) 26 | {:project {:name 1, :city 1}})) 27 | 28 | (testing "it transforms aggregations" 29 | (does-contain (:mongo (project {:select "city, MAX(income)", :group "city"})) 30 | {:project {:fields [:city :max_income] 31 | :aggregations {:max_income ["max" "income"]}}}))) 32 | 33 | (deftest test-group 34 | (testing "it makes a list of all fields to group by" 35 | (does-contain (:mongo (group {:select "state, county, COUNT()" :group "state, county"})) 36 | {:group [:state :county]}))) 37 | 38 | (deftest test-process 39 | (let [slicedef {:dimensions ["state_abbr" "county"] 40 | :metrics ["tax_returns"]} 41 | errors (comp :errors process)] 42 | (testing "it errors if you use a field in WHERE that does not exist" 43 | (is (contains? (errors {:where "foo > 0" :slicedef slicedef}) :where))) 44 | 45 | (testing "it errors if you try to ORDER BY a field that does not exist" 46 | (is (contains? (errors {:orderBy "foo" :slicedef slicedef}) :orderBy))) 47 | 48 | (testing "it does not error when you ORDER BY an aggregated field" 49 | (is (not (contains? (errors {:group "state_abbr" 50 | :select "state_abbr, COUNT()" 51 | :orderBy "count" 52 | :slicedef slicedef}) 53 | :orderBy)))))) 54 | 55 | ;; (run-tests) 56 | -------------------------------------------------------------------------------- /test/qu/test/query/parser.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.query.parser 2 | (:require [clojure.test :refer :all] 3 | [qu.test-util :refer :all] 4 | [protoflex.parse :as p] 5 | [qu.query.parser :refer :all] 6 | [clj-time.core :as time])) 7 | 8 | (defmacro has-parse-error [& body] 9 | `(try 10 | ~@body 11 | (is false) 12 | (catch Exception ex# 13 | (is (re-find #"^Parse Error" (.getMessage ex#)))))) 14 | 15 | (deftest test-value 16 | (testing "can parse numbers" 17 | (is (= (p/parse value "4.5"))) 4.5) 18 | 19 | (testing "can parse strings with single or double quotes" 20 | (is (= (p/parse value "\"hello world\"") "hello world")) 21 | (is (= (p/parse value "'hello world'") "hello world"))) 22 | 23 | (testing "can parse boolean literals" 24 | (is (= (p/parse value "true") {:bool true})) 25 | (is (= (p/parse value "false") {:bool false}))) 26 | 27 | (testing "can parse dates" 28 | (is (= (p/parse value "2013-04-01") (time/date-time 2013 4 1))) 29 | (is (= (p/parse value "1999/12/31") (time/date-time 1999 12 31))))) 30 | 31 | (deftest test-identifiers 32 | (testing "identifiers can be made up of letters, numbers, dashes, and underscores" 33 | (does= 34 | (p/parse identifier "hello") :hello 35 | (p/parse identifier "hello-world") :hello-world 36 | (p/parse identifier "HelloWorld") :HelloWorld 37 | (p/parse identifier "h3110_w0r1d") :h3110_w0r1d)) 38 | 39 | (testing "identifiers must start with a letter" 40 | (has-parse-error 41 | (p/parse identifier "3times")))) 42 | 43 | (deftest test-comparisons 44 | (testing "simple comparisons can be parsed" 45 | (does= 46 | (p/parse comparison "length > 3") {:comparison [:length :> 3]} 47 | (p/parse comparison "length < 3") {:comparison [:length :< 3]} 48 | (p/parse comparison "size != 12.5") {:comparison [:size :!= 12.5]})) 49 | 50 | (testing "IS NULL and IS NOT NULL comparisons can be parsed" 51 | (does= 52 | (p/parse comparison "length IS NULL") 53 | {:comparison [:length := nil]} 54 | 55 | (p/parse comparison "length IS NOT NULL") 56 | {:comparison [:length :!= nil]})) 57 | 58 | (testing "LIKE and ILIKE comparisons can be parsed" 59 | (does= 60 | (p/parse comparison "name LIKE 'Mar%'") 61 | {:comparison [:name :LIKE "Mar%"]} 62 | 63 | (p/parse comparison "name ILIKE 'mar%'") 64 | {:comparison [:name :ILIKE "mar%"]})) 65 | 66 | (testing "IN comparisons can be parsed" 67 | (is (= (p/parse comparison "length IN (1, 2, 3)") 68 | {:comparison [:length :IN [1 2 3]]}))) 69 | 70 | (testing "spaces are irrelevant" 71 | (is (= (p/parse comparison "length>3") 72 | {:comparison [:length :> 3]})))) 73 | 74 | 75 | (deftest test-where-expressions 76 | (testing "can be comparisons" 77 | (does= 78 | (p/parse where-expr "length > 3") {:comparison [:length :> 3]} 79 | (p/parse where-expr "tax_returns > 20000") {:comparison [:tax_returns :> 20000]})) 80 | 81 | (testing "can have NOT operators" 82 | (does= 83 | (p/parse where-expr "NOT length > 3") {:not {:comparison [:length :> 3]}} 84 | (p/parse where-expr "NOT (length > 3 AND height < 4.5)") 85 | {:not {:left {:comparison [:length :> 3]} 86 | :op :AND 87 | :right {:comparison [:height :< 4.5]}}})) 88 | 89 | (testing "can have AND and OR operators" 90 | (does= 91 | (p/parse where-expr "length > 3 AND height < 4.5") 92 | {:left {:comparison [:length :> 3]} 93 | :op :AND 94 | :right {:comparison [:height :< 4.5]}} 95 | 96 | (p/parse where-expr "length > 3 AND height < 4.5 OR name = \"Pete\"") 97 | {:left {:left {:comparison [:length :> 3]} 98 | :op :AND 99 | :right {:comparison [:height :< 4.5]}} 100 | :op :OR 101 | :right {:comparison [:name := "Pete"]}})) 102 | 103 | (testing "can parse a query with four parts" 104 | (does= 105 | (p/parse where-expr "as_of_year=2011 AND state_abbr=\"CA\" AND applicant_race_1=1 AND applicant_ethnicity=1") 106 | {:left {:left {:left {:comparison [:as_of_year := 2011]} 107 | :op :AND 108 | :right {:comparison [:state_abbr := "CA"]}} 109 | :op :AND 110 | :right {:comparison [:applicant_race_1 := 1]}} 111 | :op :AND 112 | :right {:comparison [:applicant_ethnicity := 1]}})) 113 | 114 | (testing "can have parentheses for precedence" 115 | (does= 116 | (p/parse where-expr "(length > 3 AND height < 4.5)") 117 | {:left {:comparison [:length :> 3]} 118 | :op :AND 119 | :right {:comparison [:height :< 4.5]}} 120 | 121 | (p/parse where-expr "length > 3 AND (height < 4.5 OR name = \"Pete\")") 122 | {:left {:comparison [:length :> 3]} 123 | :op :AND 124 | :right {:left {:comparison [:height :< 4.5]} 125 | :op :OR 126 | :right {:comparison [:name := "Pete"]}}}))) 127 | 128 | (deftest test-select-expressions 129 | (testing "can have one column" 130 | (is (= (p/parse select-expr "length")) [{:select :length}])) 131 | 132 | (testing "can have multiple columns" 133 | (is (= (p/parse select-expr "length, height") 134 | [{:select :length} 135 | {:select :height}]))) 136 | 137 | (testing "can have aggregations" 138 | (is (= (p/parse select-expr "state, SUM(population)") 139 | [{:select :state} 140 | {:aggregation [:SUM :population] 141 | :select :sum_population}]))) 142 | 143 | (testing "COUNT aggregations do not need an identifier" 144 | (does= 145 | (p/parse select-expr "state, COUNT()") 146 | [{:select :state} 147 | {:aggregation [:COUNT :_id] 148 | :select :count}] 149 | 150 | (p/parse select-expr "state, count()") 151 | [{:select :state} 152 | {:aggregation [:COUNT :_id] 153 | :select :count}])) 154 | 155 | (testing "aggregations are case-insensitive" 156 | (does= 157 | (p/parse select-expr "state, sum(population)") 158 | [{:select :state} 159 | {:aggregation [:SUM :population] 160 | :select :sum_population}] 161 | 162 | (p/parse select-expr "state, cOuNt(population)") 163 | [{:select :state} 164 | {:aggregation [:COUNT :population] 165 | :select :count_population}])) 166 | 167 | (testing "invalid aggregations do not work" 168 | (has-parse-error 169 | (p/parse select-expr "state, TOTAL(population)")))) 170 | 171 | (deftest test-group-expressions 172 | (testing "can have one column" 173 | (is (= (p/parse group-expr "state") [:state]))) 174 | 175 | (testing "can have multiple columns" 176 | (is (= (p/parse group-expr "state, county") [:state :county])))) 177 | 178 | ;; (run-tests) 179 | -------------------------------------------------------------------------------- /test/qu/test/query/validation.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.query.validation 2 | (:require [clojure.test :refer :all] 3 | [qu.test-util :refer :all] 4 | [qu.query :as query] 5 | [qu.query.validation :as v])) 6 | 7 | (deftest test-validate 8 | (let [slicedef {:dimensions ["state_abbr" "county" "county_code" "city" "city_abbr"] 9 | :metrics ["tax_returns"] 10 | :max-group-fields 3} 11 | metadata {:slices {:county_taxes slicedef} 12 | :concepts {:county {:properties {:population {:type "number"} 13 | :state {:type "string"}}}}} 14 | q (fn [& {:as query}] 15 | (merge {:slicedef slicedef 16 | :slice :county_taxes 17 | :metadata metadata} 18 | query)) 19 | errors (comp :errors v/validate) 20 | limit-errors (fn [query] (:errors (v/validate-max-limit query 100)))] 21 | 22 | (testing "it errors when it cannot parse SELECT" 23 | (does-contain (errors (q :select "what what")) :select)) 24 | 25 | (testing "it errors when you have an aggregation in SELECT without a GROUP" 26 | (let [query (q :select "state_abbr, SUM(tax_returns)")] 27 | (does-contain (errors query) :select) 28 | (does-not-contain (errors (assoc query :group "state_abbr")) :select))) 29 | 30 | (testing "it errors if you have an unaggregated SELECT field without it being in GROUP" 31 | (let [query (q :select "state_abbr, county, SUM(tax_returns)" 32 | :group "state_abbr")] 33 | (does-contain (errors query) :select) 34 | (does-not-contain (errors (assoc query :group "state_abbr, county")) :select))) 35 | 36 | (testing "it errors if you reference a field that is not in the slice" 37 | (does-contain (errors (q :select "foo")) :select) 38 | (does-contain (errors (q :select "state_abbr, SUM(foo)" :group "state_abbr")) :select) 39 | (does-contain (errors (q :select "foo.population")) :select)) 40 | 41 | (testing "it errors if it cannot parse GROUP" 42 | (does-contain (errors (q :select "state_abbr" :group "what what")) :group)) 43 | 44 | (testing "it errors if if the number of fields in GROUP exceeds the maximum allowed" 45 | (let [query (q :select "state_abbr, county, county_code, city, city_abbr, SUM(tax_returns)" 46 | :group "state_abbr, county, county_code, city, city_abbr") 47 | group-errors (:group (errors query))] 48 | (does-contain group-errors "Number of group fields exceeds maximum allowed (3)."))) 49 | 50 | (testing "it errors if you use GROUP without SELECT" 51 | (let [query (q :group "state_abbr")] 52 | (does-contain (errors query) :group) 53 | (does-not-contain (errors (assoc query :select "state_abbr")) :group))) 54 | 55 | (testing "it errors if you GROUP on something that is not a dimension" 56 | (let [query (q :select "tax_returns" :group "tax_returns")] 57 | (does-contain (errors query) :group))) 58 | 59 | (testing "it errors if it cannot parse WHERE" 60 | (does-contain (errors (q :where "what what")) :where)) 61 | 62 | (testing "it errors if it cannot parse ORDER BY" 63 | (does-contain (errors (q :orderBy "what what")) :orderBy) 64 | (does-not-contain (errors (q :orderBy "state_abbr DESC")) :orderBy)) 65 | 66 | (testing "it errors if limit is not an integer string" 67 | (does-contain (errors (q :limit "ten")) :limit) 68 | (does-not-contain (errors (q :limit "10")) :limit)) 69 | 70 | (testing "it errors if limit is greater than 100" 71 | (does-contain (limit-errors (q :limit "101")) :limit) 72 | (does-not-contain (limit-errors (q :limit "100")) :limit)) 73 | 74 | (testing "it errors if limit is less than 0" 75 | (does-contain (errors (q :limit "-1")) :limit) 76 | (does-not-contain (errors (q :limit "0")) :limit)) 77 | 78 | (testing "it errors if offset is not an integer string" 79 | (does-contain (errors (q :offset "ten")) :offset) 80 | (does-not-contain (errors (q :offset "10")) :offset)) 81 | 82 | (testing "it errors if page is not a positive integer string" 83 | (does-contain (errors (q :page "ten")) :page) 84 | (does-contain (errors (q :page "-1")) :page) 85 | (does-contain (errors (q :page "0")) :page) 86 | (does-not-contain (errors (q :page "10")) :page)))) 87 | 88 | ;; (run-tests) 89 | -------------------------------------------------------------------------------- /test/qu/test/query/where.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test.query.where 2 | (:require [clojure.test :refer :all] 3 | [qu.test-util :refer :all] 4 | [protoflex.parse :as p] 5 | [qu.query.where :refer [parse mongo-eval]]) 6 | (:import (java.util.regex Pattern))) 7 | 8 | (deftest test-parse 9 | (testing "can parse simple comparisons" 10 | (does= 11 | (parse "length > 3") {:comparison [:length :> 3]} 12 | (parse "name IS NULL") {:comparison [:name := nil]} 13 | (parse "name IS NOT NULL") {:comparison [:name :!= nil]})) 14 | 15 | (testing "can parse complex comparisons" 16 | (does= 17 | (parse "length > 3 AND height < 4.5") 18 | {:left {:comparison [:length :> 3]} 19 | :op :AND 20 | :right {:comparison [:height :< 4.5]}} 21 | 22 | (parse "length > 3 AND height < 4.5 OR name = \"Pete\"") 23 | {:left {:left {:comparison [:length :> 3]} 24 | :op :AND 25 | :right {:comparison [:height :< 4.5]}} 26 | :op :OR 27 | :right {:comparison [:name := "Pete"]}} 28 | 29 | (parse "length > 3 AND (height < 4.5 OR name = \"Pete\")") 30 | {:left {:comparison [:length :> 3]} 31 | :op :AND 32 | :right {:left {:comparison [:height :< 4.5]} 33 | :op :OR 34 | :right {:comparison [:name := "Pete"]}}}))) 35 | 36 | (deftest test-mongo-eval 37 | (testing "handles equality correctly" 38 | (is (= (mongo-eval (parse "length = 3")) {:length 3}))) 39 | 40 | (testing "handles non-equality comparisons" 41 | (does= 42 | (mongo-eval (parse "length < 3")) {:length {:$lt 3}} 43 | (mongo-eval (parse "length >= 3")) {:length {:$gte 3}})) 44 | 45 | (testing "handles booleans in comparisons" 46 | (does= 47 | (mongo-eval (parse "exempt = TRUE")) {:exempt true})) 48 | 49 | (testing "handles LIKE comparisons" 50 | (does-re-match 51 | "Marc" (:name (mongo-eval (parse "name LIKE 'Mar%'"))) 52 | "Markus" (:name (mongo-eval (parse "name LIKE 'Mar%'"))) 53 | "Mar" (:name (mongo-eval (parse "name LIKE 'Mar%'"))) 54 | "Clinton and Marc" (:name (mongo-eval (parse "name LIKE '%Mar%'"))) 55 | "Mick" (:name (mongo-eval (parse "name LIKE 'M__k'"))) 56 | "Mark" (:name (mongo-eval (parse "name LIKE 'M__k'"))) 57 | ".M" (:name (mongo-eval (parse "name LIKE '._'")))) 58 | 59 | (does-not-re-match 60 | "CMark" (:name (mongo-eval (parse "name LIKE 'Mar%'"))) 61 | "Mak" (:name (mongo-eval (parse "name LIKE 'M__k'"))) 62 | "CM" (:name (mongo-eval (parse "name LIKE '._'"))))) 63 | 64 | (testing "handles ILIKE comparisons" 65 | (does-re-match 66 | "Blob fish" (:name (mongo-eval (parse "name ILIKE 'blob%'"))) 67 | "AYE AYE" (:name (mongo-eval (parse "name ILIKE 'aye%ay%'"))) 68 | "jerboa ears" (:name (mongo-eval (parse "name ILIKE 'JERB%'"))) 69 | "greater pangolin is great" (:name (mongo-eval (parse "name ILIKE '%P_ng_lin%'"))) 70 | "D. melanogaster" (:name (mongo-eval (parse "name ILIKE 'D.%Melan%'")))) 71 | 72 | (does-not-re-match 73 | "goeduck clam" (:name (mongo-eval (parse "name ILIKE 'GEODUCK clam'"))) 74 | "geoduck clam" (:name (mongo-eval (parse "name ILIKE 'G._DUCK clam'"))))) 75 | 76 | 77 | (testing "handles complex comparisons" 78 | (does= 79 | (mongo-eval (parse "length > 3 AND height = 4.5")) 80 | {:$and [ {:length {:$gt 3}} {:height 4.5}]} 81 | 82 | (mongo-eval (parse "length > 3 OR height = 4.5")) 83 | {:$or [{:length {:$gt 3}} {:height 4.5}]} 84 | 85 | (mongo-eval (parse "length > 3 OR height = 4.5 OR width < 2")) 86 | {:$or [{:length {:$gt 3}} {:height 4.5} {:width {:$lt 2}}]} 87 | 88 | (mongo-eval (parse "length > 3 OR height = 4.5 OR width < 2 OR name = 'Pete'")) 89 | {:$or [{:length {:$gt 3}} {:height 4.5} {:width {:$lt 2}} {:name "Pete"}]} 90 | 91 | (mongo-eval (parse "length > 3 AND (height < 4.5 OR name = \"Pete\" OR width < 2)")) 92 | {:$and [{:length {:$gt 3}} 93 | {:$or [{:height {:$lt 4.5}} 94 | {:name "Pete"} 95 | {:width {:$lt 2}}]}]})) 96 | 97 | (testing "handles IN comparisons" 98 | (does= 99 | (mongo-eval (parse "name IN (\"Pete\", \"Sam\")")) 100 | {:name {:$in ["Pete" "Sam"]}})) 101 | 102 | (testing "handles simple comparisons with NOT" 103 | (does= 104 | (mongo-eval (parse "NOT name = \"Pete\"")) 105 | {:name {:$ne "Pete"}} 106 | 107 | (mongo-eval (parse "NOT name != \"Pete\"")) 108 | {:name "Pete"} 109 | 110 | (mongo-eval (parse "NOT length < 3")) 111 | {:length {:$not {:$lt 3}}})) 112 | 113 | (testing "handles complex comparisons with NOT and AND" 114 | (does= 115 | (mongo-eval (parse "NOT (length > 3 AND height = 4.5)")) 116 | {:$or [{:length {:$not {:$gt 3}}} {:height {:$ne 4.5}}]} 117 | 118 | (mongo-eval (parse "NOT (length > 3 AND height = 4.5 AND width < 2)")) 119 | {:$or [{:length {:$not {:$gt 3}}} {:height {:$ne 4.5}} {:width {:$not {:$lt 2}}}]})) 120 | 121 | (testing "uses $nor on complex comparisons with NOT and OR" 122 | (does= 123 | (mongo-eval (parse "NOT (length > 3 OR height = 4.5)")) 124 | {:$nor [{:length {:$gt 3}} {:height 4.5}]} 125 | 126 | (mongo-eval (parse "NOT (length > 3 OR height = 4.5 OR width < 2)")) 127 | {:$nor [{:length {:$gt 3}} {:height 4.5} {:width {:$lt 2}}]})) 128 | 129 | (testing "NOT binds tighter than AND" 130 | (does= 131 | (mongo-eval (parse "NOT length > 3 AND height = 4.5")) 132 | {:$and [{:length {:$not {:$gt 3}}} {:height 4.5}]})) 133 | 134 | (testing "handles nested comparisons with multiple NOTs" 135 | (does= 136 | (mongo-eval (parse "NOT (length > 3 OR NOT (height > 4.5 AND name IS NOT NULL))")) 137 | {:$nor [{:length {:$gt 3}} 138 | {:$or [{:height {:$not {:$gt 4.5}}} 139 | {:name nil}]}]} 140 | 141 | (mongo-eval (parse "NOT (length > 3 AND NOT (height > 4.5 AND name = \"Pete\"))")) 142 | {:$or [{:length {:$not {:$gt 3}}} 143 | {:$and [{:height {:$gt 4.5}} 144 | {:name "Pete"}]}]}))) 145 | 146 | 147 | ;; (run-tests) 148 | -------------------------------------------------------------------------------- /test/qu/test_util.clj: -------------------------------------------------------------------------------- 1 | (ns qu.test-util 2 | (:require [clojure.test :refer :all] 3 | [ring.mock.request :as mockreq] 4 | [org.httpkit.client :as client] 5 | [qu.main :as main] 6 | [qu.app :as app] 7 | [qu.app.webserver :as webserver] 8 | [qu.app.mongo :refer [new-mongo]] 9 | [qu.loader :as loader] 10 | [com.stuartsierra.component :as component])) 11 | 12 | (def port 4545) 13 | (def system (atom nil)) 14 | (def app (webserver/get-handler false)) 15 | (def test-options (-> (main/default-options) 16 | (assoc-in [:http :port] port))) 17 | 18 | (defn system-setup 19 | [test] 20 | (if @system 21 | (swap! system component/stop) 22 | (reset! system (app/new-qu-system test-options))) 23 | (swap! system component/start) 24 | (test) 25 | (swap! system component/stop)) 26 | 27 | (defn mongo-setup-fn 28 | [db] 29 | (fn [test] 30 | (let [mongo (new-mongo (main/default-mongo-options))] 31 | (component/start mongo) 32 | (loader/load-dataset db) 33 | (test) 34 | (component/stop mongo)))) 35 | 36 | (defn GET 37 | [url] 38 | (app (mockreq/request :get url))) 39 | 40 | (defn does-contain [container contained] 41 | (let [container (if (sequential? container) 42 | (set container) 43 | container)] 44 | (cond 45 | (not (coll? contained)) (is (contains? container contained)) 46 | (vector? contained) (doseq [e contained] (is (contains? container e))) 47 | :else (doseq [[k v] contained] 48 | (is (= (get container k) v)))))) 49 | 50 | (defn does-not-contain [m1 m2] 51 | (cond 52 | (keyword? m2) (is (not (contains? m1 m2))) 53 | :else (doseq [[k v] m2] 54 | (is (not (= (get m1 k) v)))))) 55 | 56 | (defmacro does= [& body] 57 | `(are [x y] (= x y) 58 | ~@body)) 59 | 60 | (defmacro does-not= [& body] 61 | `(are [x y] (not (= x y)) 62 | ~@body)) 63 | 64 | (defmacro does-re-match [& body] 65 | `(are [text pattern] (re-matches pattern text) 66 | ~@body)) 67 | 68 | (defmacro does-not-re-match [& body] 69 | `(are [text pattern] (not (re-matches pattern text)) 70 | ~@body)) 71 | 72 | (defmacro does-re-find [& body] 73 | `(are [text pattern] (re-find pattern text) 74 | ~@body)) 75 | 76 | (defmacro does-not-re-match [& body] 77 | `(are [text pattern] (not (re-find pattern text)) 78 | ~@body)) 79 | --------------------------------------------------------------------------------