├── .gitignore ├── .ruby-version ├── Gemfile ├── Gemfile.lock ├── Gemfile.lock.orig ├── LICENSE.txt ├── README.md ├── Rakefile ├── app ├── assets │ ├── javascripts │ │ └── application.js │ └── stylesheets │ │ ├── application.css │ │ ├── custom.css.scss │ │ ├── jquery-ui-1.10.1.custom.min.css │ │ └── signin.css.scss ├── controllers │ ├── api │ │ └── v1 │ │ │ ├── data_controller.rb │ │ │ ├── datapackage_resource_fields_controller.rb │ │ │ ├── datapackage_resources_controller.rb │ │ │ ├── datapackages_controller.rb │ │ │ ├── datasources_controller.rb │ │ │ └── projects_controller.rb │ ├── api_key_permissions_controller.rb │ ├── api_keys_controller.rb │ ├── application_controller.rb │ ├── datapackage_resources_controller.rb │ ├── datapackages_controller.rb │ ├── datasources_controller.rb │ └── projects_controller.rb ├── helpers │ ├── application_helper.rb │ ├── data_access_helper.rb │ ├── datapackage_helper.rb │ └── project_helper.rb ├── jobs │ └── process_csv_upload.rb ├── models │ ├── api_key.rb │ ├── api_key_permission.rb │ ├── datapackage.rb │ ├── datapackage_resource.rb │ ├── datapackage_resource_field.rb │ ├── datasource.rb │ ├── project.rb │ └── user.rb └── views │ ├── api_key_permissions │ ├── _error_messages.html.erb │ ├── _form.html.erb │ ├── index.html.erb │ └── new.html.erb │ ├── api_keys │ ├── _error_messages.html.erb │ ├── _form.html.erb │ ├── create.html.erb │ ├── edit.html.erb │ ├── index.html.erb │ ├── index_project.html.erb │ ├── new.html.erb │ ├── show.html.erb │ └── update.html.erb │ ├── datapackage_resources │ ├── show.html.erb │ └── show_old.html.erb │ ├── datasources │ ├── index.html.erb │ └── show.html.erb │ ├── devise │ ├── confirmations │ │ └── new.html.erb │ ├── mailer │ │ ├── confirmation_instructions.html.erb │ │ ├── reset_password_instructions.html.erb │ │ └── unlock_instructions.html.erb │ ├── passwords │ │ ├── edit.html.erb │ │ └── new.html.erb │ ├── registrations │ │ ├── edit.html.erb │ │ └── new.html.erb │ ├── sessions │ │ └── new.html.erb │ ├── shared │ │ └── _links.html.erb │ └── unlocks │ │ └── new.html.erb │ ├── layouts │ ├── _footer.html.erb │ ├── _header.html.erb │ ├── _shim.html.erb │ └── application.html.erb │ ├── projects │ ├── _error_messages.html.erb │ ├── _form.html.erb │ ├── _upload_datapackage.html.erb │ ├── _upload_datasources.html.erb │ ├── _upload_project_files.html.erb │ ├── api_detail.html.erb │ ├── edit.html.erb │ ├── index.html.erb │ ├── new.html.erb │ ├── partials │ │ ├── _api_detail_datatables_info.html.erb │ │ ├── _api_detail_distinct_values.html.erb │ │ ├── _api_detail_endpoints.html.erb │ │ ├── _api_detail_ordering.html.erb │ │ ├── _api_detail_paging.html.erb │ │ ├── _api_detail_project_details.html.erb │ │ ├── _api_detail_querying.html.erb │ │ ├── _api_detail_select.html.erb │ │ └── _project_details.html.erb │ └── show.html.erb │ └── shared │ └── _devise_links.html.erb ├── bin ├── bundle ├── delayed_job ├── rails ├── rake ├── setup └── spring ├── config.ru ├── config ├── application.rb ├── boot.rb ├── database.yml ├── environment.rb ├── environments │ ├── development.rb │ ├── production.rb │ └── test.rb ├── initializers │ ├── assets.rb │ ├── backtrace_silencers.rb │ ├── cookies_serializer.rb │ ├── devise.rb │ ├── filter_parameter_logging.rb │ ├── inflections.rb │ ├── mime_types.rb │ ├── mira.rb │ ├── session_store.rb │ └── wrap_parameters.rb ├── locales │ ├── devise.en.yml │ └── en.yml ├── mira_constants.rb ├── routes.rb └── secrets.yml ├── db ├── migrate │ ├── 20150318173213_create_projects.rb │ ├── 20150318181209_add_index_to_projects_name.rb │ ├── 20150320162605_create_datasources.rb │ ├── 20150320163854_add_attachment_datafile_to_datasources.rb │ ├── 20150404082247_create_delayed_jobs.rb │ ├── 20150505184421_add_file_path_to_datasources.rb │ ├── 20150506094858_add_index_datasources_table_ref.rb │ ├── 20150726062647_add_user_id_to_project.rb │ ├── 20150726062922_devise_create_users.rb │ ├── 20151020101937_create_datapackages.rb │ ├── 20151020102022_add_attachment_file_to_datapackages.rb │ ├── 20151020103351_create_datapackage_resources.rb │ ├── 20151021130254_create_datapackage_resource_schemas.rb │ ├── 20151022145248_big_integer_to_datapackage_resource_fields.rb │ ├── 20151022153557_add_quote_character_to_datapackage_resources.rb │ ├── 20151023084243_add_table_ref_to_datapackage_resources.rb │ ├── 20151030073834_add_import_status_to_datasources.rb │ ├── 20151128084949_add_add_index_to_datapackage_resource_fields.rb │ ├── 20151206044323_add_db_table_name_to_datapackage_resources.rb │ ├── 20151209215327_add_format_to_datapackage_resource_field.rb │ ├── 20151227122837_add_description_to_datapackage_resources.rb │ ├── 20160415042134_create_api_keys.rb │ ├── 20160415185504_create_api_key_permissions.rb │ ├── 20160531140633_add_private_to_datapackage_resource_fields.rb │ ├── 20160616200650_add_datapackage_resource_to_datasources.rb │ ├── 20160617073857_add_datapackage_to_datasources.rb │ └── 20160624141850_add_imported_rows_to_datasources.rb ├── schema.rb └── seeds.rb ├── lib ├── api_constraints.rb ├── load_dynamic_AR_class_with_scopes.rb ├── load_table.rb └── tasks │ ├── kill_postgres_connections.rake │ ├── mira.rake │ └── test.rake ├── project_files ├── job_logs │ └── .gitignore └── uploads │ └── .gitignore ├── public ├── 404.html ├── 422.html ├── 500.html ├── assets │ ├── .sprockets-manifest-7e63634ef7c989258404942fb40cbda3.json │ ├── .sprockets-manifest-92af3159f449125806a09a697bd61573.json │ ├── application-5b52394f4595077da6d3ca54f0a32c70fec87c62c77757dca0b38c181f961108.js │ ├── application-fa5875e27f9dfff37d42fae684ac7258f51fe64183140e578ba87737388b8e5d.css │ ├── bootstrap │ │ ├── glyphicons-halflings-regular-13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407.eot │ │ ├── glyphicons-halflings-regular-42f60659d265c1a3c30f9fa42abcbb56bd4a53af4d83d316d6dd7a36903c43e5.svg │ │ ├── glyphicons-halflings-regular-a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742.woff │ │ ├── glyphicons-halflings-regular-e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456.ttf │ │ └── glyphicons-halflings-regular-fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c.woff2 │ └── sextant │ │ └── application-572803211757a77712103db3532083399575a1c0284db973b6079f6f24f78c2e.js ├── favicon.ico ├── job_logs │ └── .gitignore ├── robots.txt └── uploads │ └── .gitignore ├── test ├── api │ └── v1 │ │ ├── data_controller_permissions_test.rb │ │ ├── data_controller_test.rb │ │ ├── datapackage_endpoints_test.rb │ │ ├── datapackage_resource_fields_endpoints_test.rb │ │ ├── datapackage_resources_endpoints_test.rb │ │ ├── datasources_endpoints_test.rb │ │ ├── projects_endpoints_test.rb │ │ └── scratch_test.rb ├── controllers │ ├── api_key_permissions_controller_test.rb │ ├── api_keys_controller_test.rb │ ├── datasources_controller_test.rb │ ├── projects_controller_test.rb │ ├── projects_controller_upload_datapackage_test.rb │ └── projects_controller_upload_datasources_test.rb ├── fixtures │ ├── api_key_permissions.yml │ ├── api_keys.yml │ ├── datapackage_resource_fields.yml │ ├── datapackage_resources.yml │ ├── datapackages.yml │ ├── projects.yml │ ├── uploads │ │ ├── bad_upload.csv │ │ ├── datapackage.json │ │ ├── datapackage │ │ │ ├── bad_delimiter_too_long │ │ │ │ └── datapackage.json │ │ │ ├── bad_field_invalid_name │ │ │ │ └── datapackage.json │ │ │ ├── bad_field_invalid_type │ │ │ │ └── datapackage.json │ │ │ ├── bad_field_not_name_and_type │ │ │ │ └── datapackage.json │ │ │ ├── bad_not_json │ │ │ │ └── datapackage.json │ │ │ ├── bad_path_empty │ │ │ │ └── datapackage.json │ │ │ ├── bad_path_missing │ │ │ │ └── datapackage.json │ │ │ ├── bad_path_not_csv │ │ │ │ └── datapackage.json │ │ │ ├── bad_path_not_string │ │ │ │ └── datapackage.json │ │ │ ├── bad_quote_char_too_long │ │ │ │ └── datapackage.json │ │ │ ├── bad_resources_empty │ │ │ │ └── datapackage.json │ │ │ ├── bad_resources_missing │ │ │ │ └── datapackage.json │ │ │ ├── bad_resources_not_array │ │ │ │ └── datapackage.json │ │ │ ├── bad_schema_missing │ │ │ │ └── datapackage.json │ │ │ ├── bad_schema_no_fields │ │ │ │ └── datapackage.json │ │ │ ├── bad_schema_not_array │ │ │ │ └── datapackage.json │ │ │ ├── bad_schema_not_hash │ │ │ │ └── datapackage.json │ │ │ ├── good │ │ │ │ └── datapackage.json │ │ │ └── with_id_column │ │ │ │ ├── datapackage.json │ │ │ │ └── datapackage.json~ │ │ ├── good_upload.csv │ │ ├── good_upload_with_tabs.csv │ │ ├── not_in_datapackage.csv │ │ ├── upload1.csv │ │ ├── upload1.txt │ │ ├── upload2.csv │ │ ├── upload2.txt │ │ └── with_id_column.csv │ └── users.yml ├── models │ ├── api_key_permission_test.rb │ ├── api_key_test.rb │ ├── datapackage_resource_field_test.rb │ ├── datapackage_resource_test.rb │ ├── datapackage_test.rb │ ├── project_test.rb │ └── user_test.rb ├── readme.md └── test_helper.rb └── tmp ├── cache └── .gitignore ├── pids └── .gitignore ├── sessions └── .gitignore ├── sockets └── .gitignore └── temp_paperclip_uploads └── .gitignore /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files for more about ignoring files. 2 | # 3 | # If you find yourself ignoring temporary files generated by your text editor 4 | # or operating system, you probably want to add a global ignore instead: 5 | # git config --global core.excludesfile '~/.gitignore_global' 6 | 7 | # Ignore bundler config. 8 | /.bundle 9 | 10 | # Ignore the default SQLite database. 11 | /db/*.sqlite3 12 | /db/*.sqlite3-journal 13 | 14 | # Ignore all logfiles and tempfiles. 15 | /log/* 16 | !/log/.keep 17 | #/tmp 18 | 19 | # DBR: ignore spring files. See https://www.railstutorial.org/book/static_pages#sec-getting_started_with_testing 20 | /spring 21 | /.idea/* 22 | docker.md 23 | notes* 24 | commands.md 25 | Dockerfile 26 | docker-compose.yml 27 | /ansible 28 | /ansible/* 29 | ansible* 30 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.3.0 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | ruby "2.3.0" 4 | 5 | 6 | gem 'rails', '4.2.5.2' 7 | gem 'pg', '~> 0.18.4' 8 | gem 'sass-rails', '~> 5.0' 9 | gem 'uglifier', '>= 1.3.0' 10 | gem 'coffee-rails', '~> 4.1.0' 11 | gem 'jquery-rails' 12 | 13 | gem 'jquery-ui-rails' 14 | gem 'jtable-rails4', '~> 0.1.1' 15 | # gem "jtable-rails4", :path => "/home/david/webdev/jtable-rails4" 16 | 17 | gem 'turbolinks' 18 | gem 'sdoc', '~> 0.4.0', group: :doc 19 | 20 | # DBR: https://www.railstutorial.org/book/filling_in_the_layout 21 | gem 'bootstrap-sass' 22 | #gem 'bootstrap-will_paginate' 23 | gem 'kaminari' 24 | gem 'api-pagination' 25 | 26 | gem 'paperclip' 27 | gem 'delayed_paperclip' 28 | 29 | gem 'aws-sdk-v1' # Needed as per (otherwise errors): http://ruby.awsblog.com/post/TxFKSK2QJE6RPZ/Upcoming-Stable-Release-of-AWS-SDK-for-Ruby-Version-2 30 | gem 'aws-sdk' 31 | gem 'delayed_job_active_record' 32 | gem 'daemons' # for start/stop/restart delayed_job 33 | gem 'sextant', :group => :development # to show routes in browser localhost:3000/rails/routes 34 | 35 | gem 'devise' 36 | 37 | # DBR: cors needed as have API and separate app. Getting errors 38 | # when doing ajax requests across different domains. 39 | gem 'rack-cors', :require => 'rack/cors' 40 | gem 'yajl-ruby' # ?not sure needed anymore? faster JSON backends. See https://github.com/rails/jbuilder#faster-json-backends 41 | 42 | gem 'therubyracer' 43 | 44 | group :development, :test do 45 | gem 'faker' 46 | gem 'pry-rails' 47 | gem 'pry-nav' 48 | 49 | # Call 'byebug' anywhere in the code to stop execution and get a debugger console 50 | gem 'byebug' 51 | 52 | # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring 53 | gem 'spring' 54 | 55 | gem 'minitest-reporters' 56 | gem 'mini_backtrace' 57 | gem 'guard' # without this you get an error when running tests. See https://github.com/guard/guard-minitest#install 58 | gem 'guard-minitest' 59 | 60 | #gem "factory_girl_rails" 61 | end 62 | 63 | group :development do 64 | # Access an IRB console on exception pages or by using <%= console %> in views 65 | gem 'web-console', '~> 3.0' 66 | 67 | end 68 | 69 | group :test do 70 | 71 | end 72 | 73 | group :production do 74 | #gem 'pg', '0.17.1' 75 | gem 'rails_12factor', '0.0.3' 76 | end 77 | 78 | 79 | # Use ActiveModel has_secure_password 80 | # gem 'bcrypt', '~> 3.1.7' 81 | 82 | # Use Unicorn as the app server 83 | # gem 'unicorn' 84 | 85 | # Use Capistrano for deployment 86 | # gem 'capistrano-rails', group: :development 87 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 David Brennan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mira 2 | 3 | Mira is a Ruby on Rails application which gives you a simple HTTP API for CSV files. 4 | 5 | ## Summary 6 | 7 | 1. You create a Mira *project*. A project is simply a home for one or more CSV files, along with a datapackage.json file. More on that now... 8 | 9 | 2. You provide Mira with information about your CSV files by uploading a datapackage.json file to the project. This file provides metadata for the CSV files you plan to upload to the project. i.e. file names, columns names and types, delimiters etc. See [here](http://data.okfn.org/doc/tabular-data-package) and [here](https://frictionlessdata.io/docs/tabular-data-package/) for more information about datapackage.json files and tabular data packages. 10 | 11 | 3. With the datapackage.json file Mira then does the following: 12 | - it creates an empty database table for each CSV file specified in the datapackage.json file. 13 | - it creates an API to these database tables which you can use to read and write data. 14 | 15 | 4. You write data to the database tables by uploading CSV files, or by using a JSON API. 16 | 17 | 5. You can query the data using simple API requests. Consider a table `mytable` in a project, with columns `col1`, `col2` and `col3`. To get rows where `col1` equals "XXX", `col2` equals "YYY" and `col3` equals "ZZZ", you could make the following `GET` request: 18 | 19 | 
 20 |   http://localhost:3000/api/projects/1/tables/mytable/data?col1_eq=XXX&col2_eq=YYY&col3_eq=ZZZ
 21 |
22 | 23 | See the demo for more details on how the data can be queried. 24 | 25 | 6. You can generate API keys to control the reading and writing data. 26 | 27 | 28 | ## Quick Start 29 | 30 | #### Pre-requisites 31 | - You're familiar with [Ruby](https://www.ruby-lang.org/en/) and [Ruby on Rails](http://rubyonrails.org/). 32 | 33 | - PostgreSQL is installed 34 | 35 | [https://www.digitalocean.com/community/tutorials/how-to-use-postgresql-with-your-ruby-on-rails-application-on-ubuntu-14-04] (https://www.digitalocean.com/community/tutorials/how-to-use-postgresql-with-your-ruby-on-rails-application-on-ubuntu-14-04) 36 | 37 | 38 | --- 39 | 40 | 1. Clone the repository 41 | 42 | 2. Run bundle 43 | 44 | bundle install 45 | 46 | 3. Update the config/database.yml file with your database credentials. Assuming you've created a user "mira" with full access to a database of the same name: 47 | 48 | default: &default 49 | adapter: postgresql 50 | encoding: unicode 51 | pool: 5 52 | host: localhost 53 | port: 5432 54 | username: mira 55 | password: **your_password_here** 56 | 57 | development: 58 | <<: *default 59 | database: mira_dev 60 | 61 | test: 62 | <<: *default 63 | database: mira_test 64 | 65 | 4. Create and migrate database, and seed database with a single admin user (email = admin@example.com and password = topsecret): 66 | 67 | rake db:create 68 | rake db:migrate 69 | rake db:seed 70 | 71 | 5. Start your local development server 72 | 73 | rails s 74 | 75 | 6. In a separate terminal start a background job to process uploaded files 76 | 77 | rake jobs:work 78 | 79 | 7. Open up the Mira homepage: 80 | 81 | [http://localhost:3000] (http://localhost:3000) 82 | 83 | 8. Download sample csv files + their datapackage.json file: 84 | 85 | [mira_sample_data.tar.gz] (https://github.com/davbre/dummy-sdtm/blob/master/output/mira_sample_data/mira_sample_data.tar.gz) 86 | or 87 | [mira_sample_data.zip] (https://github.com/davbre/dummy-sdtm/blob/master/output/mira_sample_data/mira_sample_data.zip) 88 | 89 | 9. Log in, create a new project, first upload the datapackage.json file, then the sample csv files 90 | 91 | 10. Navigate to the following address for the project's API details: 92 | 93 | [http://localhost:3000/projects/1/api-details](http://localhost:3000/projects/1/api-details) 94 | 95 | 96 | ## Extra Notes 97 | 98 | Assuming a write API key has been generated, here's how you can write data: 99 | 100 | curl -d "data[col1]=value1&data[col2]=val2" -H "X-Api-Key: 6041fa394bc84abe46ffdb71" http://localhost:3000/api/projects/1/tables/mytable/data 101 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # Add your own tasks in files placed in lib/tasks ending in .rake, 2 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 3 | 4 | require File.expand_path('../config/application', __FILE__) 5 | 6 | Rails.application.load_tasks 7 | -------------------------------------------------------------------------------- /app/assets/javascripts/application.js: -------------------------------------------------------------------------------- 1 | // This is a manifest file that'll be compiled into application.js, which will include all the files 2 | // listed below. 3 | // 4 | // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts, 5 | // or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path. 6 | // 7 | // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the 8 | // compiled file. 9 | // 10 | // Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details 11 | // about supported directives. 12 | // 13 | //= require jquery3 14 | //= require jquery_ujs 15 | //= require jquery-ui 16 | //= require turbolinks 17 | //= require_tree . 18 | //= require bootstrap-sprockets 19 | //= require jtable/jquery.jtable 20 | -------------------------------------------------------------------------------- /app/assets/stylesheets/application.css: -------------------------------------------------------------------------------- 1 | /* 2 | * This is a manifest file that'll be compiled into application.css, which will include all the files 3 | * listed below. 4 | * 5 | * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets, 6 | * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path. 7 | * 8 | * You're free to add application-wide styles to this file and they'll appear at the bottom of the 9 | * compiled file so the styles you add here take precedence over styles defined in any styles 10 | * defined in the other CSS/SCSS files in this directory. It is generally better to create a new 11 | * file per style scope. 12 | * 13 | *= require_tree . 14 | *= require_self 15 | *= require jquery-ui 16 | * require jtable/themes/metro/blue/jtable 17 | * require normalize/normalize 18 | *= require jtable/themes/jqueryui/jtable_jqueryui 19 | *= require jquery-ui-1.10.1.custom.min 20 | */ 21 | -------------------------------------------------------------------------------- /app/assets/stylesheets/custom.css.scss: -------------------------------------------------------------------------------- 1 | @import "bootstrap-sprockets"; 2 | @import "bootstrap"; 3 | 4 | /* universal */ 5 | 6 | body { 7 | padding-top: 60px; 8 | } 9 | 10 | section { 11 | overflow: auto; 12 | } 13 | 14 | textarea { 15 | resize: vertical; 16 | } 17 | 18 | .center { 19 | text-align: center; 20 | } 21 | 22 | .center h1 { 23 | margin-bottom: 10px; 24 | } 25 | 26 | /* typography */ 27 | 28 | h1, h2, h3, h4, h5, h6 { 29 | line-height: 1; 30 | } 31 | 32 | h1 { 33 | font-size: 3em; 34 | letter-spacing: -2px; 35 | margin-bottom: 30px; 36 | text-align: center; 37 | } 38 | 39 | /*h2 { 40 | font-size: 1.2em; 41 | letter-spacing: -1px; 42 | margin-bottom: 30px; 43 | text-align: center; 44 | font-weight: normal; 45 | color: #777; 46 | }*/ 47 | 48 | p { 49 | font-size: 1.1em; 50 | line-height: 1.7em; 51 | } 52 | 53 | /* header */ 54 | 55 | #logo { 56 | float: left; 57 | margin-right: 10px; 58 | font-size: 1.7em; 59 | color: #fff; 60 | text-transform: uppercase; 61 | letter-spacing: -1px; 62 | padding-top: 9px; 63 | font-weight: bold; 64 | } 65 | 66 | #logo:hover { 67 | color: #fff; 68 | text-decoration: none; 69 | } 70 | 71 | /* footer */ 72 | 73 | footer { 74 | margin-top: 45px; 75 | padding-top: 5px; 76 | border-top: 1px solid #eaeaea; 77 | color: #777; 78 | } 79 | 80 | footer a { 81 | color: #555; 82 | } 83 | 84 | footer a:hover { 85 | color: #222; 86 | } 87 | 88 | footer small { 89 | float: left; 90 | } 91 | 92 | footer ul { 93 | float: right; 94 | list-style: none; 95 | } 96 | 97 | footer ul li { 98 | float: left; 99 | margin-left: 15px; 100 | } 101 | 102 | 103 | /* mixins, variables, etc. */ 104 | 105 | $gray-medium-light: #eaeaea; 106 | 107 | @mixin box_sizing { 108 | -moz-box-sizing: border-box; 109 | -webkit-box-sizing: border-box; 110 | box-sizing: border-box; 111 | } 112 | 113 | 114 | 115 | /* miscellaneous */ 116 | 117 | .debug_dump { 118 | clear: both; 119 | float: left; 120 | width: 100%; 121 | margin-top: 45px; 122 | @include box_sizing; 123 | } 124 | 125 | 126 | /* forms */ 127 | 128 | input, textarea, /*select,*/ .uneditable-input { 129 | border: 1px solid #bbb; 130 | width: 100%; 131 | margin-bottom: 15px; 132 | @include box_sizing; 133 | } 134 | 135 | input { 136 | height: auto !important; 137 | } 138 | 139 | #error_explanation { 140 | color: red; 141 | ul { 142 | color: red; 143 | margin: 0 0 30px 0; 144 | } 145 | } 146 | 147 | .field_with_errors { 148 | @extend .has-error; 149 | .form-control { 150 | color: $state-danger-text; 151 | } 152 | } 153 | 154 | /* Projects index */ 155 | 156 | .projects { 157 | list-style: none; 158 | margin: 0; 159 | li { 160 | overflow: auto; 161 | padding: 10px 0; 162 | border-bottom: 1px solid $gray-lighter; 163 | } 164 | } 165 | 166 | /* Devise log in links */ 167 | ul.hmenu { 168 | list-style: none; 169 | margin: 0 0 2em; 170 | padding: 0; 171 | } 172 | 173 | ul.hmenu li { 174 | display: inline; 175 | } 176 | 177 | /* copied from Bootstrap, renamed and adjusted */ 178 | .dl-horizontal2 dt { 179 | float: left; 180 | width: 120px; 181 | clear: left; 182 | text-align: right; 183 | overflow: hidden; 184 | text-overflow: ellipsis; 185 | white-space: nowrap; 186 | } 187 | /* copied from Bootstrap, renamed and adjusted */ 188 | .dl-horizontal2 dd { 189 | margin-left: 130px; 190 | margin-bottom: 5px; 191 | } 192 | dt { 193 | font-weight: normal; 194 | } 195 | dd, dt { 196 | line-height: 130%; 197 | } 198 | dl { 199 | margin-top: 10; 200 | margin-bottom: 10px; 201 | } 202 | 203 | .special-mira-col { 204 | background-color: LightGray; 205 | } 206 | -------------------------------------------------------------------------------- /app/assets/stylesheets/signin.css.scss: -------------------------------------------------------------------------------- 1 | body { 2 | padding-top: 40px; 3 | padding-bottom: 40px; 4 | background-color: #eee; 5 | } 6 | 7 | .form-signin { 8 | max-width: 330px; 9 | padding: 15px; 10 | margin: 0 auto; 11 | } 12 | .form-signin .form-signin-heading, 13 | .form-signin .checkbox { 14 | margin-bottom: 10px; 15 | } 16 | .form-signin .checkbox { 17 | font-weight: normal; 18 | } 19 | .form-signin .form-control { 20 | position: relative; 21 | height: auto; 22 | -webkit-box-sizing: border-box; 23 | -moz-box-sizing: border-box; 24 | box-sizing: border-box; 25 | padding: 10px; 26 | font-size: 16px; 27 | } 28 | .form-signin .form-control:focus { 29 | z-index: 2; 30 | } 31 | .form-signin input[type="email"] { 32 | margin-bottom: -1px; 33 | border-bottom-right-radius: 0; 34 | border-bottom-left-radius: 0; 35 | } 36 | .form-signin input[type="password"] { 37 | margin-bottom: 10px; 38 | border-top-left-radius: 0; 39 | border-top-right-radius: 0; 40 | } 41 | 42 | /*.boolean, .controls{ 43 | float: left; 44 | margin: 20px; 45 | }*/ -------------------------------------------------------------------------------- /app/controllers/api/v1/datapackage_resource_fields_controller.rb: -------------------------------------------------------------------------------- 1 | module Api 2 | module V1 3 | 4 | 5 | class DatapackageResourceFieldsController < ActionController::Base 6 | 7 | include ApplicationHelper 8 | include DataAccessHelper 9 | 10 | before_action :key_authorize_read 11 | 12 | def index 13 | project = Project.find(params[:id]) 14 | datapackage = Datapackage.where(project_id: project.id).first 15 | datapackage_resource = datapackage.datapackage_resources.where(table_ref: params[:table_ref]).first 16 | resource_fields = DatapackageResourceField.where(datapackage_resource_id: datapackage_resource.id).order(:order) 17 | response_array = [] 18 | resource_fields.each do |fld| 19 | full_field_response = rename_ftype(fld.attributes) 20 | trimmed_field_response = full_field_response.select {|k,v| keep_keys.include? k } 21 | response_array << trimmed_field_response 22 | end 23 | response = response_array.as_json 24 | render json: response 25 | end 26 | 27 | def show 28 | project = Project.find(params[:id]) 29 | datapackage = Datapackage.where(project_id: project.id).first 30 | datapackage_resource = datapackage.datapackage_resources.where(table_ref: params[:table_ref]).first 31 | resource_field = DatapackageResourceField.where(datapackage_resource_id: datapackage_resource.id,name: params[:col_ref]).order(:order).first 32 | response = rename_ftype(resource_field.attributes) 33 | response = response.as_json(:only => keep_keys) 34 | render json: response 35 | end 36 | 37 | private 38 | 39 | def rename_ftype(resp_hash) 40 | resp_hash["type"] = resp_hash.delete "ftype" 41 | resp_hash 42 | end 43 | 44 | def keep_keys 45 | ["name", "type", "order", "add_index", "big_integer", "format"] 46 | end 47 | end 48 | 49 | 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /app/controllers/api/v1/datapackage_resources_controller.rb: -------------------------------------------------------------------------------- 1 | module Api 2 | module V1 3 | 4 | 5 | class DatapackageResourcesController < ActionController::Base 6 | 7 | include ApplicationHelper 8 | include DataAccessHelper 9 | 10 | before_action :key_authorize_read 11 | 12 | def index 13 | project = Project.find(params[:id]) 14 | datapackage = Datapackage.where(project_id: project.id).first 15 | datapackage_resources = datapackage.present? ? datapackage.datapackage_resources : [] 16 | paginate json: datapackage_resources, except: [:db_table_name] 17 | end 18 | 19 | def show 20 | project = Project.find(params[:id]) 21 | datapackage = Datapackage.where(project_id: project.id).first 22 | datapackage_resource = datapackage.datapackage_resources.where(table_ref: params[:table_ref]).first 23 | render json: datapackage_resource, except: [:db_table_name] 24 | end 25 | 26 | def column_index 27 | project = Project.find(params[:id]) 28 | datapackage = Datapackage.where(project_id: project.id).first 29 | datapackage_resource = DatapackageResource.where(datapackage_id: datapackage.id, table_ref: params[:table_ref]).first 30 | ar_object = get_mira_ar_table(datapackage_resource.db_table_name) 31 | render_hash = {} 32 | ar_object.columns_hash.each { |a| render_hash[a[1].name] = a[1].sql_type } 33 | render json: render_hash 34 | end 35 | 36 | def column_show 37 | project = Project.find(params[:id]) 38 | datapackage = Datapackage.where(project_id: project.id).first 39 | datapackage_resource = DatapackageResource.where(datapackage_id: datapackage.id, table_ref: params[:table_ref]).first 40 | ar_object_col = get_mira_ar_table(datapackage_resource.db_table_name).columns_hash[params[:col_ref]] 41 | render_hash = { "name" => ar_object_col.name, "type" => ar_object_col.sql_type} 42 | render json: render_hash 43 | end 44 | 45 | 46 | end 47 | 48 | 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /app/controllers/api/v1/datapackages_controller.rb: -------------------------------------------------------------------------------- 1 | module Api 2 | module V1 3 | 4 | 5 | class DatapackagesController < ActionController::Base 6 | 7 | include ApplicationHelper 8 | include DataAccessHelper 9 | 10 | before_action :key_authorize_read 11 | 12 | def show 13 | project = Project.find(params[:id]) 14 | datapackage = Datapackage.where(project_id: project.id).first 15 | render json: datapackage 16 | end 17 | 18 | end 19 | 20 | 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /app/controllers/api/v1/datasources_controller.rb: -------------------------------------------------------------------------------- 1 | module Api 2 | module V1 3 | 4 | 5 | class DatasourcesController < ActionController::Base 6 | 7 | include ApplicationHelper 8 | 9 | before_action :key_authorize_read 10 | 11 | def index 12 | datasources = Project.find(params[:id]).datasources 13 | response = datasources.as_json(:except => datasource_exclude_fields) 14 | paginate json: response 15 | end 16 | 17 | 18 | def show 19 | datasource = Project.find(params[:id]).datasources.where(datafile_file_name: params[:table_ref] + ".csv").first 20 | response = datasource.as_json(:except => datasource_exclude_fields) 21 | render json: response 22 | end 23 | 24 | 25 | private 26 | 27 | def datasource_exclude_fields 28 | exclude = user_signed_in? ? [] : ["db_table_name"] 29 | exclude 30 | end 31 | end 32 | 33 | 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /app/controllers/api/v1/projects_controller.rb: -------------------------------------------------------------------------------- 1 | module Api 2 | module V1 3 | 4 | class ProjectsController < ActionController::Base 5 | 6 | def index 7 | header_api_key = request.headers['X-Api-Key'] 8 | api_key = ApiKey.where(token: header_api_key).first 9 | # if the API key sent is a global key OR if no global key exists, then send all projects 10 | if key_has_global_permission?(api_key) || !global_permission_exists? 11 | resp = Project.all 12 | # if a global permission exists (but is not sent), then only display the projects relating to that key 13 | elsif global_permission_exists? 14 | if api_key.nil? 15 | resp = not_authorized_response 16 | else 17 | key_project_ids = get_key_project_ids(api_key) 18 | resp = Project.where(project_id: key_project_ids) 19 | paginate json: resp 20 | end 21 | else 22 | resp = { message: "No projects to list" } 23 | end 24 | # reaches here if nothing to paginate or error 25 | render json: resp 26 | 27 | end 28 | 29 | 30 | def show 31 | header_api_key = request.headers['X-Api-Key'] 32 | api_key = ApiKey.where(token: header_api_key).first 33 | key_project_ids = (api_key.present?) ? get_key_project_ids(api_key) : [] 34 | if key_has_global_permission?(api_key) || !global_permission_exists? \ 35 | || key_project_ids.include?(params[:id].to_i) 36 | resp = Project.find(params[:id]) 37 | else 38 | resp = not_authorized_response 39 | end 40 | render json: resp 41 | end 42 | 43 | 44 | private 45 | 46 | def not_authorized_response 47 | {errors: [ code: 401, message: "Not authorized." ]} 48 | end 49 | 50 | def key_has_global_permission?(api_key) 51 | return false if api_key.nil? 52 | ApiKeyPermission.where(api_key_id: api_key.id, permission_scope: 0).present? 53 | end 54 | 55 | def global_permission_exists? 56 | ApiKeyPermission.where(permission_scope: 0).present? 57 | end 58 | 59 | def get_key_project_ids(api_key) 60 | ApiKeyPermission.where(api_key_id: api_key.id, permission_scope: 1).map { |p| p.project_id } 61 | end 62 | end 63 | 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /app/controllers/api_keys_controller.rb: -------------------------------------------------------------------------------- 1 | class ApiKeysController < ApplicationController 2 | 3 | include ApplicationHelper 4 | include DatapackageHelper 5 | 6 | before_action :authenticate_user! 7 | before_action :correct_user#, only: [ :destroy, :edit, :update ] 8 | 9 | 10 | def index 11 | @keys = ApiKey.where(user_id: current_user.id).order(id: :desc).page params[:page] # kaminari 12 | end 13 | 14 | 15 | def show 16 | @key = ApiKey.find(params[:id]) 17 | end 18 | 19 | 20 | def new 21 | @key = ApiKey.new 22 | end 23 | 24 | 25 | def create 26 | @key = current_user.api_keys.build(api_key_params) 27 | @key.token = generate_api_key 28 | if @key.save 29 | flash[:success] = "New API key created." 30 | redirect_to user_api_keys_url(current_user) 31 | else 32 | render 'new' 33 | end 34 | end 35 | 36 | 37 | def edit 38 | @key = ApiKey.find(params[:id]) 39 | # @key.token = generate_api_key 40 | # @key.save 41 | end 42 | 43 | 44 | def update 45 | @key = ApiKey.find(params[:id]) 46 | if @key.update_attributes(api_key_params) 47 | flash[:success] = "API key updated" 48 | redirect_to user_api_keys_url(@user) 49 | else 50 | render 'edit' 51 | end 52 | end 53 | 54 | 55 | def destroy 56 | key = ApiKey.where(user: current_user.id, id: params[:id]).first 57 | if key.destroy 58 | flash[:success] = "API key deleted" 59 | else 60 | flash[:error] = "Failed to delete API key" 61 | end 62 | redirect_to user_api_keys_url(current_user) 63 | end 64 | 65 | 66 | def gen_new_key 67 | @key = ApiKey.find(params[:id]) 68 | @key.token = generate_api_key 69 | if @key.save 70 | flash[:success] = "New API key created: " + @key.token 71 | else 72 | flash[:error] = "Failed to generate new API key!" 73 | end 74 | redirect_to user_api_keys_url(current_user) 75 | end 76 | 77 | 78 | def index_project 79 | select_str = "api_key_permissions.id,api_key_permissions.permission" 80 | select_str += ",api_key_permissions.permission_scope,api_keys.description,api_keys.token" 81 | @proj_keys_info = ApiKeyPermission.where(project_id: [nil, params[:project_id]]).joins(:api_key).select(select_str).order(id: :desc).page params[:page] 82 | end 83 | 84 | 85 | private 86 | 87 | 88 | # Rails strong parameters 89 | def api_key_params 90 | params.require(:api_key).permit(:description) 91 | end 92 | 93 | def correct_user 94 | @user = current_user 95 | redirect_to root_url if params[:user_id] != current_user.id.to_s 96 | end 97 | 98 | def generate_api_key 99 | loop do 100 | token = SecureRandom.hex(12) 101 | break token unless ApiKey.exists?(token: token) 102 | end 103 | end 104 | 105 | end 106 | -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | class ApplicationController < ActionController::Base 2 | 3 | # Prevent CSRF attacks by raising an exception. 4 | # For APIs, you may want to use :null_session instead. 5 | 6 | # DBR going to turn this off as want to POST uploads via curl 7 | # and only currently open to an admin user 8 | # protect_from_forgery with: :exception 9 | 10 | end 11 | -------------------------------------------------------------------------------- /app/controllers/datapackage_resources_controller.rb: -------------------------------------------------------------------------------- 1 | class DatapackageResourcesController < ApplicationController 2 | 3 | include ApplicationHelper 4 | 5 | before_action :authenticate_user! 6 | # before_action :correct_user, only: [:destroy] 7 | 8 | def show 9 | @project = Project.find(params[:project_id]) 10 | @dpr = DatapackageResource.find(params[:id]) 11 | @dpr_ds = Datasource.where(datapackage_resource_id: @dpr.id).order(id: :desc).page params[:page] # kaminari 12 | db_table = get_mira_ar_table(@dpr.db_table_name) 13 | @apikey_ids = db_table.where(mira_source_type: "key").uniq.pluck(:mira_source_id) 14 | @fields = DatapackageResourceField.where(datapackage_resource_id: @dpr.id).order(:order) 15 | @tableUrl = request.base_url + "/api/projects/" + @project.id.to_s + "/tables/" + @dpr.table_ref + "/" 16 | end 17 | 18 | 19 | def delete_apikey_rows 20 | @proj = Project.find(params[:project_id]) 21 | @dpr = DatapackageResource.find(params[:id]) 22 | db_table = get_mira_ar_table(@dpr.db_table_name) 23 | num_rows_deleted = db_table.where( 24 | mira_source_type: "key", 25 | mira_source_id: params[:api_key_id]).delete_all 26 | flash[:notice] = "Deleted " + num_rows_deleted.to_s + " rows from the corresponding database table." 27 | redirect_to project_datapackage_datapackage_resource_path(@proj,@dpr) 28 | end 29 | 30 | 31 | def show_orig 32 | # /projects/:project_id/datapackage/datapackage_resources/:id 33 | # The aim is to show the datapackage resource (which gives the metadata of those files 34 | # that CAN be uploaded), and alongside this the actual files that have been uploaded 35 | @project = Project.find(params[:project_id]) 36 | @dpr = DatapackageResource.find(params[:id]) 37 | @dpr_ds = Datasource.where(datapackage_resource_id: @dpr.id) 38 | # get the unique API key IDs in the table 39 | db_table = get_mira_ar_table(@dpr.db_table_name) 40 | key_ids = db_table.uniq.pluck(:mira_source_id) 41 | end 42 | 43 | # def destroy 44 | # ds = Datasource.where(id: params[:id]).first 45 | # dp_res = DatapackageResource.where(datasource_id: ds.id).first 46 | # dp_res.delete_associated_artifacts 47 | # dp_res.clear_db_table 48 | # Datasource.find(params[:id]).destroy 49 | # redirect_to project_path(params[:project_id]) 50 | # end 51 | 52 | private 53 | def correct_user 54 | @ds = current_user.projects.find_by(id: params[:project_id]).datasources.find_by(id: params[:id]) 55 | redirect_to root_url if @ds.nil? 56 | end 57 | 58 | end 59 | -------------------------------------------------------------------------------- /app/controllers/datapackages_controller.rb: -------------------------------------------------------------------------------- 1 | class DatapackagesController < ApplicationController 2 | 3 | include ApplicationHelper 4 | include DataAccessHelper 5 | 6 | before_action :key_authorize_read, only: [:show] 7 | 8 | def show 9 | project = Project.find(params[:project_id]) 10 | @datapackage = project.datapackage 11 | send_file @datapackage.datapackage.path, :type => @datapackage.datapackage_content_type, :disposition => 'inline' 12 | end 13 | 14 | # to destroy a datapackage, we just destroy the project (because the project is 15 | # pretty much defined by the datapackage), so no need for action here 16 | 17 | end 18 | -------------------------------------------------------------------------------- /app/controllers/datasources_controller.rb: -------------------------------------------------------------------------------- 1 | class DatasourcesController < ApplicationController 2 | 3 | include ApplicationHelper 4 | include DataAccessHelper 5 | 6 | before_action :authenticate_user!, only: [:destroy] 7 | before_action :correct_user, only: [:destroy] 8 | 9 | before_action :key_authorize_read, only: [:show, :index] 10 | # before_action :key_authorize_write, only: [:destroy, :update] 11 | 12 | def show 13 | @ds = Datasource.find(params[:id]) 14 | send_file @ds.datafile.path, :type => @ds.datafile_content_type #, :disposition => 'inline' 15 | end 16 | 17 | def index 18 | @proj = Project.find(params[:project_id]) 19 | @dss = @proj.datasources.page params[:page] 20 | end 21 | 22 | def destroy 23 | @proj = Project.find(params[:project_id]) 24 | @ds = Datasource.find(params[:id]) 25 | @dpr = DatapackageResource.find(@ds.datapackage_resource_id) 26 | # delete_associated_artifacts 27 | num_rows_deleted = remove_datasource_rows_from_db_table 28 | flash[:notice] = "Deleted " + num_rows_deleted.to_s + " rows from the corresponding database table." 29 | ds_name = @ds.datafile_file_name 30 | if Datasource.find(params[:id]).destroy 31 | flash[:notice].prepend(ds_name + " successfully deleted. ") 32 | end 33 | redirect_to project_datapackage_datapackage_resource_path(@proj,@dpr) 34 | end 35 | 36 | private 37 | 38 | def correct_user 39 | @ds = current_user.projects.find_by(id: params[:project_id]).datasources.find_by(id: params[:id]) 40 | redirect_to root_url if @ds.nil? 41 | end 42 | 43 | def remove_datasource_rows_from_db_table 44 | dpr = DatapackageResource.find(@ds.datapackage_resource_id) 45 | db_table = get_mira_ar_table(dpr.db_table_name) 46 | num_rows_deleted = db_table.where(mira_source_type: "csv", mira_source_id: @ds.id).delete_all 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /app/helpers/application_helper.rb: -------------------------------------------------------------------------------- 1 | 2 | module ApplicationHelper 3 | 4 | # Returns the full title on a per-page basis. 5 | def full_title(page_title = '') 6 | base_title = "Mira" 7 | if page_title.empty? 8 | base_title 9 | else 10 | "#{page_title} | #{base_title}" 11 | end 12 | end 13 | 14 | 15 | # http://stackoverflow.com/questions/6672244/convert-ruby-string-to-nix-filename-compatible-string 16 | def friendly_filename(filename) 17 | filename.gsub(/[^\w\s_-]+/, '') 18 | .gsub(/(^|\b\s)\s+($|\s?\b)/, '\\1\\2') 19 | .gsub(/\s+/, '_') 20 | end 21 | 22 | #http://stackoverflow.com/questions/5661466/test-if-string-is-a-number-in-ruby-on-rails 23 | def is_number? string 24 | true if Float(string) rescue false 25 | end 26 | 27 | 28 | def get_mira_ar_table(table) # table is a string 29 | begin 30 | ar_table = Mira::Application.const_get(table.capitalize) 31 | rescue NameError => e 32 | load_dynamic_AR_class_with_scopes(table) 33 | end 34 | end 35 | 36 | 37 | def custom_is_string_int?(str) # http://stackoverflow.com/a/1235990/1002140 38 | /\A[-+]?\d+\z/ === str 39 | end 40 | 41 | 42 | def bootstrap_class_for(flash_type) 43 | case flash_type 44 | when "success" 45 | "alert-success" # Green 46 | when "error" 47 | "alert-danger" # Red 48 | when "alert" 49 | "alert-warning" # Yellow 50 | when "notice" 51 | "alert-info" # Blue 52 | else 53 | flash_type.to_s 54 | end 55 | end 56 | 57 | end 58 | -------------------------------------------------------------------------------- /app/helpers/data_access_helper.rb: -------------------------------------------------------------------------------- 1 | module DataAccessHelper 2 | 3 | private 4 | def key_authorize_read 5 | 6 | project_id = params[:project_id] || params[:id] 7 | 8 | # If no project key permissions set then allow read access. Otherwise check for valid permission. 9 | if no_project_permissions?(project_id) 10 | global_permission = "ok" 11 | elsif db_key.present? 12 | global_permission = ApiKeyPermission.where(api_key_id: db_key.id, permission_scope: 0).first 13 | if global_permission.nil? 14 | project_permission = ApiKeyPermission.where(api_key_id: db_key.id, permission_scope: 1, project_id: params[:id]).first 15 | else 16 | project_permission = nil 17 | end 18 | end 19 | 20 | unless global_permission || project_permission || current_user 21 | render json: resp401("for read access"), status: 401 22 | end 23 | end 24 | 25 | 26 | def key_authorize_write 27 | if db_key.present? 28 | global_permission = ApiKeyPermission.where(api_key_id: db_key.id, permission_scope: 0, permission: 1).first 29 | if global_permission.nil? 30 | project_permission = ApiKeyPermission.where(api_key_id: db_key.id, permission_scope: 1, permission: 1, project_id: params[:id]).first 31 | else 32 | project_permission = nil 33 | end 34 | end 35 | 36 | unless global_permission || project_permission || current_user 37 | render json: resp401("for write access"), status: 401 38 | end 39 | end 40 | 41 | 42 | def no_project_permissions?(project_id) 43 | # Check no global API key and no project specific key. If none => we will later allow read access 44 | ApiKeyPermission.where(project_id: nil,permission_scope: 0).empty? && ApiKeyPermission.where(project_id: project_id).empty? 45 | end 46 | 47 | 48 | def db_key 49 | header_api_key = request.headers['X-Api-Key'] 50 | ApiKey.where(token: header_api_key).first # should be unique on DB 51 | end 52 | 53 | def resp401(extra) 54 | message = "No valid API key provided." 55 | message = message.chop + " " + extra + "." if extra.present? 56 | {errors: [ code: 401, message: message]} 57 | end 58 | 59 | end 60 | -------------------------------------------------------------------------------- /app/helpers/datapackage_helper.rb: -------------------------------------------------------------------------------- 1 | module DatapackageHelper 2 | 3 | end 4 | -------------------------------------------------------------------------------- /app/jobs/process_csv_upload.rb: -------------------------------------------------------------------------------- 1 | require 'load_table' 2 | require 'net/http' 3 | 4 | 5 | # https://github.com/collectiveidea/delayed_job#custom-jobs 6 | class ProcessCsvUpload 7 | 8 | include Rails.application.routes.url_helpers 9 | include ProjectHelper 10 | 11 | def initialize(datasource_id,upload_method) 12 | @ds = Datasource.find(datasource_id) 13 | @dp = Datapackage.find(@ds.datapackage_id) 14 | mapped_csv_name = map_csv_basename(@dp, basename(@ds.datafile_file_name)) 15 | @datapackage_resource = DatapackageResource.where(datapackage_id: @dp.id, table_ref: mapped_csv_name).first 16 | @upload_method = upload_method 17 | end 18 | 19 | def job_logger 20 | log_dir = Project.find(@ds.project_id).job_log_path 21 | Dir.mkdir(log_dir) unless File.directory?(log_dir) 22 | @job_logger ||= Logger.new(@ds.logfile_path) 23 | # @job_logger ||= Logger.new("#{log_dir}/#{@ds.datafile_file_name}.log") 24 | end 25 | 26 | def max_attempts 27 | 1 28 | end 29 | 30 | def perform 31 | # puts "About to process: " + @ds.datafile_file_name 32 | job_logger.info("About to process " + @ds.datafile_file_name + " using '" + @upload_method + "' upload method") 33 | LoadTable.new(@ds, @datapackage_resource, @upload_method) 34 | end 35 | 36 | def success 37 | job_logger.info("Finished uploading " + @ds.datafile_file_name + " to the database") 38 | # we can now set the datasource_id in the datapackage_resource table as we 39 | # know it has been uploaded 40 | # @datapackage_resource.datasource_id = @ds.id 41 | if @datapackage_resource.save 42 | job_logger.info("Saved the datapackage_resource table") 43 | @ds.ok! 44 | else 45 | job_logger.error("Unexpected - failed to save the datapackage_resource table!") 46 | @ds.error! 47 | end 48 | # TODO log some upload info, number or rows, column names. 49 | end 50 | 51 | def error(job,exception) 52 | # if @datapackage_resource.save 53 | # job_logger.info("Saved the datasource_id to the datapackage_resource table") 54 | # end 55 | job_logger.error("Something went wrong while loading " + @ds.datafile_file_name + " into the database...") 56 | job_logger.error(exception) 57 | @ds.error! 58 | end 59 | 60 | end 61 | -------------------------------------------------------------------------------- /app/models/api_key.rb: -------------------------------------------------------------------------------- 1 | class ApiKey < ActiveRecord::Base 2 | belongs_to :user 3 | has_many :api_key_permissions, dependent: :destroy 4 | validates :user, presence: true # using user_id here does not enforce that user actually exists. Using user does. 5 | validates :token, presence: true, length: { is: 24 }, uniqueness: true 6 | end 7 | -------------------------------------------------------------------------------- /app/models/api_key_permission.rb: -------------------------------------------------------------------------------- 1 | class ApiKeyPermission < ActiveRecord::Base 2 | 3 | enum permission_scope: [:global, :project, :table] 4 | enum permission: [ :read, :write ] 5 | 6 | belongs_to :api_key 7 | validates :api_key, presence: true # using api_key to enforce that api_key exists (instead of using api_key_id) 8 | validates :permission, presence: true 9 | validates :permission_scope, presence: true 10 | validates :project_id, uniqueness: { scope: :api_key_id, message: "Each API key can have only one permission per project!" } 11 | end 12 | -------------------------------------------------------------------------------- /app/models/datapackage.rb: -------------------------------------------------------------------------------- 1 | class Datapackage < ActiveRecord::Base 2 | belongs_to :project 3 | has_many :datasources, dependent: :destroy 4 | has_many :datapackage_resources, dependent: :destroy 5 | validates :project_id, presence: true 6 | 7 | has_attached_file :datapackage, 8 | :path => ":rails_root/" + Rails.configuration.x.upload_path + "/project_:proj_id/:filename", 9 | :url => "/uploads/project_:proj_id/:filename" 10 | 11 | validates_attachment :datapackage, content_type: { :content_type => ["text/plain", "application/json"] } 12 | validates_attachment_file_name :datapackage, :matches => [/datapackage.json\Z/] 13 | 14 | private 15 | 16 | # Use an interpolation to get project_id into the path 17 | # https://github.com/thoughtbot/paperclip/wiki/Interpolations 18 | #http://stackoverflow.com/questions/9173920/paperclip-custom-path-with-id-belongs-to 19 | Paperclip.interpolates :proj_id do |attachment, style| 20 | attachment.instance.project_id 21 | end 22 | 23 | end 24 | -------------------------------------------------------------------------------- /app/models/datapackage_resource.rb: -------------------------------------------------------------------------------- 1 | class DatapackageResource < ActiveRecord::Base 2 | belongs_to :datapackage 3 | has_many :datapackage_resource_fields, dependent: :destroy 4 | has_many :datasources, dependent: :destroy 5 | validates :datapackage_id, presence: true 6 | validates :path, presence: true 7 | validates :delimiter, exclusion: {in: [nil]}, allow_blank: false # using this instead of presence: true as was unable to save "\t" (signifying tab-delimited) 8 | validates :quote_character, presence: true 9 | validates :table_ref, presence: true, uniqueness: { scope: :datapackage, 10 | message: "table_ref must be unique within a datapackage!" } 11 | 12 | 13 | def delete_db_table 14 | table = self.db_table_name 15 | if ActiveRecord::Base.connection.table_exists? table 16 | ActiveRecord::Base.connection.drop_table(table) 17 | end 18 | end 19 | 20 | end 21 | -------------------------------------------------------------------------------- /app/models/datapackage_resource_field.rb: -------------------------------------------------------------------------------- 1 | class DatapackageResourceField < ActiveRecord::Base 2 | belongs_to :datapackage_resource 3 | validates :datapackage_resource_id, presence: true 4 | validates :name, presence: true 5 | validates :ftype, presence: true 6 | validates :order, presence: true 7 | validates_uniqueness_of :order, scope: :datapackage_resource_id 8 | end 9 | -------------------------------------------------------------------------------- /app/models/datasource.rb: -------------------------------------------------------------------------------- 1 | class Datasource < ActiveRecord::Base 2 | 3 | belongs_to :project 4 | belongs_to :datapackage 5 | belongs_to :datapackage_resource 6 | validates :project_id, presence: true 7 | 8 | has_attached_file :datafile, 9 | :path => ":rails_root/" + Rails.configuration.x.upload_path + "/project_:proj_id/:filename", 10 | :url => "/uploads/project_:proj_id/:filename", 11 | :restricted_characters => /@/ # paperclip automatically cleans up names apparently, replacing 12 | # characters with underscores. Using this option to negate this 13 | # behaviour. See http://stackoverflow.com/questions/7328423/does-paperclip-automatically-clean-up-filenames 14 | 15 | # validates_attachment :datafile, :content_type => /\Atext\/csv/ 16 | validates_attachment :datafile, content_type: { :content_type => [/\Atext\//, "application/json"] } 17 | validates_attachment_file_name :datafile, :matches => [/csv\Z/] 18 | 19 | validate :validate_filename_unique, on: :create 20 | 21 | process_in_background :datafile # delayed_paperclip 22 | 23 | enum import_status: [ :ok, :note, :warning, :error ] 24 | 25 | before_save :set_logfile_path 26 | after_destroy :delete_associated_artifacts 27 | 28 | paginates_per 10 # kaminari 29 | 30 | # Some helper methods, useful for linking to log file 31 | def basename 32 | File.basename(self.datafile_file_name, ".*" ) 33 | end 34 | 35 | def full_upload_path 36 | File.dirname(self.datafile.path) 37 | end 38 | 39 | # def url 40 | # self.datafile.url 41 | # end 42 | 43 | def set_logfile_path 44 | self.logfile_path = Project.find(self.project_id).job_log_path + self.datafile_file_name + ".log" 45 | end 46 | 47 | 48 | private 49 | 50 | # Use an interpolation to get project_id into the path 51 | # https://github.com/thoughtbot/paperclip/wiki/Interpolations 52 | #http://stackoverflow.com/questions/9173920/paperclip-custom-path-with-id-belongs-to 53 | Paperclip.interpolates :proj_name do |attachment, style| 54 | Project.find(attachment.instance.project_id).name 55 | end 56 | 57 | Paperclip.interpolates :proj_id do |attachment, style| 58 | attachment.instance.project_id 59 | end 60 | 61 | def validate_filename_unique 62 | if Datasource.where(project_id: self.project_id, datafile_file_name: self.datafile_file_name).length > 0 63 | errors.add(:datasource, ": A file of this name has already been uploaded to this project!") 64 | end 65 | end 66 | 67 | def delete_associated_artifacts 68 | # There is no need to delete the uploaded file as this will occur on delete anyway. 69 | # delete_upload 70 | delete_logfile 71 | end 72 | 73 | def delete_logfile 74 | if self.logfile_path.present? && self.logfile_path.length>8 && logfile_path.upcase.end_with?(".LOG") 75 | File.delete(self.logfile_path) if File.exist?(self.logfile_path) 76 | end 77 | end 78 | 79 | def delete_upload 80 | upload_file_name = Datasource.find(ds.id).datafile_file_name 81 | upload = self.datapackage.project.upload_path + upload_file_name 82 | File.delete(upload) if File.exist?(upload) 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /app/models/project.rb: -------------------------------------------------------------------------------- 1 | class Project < ActiveRecord::Base 2 | 3 | belongs_to :user 4 | has_one :datapackage, dependent: :destroy, validate: true 5 | has_many :datasources, dependent: :destroy 6 | has_many :api_key_permissions, dependent: :destroy 7 | 8 | validates :name, 9 | presence: true, 10 | uniqueness: { case_sensitive: false } 11 | 12 | validates :user_id, presence: true 13 | 14 | def job_log_path 15 | Rails.configuration.x.job_log_path + "/project_" + self.id.to_s + "/" 16 | end 17 | 18 | def upload_path 19 | Rails.configuration.x.upload_path + "/project_" + self.id.to_s + "/" 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /app/models/user.rb: -------------------------------------------------------------------------------- 1 | class User < ActiveRecord::Base 2 | # Include default devise modules. Others available are: 3 | # :confirmable, :lockable, :timeoutable and :omniauthable 4 | devise :database_authenticatable, 5 | :recoverable, :rememberable, :trackable, :validatable 6 | 7 | # DBR: removed from above (only want admin user(s): :registerable, 8 | 9 | has_many :projects, dependent: :destroy 10 | has_many :api_keys, dependent: :destroy 11 | end 12 | -------------------------------------------------------------------------------- /app/views/api_key_permissions/_error_messages.html.erb: -------------------------------------------------------------------------------- 1 | <% if @permission.errors.any? %> 2 |
3 |
4 | There is <%= pluralize(@permission.errors.count, "error") %>. 5 |
6 | 11 |
12 | <% end %> 13 | -------------------------------------------------------------------------------- /app/views/api_key_permissions/_form.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | <%= form_for [@user, @key, @permission], :html => { multipart: true } do |f| %> 5 | <%= render 'error_messages' %> 6 |

Select projects:

7 | 16 |
17 |

Apply permission: read or read-write

18 | 19 | 25 | 26 |
27 | Read
28 | Read-write 29 |
30 | 31 |
32 | <%= f.submit "Save", :class => 'btn btn-primary' %> 33 | <%= link_to t('.cancel', :default => t("helpers.links.cancel")), 34 | root_url, :class => 'btn btn-mini' %> 35 |
36 | 37 | 38 | <%#= f.submit "Save changes", class: "btn btn-primary" %> 39 | <% end %> 40 |
41 |
42 | -------------------------------------------------------------------------------- /app/views/api_key_permissions/index.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'API Key Permissions') %> 2 | 3 | <% if user_signed_in? %> 4 | <%= link_to "New API Key Permission", new_user_api_key_api_key_permission_path, :class => "btn btn-primary" %> 5 |

6 | 7 |

Permissions for API Key <%= @key.token %>

8 | 9 | <%= paginate @permissions %> 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | <% @permissions.each do |p| %> 18 | 19 | 20 | 21 | <% if p.permission_scope == "global" %> 22 | 23 | <% elsif p.permission_scope == "project" %> 24 | 25 | <% end %> 26 | 28 | 29 | 30 | <% end %> 31 |
PermissionScopeProject
<%= p.permission %><%= p.permission_scope %>All projects<%= Project.find(p.project_id).name %><%= link_to "delete", [@user, @key, p] , method: :delete, 27 | data: { confirm: "You sure?" }, :class => "btn btn-xs btn-danger" %>
32 | <%= paginate @permissions %> 33 | 34 | <% end %> 35 | -------------------------------------------------------------------------------- /app/views/api_key_permissions/new.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'New API Key Permission') %> 2 |
3 |

New API Key Permission

4 | <%= render :partial => 'form' %> 5 |
6 | -------------------------------------------------------------------------------- /app/views/api_keys/_error_messages.html.erb: -------------------------------------------------------------------------------- 1 | <% if @key.errors.any? %> 2 |
3 |
4 | There is <%= pluralize(@key.errors.count, "error") %>. 5 |
6 | 11 |
12 | <% end %> 13 | -------------------------------------------------------------------------------- /app/views/api_keys/_form.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 | <%= form_for [@user, @key] do |f| %> 4 | <%= render 'error_messages' %> 5 | 6 | <%= f.label :description %> 7 | <%= f.text_area :description, rows: 4 %> 8 | 9 | 10 |
11 | <%= f.submit nil, :class => 'btn btn-primary' %> 12 | <%= link_to t('.cancel', :default => t("helpers.links.cancel")), 13 | user_api_keys_path, :class => 'btn btn-mini' %> 14 |
15 | <% end %> 16 |
17 |
18 | -------------------------------------------------------------------------------- /app/views/api_keys/create.html.erb: -------------------------------------------------------------------------------- 1 |

Booya#create

2 |

Find me in app/views/booya/create.html.erb

3 | -------------------------------------------------------------------------------- /app/views/api_keys/edit.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'Edit API Key') %> 2 | 3 |
4 | 5 |

Edit API Key

6 | 7 | <%= render :partial => 'form' %> 8 | 9 |
10 | -------------------------------------------------------------------------------- /app/views/api_keys/index.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'API Keys') %> 2 |

API Keys

3 | 4 | <% if user_signed_in? %> 5 | 6 | <%= link_to "New API Key", new_user_api_key_path, :class => "btn btn-primary" %> 7 |

8 | <%= paginate @keys %> 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | <% @keys.each do |k| %> 19 | 20 | 21 | 22 | 23 | 24 | 25 | 27 | 28 | <% end %> 29 |
DescriptionKey
<%= k.description %><%= k.token %><%= link_to "Manage Permissions", user_api_key_api_key_permissions_path(@user,k), :class => "btn btn-xs btn-info" %><%= link_to "Generate New Key", {:user_id => @user.id, :id => k.id, :controller => :api_keys, :action => :gen_new_key}, class: "btn btn-warning btn-xs" %><%= link_to "Edit API Key", edit_user_api_key_path(@user,k), :class => "btn btn-xs btn-info" %><%= link_to "delete", [@user, k] , method: :delete, 26 | data: { confirm: "You sure?" }, :class => "btn btn-xs btn-danger" %>
30 | <%= paginate @keys %> 31 | 32 | <% end %> 33 | -------------------------------------------------------------------------------- /app/views/api_keys/index_project.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'Project API Keys') %> 2 |

API Keys

3 | 4 | <% if user_signed_in? %> 5 | 6 |

7 | <%= paginate @proj_keys_info %> 8 | 9 | 10 | 13 | 14 | 15 | 16 | <% @proj_keys_info.each do |kp| %> 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | <% end %> 25 |
ID 11 | Scope 12 | DescriptionKeyPermission
<%= kp.id %><%= kp.permission_scope %><%= kp.description %><%= kp.token %><%= kp.permission %>
26 | <%= paginate @proj_keys_info %> 27 | 28 | <% end %> 29 | -------------------------------------------------------------------------------- /app/views/api_keys/new.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'New API Key') %> 2 |
3 |

New API Key

4 | <%= render :partial => 'form' %> 5 |
6 | -------------------------------------------------------------------------------- /app/views/api_keys/show.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'API Key') %> 2 | 3 |


4 | <%= link_to "Manage API key permissions", user_api_key_api_key_permissions_path(@user,@key), :class => "btn btn-primary" %> 5 |

6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
DescriptionKey
<%= @key.description %><%= @key.token %>
17 | -------------------------------------------------------------------------------- /app/views/api_keys/update.html.erb: -------------------------------------------------------------------------------- 1 |

Booya#update

2 |

Find me in app/views/booya/update.html.erb

3 | -------------------------------------------------------------------------------- /app/views/datapackage_resources/show_old.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'Datapackage Resource') %> 2 | 3 | <%= link_to "Back to project overview", project_path(@project)%> 4 |

5 | 6 | 9 |
10 |
11 |

Resource Metadata

12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 |
ItemValue
ID <%= @dpr.id %>
Description <%= @dpr.description %>
Datapackage ID <%= @dpr.datapackage_id %>
Path <%= @dpr.path %>
Format <%= @dpr.format %>
Delimitir <%= @dpr.delimiter %>
Mediatype <%= @dpr.mediatype %>
Created Datetime <%= @dpr.created_at %>
Quote Character <%= @dpr.quote_character %>
Table Reference <%= @dpr.table_ref %>
Database Table Name <%= @dpr.db_table_name %>
30 |
31 |
32 | 33 | 34 | 36 |

Datasources (files uploaded)

37 | 38 | 39 | 40 | 41 | 42 | <% @dpr_ds.each do |ds| %> 43 | 44 | 45 | 46 | 47 | 49 | 50 | <% end %> 51 | 52 |
IDFile NameUploaded Date
<%= ds.id %><%= ds.datafile_file_name %><%= ds.created_at %><%= link_to "delete", [@project,ds], method: :delete, 48 | data: { confirm: "Are you sure? This will delete any associated observations from the database." }, :class => "btn btn-xs btn-danger" %>
53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /app/views/datasources/index.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'Datasources') %> 2 |

Datasource

3 | 4 | <%= paginate @dss %> 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | <% @dss.each do |ds| %> 13 | 14 | 15 | 16 | 17 | 18 | 19 | <% end %> 20 |
IDFile NameSizeUploaded
<%= ds.id %><%=link_to ds.datafile_file_name, project_datasource_path(@proj,ds) %><%= number_to_human_size ds.datafile_file_size %><%= ds.created_at %>
21 | <%= paginate @dss %> 22 | -------------------------------------------------------------------------------- /app/views/datasources/show.html.erb: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /app/views/devise/confirmations/new.html.erb: -------------------------------------------------------------------------------- 1 |

Resend confirmation instructions

2 | 3 | <%= form_for(resource, as: resource_name, url: confirmation_path(resource_name), html: { method: :post }) do |f| %> 4 | <%= devise_error_messages! %> 5 | 6 |
7 | <%= f.label :email %>
8 | <%= f.email_field :email, autofocus: true, value: (resource.pending_reconfirmation? ? resource.unconfirmed_email : resource.email) %> 9 |
10 | 11 |
12 | <%= f.submit "Resend confirmation instructions" %> 13 |
14 | <% end %> 15 | 16 | <%= render "devise/shared/links" %> 17 | -------------------------------------------------------------------------------- /app/views/devise/mailer/confirmation_instructions.html.erb: -------------------------------------------------------------------------------- 1 |

Welcome <%= @email %>!

2 | 3 |

You can confirm your account email through the link below:

4 | 5 |

<%= link_to 'Confirm my account', confirmation_url(@resource, confirmation_token: @token) %>

6 | -------------------------------------------------------------------------------- /app/views/devise/mailer/reset_password_instructions.html.erb: -------------------------------------------------------------------------------- 1 |

Hello <%= @resource.email %>!

2 | 3 |

Someone has requested a link to change your password. You can do this through the link below.

4 | 5 |

<%= link_to 'Change my password', edit_password_url(@resource, reset_password_token: @token) %>

6 | 7 |

If you didn't request this, please ignore this email.

8 |

Your password won't change until you access the link above and create a new one.

9 | -------------------------------------------------------------------------------- /app/views/devise/mailer/unlock_instructions.html.erb: -------------------------------------------------------------------------------- 1 |

Hello <%= @resource.email %>!

2 | 3 |

Your account has been locked due to an excessive number of unsuccessful sign in attempts.

4 | 5 |

Click the link below to unlock your account:

6 | 7 |

<%= link_to 'Unlock my account', unlock_url(@resource, unlock_token: @token) %>

8 | -------------------------------------------------------------------------------- /app/views/devise/passwords/edit.html.erb: -------------------------------------------------------------------------------- 1 |

Change your password

2 | 3 | <%= form_for(resource, as: resource_name, url: password_path(resource_name), html: { method: :put }) do |f| %> 4 | <%= devise_error_messages! %> 5 | <%= f.hidden_field :reset_password_token %> 6 | 7 |
8 | <%= f.label :password, "New password" %>
9 | <% if @minimum_password_length %> 10 | (<%= @minimum_password_length %> characters minimum) 11 | <% end %>
12 | <%= f.password_field :password, autofocus: true, autocomplete: "off" %> 13 |
14 | 15 |
16 | <%= f.label :password_confirmation, "Confirm new password" %>
17 | <%= f.password_field :password_confirmation, autocomplete: "off" %> 18 |
19 | 20 |
21 | <%= f.submit "Change my password" %> 22 |
23 | <% end %> 24 | 25 | <%= render "devise/shared/links" %> 26 | -------------------------------------------------------------------------------- /app/views/devise/passwords/new.html.erb: -------------------------------------------------------------------------------- 1 |

Forgot your password?

2 | 3 | <%= form_for(resource, as: resource_name, url: password_path(resource_name), html: { method: :post }) do |f| %> 4 | <%= devise_error_messages! %> 5 | 6 |
7 | <%= f.label :email %>
8 | <%= f.email_field :email, autofocus: true %> 9 |
10 | 11 |
12 | <%= f.submit "Send me reset password instructions" %> 13 |
14 | <% end %> 15 | 16 | <%= render "devise/shared/links" %> 17 | -------------------------------------------------------------------------------- /app/views/devise/registrations/edit.html.erb: -------------------------------------------------------------------------------- 1 |

Edit <%= resource_name.to_s.humanize %>

2 | 3 | <%= form_for(resource, as: resource_name, url: registration_path(resource_name), html: { method: :put }) do |f| %> 4 | <%= devise_error_messages! %> 5 | 6 |
7 | <%= f.label :email %>
8 | <%= f.email_field :email, autofocus: true %> 9 |
10 | 11 | <% if devise_mapping.confirmable? && resource.pending_reconfirmation? %> 12 |
Currently waiting confirmation for: <%= resource.unconfirmed_email %>
13 | <% end %> 14 | 15 |
16 | <%= f.label :password %> (leave blank if you don't want to change it)
17 | <%= f.password_field :password, autocomplete: "off" %> 18 |
19 | 20 |
21 | <%= f.label :password_confirmation %>
22 | <%= f.password_field :password_confirmation, autocomplete: "off" %> 23 |
24 | 25 |
26 | <%= f.label :current_password %> (we need your current password to confirm your changes)
27 | <%= f.password_field :current_password, autocomplete: "off" %> 28 |
29 | 30 |
31 | <%= f.submit "Update" %> 32 |
33 | <% end %> 34 | 35 |

Cancel my account

36 | 37 |

Unhappy? <%= button_to "Cancel my account", registration_path(resource_name), data: { confirm: "Are you sure?" }, method: :delete %>

38 | 39 | <%= link_to "Back", :back %> 40 | -------------------------------------------------------------------------------- /app/views/devise/registrations/new.html.erb: -------------------------------------------------------------------------------- 1 |

Sign up

2 | 3 | <%= form_for(resource, as: resource_name, url: registration_path(resource_name)) do |f| %> 4 | <%= devise_error_messages! %> 5 | 6 |
7 | <%= f.label :email %>
8 | <%= f.email_field :email, autofocus: true %> 9 |
10 | 11 |
12 | <%= f.label :password %> 13 | <% if @minimum_password_length %> 14 | (<%= @minimum_password_length %> characters minimum) 15 | <% end %>
16 | <%= f.password_field :password, autocomplete: "off" %> 17 |
18 | 19 |
20 | <%= f.label :password_confirmation %>
21 | <%= f.password_field :password_confirmation, autocomplete: "off" %> 22 |
23 | 24 |
25 | <%= f.submit "Sign up" %> 26 |
27 | <% end %> 28 | 29 | <%= render "devise/shared/links" %> 30 | -------------------------------------------------------------------------------- /app/views/devise/sessions/new.html.erb: -------------------------------------------------------------------------------- 1 |
2 | <%= form_for(resource, as: resource_name, url: session_path(resource_name), :html => { :class => "form-signin"}) do |f| %> 3 |

Log in

4 |
5 | <%= f.label " "%>
6 | <%= f.email_field :email, autofocus: true, class: "form-control", required: "true", placeholder: "Email address" %> 7 |
8 | 9 |
10 | <%= f.label " " %>
11 | <%= f.password_field :password, autocomplete: "off", class: "form-control", required: "true", placeholder: "Password" %> 12 |
13 | 14 | <% if devise_mapping.rememberable? -%> 15 |
16 | <%= f.label :remember_me %> 17 | <%= f.check_box :remember_me %> 18 |
19 | <% end -%> 20 | 21 |
22 | <%= f.submit "Log in" %> 23 |
24 | <% end %> 25 | 26 | <%= render "devise/shared/links" %> 27 |
-------------------------------------------------------------------------------- /app/views/devise/shared/_links.html.erb: -------------------------------------------------------------------------------- 1 | <%- if controller_name != 'sessions' %> 2 | <%= link_to "Log in", new_session_path(resource_name) %>
3 | <% end -%> 4 | 5 | <%- if devise_mapping.registerable? && controller_name != 'registrations' %> 6 | <%= link_to "Sign up", new_registration_path(resource_name) %>
7 | <% end -%> 8 | 9 | <%- if devise_mapping.recoverable? && controller_name != 'passwords' && controller_name != 'registrations' %> 10 | <%= link_to "Forgot your password?", new_password_path(resource_name) %>
11 | <% end -%> 12 | 13 | <%- if devise_mapping.confirmable? && controller_name != 'confirmations' %> 14 | <%= link_to "Didn't receive confirmation instructions?", new_confirmation_path(resource_name) %>
15 | <% end -%> 16 | 17 | <%- if devise_mapping.lockable? && resource_class.unlock_strategy_enabled?(:email) && controller_name != 'unlocks' %> 18 | <%= link_to "Didn't receive unlock instructions?", new_unlock_path(resource_name) %>
19 | <% end -%> 20 | 21 | <%- if devise_mapping.omniauthable? %> 22 | <%- resource_class.omniauth_providers.each do |provider| %> 23 | <%= link_to "Sign in with #{provider.to_s.titleize}", omniauth_authorize_path(resource_name, provider) %>
24 | <% end -%> 25 | <% end -%> 26 | -------------------------------------------------------------------------------- /app/views/devise/unlocks/new.html.erb: -------------------------------------------------------------------------------- 1 |

Resend unlock instructions

2 | 3 | <%= form_for(resource, as: resource_name, url: unlock_path(resource_name), html: { method: :post }) do |f| %> 4 | <%= devise_error_messages! %> 5 | 6 |
7 | <%= f.label :email %>
8 | <%= f.email_field :email, autofocus: true %> 9 |
10 | 11 |
12 | <%= f.submit "Resend unlock instructions" %> 13 |
14 | <% end %> 15 | 16 | <%= render "devise/shared/links" %> 17 | -------------------------------------------------------------------------------- /app/views/layouts/_footer.html.erb: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/views/layouts/_header.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 | <%= link_to "Mira", root_path, id: "logo" %> 4 | 25 |
26 |
-------------------------------------------------------------------------------- /app/views/layouts/_shim.html.erb: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/views/layouts/application.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | <%= full_title(yield(:title)) %> 5 | <%= stylesheet_link_tag 'application', media: 'all', 6 | 'data-turbolinks-track' => true %> 7 | <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %> 8 | <%= csrf_meta_tags %> 9 | <%= yield(:custom_style) %> 10 | <%= render 'layouts/shim' %> 11 | 12 | 13 | <%= render 'layouts/header' %> 14 |
15 | 16 | 17 | 20 | 21 | <% flash.each do |message_type, message| %> 22 |
<%= message %>
23 | <% end %> 24 | 25 | <%= yield %> 26 |
27 | <%= render 'layouts/footer' %> 28 | <%= debug(params) if Rails.env.development? %> 29 | 30 | <%= yield(:custom_endofpage) %> 31 | 32 | 33 | -------------------------------------------------------------------------------- /app/views/projects/_error_messages.html.erb: -------------------------------------------------------------------------------- 1 | <% if @project.errors.any? %> 2 |
3 |
4 | There is <%= pluralize(@project.errors.count, "error") %>. 5 |
6 | 11 |
12 | <% end %> 13 | -------------------------------------------------------------------------------- /app/views/projects/_form.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 | <%# Note that mutipart: true is required here. It inserts enctype="multipart/form-data" into the form tag definition %> 4 | <%= form_for(@project, :html => { multipart: true }) do |f| %> 5 | <%= render 'error_messages' %> 6 | <%= f.label :name %> 7 | <%= f.text_field :name %> 8 | 9 | <%= f.label :description %> 10 | <%= f.text_area :description, rows: 4 %> 11 | 12 | 13 |
14 | <%= f.submit nil, :class => 'btn btn-primary' %> 15 | <%= link_to t('.cancel', :default => t("helpers.links.cancel")), 16 | projects_path, :class => 'btn btn-mini' %> 17 |
18 | 19 | 20 | <%#= f.submit "Save changes", class: "btn btn-primary" %> 21 | <% end %> 22 |
23 |
-------------------------------------------------------------------------------- /app/views/projects/_upload_datapackage.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 | 5 | <%= form_for(@project, url: {action: "upload_datapackage"} , method: "post", :html => { multipart: true }) do |f| %> 6 | 7 |
8 | <%= f.label :datapackage, :class => 'control-label' %> 9 |
10 | <%= file_field_tag "datapackage", type: :file %> 11 |
12 |
13 | 14 |
15 | <%= f.submit "Upload datapackage.json", :class => 'btn btn-primary', id: 'submit_files' %> 16 | <%= link_to t('.cancel', :default => t("helpers.links.cancel")), 17 | projects_path, :class => 'btn btn-mini' %> 18 |
19 | 20 | <% end %> 21 | 22 |
23 |
24 | -------------------------------------------------------------------------------- /app/views/projects/_upload_datasources.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 | <%= form_for(@project, url: {action: "upload_datasources"} , method: "post", :html => { multipart: true }) do |f| %> 5 | 6 |
7 | <%= f.label :csv_files, "CSV files", :class => 'control-label' %> 8 |
9 | <%= file_field_tag "datafiles[]", type: :file, multiple: true %> 10 |
11 |
12 | 13 |
14 |
15 |
16 | <%= f.submit "Quick Upload", :class => 'btn btn-primary', id: 'submit_files', name: 'quick-csv-upload' %> 17 |
18 |
19 | <%= f.submit "Slow Upload", :class => 'btn btn-primary', id: 'submit_files', name: 'slow-csv-upload' %> 20 |
21 |  * 22 |
23 | <%= link_to t('.cancel', :default => t("helpers.links.cancel")), 24 | projects_path, :class => 'btn btn-mini' %> 25 |
26 | 27 | <% end %> 28 | 29 |
30 |

31 |  * The "quick upload" bypasses 32 | ActiveRecord table models, using 33 | instead an ActiveRecord raw connection. This is much faster than the "slow upload" but is more prone to 34 | failure (i.e. when data does not exactly conform to what is specified in the datapackage.json file). If 35 | the quick upload fails, then delete and try the slow upload. 36 |

37 |
38 | 39 |
40 |
41 | -------------------------------------------------------------------------------- /app/views/projects/_upload_project_files.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 | 5 | <%= form_for(@project, url: {action: "upload_datapackage"} , method: "post", :html => { multipart: true }) do |f| %> 6 | 7 |
8 | <%= f.label :datapackage, :class => 'control-label' %> 9 |
10 | <%= file_field_tag "datapackage", type: :file %> 11 |
12 |
13 | 14 |
15 | <%= f.submit "Upload datapackage.json", :class => 'btn btn-primary', id: 'submit_files' %> 16 | <%= link_to t('.cancel', :default => t("helpers.links.cancel")), 17 | projects_path, :class => 'btn btn-mini' %> 18 |
19 | 20 | <% end %> 21 | 22 |
23 |
24 | -------------------------------------------------------------------------------- /app/views/projects/api_detail.html.erb: -------------------------------------------------------------------------------- 1 | 2 |

Project Details

3 | 4 | <%= render "projects/partials/api_detail_project_details" %> 5 | 6 | <%= render "projects/partials/api_detail_endpoints" %> 7 | 8 | <%= render "projects/partials/api_detail_distinct_values" %> 9 | 10 | <%= render "projects/partials/api_detail_querying" %> 11 | 12 | <%= render "projects/partials/api_detail_select" %> 13 | 14 | <%= render "projects/partials/api_detail_ordering" %> 15 | 16 | <%= render "projects/partials/api_detail_paging" %> 17 | 18 | <%= render "projects/partials/api_detail_datatables_info" %> 19 | -------------------------------------------------------------------------------- /app/views/projects/edit.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'Edit Project') %> 2 | 3 |
4 | 5 |

Edit Project

6 | 7 | <%= render :partial => 'form' %> 8 | 9 | <% if user_signed_in? && @project.datapackage.nil? %> 10 | <%= render :partial => 'upload_project_files' %> 11 | <% end %> 12 |
13 | -------------------------------------------------------------------------------- /app/views/projects/index.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'Projects') %> 2 |

Projects

3 | 4 | <% if user_signed_in? %> 5 | <%= link_to "New project", new_project_path, :class => "btn btn-primary" %> 6 | <%= link_to "Manage API Keys", user_api_keys_path(@user), :class => "btn btn-primary" %> 7 |

8 | <% end %> 9 | <%= paginate @projects %> 10 | 11 | 12 | 13 | 14 | 15 | 17 | 18 | 19 | <% if user_signed_in? %> 20 | 22 | 23 | 24 | <% end %> 25 | 26 | <% @projects.each do |proj| %> 27 | 28 | 29 | 33 | 39 | 53 | 54 | 68 | 69 | 70 | <% if user_signed_in? %> 71 | 72 | 74 | <% end %> 75 | 76 | <% end %> 77 |
IDNameRead
Access		Write
Access
<%= proj.id %> 30 | <%= link_to "#{proj.name}", proj %> 31 | <%#= proj.name %> 32 | 34 | API details
35 | <%#= link_to "Data", proj %>
36 | <%#= link_to proj.datapackage.datapackage_file_name, proj.datapackage.datapackage.url %> 37 | 38 | 		40 | <%-# Check if there is a global (this will have a nil key) or project specific permission applied -%> 41 | <% if @read_key_hash[nil].present? || @read_key_hash[proj.id].present? %> 42 | <% if user_signed_in? %> 43 | <%= link_to project_api_keys_path(@user,proj) do %> 44 | 45 | <% end %> 46 | <% else %> 47 | 48 | <% end %> 49 | <% else %> 50 | 51 | <% end %> 52 | 55 | <%-# Check if there is a global (this will have a nil key) or project specific permission applied -%> 56 | <% if @write_key_hash[nil].present? || @write_key_hash[proj.id].present? %> 57 | <% if user_signed_in? %> 58 | <%= link_to project_api_keys_path(@user,proj) do %> 59 | 60 | <% end %> 61 | <% else %> 62 | 63 | <% end %> 64 | <% else %> 65 | 66 | <% end %> 67 | <%= link_to "Edit Project", edit_project_path(proj), :class => "btn btn-xs btn-info" %><%= link_to "Delete", proj, method: :delete, 73 | data: { confirm: "You sure?" }, :class => "btn btn-xs btn-danger" %>
78 | <%= paginate @projects %> 79 | -------------------------------------------------------------------------------- /app/views/projects/new.html.erb: -------------------------------------------------------------------------------- 1 | <% provide(:title, 'New Project') %> 2 |
3 |

New Project

4 | <%= render :partial => 'form' %> 5 |
-------------------------------------------------------------------------------- /app/views/projects/partials/_api_detail_datatables_info.html.erb: -------------------------------------------------------------------------------- 1 |
2 |

DataTables

3 |

4 | The API is capable of responding to requests from the DataTables front-end javascript library. 5 | 6 |
Note: DataTables AJAX requests to this API must use the POST HTTP method. 7 |

8 | 9 |

10 |
11 | -------------------------------------------------------------------------------- /app/views/projects/partials/_api_detail_distinct_values.html.erb: -------------------------------------------------------------------------------- 1 |
2 |

Distinct Values

3 |

4 | Query for distinct values (text columns only): 5 | 

<%= root_url %>api/projects/<%= @project.id %>/tables/[table_name]/columns/[column_name]/distinct
6 | 7 | Results are paged. See paging. 8 |

9 | 10 |

11 |
12 | -------------------------------------------------------------------------------- /app/views/projects/partials/_api_detail_endpoints.html.erb: -------------------------------------------------------------------------------- 1 |
2 |

API Endpoints

3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | <% @dp_resources.each do |res| %> 36 | 37 | 38 | 39 | 40 | 62 | 63 | <% end %> 64 | 65 |
DescriptionHTTP MethodAPI Endpoint
All projectsGET<%= root_url %>api/projects
This projectGET<%= root_url %>api/projects/<%= @project.id %>
Project datapackageGET<%= root_url %>api/projects/<%= @project.id %>/datapackage
Project tablesGET<%= root_url %>api/projects/<%= @project.id %>/tables
Tables
        <%= res.table_ref %>GET 41 |
42 |
Detail
43 |
<%= root_url %>api/projects/<%= @project.id %>/tables/<%= res.table_ref %>
44 | 45 |
Database columns
46 |
<%= root_url %>api/projects/<%= @project.id %>/tables/<%= res.table_ref %>/columns 47 |
48 |
Single column
49 |
<%= root_url %>api/projects/<%= @project.id %>/tables/<%= res.table_ref %>/columns/[column_name] 50 |
51 | 52 |
Datapackage fields
53 |
<%= root_url %>api/projects/<%= @project.id %>/tables/<%= res.table_ref %>/datapackage/fields 54 |
55 |
Single field
56 |
<%= root_url %>api/projects/<%= @project.id %>/tables/<%= res.table_ref %>/datapackage/fields/[field_name]
57 | 58 |
Data
59 |
<%= root_url %>api/projects/<%= @project.id %>/tables/<%= res.table_ref %>/data
60 |
61 |
66 | 67 |

68 |
69 | -------------------------------------------------------------------------------- /app/views/projects/partials/_api_detail_ordering.html.erb: -------------------------------------------------------------------------------- 1 |
2 |

Ordering

3 | 4 |

5 | To apply ordering to the query, the order=[column] field-value pair is added to the query string. For example, to request rows from the mytable table, ordered by col1:

6 | 7 | 
<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?order=col1
8 | 9 |

Ascending order is applied by default. To apply descending order, the above example would become:

10 | 11 | 
<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?order=col1:desc
12 | 13 |

To order by multiple columns, delimit each column name with a comma. For example, to order by col1 and col2 :

14 | 15 | 
<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?order=col1,col2
16 | 17 |

18 |
-------------------------------------------------------------------------------- /app/views/projects/partials/_api_detail_paging.html.erb: -------------------------------------------------------------------------------- 1 |
2 | 3 |

Paging

4 |

5 | By default, the API returns <%= Rails.configuration.x.api_default_per_page %> rows per page. This can be adjusted using the per_page query field. A particular page number can be requested using the page query field, e.g. for 50 rows per page, and the 4th such page: 6 |

7 | 8 | 
<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?per_page=50&page=4&col1=XXX...
9 | 10 |

11 | The maximum allowable per_page value on this server is <%= Rails.configuration.x.api_max_per_page %> (and can be adjusted by the server administrator). 12 |

13 | Paging links are included in the response's HTTP Link Header. E.g. if you request page 4 from mytable they will look like: 14 |

15 | 
<<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?page=1>; rel="first",
<<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?page=3>; rel="prev",
<<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?page=21>; rel="last",
<<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?page=5>; rel="next"
16 | 17 |

The number of records per page and the total number of records are returned in the Records-Per-Page and Records-Total HTTP response headers. 18 | These provide what's needed to implement pagination on the client side. 19 |

20 |

21 |
-------------------------------------------------------------------------------- /app/views/projects/partials/_api_detail_project_details.html.erb: -------------------------------------------------------------------------------- 1 |
2 | <%= render 'projects/partials/project_details' %> 3 |

4 |
5 | -------------------------------------------------------------------------------- /app/views/projects/partials/_api_detail_select.html.erb: -------------------------------------------------------------------------------- 1 |
2 |

Selecting Columns

3 | 4 |

By default, responses include all columns. To request specific columns, you can use select_columns=[comma separated list of column names] in the query string:

5 | 6 | 
<%= root_url %>api/projects/<%= @project.id %>/tables/mytable/data?select_columns=col1,col4,col7
7 | 8 |

9 |
10 | 11 | -------------------------------------------------------------------------------- /app/views/projects/partials/_project_details.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 21 | 22 | 23 | 24 | 33 | 34 | 35 |
ID<%= @project.id %>
Name<%= @project.name %>
Description<%= @project.description %>
API details 19 | /projects/<%= @project.id %>/api-details
20 |
Datapackage 25 | <% if @project.datapackage.present? %> 26 | <%#= link_to @project.datapackage.datapackage_file_name, @project.datapackage.datapackage.url %> 27 | <%= @project.datapackage.datapackage_file_name %> 28 | , uploaded <%= @project.datapackage.created_at.strftime("%FT%R") %> 29 | <% else %> 30 | Upload a datapackage.json file! <%= link_to "See here", edit_project_path(@project) %> 31 | <% end %> 32 |
36 | -------------------------------------------------------------------------------- /app/views/projects/show.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Project Overview

4 |
5 | 6 | <%= render 'error_messages' %> 7 | 8 |
9 | 10 |
11 | <%= render 'projects/partials/project_details' %>
12 |
13 | 14 |
15 | 16 | 17 | <% if @datapackage.present? %> 18 | 19 | 20 | 21 | 22 | 23 | 24 | <% if user_signed_in? %> 25 | 26 | 27 | 28 | 29 | 30 | <% end %> 31 | 32 | 33 | 34 | <% @datapackage.datapackage_resources.sort_by { |obj| obj.table_ref }.each do |res| %> 35 | 36 | 37 | 38 | 41 | 42 | <% if user_signed_in? %> 43 | 44 | 45 | 46 | 47 | 48 | 53 | 54 | 62 | 63 | 74 | <% end %> 75 | 76 | 77 | <% end %> 78 | 79 |
Datasources
(as per datapackage.json)		Database
Table		Number
of Rows		Browse/
update data
39 | <%= res.path %> 40 | <%= res.db_table_name %><%= get_mira_ar_table(res.db_table_name.capitalize).count %> 49 | <%= link_to project_datapackage_datapackage_resource_path(@project,res) do %> 50 | 51 | <% end %> 52 | 55 | <% num_uploads=Datasource.where(datapackage_resource_id: res.id).count %> 56 | <% if num_uploads == 0 %> 57 | 0 files uploaded 58 | <% else %> 59 | <%= link_to pluralize(num_uploads,"file") + " uploaded", project_datapackage_datapackage_resource_path(@project,res) %> 60 | <% end %> 61 | 64 | <% key_count_hash = get_mira_ar_table(res.db_table_name.capitalize).where(mira_source_type: "key").group(:mira_source_id).count %> 65 | <% num_apikeys = key_count_hash.length %> 66 | <%= key_count_hash.values.inject(0){|sum,x| sum + x } %> rows from 67 | <% if num_apikeys == 0 %> 68 | 0 API keys 69 | <% else %> 70 | <%= link_to pluralize(num_apikeys,"API key"), project_datapackage_datapackage_resource_path(@project,res) %> 71 | 72 | <% end %> 73 |
80 | 81 | 82 | 83 | 84 | <% else %> 85 |

A datapackage.json file has not yet been uploaded and processed.

86 | <% end %> 87 | 88 | 89 | 90 |
91 |
92 |
93 | 94 | <% if user_signed_in? %> 95 | <% if !@datapackage %> 96 | <%= render :partial => 'upload_datapackage' %> 97 | <% else %> 98 | <%= render :partial => 'upload_datasources' %> 99 | <% end %> 100 | <% end %> 101 | 102 | 103 | 104 | <% provide(:custom_endofpage) do %> 105 | 121 | <% end %> 122 | -------------------------------------------------------------------------------- /app/views/shared/_devise_links.html.erb: -------------------------------------------------------------------------------- 1 |
2 |

3 | <% if user_signed_in? %> 4 | Logged in as <%= current_user.email %>. 5 | <%#= link_to 'Edit profile', edit_user_registration_path, :class => 'navbar-link' %> | 6 | <%= link_to "Logout", destroy_user_session_path, method: :delete, :class => 'navbar-link' %> 7 | <% else %> 8 | <%#= link_to "Sign up", new_user_registration_path, :class => 'navbar-link' %> 9 | 10 | <% if Rails.env.production? %> 11 | <%= link_to "Admin", new_user_session_url(:protocol => "https"), :class => 'navbar-link', :id => "admin-login-link" %> 12 | <% else %> 13 | <%= link_to "Admin", new_user_session_path, :class => 'navbar-link', :id => "admin-login-link" %> 14 | <% end %> 15 | 16 | <% end %> 17 |

18 | -------------------------------------------------------------------------------- /bin/bundle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 3 | load Gem.bin_path('bundler', 'bundle') 4 | -------------------------------------------------------------------------------- /bin/delayed_job: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment')) 4 | require 'delayed/command' 5 | Delayed::Command.new(ARGV).daemonize 6 | -------------------------------------------------------------------------------- /bin/rails: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | begin 3 | load File.expand_path("../spring", __FILE__) 4 | rescue LoadError 5 | end 6 | APP_PATH = File.expand_path('../../config/application', __FILE__) 7 | require_relative '../config/boot' 8 | require 'rails/commands' 9 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | begin 3 | load File.expand_path("../spring", __FILE__) 4 | rescue LoadError 5 | end 6 | require_relative '../config/boot' 7 | require 'rake' 8 | Rake.application.run 9 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'pathname' 3 | 4 | # path to your application root. 5 | APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) 6 | 7 | Dir.chdir APP_ROOT do 8 | # This script is a starting point to setup your application. 9 | # Add necessary setup steps to this file: 10 | 11 | puts "== Installing dependencies ==" 12 | system "gem install bundler --conservative" 13 | system "bundle check || bundle install" 14 | 15 | # puts "\n== Copying sample files ==" 16 | # unless File.exist?("config/database.yml") 17 | # system "cp config/database.yml.sample config/database.yml" 18 | # end 19 | 20 | puts "\n== Preparing database ==" 21 | system "bin/rake db:setup" 22 | 23 | puts "\n== Removing old logs and tempfiles ==" 24 | system "rm -f log/*" 25 | system "rm -rf tmp/cache" 26 | 27 | puts "\n== Restarting application server ==" 28 | system "touch tmp/restart.txt" 29 | end 30 | -------------------------------------------------------------------------------- /bin/spring: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # This file loads spring without using Bundler, in order to be fast. 4 | # It gets overwritten when you run the `spring binstub` command. 5 | 6 | unless defined?(Spring) 7 | require "rubygems" 8 | require "bundler" 9 | 10 | if match = Bundler.default_lockfile.read.match(/^GEM$.*?^ (?: )*spring \((.*?)\)$.*?^$/m) 11 | Gem.paths = { "GEM_PATH" => [Bundler.bundle_path.to_s, *Gem.path].uniq } 12 | gem "spring", match[1] 13 | require "spring/binstub" 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | # This file is used by Rack-based servers to start the application. 2 | 3 | require ::File.expand_path('../config/environment', __FILE__) 4 | run Rails.application 5 | -------------------------------------------------------------------------------- /config/application.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../boot', __FILE__) 2 | 3 | require 'rails/all' 4 | 5 | # Require the gems listed in Gemfile, including any gems 6 | # you've limited to :test, :development, or :production. 7 | Bundler.require(*Rails.groups) 8 | 9 | module Mira 10 | class Application < Rails::Application 11 | # Settings in config/environments/* take precedence over those specified here. 12 | # Application configuration should go into files in config/initializers 13 | # -- all .rb files in that directory are automatically loaded. 14 | 15 | # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone. 16 | # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC. 17 | # config.time_zone = 'Central Time (US & Canada)' 18 | 19 | # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. 20 | # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] 21 | # config.i18n.default_locale = :de 22 | 23 | # Do not swallow errors in after_commit/after_rollback callbacks. 24 | config.active_record.raise_in_transactional_callbacks = true 25 | 26 | 27 | ########################################################### 28 | ########################################################### 29 | 30 | # DBR: as per https://github.com/collectiveidea/delayed_job 31 | config.active_job.queue_adapter = :delayed_job 32 | config.x.db_table_prefix = 'Xy' 33 | 34 | # DBR: paging max 35 | config.x.api_default_per_page = 25; 36 | config.x.api_max_per_page = 200; 37 | 38 | # DBR: job logs 39 | # config.x.job_log_path = "#{Rails.root}/project_files/job_logs" 40 | config.x.job_log_path = "project_files/job_logs" 41 | 42 | # config.x.upload_path = "#{Rails.root}/project_files/uploads" 43 | config.x.upload_path = "project_files/uploads" 44 | 45 | # This is not exactly required but decided to make it explicit 46 | # as we can see below that we are :expose'ing some of these 47 | # headers in the rack-cors section 48 | ApiPagination.configure do |config| 49 | config.paginator = :kaminari # or :will_paginate 50 | config.total_header = 'Records-Total' 51 | config.per_page_header = 'Records-Per-Page' 52 | end 53 | 54 | # DBR: see https://devmynd.com/blog/2014-7-rails-ember-js-with-the-ember-cli-redux-part-1-the-api-and-cms-with-ruby-on-rails 55 | # http://www.adobe.com/devnet/archive/html5/articles/understanding-cross-origin-resource-sharing-cors.html 56 | #"Rack::Cors" 57 | config.middleware.insert_before "ActionDispatch::Static", "Rack::Cors", :debug => true, :logger => (-> { Rails.logger }) do 58 | allow do 59 | origins '*' 60 | resource '*', 61 | :headers => :any, 62 | :expose => ['Records-Total','Records-Per-Page'], # the total which will be used for pagination 63 | #:expose => ['X-User-Authentication-Token', 'X-User-Id'], 64 | :methods => [:get, :post, :options, :patch, :delete] 65 | end 66 | end 67 | 68 | 69 | config.gzip_compression = true 70 | 71 | # DBR: want to save datapackage.json. Ran into issue where saving the file was causing 72 | # validation errors ("has contents that are not what they are reported to be"). Came across 73 | # this same issue and solution: 74 | # https://github.com/thoughtbot/paperclip/issues/1477#issuecomment-66522989 75 | Paperclip.options[:content_type_mappings] = { 76 | :json => "text/plain" 77 | } 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /config/boot.rb: -------------------------------------------------------------------------------- 1 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 2 | 3 | require 'bundler/setup' # Set up gems listed in the Gemfile. 4 | -------------------------------------------------------------------------------- /config/environment.rb: -------------------------------------------------------------------------------- 1 | # Load the Rails application. 2 | require File.expand_path('../application', __FILE__) 3 | 4 | # Initialize the Rails application. 5 | Rails.application.initialize! 6 | 7 | ActiveRecord::SchemaDumper.ignore_tables = [ /^#{Rails.configuration.x.db_table_prefix.downcase}/ ] -------------------------------------------------------------------------------- /config/environments/development.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # In the development environment your application's code is reloaded on 5 | # every request. This slows down response time but is perfect for development 6 | # since you don't have to restart the web server when you make code changes. 7 | config.cache_classes = false 8 | 9 | # Do not eager load code on boot. 10 | config.eager_load = false 11 | 12 | # Show full error reports and disable caching. 13 | config.consider_all_requests_local = true 14 | config.action_controller.perform_caching = false 15 | 16 | # Don't care if the mailer can't send. 17 | config.action_mailer.raise_delivery_errors = false 18 | 19 | # Print deprecation notices to the Rails logger. 20 | config.active_support.deprecation = :log 21 | 22 | # Raise an error on page load if there are pending migrations. 23 | config.active_record.migration_error = :page_load 24 | 25 | # Debug mode disables concatenation and preprocessing of assets. 26 | # This option may cause significant delays in view rendering with a large 27 | # number of complex assets. 28 | config.assets.debug = true 29 | 30 | # Asset digests allow you to set far-future HTTP expiration dates on all assets, 31 | # yet still be able to expire them through the digest params. 32 | config.assets.digest = true 33 | 34 | # Adds additional error checking when serving assets at runtime. 35 | # Checks for improperly declared sprockets dependencies. 36 | # Raises helpful error messages. 37 | config.assets.raise_runtime_errors = true 38 | 39 | # Raises error for missing translations 40 | # config.action_view.raise_on_missing_translations = true 41 | 42 | # DBR: whitelist private network. Couldn't access from other local machine.... 43 | # http://stackoverflow.com/questions/29417328/how-to-disable-cannot-render-console-from-on-rails 44 | config.web_console.whitelisted_ips = '192.168.0.0/16' 45 | 46 | # DBR trying to get delayed job to communicate back to server 47 | Rails.application.routes.default_url_options[:host] = 'localhost:3000' 48 | 49 | #Rails.logger = Logger.new(STDOUT) 50 | end 51 | -------------------------------------------------------------------------------- /config/environments/production.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # Code is not reloaded between requests. 5 | config.cache_classes = true 6 | 7 | # Eager load code on boot. This eager loads most of Rails and 8 | # your application in memory, allowing both threaded web servers 9 | # and those relying on copy on write to perform better. 10 | # Rake tasks automatically ignore this option for performance. 11 | config.eager_load = true 12 | 13 | # Full error reports are disabled and caching is turned on. 14 | config.consider_all_requests_local = false 15 | config.action_controller.perform_caching = true 16 | 17 | # Enable Rack::Cache to put a simple HTTP cache in front of your application 18 | # Add `rack-cache` to your Gemfile before enabling this. 19 | # For large-scale production use, consider using a caching reverse proxy like 20 | # NGINX, varnish or squid. 21 | # config.action_dispatch.rack_cache = true 22 | 23 | # Disable serving static files from the `/public` folder by default since 24 | # Apache or NGINX already handles this. 25 | config.serve_static_files = ENV['RAILS_SERVE_STATIC_FILES'].present? 26 | 27 | # Compress JavaScripts and CSS. 28 | config.assets.js_compressor = :uglifier 29 | # config.assets.css_compressor = :sass 30 | 31 | # Do not fallback to assets pipeline if a precompiled asset is missed. 32 | config.assets.compile = false 33 | 34 | # Asset digests allow you to set far-future HTTP expiration dates on all assets, 35 | # yet still be able to expire them through the digest params. 36 | config.assets.digest = true 37 | 38 | # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb 39 | 40 | # Specifies the header that your server uses for sending files. 41 | # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache 42 | # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX 43 | 44 | # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. 45 | #config.force_ssl = true 46 | 47 | # Use the lowest log level to ensure availability of diagnostic information 48 | # when problems arise. 49 | config.log_level = :debug 50 | 51 | # Prepend all log lines with the following tags. 52 | # config.log_tags = [ :subdomain, :uuid ] 53 | 54 | # Use a different logger for distributed setups. 55 | # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) 56 | 57 | # Use a different cache store in production. 58 | # config.cache_store = :mem_cache_store 59 | 60 | # Enable serving of images, stylesheets, and JavaScripts from an asset server. 61 | # config.action_controller.asset_host = 'http://assets.example.com' 62 | 63 | # Ignore bad email addresses and do not raise email delivery errors. 64 | # Set this to true and configure the email server for immediate delivery to raise delivery errors. 65 | # config.action_mailer.raise_delivery_errors = false 66 | 67 | # Enable locale fallbacks for I18n (makes lookups for any locale fall back to 68 | # the I18n.default_locale when a translation cannot be found). 69 | config.i18n.fallbacks = true 70 | 71 | # Send deprecation notices to registered listeners. 72 | config.active_support.deprecation = :notify 73 | 74 | # Use default logging formatter so that PID and timestamp are not suppressed. 75 | config.log_formatter = ::Logger::Formatter.new 76 | 77 | # Do not dump schema after migrations. 78 | config.active_record.dump_schema_after_migration = false 79 | 80 | # DBR trying to get delayed job to communicate back to server via GET request 81 | Rails.application.routes.default_url_options[:host] = 'localhost' # local to database, i.e. on same server 82 | 83 | end 84 | -------------------------------------------------------------------------------- /config/environments/test.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # The test environment is used exclusively to run your application's 5 | # test suite. You never need to work with it otherwise. Remember that 6 | # your test database is "scratch space" for the test suite and is wiped 7 | # and recreated between test runs. Don't rely on the data there! 8 | config.cache_classes = true 9 | 10 | # Do not eager load code on boot. This avoids loading your whole application 11 | # just for the purpose of running a single test. If you are using a tool that 12 | # preloads Rails for running tests, you may have to set it to true. 13 | config.eager_load = false 14 | 15 | # Configure static file server for tests with Cache-Control for performance. 16 | config.serve_static_files = true 17 | config.static_cache_control = 'public, max-age=3600' 18 | 19 | # Show full error reports and disable caching. 20 | config.consider_all_requests_local = true 21 | config.action_controller.perform_caching = false 22 | 23 | # Raise exceptions instead of rendering exception templates. 24 | config.action_dispatch.show_exceptions = false 25 | 26 | # Disable request forgery protection in test environment. 27 | config.action_controller.allow_forgery_protection = false 28 | 29 | # Tell Action Mailer not to deliver emails to the real world. 30 | # The :test delivery method accumulates sent emails in the 31 | # ActionMailer::Base.deliveries array. 32 | config.action_mailer.delivery_method = :test 33 | 34 | # Randomize the order test cases are executed. 35 | config.active_support.test_order = :random 36 | 37 | # Print deprecation notices to the stderr. 38 | config.active_support.deprecation = :stderr 39 | 40 | # Raises error for missing translations 41 | # config.action_view.raise_on_missing_translations = true 42 | end 43 | -------------------------------------------------------------------------------- /config/initializers/assets.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Version of your assets, change this if you want to expire all your assets. 4 | Rails.application.config.assets.version = '1.0' 5 | 6 | # Add additional assets to the asset load path 7 | # Rails.application.config.assets.paths << Emoji.images_path 8 | 9 | # Precompile additional assets. 10 | # application.js, application.css, and all non-JS/CSS in app/assets folder are already added. 11 | # Rails.application.config.assets.precompile += %w( search.js ) 12 | -------------------------------------------------------------------------------- /config/initializers/backtrace_silencers.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. 4 | # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } 5 | 6 | # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. 7 | # Rails.backtrace_cleaner.remove_silencers! 8 | -------------------------------------------------------------------------------- /config/initializers/cookies_serializer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.action_dispatch.cookies_serializer = :json 4 | -------------------------------------------------------------------------------- /config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Configure sensitive parameters which will be filtered from the log file. 4 | Rails.application.config.filter_parameters += [:password] 5 | -------------------------------------------------------------------------------- /config/initializers/inflections.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new inflection rules using the following format. Inflections 4 | # are locale specific, and you may define rules for as many different 5 | # locales as you wish. All of these examples are active by default: 6 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 7 | # inflect.plural /^(ox)$/i, '\1en' 8 | # inflect.singular /^(ox)en/i, '\1' 9 | # inflect.irregular 'person', 'people' 10 | # inflect.uncountable %w( fish sheep ) 11 | # end 12 | 13 | # These inflection rules are supported but not enabled by default: 14 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 15 | # inflect.acronym 'RESTful' 16 | # end 17 | -------------------------------------------------------------------------------- /config/initializers/mime_types.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new mime types for use in respond_to blocks: 4 | # Mime::Type.register "text/richtext", :rtf 5 | -------------------------------------------------------------------------------- /config/initializers/mira.rb: -------------------------------------------------------------------------------- 1 | 2 | require 'load_dynamic_AR_class_with_scopes' 3 | 4 | 5 | # Prior to a migration the project table does not exist. Without checking 6 | # for it, it causes errors during an initial migration (i.e. create, migrate), 7 | # because you are referring to a table not yet created! 8 | if ActiveRecord::Base.connection.table_exists? 'projects' 9 | 10 | valid_project_prefixes = Project.ids.map do |id| 11 | Rails.application.config.x.db_table_prefix.downcase + id.to_s + "_" 12 | end 13 | 14 | api_tables = ActiveRecord::Base.connection.tables.select { |a| 15 | valid_project_prefixes.any? { |z| a.starts_with? z} 16 | } 17 | 18 | # create scopes on Project model to allow metadata search 19 | load_dynamic_AR_class_with_scopes("project") 20 | 21 | # Create a model for each of the data tables 22 | api_tables.each do |table| 23 | 24 | load_dynamic_AR_class_with_scopes(table) 25 | 26 | end 27 | 28 | 29 | end 30 | 31 | # map datapackage types to activerecord types 32 | DATAPACKAGE_TYPE_MAP = { 33 | "boolean" => "boolean", 34 | "integer" => "integer", 35 | "number" => "float", 36 | "float" => "float", 37 | "geopoint" => "float", # seen in airport-codes dataset although have not found any documentation for it! 38 | "datetime" => "datetime", 39 | "date" => "date", 40 | "time" => "time", 41 | "string" => "text", 42 | "null" => "text" 43 | } 44 | 45 | BIG_INTEGER_LIMIT = 2147483647 # 2^31-1 46 | 47 | MIRA_EXTRA_VARIABLE_MAP = { 48 | "mira_created_at" => "datetime", 49 | "mira_source_id" => "integer", 50 | "mira_source_type" => "text" 51 | } 52 | -------------------------------------------------------------------------------- /config/initializers/session_store.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.session_store :cookie_store, key: '_mira_session' 4 | -------------------------------------------------------------------------------- /config/initializers/wrap_parameters.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # This file contains settings for ActionController::ParamsWrapper which 4 | # is enabled by default. 5 | 6 | # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array. 7 | ActiveSupport.on_load(:action_controller) do 8 | wrap_parameters format: [:json] if respond_to?(:wrap_parameters) 9 | end 10 | 11 | # To enable root element in JSON for ActiveRecord objects. 12 | # ActiveSupport.on_load(:active_record) do 13 | # self.include_root_in_json = true 14 | # end 15 | -------------------------------------------------------------------------------- /config/locales/devise.en.yml: -------------------------------------------------------------------------------- 1 | # Additional translations at https://github.com/plataformatec/devise/wiki/I18n 2 | 3 | en: 4 | devise: 5 | confirmations: 6 | confirmed: "Your email address has been successfully confirmed." 7 | send_instructions: "You will receive an email with instructions for how to confirm your email address in a few minutes." 8 | send_paranoid_instructions: "If your email address exists in our database, you will receive an email with instructions for how to confirm your email address in a few minutes." 9 | failure: 10 | already_authenticated: "You are already signed in." 11 | inactive: "Your account is not activated yet." 12 | invalid: "Invalid %{authentication_keys} or password." 13 | locked: "Your account is locked." 14 | last_attempt: "You have one more attempt before your account is locked." 15 | not_found_in_database: "Invalid %{authentication_keys} or password." 16 | timeout: "Your session expired. Please sign in again to continue." 17 | unauthenticated: "You need to sign in or sign up before continuing." 18 | unconfirmed: "You have to confirm your email address before continuing." 19 | mailer: 20 | confirmation_instructions: 21 | subject: "Confirmation instructions" 22 | reset_password_instructions: 23 | subject: "Reset password instructions" 24 | unlock_instructions: 25 | subject: "Unlock instructions" 26 | omniauth_callbacks: 27 | failure: "Could not authenticate you from %{kind} because \"%{reason}\"." 28 | success: "Successfully authenticated from %{kind} account." 29 | passwords: 30 | no_token: "You can't access this page without coming from a password reset email. If you do come from a password reset email, please make sure you used the full URL provided." 31 | send_instructions: "You will receive an email with instructions on how to reset your password in a few minutes." 32 | send_paranoid_instructions: "If your email address exists in our database, you will receive a password recovery link at your email address in a few minutes." 33 | updated: "Your password has been changed successfully. You are now signed in." 34 | updated_not_active: "Your password has been changed successfully." 35 | registrations: 36 | destroyed: "Bye! Your account has been successfully cancelled. We hope to see you again soon." 37 | signed_up: "Welcome! You have signed up successfully." 38 | signed_up_but_inactive: "You have signed up successfully. However, we could not sign you in because your account is not yet activated." 39 | signed_up_but_locked: "You have signed up successfully. However, we could not sign you in because your account is locked." 40 | signed_up_but_unconfirmed: "A message with a confirmation link has been sent to your email address. Please follow the link to activate your account." 41 | update_needs_confirmation: "You updated your account successfully, but we need to verify your new email address. Please check your email and follow the confirm link to confirm your new email address." 42 | updated: "Your account has been updated successfully." 43 | sessions: 44 | signed_in: "Signed in successfully." 45 | signed_out: "Signed out successfully." 46 | already_signed_out: "Signed out successfully." 47 | unlocks: 48 | send_instructions: "You will receive an email with instructions for how to unlock your account in a few minutes." 49 | send_paranoid_instructions: "If your account exists, you will receive an email with instructions for how to unlock it in a few minutes." 50 | unlocked: "Your account has been unlocked successfully. Please sign in to continue." 51 | errors: 52 | messages: 53 | already_confirmed: "was already confirmed, please try signing in" 54 | confirmation_period_expired: "needs to be confirmed within %{period}, please request a new one" 55 | expired: "has expired, please request a new one" 56 | not_found: "not found" 57 | not_locked: "was not locked" 58 | not_saved: 59 | one: "1 error prohibited this %{resource} from being saved:" 60 | other: "%{count} errors prohibited this %{resource} from being saved:" 61 | -------------------------------------------------------------------------------- /config/locales/en.yml: -------------------------------------------------------------------------------- 1 | # Files in the config/locales directory are used for internationalization 2 | # and are automatically loaded by Rails. If you want to use locales other 3 | # than English, add the necessary files in this directory. 4 | # 5 | # To use the locales, use `I18n.t`: 6 | # 7 | # I18n.t 'hello' 8 | # 9 | # In views, this is aliased to just `t`: 10 | # 11 | # <%= t('hello') %> 12 | # 13 | # To use a different locale, set it with `I18n.locale`: 14 | # 15 | # I18n.locale = :es 16 | # 17 | # This would use the information in config/locales/es.yml. 18 | # 19 | # To learn more, please read the Rails Internationalization guide 20 | # available at http://guides.rubyonrails.org/i18n.html. 21 | 22 | en: 23 | hello: "Hello world" 24 | -------------------------------------------------------------------------------- /config/mira_constants.rb: -------------------------------------------------------------------------------- 1 | RESERVED_COLUMN_NAMES = ['order', 'page', 'per_page'] + MIRA_EXTRA_VARIABLE_MAP.keys 2 | -------------------------------------------------------------------------------- /config/routes.rb: -------------------------------------------------------------------------------- 1 | require 'api_constraints' 2 | 3 | Rails.application.routes.draw do 4 | 5 | devise_for :users 6 | root "projects#index" 7 | 8 | 9 | resources :user do 10 | resources :api_keys do 11 | resources :api_key_permissions 12 | end 13 | end 14 | 15 | # Custom routes relating to API keys 16 | get "user/:user_id/api_keys/:id/generate-new-api-key" => "api_keys#gen_new_key" 17 | get "user/:user_id/api_keys/projects/:project_id" => "api_keys#index_project", as: :project_api_keys 18 | delete "projects/:project_id/datapackage/datapackage_resources/:id/api_keys/:api_key_id" => "datapackage_resources#delete_apikey_rows" 19 | 20 | # UI routes 21 | resources :projects do 22 | resources :datasources 23 | resource :datapackage do 24 | resources :datapackage_resources 25 | end 26 | end 27 | 28 | 29 | # custom project routes 30 | post "projects/:id/upload_datasources" => "projects#upload_datasources" 31 | post "projects/:id/upload_datapackage" => "projects#upload_datapackage" 32 | get "projects/:id/api-details" => "projects#api_detail" 33 | 34 | 35 | # API routes 36 | namespace :api, defaults: {format: 'json'} do 37 | # /api/... API:: 38 | scope module: :v1, constraints: ApiConstraints.new(version: 1, default: true) do 39 | # Projects 40 | get "projects" => "projects#index" 41 | get "projects/:id" => "projects#show" 42 | 43 | # Datapackage file 44 | get "projects/:id/datapackage" => "datapackages#show" 45 | 46 | # Datapackage resource metadata combined with some information from datasources (e.g. public_url) 47 | get "projects/:id/tables" => "datapackage_resources#index" 48 | get "projects/:id/tables/:table_ref" => "datapackage_resources#show" 49 | get "projects/:id/tables/:table_ref/columns" => "datapackage_resources#column_index" 50 | get "projects/:id/tables/:table_ref/columns/:col_ref" => "datapackage_resources#column_show" 51 | # Expose datapackage field metadata 52 | get "projects/:id/tables/:table_ref/datapackage/fields" => "datapackage_resource_fields#index" 53 | get "projects/:id/tables/:table_ref/datapackage/fields/:col_ref" => "datapackage_resource_fields#show" 54 | 55 | # Uploads (via datasources table) 56 | get "projects/:id/uploads" => "datasources#index" 57 | get "projects/:id/uploads/:table_ref" => "datasources#show" 58 | 59 | get "projects/:id/tables/:table_ref/data" => "data#index" 60 | get "projects/:id/tables/:table_ref/recline/data" => "data#recline" 61 | get ":db_table/metadata/search" => "data#index" # e.g. for searching project metadata 62 | # Distinct values 63 | get "projects/:id/tables/:table_ref/columns/:col_ref/distinct" => "data#distinct" 64 | 65 | # generic CRUD 66 | post "projects/:id/tables/:table_ref/data" => "data#create" 67 | get "projects/:id/tables/:table_ref/data/:data_id" => "data#show" 68 | patch "projects/:id/tables/:table_ref/data/:data_id" => "data#update" 69 | delete "projects/:id/tables/:table_ref/data/:data_id" => "data#destroy" 70 | 71 | # datatables CRUD 72 | post "projects/:id/tables/:table_ref/datatables" => "data#datatables", 73 | :via => [:post], 74 | :constraints => lambda { |request| (request.params.has_key?(:draw) && request.params.has_key?(:start) && request.params.has_key?(:length)) } 75 | post "projects/:id/tables/:table_ref/datatables/editor" => "data#datatables_editor" 76 | 77 | end 78 | 79 | end 80 | 81 | # table = "xy41_97" 82 | # table_klass = ActiveRecord::Base.const_get "#{table}".capitalize 83 | # project_number, datasource_number = table.sub(Rails.configuration.x.db_table_prefix.downcase,"").split("_").map { |s| s.to_i } 84 | # table_ref = DatapackageResource.where(datasource_id: datasource_number).first.table_ref 85 | # 86 | # puts "Adding routes for project " + project_number.to_s + ", table " + table_ref 87 | # get_route = "/api/projects/" + project_number.to_s + "/tables/" + table_ref + "/:id" 88 | # binding.pry 89 | # get get_route, :to => "projects#index" 90 | 91 | end 92 | -------------------------------------------------------------------------------- /config/secrets.yml: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Your secret key is used for verifying the integrity of signed cookies. 4 | # If you change this key, all old signed cookies will become invalid! 5 | 6 | # Make sure the secret is at least 30 characters and all random, 7 | # no regular words or you'll be exposed to dictionary attacks. 8 | # You can use `rake secret` to generate a secure secret key. 9 | 10 | # Make sure the secrets in this file are kept private 11 | # if you're sharing your code publicly. 12 | 13 | development: 14 | secret_key_base: eef0c418766153736f2398421eaa47d2a5585d8088edad87c78b0f9e019d24686476ea1f8b7c322fa6446f347452f20a0152ed40a983c926fadefbbaa15293e4 15 | 16 | test: 17 | secret_key_base: 5b9a665adb5aca00354dc334251d040c729e4cac3d60dc39dbd87645521e6a9d01eddeedfc31910ee174aaebcfe84beddbb2a23c67bc6767c706e2596822f68b 18 | 19 | # Do not keep production secrets in the repository, 20 | # instead read values from the environment. 21 | production: 22 | secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> 23 | 24 | -------------------------------------------------------------------------------- /db/migrate/20150318173213_create_projects.rb: -------------------------------------------------------------------------------- 1 | class CreateProjects < ActiveRecord::Migration 2 | def change 3 | create_table :projects do |t| 4 | t.text :name 5 | t.text :description 6 | 7 | t.timestamps null: false 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20150318181209_add_index_to_projects_name.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToProjectsName < ActiveRecord::Migration 2 | def change 3 | add_index :projects, :name, unique: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20150320162605_create_datasources.rb: -------------------------------------------------------------------------------- 1 | class CreateDatasources < ActiveRecord::Migration 2 | def change 3 | create_table :datasources do |t| 4 | t.references :project, index: true 5 | 6 | t.timestamps null: false 7 | end 8 | add_foreign_key :datasources, :projects 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20150320163854_add_attachment_datafile_to_datasources.rb: -------------------------------------------------------------------------------- 1 | class AddAttachmentDatafileToDatasources < ActiveRecord::Migration 2 | def self.up 3 | change_table :datasources do |t| 4 | t.attachment :datafile 5 | end 6 | end 7 | 8 | def self.down 9 | remove_attachment :datasources, :datafile 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20150404082247_create_delayed_jobs.rb: -------------------------------------------------------------------------------- 1 | class CreateDelayedJobs < ActiveRecord::Migration 2 | def self.up 3 | create_table :delayed_jobs, force: true do |table| 4 | table.integer :priority, default: 0, null: false # Allows some jobs to jump to the front of the queue 5 | table.integer :attempts, default: 0, null: false # Provides for retries, but still fail eventually. 6 | table.text :handler, null: false # YAML-encoded string of the object that will do work 7 | table.text :last_error # reason for last failure (See Note below) 8 | table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future. 9 | table.datetime :locked_at # Set when a client is working on this object 10 | table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead) 11 | table.string :locked_by # Who is working on this object (if locked) 12 | table.string :queue # The name of the queue this job is in 13 | table.timestamps null: true 14 | end 15 | 16 | add_index :delayed_jobs, [:priority, :run_at], name: "delayed_jobs_priority" 17 | end 18 | 19 | def self.down 20 | drop_table :delayed_jobs 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /db/migrate/20150505184421_add_file_path_to_datasources.rb: -------------------------------------------------------------------------------- 1 | class AddFilePathToDatasources < ActiveRecord::Migration 2 | def change 3 | add_column :datasources, :logfile_path, :text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20150506094858_add_index_datasources_table_ref.rb: -------------------------------------------------------------------------------- 1 | class AddIndexDatasourcesTableRef < ActiveRecord::Migration 2 | def change 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /db/migrate/20150726062647_add_user_id_to_project.rb: -------------------------------------------------------------------------------- 1 | class AddUserIdToProject < ActiveRecord::Migration 2 | def change 3 | add_column :projects, :user_id, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20150726062922_devise_create_users.rb: -------------------------------------------------------------------------------- 1 | class DeviseCreateUsers < ActiveRecord::Migration 2 | def change 3 | create_table(:users) do |t| 4 | ## Database authenticatable 5 | t.string :email, null: false, default: "" 6 | t.string :encrypted_password, null: false, default: "" 7 | 8 | ## Recoverable 9 | t.string :reset_password_token 10 | t.datetime :reset_password_sent_at 11 | 12 | ## Rememberable 13 | t.datetime :remember_created_at 14 | 15 | ## Trackable 16 | t.integer :sign_in_count, default: 0, null: false 17 | t.datetime :current_sign_in_at 18 | t.datetime :last_sign_in_at 19 | t.inet :current_sign_in_ip 20 | t.inet :last_sign_in_ip 21 | 22 | ## Confirmable 23 | # t.string :confirmation_token 24 | # t.datetime :confirmed_at 25 | # t.datetime :confirmation_sent_at 26 | # t.string :unconfirmed_email # Only if using reconfirmable 27 | 28 | ## Lockable 29 | # t.integer :failed_attempts, default: 0, null: false # Only if lock strategy is :failed_attempts 30 | # t.string :unlock_token # Only if unlock strategy is :email or :both 31 | # t.datetime :locked_at 32 | 33 | 34 | t.timestamps null: false 35 | end 36 | 37 | add_index :users, :email, unique: true 38 | add_index :users, :reset_password_token, unique: true 39 | # add_index :users, :confirmation_token, unique: true 40 | # add_index :users, :unlock_token, unique: true 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /db/migrate/20151020101937_create_datapackages.rb: -------------------------------------------------------------------------------- 1 | class CreateDatapackages < ActiveRecord::Migration 2 | def change 3 | create_table :datapackages do |t| 4 | t.references :project, index: true 5 | t.text :public_url 6 | t.timestamps null: false 7 | end 8 | add_foreign_key :datapackages, :projects 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20151020102022_add_attachment_file_to_datapackages.rb: -------------------------------------------------------------------------------- 1 | class AddAttachmentFileToDatapackages < ActiveRecord::Migration 2 | def self.up 3 | change_table :datapackages do |t| 4 | t.attachment :datapackage 5 | end 6 | end 7 | 8 | def self.down 9 | remove_attachment :datapackages, :datapackage 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20151020103351_create_datapackage_resources.rb: -------------------------------------------------------------------------------- 1 | class CreateDatapackageResources < ActiveRecord::Migration 2 | def change 3 | create_table :datapackage_resources do |t| 4 | t.integer :datapackage_id 5 | t.text :path 6 | t.text :format 7 | t.text :delimiter 8 | t.text :mediatype 9 | 10 | t.timestamps null: false 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20151021130254_create_datapackage_resource_schemas.rb: -------------------------------------------------------------------------------- 1 | class CreateDatapackageResourceSchemas < ActiveRecord::Migration 2 | def change 3 | create_table :datapackage_resource_fields do |t| 4 | t.references :datapackage_resource, index: true 5 | t.text :name 6 | t.text :ftype # seems type is reserved word 7 | t.integer :order 8 | 9 | t.timestamps null: false 10 | end 11 | add_foreign_key :datapackage_resource_fields, :datapackage_resources 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /db/migrate/20151022145248_big_integer_to_datapackage_resource_fields.rb: -------------------------------------------------------------------------------- 1 | class BigIntegerToDatapackageResourceFields < ActiveRecord::Migration 2 | def change 3 | add_column :datapackage_resource_fields, :big_integer, :boolean 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20151022153557_add_quote_character_to_datapackage_resources.rb: -------------------------------------------------------------------------------- 1 | class AddQuoteCharacterToDatapackageResources < ActiveRecord::Migration 2 | def change 3 | add_column :datapackage_resources, :quote_character, :text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20151023084243_add_table_ref_to_datapackage_resources.rb: -------------------------------------------------------------------------------- 1 | class AddTableRefToDatapackageResources < ActiveRecord::Migration 2 | def change 3 | add_column :datapackage_resources, :table_ref, :text 4 | add_index :datapackage_resources, :table_ref 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20151030073834_add_import_status_to_datasources.rb: -------------------------------------------------------------------------------- 1 | class AddImportStatusToDatasources < ActiveRecord::Migration 2 | def change 3 | add_column :datasources, :import_status, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20151128084949_add_add_index_to_datapackage_resource_fields.rb: -------------------------------------------------------------------------------- 1 | class AddAddIndexToDatapackageResourceFields < ActiveRecord::Migration 2 | def change 3 | add_column :datapackage_resource_fields, :add_index, :boolean, :default => true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20151206044323_add_db_table_name_to_datapackage_resources.rb: -------------------------------------------------------------------------------- 1 | class AddDbTableNameToDatapackageResources < ActiveRecord::Migration 2 | def change 3 | add_column :datapackage_resources, :db_table_name, :text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20151209215327_add_format_to_datapackage_resource_field.rb: -------------------------------------------------------------------------------- 1 | class AddFormatToDatapackageResourceField < ActiveRecord::Migration 2 | def change 3 | add_column :datapackage_resource_fields, :format, :text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20151227122837_add_description_to_datapackage_resources.rb: -------------------------------------------------------------------------------- 1 | class AddDescriptionToDatapackageResources < ActiveRecord::Migration 2 | def change 3 | add_column :datapackage_resources, :description, :text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160415042134_create_api_keys.rb: -------------------------------------------------------------------------------- 1 | class CreateApiKeys < ActiveRecord::Migration 2 | def change 3 | create_table :api_keys do |t| 4 | t.integer :user_id 5 | t.string :token 6 | t.text :description 7 | t.timestamps null: false 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20160415185504_create_api_key_permissions.rb: -------------------------------------------------------------------------------- 1 | class CreateApiKeyPermissions < ActiveRecord::Migration 2 | def change 3 | create_table :api_key_permissions do |t| 4 | t.integer :api_key_id 5 | t.integer :permission_scope 6 | t.integer :permission 7 | t.integer :project_id 8 | t.integer :datapackage_resource_id 9 | t.string :db_table_name 10 | 11 | t.timestamps null: false 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /db/migrate/20160531140633_add_private_to_datapackage_resource_fields.rb: -------------------------------------------------------------------------------- 1 | class AddPrivateToDatapackageResourceFields < ActiveRecord::Migration 2 | def change 3 | add_column :datapackage_resource_fields, :private, :boolean, :default => false 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160616200650_add_datapackage_resource_to_datasources.rb: -------------------------------------------------------------------------------- 1 | class AddDatapackageResourceToDatasources < ActiveRecord::Migration 2 | def change 3 | add_reference :datasources, :datapackage_resource, index: true, foreign_key: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160617073857_add_datapackage_to_datasources.rb: -------------------------------------------------------------------------------- 1 | class AddDatapackageToDatasources < ActiveRecord::Migration 2 | def change 3 | add_reference :datasources, :datapackage, index: true, foreign_key: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160624141850_add_imported_rows_to_datasources.rb: -------------------------------------------------------------------------------- 1 | class AddImportedRowsToDatasources < ActiveRecord::Migration 2 | def change 3 | add_column :datasources, :imported_rows, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/seeds.rb: -------------------------------------------------------------------------------- 1 | 2 | admin_email = 'admin@example.com' 3 | 4 | u = User.where(email: admin_email) 5 | if u.empty? 6 | user = User.create! :email => admin_email, :password => 'topsecret', :password_confirmation => 'topsecret' 7 | end 8 | 9 | -------------------------------------------------------------------------------- /lib/api_constraints.rb: -------------------------------------------------------------------------------- 1 | class ApiConstraints 2 | def initialize(options) 3 | @version = options[:version] 4 | @default = options[:default] 5 | end 6 | 7 | def matches?(req) 8 | @default || req.headers['Accept'].include?("application/vnd.example.v#{@version}") 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/load_table.rb: -------------------------------------------------------------------------------- 1 | require 'csv' 2 | require 'tempfile' 3 | require 'load_dynamic_AR_class_with_scopes' 4 | 5 | class LoadTable 6 | 7 | include ApplicationHelper 8 | include ProjectHelper 9 | 10 | attr_reader :table_name, :column_list, :column_type_hash 11 | 12 | 13 | def initialize(datasource, datapackage_resource, upload_method) 14 | @ds = datasource 15 | # @datapackage_resources = DatapackageResource.where(datapackage_id: @ds.datapackage_id) 16 | @datapackage_resource = datapackage_resource 17 | load_logger.info("Initialising load of #{@ds.datafile_file_name}") 18 | @column_metadata = DatapackageResourceField.where(datapackage_resource_id: @datapackage_resource.id) 19 | @csv_file = File.open(@ds.datafile.path) 20 | @upload_method = upload_method 21 | 22 | upload_to_db_table 23 | 24 | end 25 | 26 | 27 | private 28 | 29 | def load_logger 30 | log_dir = Project.find(@ds.project_id).job_log_path 31 | Dir.mkdir(log_dir) unless File.directory?(log_dir) 32 | @load_logger ||= Logger.new(@ds.logfile_path) 33 | # @load_logger ||= Logger.new("#{log_dir}/#{@ds.datafile_file_name}.log") 34 | end 35 | 36 | 37 | def new_col_name(name) 38 | # NOTE: better not to try maintaining case of variables as uppercase column names 39 | # leads to problems later when creating scopes dynamically (ruby starts looking for constants). 40 | name.parameterize.underscore 41 | end 42 | 43 | 44 | def upload_to_db_table 45 | if @upload_method == "quick" 46 | quick_upload_to_db_table 47 | else 48 | slow_upload_to_db_table 49 | end 50 | end 51 | 52 | 53 | def slow_upload_to_db_table 54 | ar_table_klass = get_mira_ar_table(@datapackage_resource.db_table_name) 55 | uploaded_row_count = 0 56 | CSV.foreach(@csv_file, 57 | :headers => true, 58 | :header_converters => lambda { |h| new_col_name(h) }) do |row| 59 | next if row.to_s.strip.size == 0 60 | if ar_table_klass.create!(row.to_hash) 61 | uploaded_row_count += 1 62 | else 63 | load_logger.error("Failed to upload row " + (uploaded_row_count + 1).to_s ) 64 | end 65 | end 66 | save_row_count(uploaded_row_count) 67 | end 68 | 69 | 70 | def quick_upload_to_db_table 71 | # columns in correct order 72 | column_names = @column_metadata.sort{ |a,b| a.order <=> b.order }.map{ |c| new_col_name(c.name) } 73 | extra_column_string = ',"' + MIRA_EXTRA_VARIABLE_MAP.keys.join('","') + '"' 74 | column_string = "\"#{column_names.join('","')}\"" + extra_column_string 75 | 76 | # now add our extra variables on to our string 77 | dlm = @datapackage_resource.delimiter 78 | mira_created_at = Time.now.iso8601 79 | mira_source_id = @ds.id 80 | mira_source_type = "csv" 81 | extra_columns = dlm + mira_created_at + dlm + mira_source_id.to_s + dlm + mira_source_type 82 | 83 | csv_options = "DELIMITER '#{@datapackage_resource.delimiter}' CSV" 84 | skip_header_line = @csv_file.gets 85 | uploaded_row_count = 0 86 | # https://github.com/theSteveMitchell/postgres_upsert 87 | ActiveRecord::Base.connection.raw_connection.copy_data "COPY #{@datapackage_resource.db_table_name} (#{column_string}) FROM STDIN #{csv_options} QUOTE '#{@datapackage_resource.quote_character}'" do 88 | while line = @csv_file.gets do 89 | next if line.strip.size == 0 90 | putline = line.split("\n")[0] + extra_columns + "\n" 91 | ActiveRecord::Base.connection.raw_connection.put_copy_data putline 92 | uploaded_row_count += 1 93 | end 94 | end 95 | save_row_count(uploaded_row_count) 96 | end 97 | 98 | def save_row_count(uploaded_row_count) 99 | @ds.imported_rows = uploaded_row_count 100 | @ds.save 101 | load_logger.info("Imported " + uploaded_row_count.to_s + " rows of data.") 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /lib/tasks/kill_postgres_connections.rake: -------------------------------------------------------------------------------- 1 | # http://stackoverflow.com/a/5750734/1002140 2 | # Relates to error: 3 | # PG::ObjectInUse: ERROR: database "mira_dev" is being accessed by other users 4 | task :kill_postgres_connections => :environment do 5 | suff = "dev" if "#{Rails.env}" == "development" 6 | db_name = "#{File.basename(Rails.root)}_"+ suff 7 | sh = < :kill_postgres_connections 19 | -------------------------------------------------------------------------------- /lib/tasks/mira.rake: -------------------------------------------------------------------------------- 1 | namespace :mira do 2 | namespace :dev do 3 | 4 | task :clear_logs do 5 | system ("find #{Rails.root.to_s}/project_files/job_logs/ -name \"*.log\" -type f -delete") 6 | system("find #{Rails.root.to_s}/project_files/job_logs/ -depth -type d -empty -exec rmdir {} \\;") 7 | system("> #{Rails.root}/log/development.log") 8 | end 9 | 10 | task :clear_uploads do 11 | system ("find #{Rails.root.to_s}/project_files/uploads/ -name \"*.csv\" -type f -delete") 12 | system ("find #{Rails.root.to_s}/project_files/uploads/ -name \"*.json\" -type f -delete") 13 | system("find #{Rails.root.to_s}/project_files/uploads/ -depth -type d -empty -exec rmdir {} \\;") 14 | end 15 | 16 | 17 | desc "Reset dev database and clear out uploads and logs" 18 | task :reset do 19 | system("sudo service postgresql restart") 20 | begin 21 | Rake::Task["db:drop"].invoke 22 | rescue Exception => e 23 | puts "Database not dropped: " + e.message 24 | end 25 | Rake::Task["db:create"].invoke 26 | Rake::Task["db:migrate"].invoke 27 | Rake::Task["db:seed"].invoke 28 | Rake::Task["mira:dev:clear_logs"].invoke 29 | Rake::Task["mira:dev:clear_uploads"].invoke 30 | end 31 | 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/tasks/test.rake: -------------------------------------------------------------------------------- 1 | 2 | # require 'rake/testtask' 3 | namespace :test do 4 | 5 | Rails::TestTask.new("api") do |t| 6 | t.pattern = "test/api/v1/*_test.rb" 7 | end 8 | 9 | Rails::TestTask.new("controllers") do |t| 10 | t.pattern = "test/controllers/*_test.rb" 11 | end 12 | 13 | Rails::TestTask.new("models") do |t| 14 | t.pattern = "test/models/*_test.rb" 15 | end 16 | 17 | # Rails::TestTask.new("all") do 18 | # Rake::Task["test:api"].invoke 19 | # Rake::Task["test:controllers"].invoke 20 | # Rake::Task["test:models"].invoke 21 | # end 22 | end 23 | -------------------------------------------------------------------------------- /project_files/job_logs/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /project_files/uploads/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /public/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The page you were looking for doesn't exist (404) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The page you were looking for doesn't exist.

62 |

You may have mistyped the address or the page may have moved.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /public/422.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The change you wanted was rejected (422) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The change you wanted was rejected.

62 |

Maybe you tried to change something you didn't have access to.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /public/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | We're sorry, but something went wrong (500) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

We're sorry, but something went wrong.

62 |
63 |

If you are the application owner check the logs for more information.

64 |
65 | 66 | 67 | -------------------------------------------------------------------------------- /public/assets/.sprockets-manifest-7e63634ef7c989258404942fb40cbda3.json: -------------------------------------------------------------------------------- 1 | {"files":{"application-5b52394f4595077da6d3ca54f0a32c70fec87c62c77757dca0b38c181f961108.js":{"logical_path":"application.js","mtime":"2015-07-26T09:02:54+01:00","size":328446,"digest":"5b52394f4595077da6d3ca54f0a32c70fec87c62c77757dca0b38c181f961108","integrity":"sha256-W1I5T0WVB32m08pU8KMscP7IfGLHd1fcoLOMGB+WEQg="},"application-fa5875e27f9dfff37d42fae684ac7258f51fe64183140e578ba87737388b8e5d.css":{"logical_path":"application.css","mtime":"2015-07-27T11:27:46+01:00","size":337507,"digest":"fa5875e27f9dfff37d42fae684ac7258f51fe64183140e578ba87737388b8e5d","integrity":"sha256-+lh14n+d//N9QvrmhKxyWPUf5kGDFA5Xi6h3NziLjl0="},"sextant/application-572803211757a77712103db3532083399575a1c0284db973b6079f6f24f78c2e.js":{"logical_path":"sextant/application.js","mtime":"2015-04-13T11:46:44+01:00","size":770,"digest":"572803211757a77712103db3532083399575a1c0284db973b6079f6f24f78c2e","integrity":"sha256-VygDIRdXp3cSED2zUyCDOZV1ocAoTblztgefbyT3jC4="},"bootstrap/glyphicons-halflings-regular-13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407.eot":{"logical_path":"bootstrap/glyphicons-halflings-regular.eot","mtime":"2015-07-14T14:10:41+01:00","size":20127,"digest":"13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407","integrity":"sha256-E2NNqH2eI/jD7ZEIzhck0YOjmtBy5z4bPYy/ZG0tBAc="},"bootstrap/glyphicons-halflings-regular-42f60659d265c1a3c30f9fa42abcbb56bd4a53af4d83d316d6dd7a36903c43e5.svg":{"logical_path":"bootstrap/glyphicons-halflings-regular.svg","mtime":"2015-07-14T14:10:41+01:00","size":108738,"digest":"42f60659d265c1a3c30f9fa42abcbb56bd4a53af4d83d316d6dd7a36903c43e5","integrity":"sha256-QvYGWdJlwaPDD5+kKry7Vr1KU69Ng9MW1t16NpA8Q+U="},"bootstrap/glyphicons-halflings-regular-e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456.ttf":{"logical_path":"bootstrap/glyphicons-halflings-regular.ttf","mtime":"2015-07-14T14:10:41+01:00","size":45404,"digest":"e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456","integrity":"sha256-45UEQJN1fYKvyxOJV9BqHqk2G9zwtELQahioBRr1dFY="},"bootstrap/glyphicons-halflings-regular-a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742.woff":{"logical_path":"bootstrap/glyphicons-halflings-regular.woff","mtime":"2015-07-14T14:10:41+01:00","size":23424,"digest":"a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742","integrity":"sha256-omOU9+3hAMoRjv8u2ghZYnWpg5uVnCJuFUOVV6WoB0I="},"bootstrap/glyphicons-halflings-regular-fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c.woff2":{"logical_path":"bootstrap/glyphicons-halflings-regular.woff2","mtime":"2015-07-14T14:10:41+01:00","size":18028,"digest":"fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c","integrity":"sha256-/hhdEaSWdokNR7t4MxKgzaWkTEA5IUCU55V7TAQO8Rw="}},"assets":{"application.js":"application-5b52394f4595077da6d3ca54f0a32c70fec87c62c77757dca0b38c181f961108.js","application.css":"application-fa5875e27f9dfff37d42fae684ac7258f51fe64183140e578ba87737388b8e5d.css","sextant/application.js":"sextant/application-572803211757a77712103db3532083399575a1c0284db973b6079f6f24f78c2e.js","bootstrap/glyphicons-halflings-regular.eot":"bootstrap/glyphicons-halflings-regular-13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407.eot","bootstrap/glyphicons-halflings-regular.svg":"bootstrap/glyphicons-halflings-regular-42f60659d265c1a3c30f9fa42abcbb56bd4a53af4d83d316d6dd7a36903c43e5.svg","bootstrap/glyphicons-halflings-regular.ttf":"bootstrap/glyphicons-halflings-regular-e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456.ttf","bootstrap/glyphicons-halflings-regular.woff":"bootstrap/glyphicons-halflings-regular-a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742.woff","bootstrap/glyphicons-halflings-regular.woff2":"bootstrap/glyphicons-halflings-regular-fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c.woff2"}} -------------------------------------------------------------------------------- /public/assets/.sprockets-manifest-92af3159f449125806a09a697bd61573.json: -------------------------------------------------------------------------------- 1 | {"files":{"application-5b52394f4595077da6d3ca54f0a32c70fec87c62c77757dca0b38c181f961108.js":{"logical_path":"application.js","mtime":"2015-07-26T09:02:54+01:00","size":328446,"digest":"5b52394f4595077da6d3ca54f0a32c70fec87c62c77757dca0b38c181f961108","integrity":"sha256-W1I5T0WVB32m08pU8KMscP7IfGLHd1fcoLOMGB+WEQg="},"application-fa5875e27f9dfff37d42fae684ac7258f51fe64183140e578ba87737388b8e5d.css":{"logical_path":"application.css","mtime":"2015-07-27T11:27:46+01:00","size":337507,"digest":"fa5875e27f9dfff37d42fae684ac7258f51fe64183140e578ba87737388b8e5d","integrity":"sha256-+lh14n+d//N9QvrmhKxyWPUf5kGDFA5Xi6h3NziLjl0="},"sextant/application-572803211757a77712103db3532083399575a1c0284db973b6079f6f24f78c2e.js":{"logical_path":"sextant/application.js","mtime":"2015-04-13T11:46:44+01:00","size":770,"digest":"572803211757a77712103db3532083399575a1c0284db973b6079f6f24f78c2e","integrity":"sha256-VygDIRdXp3cSED2zUyCDOZV1ocAoTblztgefbyT3jC4="},"bootstrap/glyphicons-halflings-regular-13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407.eot":{"logical_path":"bootstrap/glyphicons-halflings-regular.eot","mtime":"2015-07-14T14:10:41+01:00","size":20127,"digest":"13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407","integrity":"sha256-E2NNqH2eI/jD7ZEIzhck0YOjmtBy5z4bPYy/ZG0tBAc="},"bootstrap/glyphicons-halflings-regular-42f60659d265c1a3c30f9fa42abcbb56bd4a53af4d83d316d6dd7a36903c43e5.svg":{"logical_path":"bootstrap/glyphicons-halflings-regular.svg","mtime":"2015-07-14T14:10:41+01:00","size":108738,"digest":"42f60659d265c1a3c30f9fa42abcbb56bd4a53af4d83d316d6dd7a36903c43e5","integrity":"sha256-QvYGWdJlwaPDD5+kKry7Vr1KU69Ng9MW1t16NpA8Q+U="},"bootstrap/glyphicons-halflings-regular-e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456.ttf":{"logical_path":"bootstrap/glyphicons-halflings-regular.ttf","mtime":"2015-07-14T14:10:41+01:00","size":45404,"digest":"e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456","integrity":"sha256-45UEQJN1fYKvyxOJV9BqHqk2G9zwtELQahioBRr1dFY="},"bootstrap/glyphicons-halflings-regular-a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742.woff":{"logical_path":"bootstrap/glyphicons-halflings-regular.woff","mtime":"2015-07-14T14:10:41+01:00","size":23424,"digest":"a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742","integrity":"sha256-omOU9+3hAMoRjv8u2ghZYnWpg5uVnCJuFUOVV6WoB0I="},"bootstrap/glyphicons-halflings-regular-fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c.woff2":{"logical_path":"bootstrap/glyphicons-halflings-regular.woff2","mtime":"2015-07-14T14:10:41+01:00","size":18028,"digest":"fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c","integrity":"sha256-/hhdEaSWdokNR7t4MxKgzaWkTEA5IUCU55V7TAQO8Rw="}},"assets":{"application.js":"application-5b52394f4595077da6d3ca54f0a32c70fec87c62c77757dca0b38c181f961108.js","application.css":"application-fa5875e27f9dfff37d42fae684ac7258f51fe64183140e578ba87737388b8e5d.css","sextant/application.js":"sextant/application-572803211757a77712103db3532083399575a1c0284db973b6079f6f24f78c2e.js","bootstrap/glyphicons-halflings-regular.eot":"bootstrap/glyphicons-halflings-regular-13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407.eot","bootstrap/glyphicons-halflings-regular.svg":"bootstrap/glyphicons-halflings-regular-42f60659d265c1a3c30f9fa42abcbb56bd4a53af4d83d316d6dd7a36903c43e5.svg","bootstrap/glyphicons-halflings-regular.ttf":"bootstrap/glyphicons-halflings-regular-e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456.ttf","bootstrap/glyphicons-halflings-regular.woff":"bootstrap/glyphicons-halflings-regular-a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742.woff","bootstrap/glyphicons-halflings-regular.woff2":"bootstrap/glyphicons-halflings-regular-fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c.woff2"}} -------------------------------------------------------------------------------- /public/assets/bootstrap/glyphicons-halflings-regular-13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davbre/mira/f9dde315b11c5d26329d200869672a5d3c9b304a/public/assets/bootstrap/glyphicons-halflings-regular-13634da87d9e23f8c3ed9108ce1724d183a39ad072e73e1b3d8cbf646d2d0407.eot -------------------------------------------------------------------------------- /public/assets/bootstrap/glyphicons-halflings-regular-a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davbre/mira/f9dde315b11c5d26329d200869672a5d3c9b304a/public/assets/bootstrap/glyphicons-halflings-regular-a26394f7ede100ca118eff2eda08596275a9839b959c226e15439557a5a80742.woff -------------------------------------------------------------------------------- /public/assets/bootstrap/glyphicons-halflings-regular-e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davbre/mira/f9dde315b11c5d26329d200869672a5d3c9b304a/public/assets/bootstrap/glyphicons-halflings-regular-e395044093757d82afcb138957d06a1ea9361bdcf0b442d06a18a8051af57456.ttf -------------------------------------------------------------------------------- /public/assets/bootstrap/glyphicons-halflings-regular-fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davbre/mira/f9dde315b11c5d26329d200869672a5d3c9b304a/public/assets/bootstrap/glyphicons-halflings-regular-fe185d11a49676890d47bb783312a0cda5a44c4039214094e7957b4c040ef11c.woff2 -------------------------------------------------------------------------------- /public/assets/sextant/application-572803211757a77712103db3532083399575a1c0284db973b6079f6f24f78c2e.js: -------------------------------------------------------------------------------- 1 | function each(elems, func) { 2 | if (!elems instanceof Array) { elems = [elems]; } 3 | for (var i = elems.length; i--; ) { 4 | func(elems[i]); 5 | } 6 | } 7 | 8 | function setValOn(elems, val) { 9 | each(elems, function(elem) { 10 | elem.innerHTML = val; 11 | }); 12 | } 13 | 14 | function onClick(elems, func) { 15 | each(elems, function(elem) { 16 | elem.onclick = func; 17 | }); 18 | } 19 | 20 | // Enables functionality to toggle between `_path` and `_url` helper suffixes 21 | function setupRouteToggleHelperLinks() { 22 | var toggleLinks = document.querySelectorAll('#route_table [data-route-helper]'); 23 | onClick(toggleLinks, function(){ 24 | var helperTxt = this.getAttribute("data-route-helper"); 25 | var helperElems = document.querySelectorAll('[data-route-name] span.helper'); 26 | setValOn(helperElems, helperTxt); 27 | }); 28 | } 29 | ; 30 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davbre/mira/f9dde315b11c5d26329d200869672a5d3c9b304a/public/favicon.ico -------------------------------------------------------------------------------- /public/job_logs/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | # 3 | # To ban all spiders from the entire site uncomment the next two lines: 4 | # User-agent: * 5 | # Disallow: / 6 | -------------------------------------------------------------------------------- /public/uploads/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore -------------------------------------------------------------------------------- /test/api/v1/data_controller_permissions_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class Api::V1::DataControllerTest < ActionController::TestCase 4 | 5 | include Devise::TestHelpers 6 | 7 | 8 | setup do 9 | sign_in users(:one) 10 | # @project = projects(:one) 11 | @user = users(:one) 12 | @project = @user.projects.build(name: "Upload test project", description: "Upload test project description") 13 | @project.save 14 | @uploads = ["good_upload"] 15 | upload_to_project(@controller,@project, @uploads, "uploads/datapackage/good/datapackage.json") # just upload datapackage file 16 | @last_dpr = DatapackageResource.last 17 | @test_table = Mira::Application.const_get(@last_dpr.db_table_name.capitalize.to_sym) 18 | end 19 | 20 | test "should return 200 success when reading row data with no API key set" do 21 | # get first row of data 22 | get :show, :id => @project.id, :table_ref => @uploads[0], :data_id => 1 23 | assert_response :success 24 | end 25 | 26 | 27 | test "should return a row of data when no API key set" do 28 | sign_in users(:one) 29 | csv_file = fixture_file_upload("uploads/" + @uploads[0] + ".csv", "text/plain") 30 | first_row = IO.readlines(csv_file)[1] 31 | first_row_compare_array = first_row.split(",").map { |e| e.gsub("\"","").gsub("\n","").downcase } 32 | num_extra_cols = MIRA_EXTRA_VARIABLE_MAP.keys.length 33 | # get first row of data 34 | get :show, :id => @project.id, :table_ref => @uploads[0], :data_id => 1 35 | response_compare_array = JSON.parse(@response.body).except("id").values.map {|e| e.to_s.downcase } 36 | assert_equal first_row_compare_array, response_compare_array[0..-1*(num_extra_cols+1)] 37 | end 38 | 39 | 40 | test "should not be able to read or write data when global API key set" do 41 | sign_out users(:one) 42 | # read permissions restricted for both :read and :write 43 | # Cross reference the api data actions. We loop over the API key scopes and 44 | # then permissions, testing each endpoint. 45 | [:global, :project].each do |scope| 46 | 47 | project_id = (scope == :global) ? nil : @project.id 48 | 49 | [:read, :write].each_with_index do |perm,ndx| 50 | 51 | new_key = ApiKey.new(user_id: @user.id, token: ndx.to_s[0]*24, description: "New API key") 52 | new_key.save 53 | permission = ApiKeyPermission.new(api_key_id: new_key.id, \ 54 | permission_scope: scope, \ 55 | permission: perm, 56 | project_id: project_id) 57 | permission.save 58 | 59 | # just make sure we are referencing the correct data! 60 | assert_equal @last_dpr.table_ref, @uploads[0] 61 | 62 | # check read endpoints 63 | get :index, :id => @project.id, :table_ref => @uploads[0] 64 | assert_response :unauthorized 65 | 66 | # post :datatables, :id => @project.id, :table_ref => @uploads[0] 67 | # assert_response :unauthorized 68 | 69 | get :show, :id => @project.id, :table_ref => @uploads[0], :data_id => 1 70 | assert_response :unauthorized 71 | 72 | get :distinct, :id => @project.id, :table_ref => @uploads[0], :col_ref => "age" 73 | assert_response :unauthorized 74 | 75 | # check write endpoints 76 | count_before = @test_table.count 77 | post :create, :id => @project.id, :table_ref => @uploads[0] 78 | count_after = @test_table.count 79 | assert_response :unauthorized 80 | assert_equal count_before, count_after 81 | 82 | patch :update, :id => @project.id, :table_ref => @uploads[0], :data_id => 1 83 | assert_response :unauthorized 84 | 85 | count_before = @test_table.count 86 | delete :destroy, :id => @project.id, :table_ref => @uploads[0], :data_id => 1 87 | count_after = @test_table.count 88 | assert_response :unauthorized 89 | assert_equal count_before, count_after 90 | 91 | 92 | end 93 | 94 | end 95 | end 96 | 97 | test "should be able to write/update/delete data when using global write API key" do 98 | skip 99 | end 100 | 101 | test "should be able to write/update/delete data when using project write API key" do 102 | skip 103 | end 104 | 105 | end 106 | -------------------------------------------------------------------------------- /test/api/v1/datapackage_endpoints_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class Api::V1::DatapackageEndpointsTest < ActionController::TestCase 4 | 5 | setup do 6 | @controller = Api::V1::DatapackagesController.new # See http://stackoverflow.com/a/7743176 7 | sign_in users(:one) 8 | @user = users(:one) 9 | @project = @user.projects.build(name: "Upload test project", description: "Upload test project description") 10 | @project.save 11 | @datapackage = Datapackage.new(project_id: @project.id, public_url: "dummy/url/datapackage.json") 12 | @datapackage.save 13 | end 14 | 15 | def teardown 16 | Project.find(@project.id).destroy 17 | end 18 | 19 | test "projects/:id/datapackage endpoint should return datapackage data" do 20 | get :show, :id => @project.id 21 | json_response = JSON.parse(response.body) 22 | assert_response :success 23 | assert_equal @datapackage.id, json_response["id"] 24 | assert_equal @project.id, json_response["project_id"] 25 | assert_equal @datapackage.public_url, json_response["public_url"] 26 | end 27 | 28 | 29 | end 30 | -------------------------------------------------------------------------------- /test/api/v1/datapackage_resource_fields_endpoints_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class Api::V1::DatapackageResourceFieldsEndpointsTest < ActionController::TestCase 4 | 5 | setup do 6 | @controller = Api::V1::DatapackageResourceFieldsController.new # See http://stackoverflow.com/a/7743176 7 | sign_in users(:one) 8 | @user = users(:one) 9 | @project = @user.projects.build(name: "Upload test datapackage resources", description: "Upload test project description") 10 | @project.save 11 | upload_to_project(@controller, @project, [], "uploads/datapackage/good/datapackage.json") # just upload datapackage file 12 | @dp_file_json = JSON.parse(File.read(@dp_file)) 13 | end 14 | 15 | def teardown 16 | Project.find(@project.id).destroy 17 | end 18 | 19 | test "projects/:id/tables/:table_ref/datapackage/fields endpoint should show datapackage resource fields" do 20 | # mimic json response using datapackage file 21 | @dp_file_json["resources"].each do |res| 22 | mimic_json = [] 23 | table_ref = res["path"].split(".").first 24 | get :index, :id => @project.id, :table_ref => table_ref 25 | json_response = JSON.parse(response.body) 26 | assert_response :success 27 | assert_not_nil json_response 28 | res["schema"]["fields"].each_with_index.map do |f,i| 29 | field_hash = {} 30 | field_hash["name"] = f["name"] 31 | field_hash["type"] = f["type"] 32 | field_hash["order"] = i + 1 33 | field_hash["format"] = nil 34 | field_hash["add_index"] = true 35 | field_hash["big_integer"] = nil 36 | if f.has_key? "constraints" 37 | if f["constraints"].has_key? "maximum" 38 | field_hash["big_integer"] = true if f["constraints"]["maximum"].to_i > BIG_INTEGER_LIMIT 39 | end 40 | end 41 | mimic_json << field_hash 42 | end 43 | assert_equal mimic_json, json_response 44 | end 45 | end 46 | 47 | test "projects/:id/tables/:table_ref/datapackage/fields/:col_ref endpoint should show datapackage resource field" do 48 | # mimic json response using datapackage file 49 | @dp_file_json["resources"].each do |res| 50 | table_ref = res["path"].split(".").first 51 | res["schema"]["fields"].each_with_index.map do |f,i| 52 | get :show, :id => @project.id, :table_ref => table_ref, :col_ref => f["name"] 53 | json_response = JSON.parse(response.body) 54 | assert_response :success 55 | assert_not_nil json_response 56 | field_hash = {} 57 | field_hash["name"] = f["name"] 58 | field_hash["type"] = f["type"] 59 | field_hash["order"] = i + 1 60 | field_hash["format"] = nil 61 | field_hash["add_index"] = true 62 | field_hash["big_integer"] = nil 63 | if f.has_key? "constraints" 64 | if f["constraints"].has_key? "maximum" 65 | field_hash["big_integer"] = true if f["constraints"]["maximum"].to_i > BIG_INTEGER_LIMIT 66 | end 67 | end 68 | assert_equal field_hash, json_response 69 | end 70 | 71 | end 72 | end 73 | 74 | end 75 | -------------------------------------------------------------------------------- /test/api/v1/datasources_endpoints_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | # require_relative '../../app/controllers/api/v1/projects_controller' 3 | # require 'minitest/spec' 4 | 5 | class Api::V1::DatasourcesEndpointsTest < ActionController::TestCase 6 | 7 | include Devise::TestHelpers 8 | 9 | setup do 10 | Delayed::Worker.delay_jobs = false # turn off queuing 11 | sign_in users(:one) 12 | @controller = Api::V1::DatapackageResourcesController.new # See http://stackoverflow.com/a/7743176 13 | @user = users(:one) 14 | @project = @user.projects.build(name: "Upload test project", description: "Upload test project description") 15 | @project.save 16 | @uploads = ["upload1","upload2"] 17 | 18 | dpfile = "uploads/datapackage/good/datapackage.json" 19 | @dp = JSON.parse(File.read(Rails.root.join("test/fixtures/", dpfile))) 20 | 21 | upload_to_project(@controller,@project, @uploads, dpfile) # just upload datapackage file 22 | 23 | @dp_file_json = JSON.parse(File.read(@dp_file)) 24 | end 25 | 26 | def teardown 27 | Project.find(@project.id).destroy 28 | end 29 | 30 | # api/project/:id/tables 31 | test "Enpoint api/projects/:id/tables - response ok" do 32 | get :index, :id => @project.id 33 | assert_response :success 34 | end 35 | 36 | test "Endpoint api/projects/:id/tables - response contains same number of tables as specified in datapackage.json" do 37 | get :index, :id => @project.id 38 | json_response = JSON.parse(response.body) 39 | assert json_response.length == @dp["resources"].length 40 | end 41 | 42 | test "Endpoint api/projects/:id/tables - response contains each table uploaded" do 43 | get :index, :id => @project.id 44 | json_response = JSON.parse(response.body) 45 | json_response_tables = json_response.map { |a| a["table_ref"] } 46 | # binding.pry 47 | @uploads.each do |u| 48 | assert_includes json_response_tables, u 49 | end 50 | end 51 | 52 | test "Endpoint api/projects/:id/tables - each upload contains reference to correct datapackage" do 53 | get :index, :id => @project.id 54 | json_response = JSON.parse(response.body) 55 | @uploads.each do |upl| 56 | upload_csv_search = json_response.detect{ |a| a["table_ref"] == upl } 57 | assert_equal upload_csv_search["datapackage_id"], @project.datapackage.id 58 | end 59 | end 60 | 61 | # api/project/:id/uploads/:table_ref 62 | test "Endpoint api/projects/:id/tables/:table_ref - response ok" do 63 | @uploads.each do |upl| 64 | get :show, :id => @project.id, :table_ref => upl 65 | assert_response :success 66 | end 67 | end 68 | 69 | # this test saves from repeating the group tests on the individual table endpoints 70 | test "Endpoint api/projects/:id/tables/:table_ref - table details same from individual and group endpoints" do 71 | get :index, :id => @project.id 72 | all_tables_json_response = JSON.parse(response.body) 73 | @uploads.each do |upl| 74 | get :show, :id => @project.id, :table_ref => upl 75 | individual_table_json_response = JSON.parse(response.body) 76 | group_csv_search = all_tables_json_response.detect{ |a| a["table_ref"] == upl } 77 | assert_equal group_csv_search, individual_table_json_response.except("row_count") 78 | end 79 | end 80 | 81 | end 82 | -------------------------------------------------------------------------------- /test/api/v1/projects_endpoints_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class Api::V1::ProjectsEndpointsTest < ActionController::TestCase 4 | 5 | setup do 6 | @controller = Api::V1::ProjectsController.new # See http://stackoverflow.com/a/7743176 7 | sign_in users(:one) 8 | # @project = projects(:one) 9 | @user = users(:one) 10 | @project = @user.projects.build(name: "Upload test project", description: "Upload test project description") 11 | @project.save 12 | 13 | # @upload = "upload1" 14 | # dp_file = fixture_file_upload("uploads/datapackage.json", "application/json") 15 | # csv_file = fixture_file_upload("uploads/" + @upload + ".csv", "text/plain") 16 | # 17 | # dp = @project.datasources.create(datafile: File.open(dp_file), datafile_file_name: "datapackage.json") 18 | # dp.save 19 | # 20 | # ds = @project.datasources.create(datafile: csv_file, datafile_file_name: @upload + ".csv", datapackage_id: dp.id) 21 | # ds.save 22 | # ds.db_table_name = Rails.configuration.x.db_table_prefix.downcase + ds.project_id.to_s + "_" + ds.id.to_s 23 | # ds.save 24 | # 25 | # datapackage = JSON.parse(File.read(dp_file.tempfile.path)) 26 | # ProcessCsvUpload.new(ds.id).perform 27 | end 28 | 29 | def teardown 30 | Project.find(@project.id).destroy 31 | end 32 | 33 | test "projects/ endpoint should return list of projects including newly created one" do 34 | get :index #, {}, { "Accept" => "application/json" } 35 | json_response = JSON.parse(response.body) 36 | json_project = json_response.detect{ |a| a["id"] == @project.id } 37 | assert_response :success 38 | refute_nil json_project 39 | assert json_project["name"] == @project.name && json_project["description"] == @project.description \ 40 | && json_project["id"] == @project.id && json_project["user_id"] == @user.id 41 | end 42 | 43 | test "projects/:id endpoint should return newly created project" do 44 | get :show, :id => @project.id 45 | json_project = JSON.parse(response.body) 46 | assert json_project["name"] == @project.name && json_project["description"] == @project.description \ 47 | && json_project["id"] == @project.id && json_project["user_id"] == @user.id 48 | end 49 | 50 | 51 | test "should not be able to read project metadata when API key set" do 52 | # this is not currently the case! I.e. project metadata can be read 53 | [:global, :project].each do |scope| 54 | 55 | project_id = (scope == :global) ? nil : @project.id 56 | 57 | [:read, :write].each_with_index do |perm,ndx| 58 | new_key = ApiKey.new(user_id: @user.id, token: ndx.to_s[0]*24, description: "New API key") 59 | new_key.save 60 | permission = ApiKeyPermission.new(api_key_id: new_key.id, \ 61 | permission_scope: scope, \ 62 | permission: perm, 63 | project_id: project_id) 64 | permission.save 65 | get :index 66 | assert_equal 401, JSON.parse(response.body)["errors"][0]["code"] 67 | end 68 | end 69 | end 70 | 71 | end 72 | -------------------------------------------------------------------------------- /test/api/v1/scratch_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davbre/mira/f9dde315b11c5d26329d200869672a5d3c9b304a/test/api/v1/scratch_test.rb -------------------------------------------------------------------------------- /test/controllers/api_keys_controller_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class ApiKeysControllerTest < ActionController::TestCase 4 | 5 | setup do 6 | @user1 = users(:one) 7 | @key = api_keys(:one) 8 | sign_in users(:one) 9 | end 10 | 11 | # index 12 | test "should not get INDEX if signed out" do 13 | sign_out users(:one) 14 | get :index, :user_id => @user1.id 15 | assert_redirected_to new_user_session_path 16 | end 17 | 18 | test "should get INDEX if signed in" do 19 | get :index, :user_id => @user1.id 20 | assert_response :success 21 | end 22 | 23 | # show 24 | test "should not get SHOW if signed out" do 25 | sign_out users(:one) 26 | get :show, :user_id => @user1.id, id: @key.id 27 | assert_redirected_to new_user_session_path 28 | end 29 | 30 | test "should get SHOW if signed in" do 31 | get :show, :user_id => @user1.id, id: @key.id 32 | assert_response :success 33 | end 34 | 35 | # new 36 | test "should not get NEW if signed out" do 37 | sign_out users(:one) 38 | get :new, :user_id => @user1.id 39 | assert_redirected_to new_user_session_path 40 | end 41 | 42 | test "should get NEW if signed in" do 43 | get :new, :user_id => @user1.id 44 | assert_response :success 45 | end 46 | 47 | # create 48 | test "should not CREATE if signed out" do 49 | sign_out users(:one) 50 | get :new, :user_id => @user1.id 51 | assert_redirected_to new_user_session_path 52 | end 53 | 54 | test "should CREATE if signed in" do 55 | assert_difference('ApiKey.count',1) do 56 | post :create, :user_id => @user1.id \ 57 | , api_key: { description: "New API key", \ 58 | token: "123456789012345678901234" } 59 | end 60 | assert_redirected_to user_api_keys_path 61 | end 62 | 63 | # edit 64 | test "should not get EDIT if signed out" do 65 | sign_out users(:one) 66 | get :edit, :user_id => @user1.id, id: @key.id 67 | assert_redirected_to new_user_session_path 68 | end 69 | 70 | test "should get EDIT if signed in" do 71 | get :edit, :user_id => @user1.id, id: @key.id 72 | assert_response :success 73 | end 74 | 75 | # update 76 | test "should not UPDATE if signed out" do 77 | sign_out users(:one) 78 | new_desc = "New API key description" 79 | patch :update,:user_id => @user1.id, :id => @key.id, api_key: { description: new_desc } 80 | assert_not_equal(ApiKey.find(@key.id).description, new_desc) 81 | assert_redirected_to new_user_session_path 82 | end 83 | 84 | test "should UPDATE if signed in" do 85 | new_desc = "New API key description" 86 | patch :update,:user_id => @user1.id, :id => @key.id, api_key: { description: new_desc } 87 | assert_equal(ApiKey.find(@key.id).description, new_desc) 88 | end 89 | 90 | # destroy 91 | test "should not DESTROY if signed out" do 92 | sign_out users(:one) 93 | assert_no_difference('users(:one).api_keys.count') do 94 | delete :destroy, :user_id => @user1.id, :id => @key.id 95 | end 96 | assert_redirected_to new_user_session_path 97 | end 98 | 99 | test "should DESTROY if signed in" do 100 | assert_difference('users(:one).api_keys.count', -1) do 101 | delete :destroy, :user_id => @user1.id, :id => @key.id 102 | end 103 | assert_redirected_to user_api_keys_path 104 | end 105 | 106 | end 107 | -------------------------------------------------------------------------------- /test/controllers/datasources_controller_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class DatasourcesControllerTest < ActionController::TestCase 4 | 5 | setup do 6 | Delayed::Worker.delay_jobs = false # turn off queuing 7 | sign_in users(:one) 8 | @user = users(:one) 9 | @project = @user.projects.build(name: "Upload test project", description: "Upload test project description") 10 | @project.save 11 | @upload_files = ["upload1","upload2"] 12 | upload_to_project(@controller,@project,@upload_files, "uploads/datapackage.json") 13 | end 14 | 15 | def teardown 16 | Project.find(@project.id).destroy 17 | end 18 | 19 | # destroy 20 | test "should destroy datasource if signed in and owner of project" do 21 | destroy_csv = @upload_files[0] 22 | relevant_datasource = @project.datasources.where(datafile_file_name: destroy_csv + ".csv").first 23 | assert_difference('Project.find(' + @project.id.to_s + ')' + '.datasources.count', -1) do 24 | delete :destroy, project_id: @project, id: relevant_datasource.id 25 | end 26 | assert_empty Datasource.where(id: relevant_datasource.id) 27 | end 28 | 29 | test "should not destroy datasource if signed out" do 30 | sign_out users(:one) 31 | destroy_csv = @upload_files[0] 32 | relevant_datasource = @project.datasources.where(datafile_file_name: destroy_csv + ".csv").first 33 | assert_no_difference('Project.find(' + @project.id.to_s + ')' + '.datasources.count', -1) do 34 | delete :destroy, project_id: @project, id: relevant_datasource.id 35 | end 36 | assert_redirected_to new_user_session_path 37 | end 38 | 39 | test "should not destroy datasource if not owner" do 40 | sign_out users(:one) 41 | sign_out users(:two) 42 | destroy_csv = @upload_files[0] 43 | relevant_datasource = @project.datasources.where(datafile_file_name: destroy_csv + ".csv").first 44 | assert_no_difference('Project.find(' + @project.id.to_s + ')' + '.datasources.count', -1) do 45 | delete :destroy, project_id: @project, id: relevant_datasource.id 46 | end 47 | assert_redirected_to new_user_session_path 48 | end 49 | 50 | test "destroy datasource should not drop associated database table" do 51 | destroy_csv = @upload_files[0] 52 | relevant_datasource = Datasource.where(datapackage_id: @datapackage.id, datafile_file_name: destroy_csv + ".csv").first 53 | relevant_db_table_name = @project.datapackage.datapackage_resources.where(table_ref: destroy_csv).first.db_table_name 54 | assert ActiveRecord::Base.connection.table_exists? relevant_db_table_name 55 | delete :destroy, project_id: @project, id: relevant_datasource.id 56 | assert ActiveRecord::Base.connection.table_exists? relevant_db_table_name 57 | end 58 | 59 | test "should delete associated upload" do 60 | destroy_csv = @upload_files[0] 61 | relevant_datasource = @project.datasources.where(datafile_file_name: destroy_csv + ".csv").first 62 | # assert file exists, delete, then refute it exists 63 | assert File.file?(@project.upload_path + relevant_datasource.datafile_file_name) 64 | delete :destroy, project_id: @project, id: relevant_datasource.id 65 | refute File.file?(@project.job_log_path + relevant_datasource.datafile_file_name) 66 | end 67 | 68 | test "should delete associated log file" do 69 | destroy_csv = @upload_files[0] 70 | relevant_datasource = @project.datasources.where(datafile_file_name: destroy_csv + ".csv").first 71 | # assert file exists, delete, then refute it exists 72 | assert File.file?(@project.job_log_path + relevant_datasource.datafile_file_name + ".log") 73 | delete :destroy, project_id: @project, id: relevant_datasource.id 74 | refute File.file?(@project.job_log_path + relevant_datasource.datafile_file_name + ".log") 75 | end 76 | 77 | test "should be able to download uploaded csv files when no API key set" do 78 | skip 79 | end 80 | 81 | test "should not be able to download uploaded csv files when API key set" do 82 | skip 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /test/fixtures/api_key_permissions.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | one: 4 | api_key: one 5 | project_id: one 6 | permission_scope: :project 7 | permission: :read 8 | 9 | two: 10 | api_key: one 11 | project_id: two 12 | permission_scope: :project 13 | permission: :write 14 | 15 | # one: 16 | # token_id: 1 17 | # scope: 1 18 | # project_id: 1 19 | # datapackage_resource_id: 1 20 | # db_table_name: MyString 21 | # permission: 1 22 | # 23 | # two: 24 | # token_id: 1 25 | # scope: 1 26 | # project_id: 1 27 | # datapackage_resource_id: 1 28 | # db_table_name: MyString 29 | # permission: 1 30 | -------------------------------------------------------------------------------- /test/fixtures/api_keys.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | one: 4 | user: one 5 | token: 123456789012345678901111 6 | description: Test API key 1 7 | 8 | two: 9 | user: one 10 | token: 123456789012345678902222 11 | description: Test API key 2 12 | -------------------------------------------------------------------------------- /test/fixtures/datapackage_resource_fields.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | one: 4 | datapackage_resource_id: 1 5 | name: MyText 6 | ftype: MyText 7 | 8 | two: 9 | datapackage_resource_id: 1 10 | name: MyText 11 | ftype: MyText 12 | -------------------------------------------------------------------------------- /test/fixtures/datapackage_resources.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | one: 4 | datapackage_id: 1 5 | path: MyText 6 | format: MyText 7 | delimiter: "," 8 | mediatype: MyText 9 | quote_character: '"' 10 | table_ref: MyText 11 | 12 | two: 13 | datapackage_id: 1 14 | path: MyText 15 | format: MyText 16 | delimiter: "," 17 | mediatype: MyText 18 | quote_character: '"' 19 | table_ref: MyText 20 | -------------------------------------------------------------------------------- /test/fixtures/datapackages.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | one: 4 | project_id: 1 5 | 6 | two: 7 | project_id: 2 8 | -------------------------------------------------------------------------------- /test/fixtures/projects.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | one: 4 | name: Project 1 5 | description: Test project 1 description 6 | user: one 7 | 8 | two: 9 | name: Test Project 2 10 | description: Test project 2 description 11 | user: one 12 | 13 | three: 14 | name: Test Project 3 15 | description: Test project 3 description 16 | user: two 17 | -------------------------------------------------------------------------------- /test/fixtures/uploads/bad_upload.csv: -------------------------------------------------------------------------------- 1 | a_string,an_integer 2 | Ok,1 3 | Ok,2 4 | Ok,3 5 | Not Ok,NOT AN INTEGER!!! 6 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_delimiter_too_long/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": ",," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "felds": [ 14 | { 15 | "name": "studyid", 16 | "type": "string" 17 | }, 18 | { 19 | "name": "usubjid", 20 | "type": "string" 21 | } 22 | ] 23 | } 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_field_invalid_name/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "fields": [ 14 | { 15 | "name": "invalid!!", 16 | "type": "string" 17 | }, 18 | { 19 | "name": "usubjid", 20 | "type": "string" 21 | } 22 | ] 23 | } 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_field_invalid_type/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "fields": [ 14 | { 15 | "name": "studyid", 16 | "type": "strong" 17 | }, 18 | { 19 | "name": "usubjid", 20 | "type": "strung" 21 | } 22 | ] 23 | } 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_field_not_name_and_type/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "fields": [ 14 | { 15 | "nombre": "studyid", 16 | "typo": "string" 17 | }, 18 | { 19 | "name": "usubjid", 20 | "type": "string" 21 | } 22 | ] 23 | } 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_not_json/datapackage.json: -------------------------------------------------------------------------------- 1 | This is not json! 2 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_path_empty/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "fields": [ 14 | { 15 | "name": "studyid", 16 | "type": "string" 17 | } 18 | ] 19 | } 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_path_missing/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "format": "csv", 7 | "dialect": { 8 | "delimiter": "," 9 | }, 10 | "mediatype": "text/csv", 11 | "schema": { 12 | "fields": [ 13 | { 14 | "name": "studyid", 15 | "type": "string" 16 | }, 17 | { 18 | "name": "usubjid", 19 | "type": "string" 20 | }, 21 | { 22 | "name": "aeterm", 23 | "type": "string" 24 | } 25 | ] 26 | } 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_path_not_csv/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "notcsv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "fields": [ 14 | { 15 | "name": "studyid", 16 | "type": "string" 17 | } 18 | ] 19 | } 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_path_not_string/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": { "not_string": 1}, 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "fields": [ 14 | { 15 | "name": "studyid", 16 | "type": "string" 17 | } 18 | ] 19 | } 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_quote_char_too_long/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": ",", 10 | "quote": "''", 11 | }, 12 | "mediatype": "text/csv", 13 | "schema": { 14 | "fields": [ 15 | { 16 | "name": "studyid", 17 | "type": "string" 18 | }, 19 | { 20 | "name": "usubjid", 21 | "type": "string" 22 | } 23 | ] 24 | } 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_resources_empty/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_resources_missing/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | } 5 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_resources_not_array/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": { 5 | non_array: "well hey" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_schema_missing/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "fields": [ 14 | { 15 | "name": "studyid", 16 | "type": "string" 17 | }, 18 | { 19 | "name": "usubjid", 20 | "type": "string" 21 | } 22 | ] 23 | } 24 | }, 25 | { 26 | "path": "upload2.csv", 27 | "format": "csv", 28 | "dialect": { 29 | "delimiter": "," 30 | }, 31 | "mediatype": "text/csv" 32 | } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_schema_no_fields/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "felds": [ 14 | { 15 | "name": "studyid", 16 | "type": "string" 17 | }, 18 | { 19 | "name": "usubjid", 20 | "type": "string" 21 | } 22 | ] 23 | } 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_schema_not_array/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": { 13 | "fields": { 14 | "not_array": [ 15 | { 16 | "name": "studyid", 17 | "type": "string" 18 | }, 19 | { 20 | "name": "usubjid", 21 | "type": "string" 22 | } 23 | ] 24 | } 25 | } 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /test/fixtures/uploads/datapackage/bad_schema_not_hash/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "--NAME--", 3 | "title": "--TITLE--", 4 | "resources": [ 5 | { 6 | "path": "upload1.csv", 7 | "format": "csv", 8 | "dialect": { 9 | "delimiter": "," 10 | }, 11 | "mediatype": "text/csv", 12 | "schema": [{ 13 | "fields": [ 14 | { 15 | "name": "studyid", 16 | "type": "string" 17 | }, 18 | { 19 | "name": "usubjid", 20 | "type": "string" 21 | } 22 | ] 23 | }] 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /test/fixtures/uploads/good_upload.csv: -------------------------------------------------------------------------------- 1 | name,age,dob,score,longid,boolfl 2 | "",23,2015-12-01,12.145,233128160005435000,TRUE 3 | Paul,,2015-12-02,1.123,206972033856437000,FALSE 4 | Peter,38,,123.434,915600704634562000,FALSE 5 | Maria,56,2015-12-04,,889804187696427000,FALSE 6 | Marie,64,2015-12-05,12.1,,TRUE 7 | Marina,80,,,154044685233384000, 8 | Maire,89,2015-12-07,224.3,,TRUE 9 | Mary,67,2015-12-08,225.3,764984323922545000,FALSE 10 | Pablo,,,226.3,573851879779249000,FALSE 11 | David,56,2015-12-10,227.3,,FALSE 12 | Cath,54,2015-12-11,228.3,870384820504114000,TRUE 13 | Kathy,23,2015-12-12,229.3,892285172780976000,TRUE 14 | Katrina,45,2015-12-13,230.3,,FALSE 15 | Kate,28,2015-12-14,231.3,914699918683618000,TRUE 16 | Katey,59,,232.3,506865238072351000,TRUE 17 | Joey,73,2015-12-16,233.3,966056652041152000,TRUE 18 | John,23,2015-12-01,12.145,,TRUE 19 | Paul,74,2015-12-02,1.123,777476104721427000,FALSE 20 | Petieneer,38,2015-12-03,123.434,212840031180531000,FALSE 21 | ,,2015-12-04,,,FALSE 22 | Marie,64,2015-12-05,12.1,529360551666468000,TRUE 23 | ,80,2015-12-06,223.3,303913316363469000,FALSE 24 | ,,,224.3,317677169572562000,TRUE 25 | Mary,67,2015-12-08,225.3,,FALSE 26 | Middle-of-the-road,40,2015-12-08,100.1,519105711625889000,FALSE 27 | David,56,2015-12-10,227.3,398686256259680000,FALSE 28 | Catieneh,54,2015-12-11,228.3,295063527394086000,TRUE 29 | Kathy,23,2015-12-12,,370350350392982000, 30 | Kattienerina,45,2015-12-13,230.3,487410659343004000, 31 | Kate,28,2015-12-14,231.3,787732418114319000,TRUE 32 | Katey,59,2015-12-15,232.3,913300037337467000,TRUE 33 | principioJoey,73,2015-12-16,233.3,620538819348440000,TRUE 34 | ,23,2015-12-01,12.145,894704451970756000,TRUE 35 | Paul,,2015-12-02,1.123,828900741972029000,FALSE 36 | Peterfin,38,,123.434,510343007044867000,FALSE 37 | Maria,56,2015-12-04,,754851576127112000,FALSE 38 | Marie,64,2015-12-05,12.1,882945427205414000,TRUE 39 | Marinafin,80,2015-12-06,223.3,, 40 | Maire,89,2015-12-07,224.3,519835900049657000,TRUE 41 | Mary_fin,67,2015-12-08,225.3,456797306984663000,FALSE 42 | Pablo,34,2015-12-09,226.3,524714343715459000,FALSE 43 | David,56,2015-12-10,227.3,,FALSE 44 | Cath,54,2015-12-11,228.3,349082480184734000,TRUE 45 | Kathy,23,2015-12-12,229.3,493680140189827000,TRUE 46 | Katrina,45,2015-12-13,230.3,304920639656484000,FALSE 47 | Kate,28,2015-12-14,231.3,755947718257085000,TRUE 48 | Katey,59,2015-12-15,232.3,459736792230979000,TRUE 49 | Joey,73,2015-12-16,233.3,792524431366473000,TRUE 50 | John,23,2015-12-01,12.145,533423822978511000,TRUE 51 | principioPaul,74,2015-12-02,1.123,572576823411509000,FALSE 52 | principioPeter,38,2015-12-03,123.434,858248389232904000, 53 | principioMaria,56,2015-12-04,534.234,962784374644980000,FALSE 54 | principioMarie,64,2015-12-05,12.1,776490139774978000, 55 | Marina,80,2015-12-06,223.3,175925559224561000,FALSE 56 | Maire,89,2015-12-07,224.3,754274492571130000,TRUE 57 | Mary,67,2015-12-08,225.3,495595851400867000,FALSE 58 | Pablo,34,2015-12-09,226.3,, 59 | David,56,2015-12-10,227.3,949338020384312000,FALSE 60 | Cath,54,2015-12-11,228.3,765946202212945000,TRUE 61 | Kathy,23,2015-12-12,229.3,862022475246340000,TRUE 62 | Katrina,45,2015-12-13,230.3,737070438498631000,FALSE 63 | Kate,28,2015-12-14,231.3,679246239550412000,TRUE 64 | Katey,59,2015-12-15,232.3,482561294594780000,TRUE 65 | Joey,73,2015-12-16,233.3,631774890888482000,TRUE 66 | -------------------------------------------------------------------------------- /test/fixtures/uploads/good_upload_with_tabs.csv: -------------------------------------------------------------------------------- 1 | name age dob score longid boolfl 2 | John\t23\t2015-12-01\t12.145\t233128160005435000\tTRUE 3 | Paul\t\t2015-12-02\t1.123\t206972033856437000\tFALSE 4 | Peter\t38\t\t123.434\t915600704634562000\tFALSE 5 | -------------------------------------------------------------------------------- /test/fixtures/uploads/not_in_datapackage.csv: -------------------------------------------------------------------------------- 1 | name,age,dob,score,longid,boolfl 2 | John,23,2015-12-01,12.145,1000000000000000000000000000,TRUE 3 | Paul,74,2015-12-02,1.123,1000000000000000000000000000,FALSE 4 | Peter,38,2015-12-03,123.434,1000000000000000000000000000,FALSE 5 | Maria,56,2015-12-04,534.234,1000000000000000000000000000,FALSE 6 | Marie,64,2015-12-05,12.1,1000000000000000000000000000,TRUE 7 | Marina,80,2015-12-06,223.3,1000000000000000000000000000,FALSE 8 | Maire,89,2015-12-07,224.3,1000000000000000000000000000,TRUE 9 | Mary,67,2015-12-08,225.3,1000000000000000000000000000,FALSE 10 | Pablo,34,2015-12-09,226.3,1000000000000000000000000000,FALSE 11 | David,56,2015-12-10,227.3,1000000000000000000000000000,FALSE 12 | Cath,54,2015-12-11,228.3,1000000000000000000000000000,TRUE 13 | Kathy,23,2015-12-12,229.3,1000000000000000000000000000,TRUE 14 | Katrina,45,2015-12-13,230.3,1000000000000000000000000000,FALSE 15 | Kate,28,2015-12-14,231.3,1000000000000000000000000000,TRUE 16 | Katey,59,2015-12-15,232.3,1000000000000000000000000000,TRUE 17 | Joey,73,2015-12-16,233.3,1000000000000000000000000000,TRUE 18 | -------------------------------------------------------------------------------- /test/fixtures/users.yml: -------------------------------------------------------------------------------- 1 | one: 2 | email: user1@testing.com 3 | encrypted_password: <%= Devise::Encryptor.digest(User, 'mypassword') %> 4 | 5 | two: 6 | email: user2@testing.com 7 | encrypted_password: <%= Devise::Encryptor.digest(User, 'mypassword') %> 8 | -------------------------------------------------------------------------------- /test/models/api_key_permission_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class ApiKeyPermissionTest < ActiveSupport::TestCase 4 | 5 | setup do 6 | @key = api_keys(:one) 7 | end 8 | 9 | test "should save with an existant API key id + permission + scope = all + NO project_id " do 10 | scope = ApiKeyPermission.permission_scopes[:global] 11 | perm = ApiKeyPermission.permissions[:read] 12 | akp = ApiKeyPermission.new(api_key_id: @key.id, permission_scope: scope, permission: perm, project_id: nil) 13 | assert akp.valid? 14 | end 15 | 16 | test "should save with an existant API key id + permission + scope = project + project_id " do 17 | scope = ApiKeyPermission.permission_scopes[:global] 18 | perm = ApiKeyPermission.permissions[:read] 19 | akp = ApiKeyPermission.new(api_key_id: @key.id, permission_scope: scope, permission: perm, project_id: projects(:one)) 20 | assert akp.valid? 21 | end 22 | 23 | test "should NOT save with an existant API key id + permission + WITHOUT scope + project_id " do 24 | perm = ApiKeyPermission.permissions[:read] 25 | akp = ApiKeyPermission.new(api_key_id: @key.id, permission_scope: nil, permission: perm, project_id: projects(:one)) 26 | assert_not akp.valid? 27 | end 28 | 29 | test "should NOT save with an existant API key id + WITHOUT permission + scope + project_id " do 30 | scope = ApiKeyPermission.permission_scopes[:global] 31 | akp = ApiKeyPermission.new(api_key_id: @key.id, permission_scope: scope, permission: nil, project_id: projects(:one)) 32 | assert_not akp.valid? 33 | end 34 | 35 | test "should NOT save when permission already exists" do 36 | project = projects(:one) 37 | scope = ApiKeyPermission.permission_scopes[:project] 38 | perm = ApiKeyPermission.permissions[:read] 39 | akp = ApiKeyPermission.new(api_key_id: @key.id, permission_scope: scope, permission: perm, project_id: project) 40 | akp.save 41 | akp_same = ApiKeyPermission.new(api_key_id: @key.id, permission_scope: scope, permission: perm, project_id: project) 42 | assert_not akp_same.valid? 43 | end 44 | 45 | end 46 | -------------------------------------------------------------------------------- /test/models/api_key_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class ApiKeyTest < ActiveSupport::TestCase 4 | 5 | setup do 6 | @ok_token = "123456789012345678901234" 7 | end 8 | 9 | test "should save unique token of length 24" do 10 | k = ApiKey.new(user_id: users(:one).id, token: @ok_token) 11 | assert k.save 12 | end 13 | 14 | test "should not save same token twice" do 15 | k1 = ApiKey.new(user_id: users(:one).id, token: @ok_token) 16 | k1.save 17 | k2 = ApiKey.new(user_id: users(:one).id, token: @ok_token) 18 | assert_not k2.save 19 | end 20 | 21 | test "should not save without a user" do 22 | k = ApiKey.new(token: @ok_token) 23 | assert_not k.save 24 | end 25 | 26 | test "should not save without an api token" do 27 | k = ApiKey.new(user_id: users(:one).id, token: "") 28 | assert_not k.save 29 | end 30 | 31 | test "should not save with a non-existant user id" do 32 | k = ApiKey.new(user_id: 999999, token: @ok_token) 33 | assert_not k.save 34 | end 35 | 36 | test "should not save token with length not 24" do 37 | short_token = "1234" 38 | long_token = "12345678901234567890123456789012345678901234567890" 39 | k1 = ApiKey.new(user_id: users(:one).id, token: short_token) 40 | k2 = ApiKey.new(user_id: users(:one).id, token: long_token) 41 | assert_not k1.save 42 | assert_not k2.save 43 | end 44 | 45 | end 46 | -------------------------------------------------------------------------------- /test/models/datapackage_resource_field_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class DatapackageResourceFieldTest < ActiveSupport::TestCase 4 | 5 | setup do 6 | @datapackage_resource = datapackage_resources(:one) 7 | @datapackage_resource_field = DatapackageResourceField.new( 8 | datapackage_resource_id: @datapackage_resource.id, 9 | ftype: "integer", 10 | name: "testvar", 11 | order: 1 12 | ) 13 | end 14 | 15 | test "should save with all required attributes" do 16 | assert @datapackage_resource_field.save 17 | end 18 | 19 | test "should not save without a datapackage_resource_id" do 20 | @datapackage_resource_field.datapackage_resource_id = nil 21 | refute @datapackage_resource_field.save 22 | end 23 | 24 | test "should not save without a ftype" do 25 | @datapackage_resource_field.ftype = nil 26 | refute @datapackage_resource_field.save 27 | end 28 | 29 | test "should not save without a name" do 30 | @datapackage_resource_field.name = nil 31 | refute @datapackage_resource_field.save 32 | end 33 | 34 | test "should not save without a order" do 35 | @datapackage_resource_field.order = nil 36 | refute @datapackage_resource_field.save 37 | end 38 | 39 | test "should not save two fields with the same order" do 40 | @datapackage_resource_field.save 41 | same_order = @datapackage_resource_field.order 42 | field2 = DatapackageResourceField.new( 43 | datapackage_resource_id: @datapackage_resource.id, 44 | ftype: "string", 45 | name: "another_testvar", 46 | order: same_order) 47 | refute field2.save 48 | end 49 | 50 | end 51 | -------------------------------------------------------------------------------- /test/models/datapackage_resource_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class DatapackageResourceTest < ActiveSupport::TestCase 4 | 5 | setup do 6 | @project = projects(:one) 7 | @datapackage = datapackages(:one) 8 | @datapackage_resource = DatapackageResource.new( 9 | datapackage_id: @datapackage.id, 10 | path: "test.csv", 11 | delimiter: ",", 12 | quote_character: '"', 13 | table_ref: "test" 14 | ) 15 | end 16 | 17 | test "should save with all required attributes" do 18 | assert @datapackage_resource.save 19 | end 20 | 21 | test "should not save without a datapackage_id" do 22 | @datapackage_resource.datapackage_id = nil 23 | refute @datapackage_resource.save 24 | end 25 | 26 | test "should not save without a path" do 27 | @datapackage_resource.path = nil 28 | refute @datapackage_resource.save 29 | end 30 | 31 | test "should not save without a delimiter" do 32 | @datapackage_resource.delimiter = nil 33 | refute @datapackage_resource.save 34 | end 35 | 36 | test "should not save without a quote_character" do 37 | @datapackage_resource.quote_character = nil 38 | refute @datapackage_resource.save 39 | end 40 | 41 | test "should not save without a table_ref" do 42 | @datapackage_resource.table_ref = nil 43 | refute @datapackage_resource.save 44 | end 45 | 46 | 47 | end 48 | -------------------------------------------------------------------------------- /test/models/datapackage_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class DatapackageTest < ActiveSupport::TestCase 4 | 5 | setup do 6 | @project = projects(:one) 7 | end 8 | 9 | test "should not save without a project_id" do 10 | datapackage = Datapackage.new 11 | assert_not datapackage.save 12 | end 13 | 14 | test "should save with a user and a project id" do 15 | datapackage = Datapackage.new(project_id: @project.id) 16 | assert datapackage.save 17 | end 18 | 19 | # Want to enforce that there can only be one datapackage per project. It 20 | # seems you can't enforce this at the database level. So will cover in the 21 | # controller. 22 | 23 | end 24 | -------------------------------------------------------------------------------- /test/models/project_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class ProjectTest < ActiveSupport::TestCase 4 | 5 | test "should not save without a user" do 6 | # user = users(:one) 7 | # project = user.project.build( 8 | project = Project.new(name: "No user project", description: "No user project description") 9 | assert_not project.save 10 | end 11 | 12 | test "should save with a user, unique name and a description" do 13 | user = users(:one) 14 | project = user.projects.build(name: "Unique name", description: "Test project description") 15 | assert project.save 16 | end 17 | 18 | test "should not save without a project name" do 19 | user = users(:one) 20 | project = user.projects.build(name: "", description: "Test project description") 21 | assert_not project.save 22 | end 23 | 24 | test "should save a long name" do 25 | user = users(:one) 26 | name65 = "a" * 2049 27 | project = user.projects.build(name: name65, description: "Test project description") 28 | assert project.save 29 | end 30 | 31 | test "should not save with non-unique name" do 32 | user = users(:one) 33 | project1 = user.projects.build(name: "Duplicate", description: "Test project description") 34 | project2 = user.projects.build(name: "Duplicate", description: "Test project description") 35 | project1.save 36 | assert_not project2.save 37 | end 38 | 39 | test "should save with unique name and no description" do 40 | user = users(:one) 41 | project = user.projects.build(name: "Unique name", description: "") 42 | assert project.save 43 | end 44 | 45 | end 46 | -------------------------------------------------------------------------------- /test/models/user_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class UserTest < ActiveSupport::TestCase 4 | # test "the truth" do 5 | # assert true 6 | # end 7 | end 8 | -------------------------------------------------------------------------------- /test/readme.md: -------------------------------------------------------------------------------- 1 | 2 | Run all tests: 3 | rake test 4 | 5 | Run tests in a folder: 6 | rake test test/models/*_test.rb 7 | 8 | Run tests in single file: 9 | rake test test/models/project_test.rb 10 | 11 | Run single test: 12 | ruby -I test test/controllers/datasources_controller_test.rb -n "test_name_of_test" 13 | 14 | 15 | 16 | # Possible new tests... 17 | 18 | do not show uploads / datapackages when not logged in?? 19 | 20 | test that private fields are not returned from API. 21 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | ENV['RAILS_ENV'] ||= 'test' 2 | require File.expand_path('../../config/environment', __FILE__) 3 | require 'rails/test_help' 4 | require 'net/http' 5 | 6 | class ActiveSupport::TestCase 7 | # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. 8 | fixtures :all 9 | 10 | include ApplicationHelper 11 | include ProjectHelper 12 | include DataAccessHelper 13 | # Add more helper methods to be used by all tests here... 14 | 15 | def map_datapackage_column_types(datapackage_json, csv_name) 16 | csv_dp_detail = datapackage_json["resources"].detect{ |a| a["path"] == csv_name } 17 | dp_column_types = {} 18 | csv_dp_detail["schema"]["fields"].each do |sf| 19 | dp_column_types[sf["name"]] = DATAPACKAGE_TYPE_MAP[sf["type"]] 20 | end 21 | dp_column_types 22 | end 23 | 24 | def default_page_size 25 | Rails.application.config.x.api_default_per_page 26 | end 27 | 28 | 29 | def upload_to_project(controller,project,file_names,datapackage_file = "uploads/datapackage.json") 30 | Delayed::Worker.delay_jobs = false # turn off queuing 31 | @dp_file = fixture_file_upload(datapackage_file, "application/json") 32 | 33 | # The call to the "post" method would fail when called outside the context of a ProjectsController 34 | # so we change it temporarily. This allows us to upload the datapackage from tests not relating directly 35 | # to project actions. See http://stackoverflow.com/a/7743176/1002140 36 | orig_controller = controller 37 | @controller = ProjectsController.new 38 | post :upload_datapackage, id: project.id, :datapackage => @dp_file 39 | @controller = orig_controller 40 | 41 | @uploads = file_names 42 | @datapackage = Datapackage.last 43 | 44 | @uploads.each do |upl| 45 | csv_file = fixture_file_upload("uploads/" + upl + ".csv", "text/plain") 46 | # need the datapackage_resource_id to mimic what's going on in the controller. 47 | dpr_name = map_csv_basename(@datapackage, upl) 48 | dpr = DatapackageResource.where(datapackage_id: @datapackage.id, table_ref: dpr_name).first 49 | ds = project.datasources.create(datafile: csv_file, datafile_file_name: upl + ".csv", datapackage_id: @datapackage.id, datapackage_resource_id: dpr.id) 50 | ds.save 51 | Delayed::Job.enqueue ProcessCsvUpload.new(ds.id,"quick") 52 | end 53 | end 54 | 55 | def csv_line_count(txtfile) 56 | csv_file = fixture_file_upload("uploads/" + txtfile + ".csv", "text/plain") 57 | row_count = File.open(csv_file,"r").readlines.size 58 | row_count 59 | end 60 | 61 | end 62 | 63 | class ActionController::TestCase 64 | include Devise::TestHelpers 65 | end 66 | -------------------------------------------------------------------------------- /tmp/cache/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore -------------------------------------------------------------------------------- /tmp/pids/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore -------------------------------------------------------------------------------- /tmp/sessions/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore -------------------------------------------------------------------------------- /tmp/sockets/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore -------------------------------------------------------------------------------- /tmp/temp_paperclip_uploads/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore --------------------------------------------------------------------------------