├── robots.txt ├── .gitignore ├── public └── images │ ├── download.png │ └── copyToClip.png ├── migrations ├── 007_remove_rdataversion.rb ├── 003_add_record_id_to_resources.rb ├── 002_add_sourcetype_to_input_sources.rb ├── 004_add_dispatchclass_to_rdatapaths.rb ├── 005_add_preparerclass_to_resources.rb ├── 006_remove_unneded_columns.rb ├── 009_lengthen_description.rb ├── 008_add_pazar_prefix.rb ├── 010_lengthen_dataprovider.rb └── 001_add_test.rb ├── Gemfile ├── views ├── robots.erb ├── error.erb ├── listing.erb ├── resultsPage.erb └── index.erb ├── config.yml.example ├── config.ru ├── logging_init.rb ├── Gemfile.lock ├── db_init.rb ├── logging_schema.rb ├── convert_db.rb ├── test3.rb ├── test2.rb ├── read_s3_logs.rb ├── schema_sequel.rb ├── README.md ├── models.rb ├── mongo2sequel.rb ├── LICENSE └── app.rb /robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.sqlite3 2 | bak.sqlite3 3 | logs/* 4 | config.yml -------------------------------------------------------------------------------- /public/images/download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bioconductor/AnnotationHubServer3.0/master/public/images/download.png -------------------------------------------------------------------------------- /public/images/copyToClip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Bioconductor/AnnotationHubServer3.0/master/public/images/copyToClip.png -------------------------------------------------------------------------------- /migrations/007_remove_rdataversion.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | drop_column :resources, :rdataversion 4 | end 5 | end -------------------------------------------------------------------------------- /migrations/003_add_record_id_to_resources.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | add_column :resources, :record_id, Integer 4 | end 5 | end -------------------------------------------------------------------------------- /migrations/002_add_sourcetype_to_input_sources.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | add_column :input_sources, :sourcetype, String 4 | end 5 | end -------------------------------------------------------------------------------- /migrations/004_add_dispatchclass_to_rdatapaths.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | add_column :rdatapaths, :dispatchclass, String 4 | end 5 | end -------------------------------------------------------------------------------- /migrations/005_add_preparerclass_to_resources.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | add_column :resources, :preparerclass, String 4 | end 5 | end -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gem "aws-sdk" 3 | gem "sinatra" 4 | gem "mysql2" 5 | gem "sequel" 6 | gem "pry" 7 | gem "sqlite3" 8 | gem "shotgun" 9 | -------------------------------------------------------------------------------- /views/robots.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | robots.txt file 4 | 5 | 6 | 7 | <% File.open('robots.txt').each do |line| %> 8 | <%= line %> 9 |
10 | <% end %> 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /config.yml.example: -------------------------------------------------------------------------------- 1 | # copy this to config.yml and edit that to match your system 2 | mysql_url: "mysql://user:password@host/dbname" 3 | sqlite_filename: "mydatabase.sqlite3" 4 | admin_password: "some_password" 5 | logging_url: "mysql://username:password@host/logging_database_name" 6 | -------------------------------------------------------------------------------- /migrations/006_remove_unneded_columns.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | drop_column :input_sources, :sourcefile 4 | drop_column :rdatapaths, :rdatasize 5 | drop_column :rdatapaths, :rdatamd5 6 | drop_column :rdatapaths, :rdatalastmodifieddate 7 | end 8 | end -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'sinatra' 3 | 4 | require './app.rb' 5 | 6 | set :environment, ENV['RACK_ENV'].to_sym 7 | set :app_file, 'app.rb' 8 | disable :run 9 | 10 | log = File.new("logs/sinatra.log", "a") 11 | 12 | $stderr.reopen(log) 13 | $stderr.sync = true 14 | 15 | run Sinatra::Application -------------------------------------------------------------------------------- /migrations/009_lengthen_description.rb: -------------------------------------------------------------------------------- 1 | basedir = File.dirname(__FILE__) 2 | 3 | require_relative "../models.rb" 4 | 5 | Sequel.migration do 6 | up do 7 | DB.set_column_type :resources, :description, String, :text => true 8 | end 9 | 10 | down do 11 | DB.set_column_type :resources, :description, String, :size => 255 12 | end 13 | end -------------------------------------------------------------------------------- /migrations/008_add_pazar_prefix.rb: -------------------------------------------------------------------------------- 1 | basedir = File.dirname(__FILE__) 2 | 3 | require_relative "../models.rb" 4 | 5 | Sequel.migration do 6 | up do 7 | lp = LocationPrefix.create(location_prefix: "http://www.pazar.info/") 8 | end 9 | 10 | down do 11 | DB[:location_prefixes].where(location_prefix: "http://www.pazar.info/").delete 12 | end 13 | end -------------------------------------------------------------------------------- /migrations/010_lengthen_dataprovider.rb: -------------------------------------------------------------------------------- 1 | basedir = File.dirname(__FILE__) 2 | 3 | require_relative "../models.rb" 4 | 5 | Sequel.migration do 6 | up do 7 | DB.set_column_type :resources, :dataprovider, String, :text => true 8 | end 9 | 10 | down do 11 | DB.set_column_type :resources, :dataprovider, String, :size => 255 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /logging_init.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'sequel' 3 | require 'yaml' 4 | 5 | 6 | unless defined? LOGGING_DB 7 | basedir = File.dirname(__FILE__) 8 | config = YAML.load_file("#{basedir}/config.yml") 9 | if config.has_key? "logging_url" 10 | LOGGING_DB = Sequel.connect config["logging_url"] 11 | else 12 | LOGGING_DB = Sequel.sqlite 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /views/error.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | ERROR 4 | 5 | 6 | 7 |

ERROR

8 | 9 |

<%=message%>

10 | 11 | 17 | 18 |
19 | <%=offenders%> 20 |
21 | 22 |

<%=helper%>

23 | 24 |
25 | <%=helper2%> 26 |
27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /migrations/001_add_test.rb: -------------------------------------------------------------------------------- 1 | # invoke migrations with e.g.: 2 | # sequel -m migrations/ sqlite:///Users/dante/dev/github/AnnotationHubServer3.0/ahtest.sqlite3 3 | # this would seem even easier, but it doesn't work (or works on an ephemeral in-memory database): 4 | # sequel -m migrations/ sqlite:annotationhub.sqlite3 5 | # you can interpolate $(pwd) into the command though: 6 | # sequel -m migrations/ sqlite://$(pwd)/annotationhub.sqlite3 7 | # for mysql do this: 8 | # sequel -m migrations/ mysql://username:password@host/dbname 9 | # Add -E before the url for more logging/debugging info 10 | Sequel.migration do 11 | up do 12 | # create_table(:test) do 13 | # primary_key :id 14 | # String :name, :null=>false 15 | # end 16 | end 17 | 18 | down do 19 | # drop_table(:test) 20 | end 21 | end -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | aws-sdk (3.0.1) 5 | aws-sdk-resources (~> 3) 6 | coderay (1.1.2) 7 | jmespath (1.4.0) 8 | method_source (0.9.2) 9 | mustermann (1.1.1) 10 | ruby2_keywords (~> 0.0.1) 11 | mysql2 (0.5.3) 12 | pry (0.12.2) 13 | coderay (~> 1.1.0) 14 | method_source (~> 0.9.0) 15 | rack (2.2.2) 16 | rack-protection (2.0.8.1) 17 | rack 18 | ruby2_keywords (0.0.2) 19 | sequel (5.29.0) 20 | shotgun (0.9.2) 21 | rack (>= 1.0) 22 | sinatra (2.0.8.1) 23 | mustermann (~> 1.0) 24 | rack (~> 2.0) 25 | rack-protection (= 2.0.8.1) 26 | tilt (~> 2.0) 27 | sqlite3 (1.4.2) 28 | tilt (2.0.10) 29 | 30 | PLATFORMS 31 | ruby 32 | 33 | DEPENDENCIES 34 | aws-sdk 35 | mysql2 36 | pry 37 | sequel 38 | shotgun 39 | sinatra 40 | sqlite3 41 | 42 | BUNDLED WITH 43 | 2.1.4 44 | -------------------------------------------------------------------------------- /db_init.rb: -------------------------------------------------------------------------------- 1 | require 'sequel' 2 | require 'yaml' 3 | require 'logger' 4 | 5 | ## This script expects these values to be set in config.yml: 6 | ## 'ahs_database_type' 7 | ## 'mysql_url' 8 | ## 'sqlite_filename' 9 | 10 | unless defined? DB 11 | basedir = File.dirname(__FILE__) 12 | config = YAML.load_file("#{basedir}/config.yml") 13 | ENV['AHS_DATABASE_TYPE']=config['ahs_database_type'] 14 | 15 | if `hostname` =~ /^ip-/ and ENV['AHS_DATABASE_TYPE'].nil? 16 | ENV['AHS_DATABASE_TYPE']='mysql' 17 | end 18 | 19 | mode = nil 20 | if ENV['AHS_DATABASE_TYPE'].nil? or 21 | !(["mysql", "sqlite"].include? ENV['AHS_DATABASE_TYPE']) 22 | puts "environment variable AHS_DATABASE_TYPE must be set to" 23 | puts "mysql or sqlite." 24 | exit 25 | else 26 | mode = ENV['AHS_DATABASE_TYPE'].to_sym 27 | end 28 | 29 | url = nil 30 | if mode == :mysql 31 | url = config['mysql_url'] 32 | else 33 | url = "sqlite://#{basedir}/#{config['sqlite_filename']}" 34 | end 35 | 36 | DB = Sequel.connect(url) 37 | end 38 | 39 | DB.loggers << Logger.new($stdout) 40 | -------------------------------------------------------------------------------- /logging_schema.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'sequel' 4 | require 'yaml' 5 | 6 | basedir = File.dirname(__FILE__) 7 | config = YAML.load_file("#{basedir}/config.yml") 8 | 9 | dbname = config['logging_url'].split('/').last 10 | _DB0 = Sequel.connect config['logging_url'] 11 | _DB0.run "drop database if exists #{dbname};" 12 | _DB0.run "create database #{dbname};" 13 | _DB0.run "use #{dbname};" 14 | 15 | 16 | require_relative './logging_init' 17 | 18 | 19 | LOGGING_DB.create_table! :s3_log_files do 20 | primary_key :id 21 | String :filename, {:null => false, :unique => true} 22 | index :filename, :unique=>true 23 | end 24 | 25 | # FIXME add appropriate indexes 26 | 27 | LOGGING_DB.create_table! :log_entries do 28 | primary_key :id 29 | DateTime :timestamp 30 | String :remote_ip 31 | String :url 32 | String :request_uri 33 | String :http_status 34 | String :s3_error_code 35 | TrueClass :is_s3, {:null => false} 36 | Integer :bytes_sent 37 | Integer :object_size 38 | Integer :total_time 39 | Integer :turn_around_time 40 | String :referrer 41 | String :user_agent 42 | String :s3_version_id 43 | Integer :s3_log_file_id 44 | Integer :id_fetched 45 | Integer :resource_fetched 46 | end 47 | -------------------------------------------------------------------------------- /convert_db.rb: -------------------------------------------------------------------------------- 1 | ## Run me via cron. 2 | ## 'AHS_DATABASE_TYPE' must be set in crontab 3 | 4 | require './db_init.rb' 5 | require 'fileutils' 6 | require 'sequel' 7 | 8 | @basedir = File.dirname(__FILE__) 9 | 10 | timestamp = DB[:timestamp].first[:timestamp] 11 | 12 | cachefile = "#{@basedir}/dbtimestamp.cache" 13 | 14 | @config = YAML.load_file("#{@basedir}/config.yml") 15 | 16 | url2 = @config['mysql_url'].sub("annotationhub", "information_schema") 17 | DB2 = Sequel.connect(url2) 18 | 19 | table_created_at = DB2[:tables].where(:table_schema => 'annotationhub').max(:create_time) 20 | 21 | def convert_db() 22 | mysql2_url = @config['mysql_url'].sub(/^mysql:/, "mysql2:") 23 | outfile = "#{@basedir}/#{@config['sqlite_filename']}" 24 | outfile_tmp = outfile + "_tmp" 25 | FileUtils.rm_rf outfile_tmp 26 | res = `sequel #{mysql2_url} -C sqlite://#{outfile_tmp}` 27 | FileUtils.rm_rf outfile 28 | FileUtils.mv outfile_tmp, outfile 29 | #puts "does it exist? #{File.exists? outfile}" 30 | end 31 | 32 | if (File.exists?(cachefile)) 33 | cached_time = Time.parse(File.readlines(cachefile).first) 34 | if (timestamp > cached_time or table_created_at > cached_time) 35 | convert_db() 36 | end 37 | else 38 | convert_db() 39 | end 40 | 41 | f = File.open(cachefile, "w") 42 | f.write(timestamp.to_s) 43 | f.close 44 | -------------------------------------------------------------------------------- /test3.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'pp' 4 | 5 | line = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be mybucket [06/Feb/2014:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 3E57427F3EXAMPLE REST.GET.VERSIONING - "GET /mybucket?versioning HTTP/1.1" 200 - 113 - 7 - "-" "S3Console/0.4" -' 6 | 7 | chars = line.split "" 8 | 9 | first = true 10 | in_delimited = false 11 | 12 | 13 | out = [] 14 | tmp = "" 15 | 16 | chars.each_with_index do |char, idx| 17 | if first 18 | first = false 19 | if ['[', '"'].include? char 20 | in_delimited = true 21 | end_delim = (char == '[') ? ']' : '"' 22 | next 23 | end 24 | end 25 | if in_delimited 26 | if char == end_delim 27 | out << tmp 28 | tmp = "" 29 | in_delimited = false 30 | skip = true 31 | else 32 | tmp += char 33 | end 34 | end 35 | if skip 36 | skip = false 37 | first = true 38 | next 39 | end 40 | if !in_delimited and char != " " 41 | tmp += char 42 | elsif !in_delimited and char == " " 43 | out << tmp 44 | tmp = "" 45 | next 46 | end 47 | if idx == (chars.length() -1) 48 | out << tmp 49 | end 50 | end 51 | 52 | pp out 53 | puts out.length -------------------------------------------------------------------------------- /test2.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | line = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be mybucket [06/Feb/2014:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 3E57427F3EXAMPLE REST.GET.VERSIONING - "GET /mybucket?versioning HTTP/1.1" 200 - 113 - 7 - "-" "S3Console/0.4" -' 4 | lin0 = '13784cc045aa79ecb3e164200025c7fc9205241e61ba76f40d486de657f5df52 annotationhub [19/Nov/2013:23:34:51 +0000] 140.107.151.128 - 8151DFA13868F466 REST.GET.OBJECT release-69/fasta/callithrix_jacchus/ncrna/Callithrix_jacchus.C_jacchus3.2.1.69.ncrna.fa_0.0.1.json "GET /release-69/fasta/callithrix_jacchus/ncrna/Callithrix_jacchus.C_jacchus3.2.1.69.ncrna.fa_0.0.1.json HTTP/1.1" 403 AccessDenied 231 - 7 - "-" "curl/7.30.0" -' 5 | ss = line.split " " 6 | 7 | out = [] 8 | 9 | in_delimited = false 10 | 11 | count = 0 12 | loop do 13 | seg = ss[count] 14 | if seg.start_with? "[" or seg.start_with? '"' 15 | delim = seg.split("")[0] 16 | if delim == '[' 17 | end_delim = ']' 18 | else 19 | end_delim = '"' 20 | end 21 | tmp = "" 22 | loop do 23 | tmp = tmp + seg 24 | if tmp.end_with? end_delim 25 | tmp = tmp[0..-2] 26 | out << tmp 27 | next 28 | end 29 | count += 1 30 | seg = ss[count] 31 | break if 32 | end 33 | else 34 | out << ss 35 | count += 1 36 | end 37 | end 38 | 39 | require 'pp' 40 | pp out 41 | -------------------------------------------------------------------------------- /views/listing.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | BiocHub Server Results 4 | 5 | 6 | 7 |


8 | 9 | 37 | 38 | 39 | 40 | 41 | 42 | 43 |
44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | <% values.each do |row| %> 54 | 55 | <% if row.is_a?(String) %> 56 | <% temp = row.gsub(" ", "%20") %> 57 | 58 | <% else %> 59 | 60 | <% end %> 61 | 62 | 63 | <% end %> 64 | 65 |
<%=message%>
><%= row %>><%= row %>
66 |
67 | 68 | 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /views/resultsPage.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | BiocHub Server Results 4 | 5 | 6 | 7 | 37 | 38 | 39 | 40 | 41 | 42 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | <% result.each do |row| %> 72 | 73 | 74 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | <% end %> 84 | 85 | 86 |
ah_idDownload FileCopy R CodeTitleDescription
<%=row[:ah_id] %>Download <%=row[:ah_id] %>"]] id="<%=row[:ah_id]%>"><%= row[:title] %><%= row[:description] %>
87 | 88 | 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /read_s3_logs.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'aws' 4 | require 'yaml' 5 | require 'sequel' 6 | 7 | require_relative './logging_init' 8 | 9 | @rgo = Regexp.new('REST\.GET\.OBJECT') 10 | 11 | def parse(line, file) 12 | return nil unless line.match(@rgo) 13 | unless instance_variable_defined?(:@regex) 14 | #13784cc045aa79ecb3e164200025c7fc9205241e61ba76f40d486de657f5df52 annotationhub [19/Nov/2013:23:34:51 +0000] 140.107.151.128 - 8151DFA13868F466 REST.GET.OBJECT release-69/fasta/callithrix_jacchus/ncrna/Callithrix_jacchus.C_jacchus3.2.1.69.ncrna.fa_0.0.1.json "GET /release-69/fasta/callithrix_jacchus/ncrna/Callithrix_jacchus.C_jacchus3.2.1.69.ncrna.fa_0.0.1.json HTTP/1.1" 403 AccessDenied 231 - 7 - "-" "curl/7.30.0" - 15 | #@regex = Regexp.new('(?\S+) (?\S+) (?