├── .ruby-version ├── .ruby-gemset ├── spec ├── spec.opts ├── data │ ├── APIC-1-image.mp3 │ ├── APIC-2-images.mp3 │ ├── PIC-1-image.mp3 │ └── PIC-2-images.mp3 ├── thingfish │ ├── metastore │ │ └── memory_spec.rb │ ├── datastore │ │ └── memory_spec.rb │ ├── processor │ │ └── sha256_spec.rb │ ├── processor_spec.rb │ ├── mixins_spec.rb │ ├── datastore_spec.rb │ ├── metastore_spec.rb │ └── handler_spec.rb ├── thingfish_spec.rb └── helpers.rb ├── checksum └── thingfish-0.8.0.gem.sha512 ├── .simplecov ├── Procfile ├── Rakefile ├── lib ├── strelka │ ├── apps.rb │ ├── app │ │ └── metadata.rb │ └── httprequest │ │ └── metadata.rb ├── thingfish.rb └── thingfish │ ├── processor │ └── sha256.rb │ ├── mixins.rb │ ├── datastore.rb │ ├── metastore.rb │ ├── processor.rb │ ├── datastore │ └── memory.rb │ ├── metastore │ └── memory.rb │ ├── spechelpers.rb │ ├── behaviors.rb │ └── handler.rb ├── .hgignore ├── gem.deps.rb ├── bin └── thingfish ├── .hoerc ├── experiments ├── event_grabber.rb ├── NEW.md ├── Processors.rdoc └── ragel_search_parsing.pl ├── .hgtags ├── .tm_properties ├── etc ├── thingfish.conf.example └── mongrel2-config.rb ├── .pryrc ├── History.md ├── Manifest.txt ├── certs ├── ged.pem └── mahlon.pem ├── LICENSE ├── .hgsigs ├── thingfish.gemspec └── README.md /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.0 2 | -------------------------------------------------------------------------------- /.ruby-gemset: -------------------------------------------------------------------------------- 1 | thingfish 2 | -------------------------------------------------------------------------------- /spec/spec.opts: -------------------------------------------------------------------------------- 1 | -f 2 | s 3 | -c 4 | -Du 5 | -------------------------------------------------------------------------------- /spec/data/APIC-1-image.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ged/ThingFish/HEAD/spec/data/APIC-1-image.mp3 -------------------------------------------------------------------------------- /spec/data/APIC-2-images.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ged/ThingFish/HEAD/spec/data/APIC-2-images.mp3 -------------------------------------------------------------------------------- /spec/data/PIC-1-image.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ged/ThingFish/HEAD/spec/data/PIC-1-image.mp3 -------------------------------------------------------------------------------- /spec/data/PIC-2-images.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ged/ThingFish/HEAD/spec/data/PIC-2-images.mp3 -------------------------------------------------------------------------------- /checksum/thingfish-0.8.0.gem.sha512: -------------------------------------------------------------------------------- 1 | eb13fe1e8afd710716543fe4c2b637b7934f9a7d4643210a06c4cf1c4cefc19ebde9b1a8c6bc87b98b5d7433ae7c0d7fdc1044b85e5f2a23a02da7647e73dbe3 -------------------------------------------------------------------------------- /.simplecov: -------------------------------------------------------------------------------- 1 | $stderr.puts "\n\n>>> Enabling coverage report.\n\n" 2 | SimpleCov.start do 3 | add_filter 'spec' 4 | add_group "Needing tests" do |file| 5 | file.covered_percent < 90 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | # Foreman procfile 2 | mongrel: m2sh.rb -c etc/mongrel2.sqlite start 3 | thingfish: ruby -Ilib:../Thingfish-Datastore-Filesystem/lib:../Mongrel2/lib bin/thingfish etc/thingfish.conf 4 | 5 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby -S rake 2 | 3 | require 'rake/deveiate' 4 | 5 | Rake::DevEiate.setup( 'thingfish' ) do |project| 6 | project.publish_to = 'deveiate:/usr/local/www/public/code' 7 | end 8 | 9 | -------------------------------------------------------------------------------- /lib/strelka/apps.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'strelka/discovery' 5 | 6 | Strelka::Discovery.register_apps( 7 | 'thingfish' => 'thingfish/handler.rb' 8 | ) 9 | 10 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | ^pkg/ 2 | ^tags 3 | ^ChangeLog$ 4 | etc/thingfish.conf$ 5 | var/ 6 | run/ 7 | tmp/ 8 | \.sqlite$ 9 | logs/ 10 | .idea 11 | coverage/ 12 | \.paw$ 13 | doc/ 14 | ^spec/\.status$ 15 | ^gem\.deps\.rb\.lock$ 16 | -------------------------------------------------------------------------------- /gem.deps.rb: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org/' 2 | 3 | gem 'strelka', '~> 0.14' 4 | 5 | group :development do 6 | gem 'rake-deveiate', '~> 0.10' 7 | gem 'rdoc-generator-fivefish', '~> 0.4' 8 | gem 'simplecov', '~> 0.18' 9 | end 10 | 11 | -------------------------------------------------------------------------------- /bin/thingfish: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'strelka' 5 | require 'thingfish/handler' 6 | 7 | configpath = ARGV.shift || 'etc/thingfish.conf' 8 | 9 | Strelka.load_config( configpath ) 10 | Thingfish::Handler.run 11 | -------------------------------------------------------------------------------- /.hoerc: -------------------------------------------------------------------------------- 1 | --- 2 | exclude: !ruby/regexp /tmp$|\.(hg|gems|hoerc|DS_Store|rvmrc|irbrc|pryrc|tm_.*)|(logs|docs|coverage|experiments|integration|manual|plugins|misc|features|tmp\w*|wiki|\w+-\d+\.\d+\.\d+)\/|\.(bundle|sqlite|ru|graffle|ldif|log|paw|sublime-\w+|pid|pem)|tmtags|etc/.*\.(sqlite|conf)$/ 3 | -------------------------------------------------------------------------------- /experiments/event_grabber.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | 4 | require 'zmq' 5 | 6 | ctx = ZMQ::Context.new 7 | sock = ctx.socket( ZMQ::SUB ) 8 | sock.subscribe( '' ) 9 | sock.connect( 'tcp://127.0.0.1:3475' ) 10 | #sock.connect( 'epgm://eth0:226.1.1.1:3475' ) 11 | 12 | while msg = sock.recv 13 | puts msg 14 | end 15 | 16 | -------------------------------------------------------------------------------- /spec/thingfish/metastore/memory_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../../helpers' 4 | 5 | require 'securerandom' 6 | require 'rspec' 7 | require 'thingfish/metastore/memory' 8 | require 'thingfish/behaviors' 9 | 10 | 11 | RSpec.describe Thingfish::Metastore::Memory do 12 | 13 | it_behaves_like "a Thingfish metastore" 14 | 15 | end 16 | 17 | # vim: set nosta noet ts=4 sw=4 ft=rspec: 18 | -------------------------------------------------------------------------------- /.hgtags: -------------------------------------------------------------------------------- 1 | 51f57cc59523d4bba0d8f7e4aea422b4094d7804 0.3.0 2 | f935efac757ca4354c1d9b681541cea64a0a97c6 pre-strelka-cleanup 3 | a5f9bfaa7f2c403bc9d23cecd3fcc09632d5dcbe v0.5.0.pre20161103181816 4 | ffebd28866d6b8e8ab4f7397c035bcf26d043862 v0.5.0 5 | 7494de0ded2d6ad261c72ee5a1b036b0f2dace12 v0.5.1 6 | dfcb639cbbe79fecf75466f941ec39b9c80d7247 v0.6.0 7 | c8bfee4b3f805954823e33c489cf1903e82cabb5 v0.7.0 8 | 9b28c599413a1ac17e8d725793400af584176f5c v0.8.0 9 | -------------------------------------------------------------------------------- /spec/thingfish/datastore/memory_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../../helpers' 4 | 5 | require 'rspec' 6 | require 'thingfish/datastore' 7 | require 'thingfish/behaviors' 8 | 9 | 10 | RSpec.describe Thingfish::Datastore, "memory" do 11 | 12 | let( :store ) { Thingfish::Datastore.create(:memory) } 13 | 14 | 15 | it_behaves_like "a Thingfish datastore" 16 | 17 | end 18 | 19 | # vim: set nosta noet ts=4 sw=4 ft=rspec: 20 | -------------------------------------------------------------------------------- /.tm_properties: -------------------------------------------------------------------------------- 1 | # Settings 2 | projectDirectory = "$CWD" 3 | windowTitle = "${CWD/^.*\///} «$TM_DISPLAYNAME»" 4 | excludeInFileChooser = "{$exclude,.hg}" 5 | 6 | TM_RUBY = "$HOME/.rvm/bin/rvm-auto-ruby" 7 | 8 | RUBYLIB = "/Users/ged/source/ruby/rspec-formatter-webkit/lib" 9 | TM_RSPEC_OPTS = '-rrspec/core/formatters/webkit' 10 | TM_RSPEC_FORMATTER = 'RSpec::Core::Formatters::WebKit' 11 | 12 | [ source.ruby ] 13 | softTabs = false 14 | tabSize = 4 15 | 16 | 17 | -------------------------------------------------------------------------------- /spec/thingfish_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative 'helpers' 4 | 5 | require 'rspec' 6 | require 'thingfish' 7 | 8 | 9 | RSpec.describe Thingfish do 10 | 11 | it "returns a version string if asked" do 12 | expect( described_class.version_string ).to match( /\w+ [\d.]+/ ) 13 | end 14 | 15 | 16 | it "returns a version string with a build number if asked" do 17 | expect( described_class.version_string(true) ). 18 | to match(/\w+ [\d.]+ \(build [[:xdigit:]]+\)/) 19 | end 20 | 21 | end 22 | 23 | # vim: set nosta noet ts=4 sw=4 ft=rspec: 24 | -------------------------------------------------------------------------------- /etc/thingfish.conf.example: -------------------------------------------------------------------------------- 1 | --- 2 | logging: 3 | __default__: debug STDERR 4 | thingfish: info (color) 5 | 6 | # Thingfish specific configuration. 7 | # 8 | thingfish: 9 | datastore: memory 10 | metastore: memory 11 | 12 | # The path to the Mongrel2 config database. 13 | # 14 | mongrel2: 15 | configdb: example/mongrel2.sqlite 16 | 17 | # Strelka configuration knobs that influence Thingfish's handler. 18 | # 19 | app: 20 | devmode: false 21 | app_glob_pattern: '{apps,handlers}/**/*' 22 | local_data_dirs: data/* 23 | multipartparser: 24 | bufsize: 524288 25 | spooldir: /var/folders/1f/6ymhh79s0n3gjdw16fj7tp480000gp/T/strelka-mimeparts 26 | 27 | -------------------------------------------------------------------------------- /.pryrc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby -*- ruby -*- 2 | 3 | require 'loggability' 4 | require 'pathname' 5 | 6 | $LOAD_PATH.unshift( 'lib' ) 7 | $LOAD_PATH.unshift( '../Strelka/lib' ) 8 | $LOAD_PATH.unshift( '../Mongrel2/lib' ) 9 | 10 | begin 11 | require 'thingfish' 12 | require 'thingfish/handler' 13 | 14 | if File.exist?( 'etc/thingfish.conf' ) 15 | $stderr.puts 'Installing the config in etc/thingfish.conf...' 16 | Strelka.load_config( 'etc/thingfish.conf' ) 17 | end 18 | 19 | Loggability.level = :debug 20 | Loggability.format_with( :color ) 21 | 22 | rescue Exception => e 23 | $stderr.puts "Ack! Thingfish libraries failed to load: #{e.message}\n\t" + 24 | e.backtrace.join( "\n\t" ) 25 | end 26 | 27 | 28 | -------------------------------------------------------------------------------- /lib/thingfish.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'loggability' 4 | 5 | # Network-accessable datastore service 6 | module Thingfish 7 | extend Loggability 8 | 9 | 10 | # Loggability API -- log all Thingfish-related stuff to a separate logger 11 | log_as :thingfish 12 | 13 | 14 | # Package version 15 | VERSION = '0.8.0' 16 | 17 | # Version control revision 18 | REVISION = %q$Revision$ 19 | 20 | 21 | ### Get the library version. If +include_buildnum+ is true, the version string will 22 | ### include the VCS rev ID. 23 | def self::version_string( include_buildnum=false ) 24 | vstring = "%s %s" % [ self.name, VERSION ] 25 | vstring << " (build %s)" % [ REVISION[/: ([[:xdigit:]]+)/, 1] || '0' ] if include_buildnum 26 | return vstring 27 | end 28 | 29 | 30 | end # module Thingfish 31 | 32 | # vim: set nosta noet ts=4 sw=4: 33 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | # Release History for thingfish 2 | 3 | --- 4 | 5 | ## v0.8.0 [2021-01-02] Michael Granger 6 | 7 | Improvements: 8 | 9 | - Several bugfixes 10 | - Support for newer Rubies 11 | 12 | 13 | ## v0.7.0 [2017-09-13] Michael Granger 14 | 15 | Improvements: 16 | 17 | - Explicitly set the HTTP status when returning resources. 18 | - Add Last-Modified/If-None-Match support. 19 | - Move the configure() method to help support deferred configuration. 20 | 21 | 22 | ## v0.6.0 [2017-01-16] Mahlon E. Smith 23 | 24 | Housekeeping: 25 | 26 | - Bump Configurability dependency. 27 | - Migrate away from .rvmrc. 28 | 29 | 30 | ## v0.5.1 [2016-11-14] Michael Granger 31 | 32 | Enhancements: 33 | 34 | - Add support for Strelka app discovery. 35 | 36 | Bugfixes: 37 | 38 | - Remove the processor daemon for now 39 | - Documentation fixes 40 | 41 | 42 | ## v0.5.0 [2016-11-14] Michael Granger 43 | 44 | First public release. 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /Manifest.txt: -------------------------------------------------------------------------------- 1 | .simplecov 2 | bin/thingfish 3 | History.md 4 | lib/strelka/app/metadata.rb 5 | lib/strelka/apps.rb 6 | lib/strelka/httprequest/metadata.rb 7 | lib/thingfish.rb 8 | lib/thingfish/behaviors.rb 9 | lib/thingfish/datastore.rb 10 | lib/thingfish/datastore/memory.rb 11 | lib/thingfish/handler.rb 12 | lib/thingfish/metastore.rb 13 | lib/thingfish/metastore/memory.rb 14 | lib/thingfish/mixins.rb 15 | lib/thingfish/processor.rb 16 | lib/thingfish/processor/sha256.rb 17 | lib/thingfish/spechelpers.rb 18 | LICENSE 19 | Rakefile 20 | README.md 21 | spec/data/APIC-1-image.mp3 22 | spec/data/APIC-2-images.mp3 23 | spec/data/PIC-1-image.mp3 24 | spec/data/PIC-2-images.mp3 25 | spec/helpers.rb 26 | spec/spec.opts 27 | spec/thingfish/datastore/memory_spec.rb 28 | spec/thingfish/datastore_spec.rb 29 | spec/thingfish/handler_spec.rb 30 | spec/thingfish/metastore/memory_spec.rb 31 | spec/thingfish/metastore_spec.rb 32 | spec/thingfish/mixins_spec.rb 33 | spec/thingfish/processor/sha256_spec.rb 34 | spec/thingfish/processor_spec.rb 35 | spec/thingfish_spec.rb 36 | -------------------------------------------------------------------------------- /lib/strelka/app/metadata.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'strelka' 5 | require 'strelka/plugins' 6 | require 'strelka/httprequest/metadata' 7 | 8 | require 'thingfish' 9 | 10 | 11 | # A Strelka plugin for setting up requests to be able to carry Thingfish metadata 12 | # with it. 13 | module Strelka::App::Metadata 14 | extend Strelka::Plugin 15 | 16 | 17 | run_outside :routing, :filters 18 | run_inside :templating, :parameters 19 | 20 | 21 | ### Extension callback -- extend the HTTPRequest classes with Metadata 22 | ### support when this plugin is loaded. 23 | def self::included( object ) 24 | self.log.debug "Extending Request with Metadata mixins" 25 | Strelka::HTTPRequest.class_eval { include Strelka::HTTPRequest::Metadata } 26 | super 27 | end 28 | 29 | 30 | ### Start content-negotiation when the response has returned. 31 | def handle_request( request ) 32 | self.log.debug "[:metadata] Attaching Thingfish metadata to request." 33 | super 34 | end 35 | 36 | 37 | end # module Strelka::App::Metadata 38 | 39 | -------------------------------------------------------------------------------- /lib/thingfish/processor/sha256.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'digest/sha2' 5 | 6 | require 'thingfish' unless defined?( Thingfish ) 7 | require 'thingfish/processor' unless defined?( Thingfish::Processor ) 8 | 9 | 10 | # Calculate and store a sha256 checksum for a resource. 11 | class Thingfish::Processor::SHA256 < Thingfish::Processor 12 | extend Loggability 13 | 14 | # The chunk size to read 15 | CHUNK_SIZE = 32 * 1024 16 | 17 | # Loggability API -- log to the :thingfish logger 18 | log_to :thingfish 19 | 20 | # The list of handled types 21 | handled_types '*/*' 22 | 23 | 24 | ### Synchronous processor API -- generate a checksum during upload. 25 | def on_request( request ) 26 | request.add_metadata( :checksum => self.checksum(request.body) ) 27 | request.related_resources.each_pair do |io, metadata| 28 | metadata[ :checksum ] = self.checksum( io ) 29 | end 30 | end 31 | 32 | 33 | ######### 34 | protected 35 | ######### 36 | 37 | ### Given an +io+, return a sha256 checksum of it's contents. 38 | def checksum( io ) 39 | digest = Digest::SHA256.new 40 | buf = String.new 41 | 42 | while io.read( CHUNK_SIZE, buf ) 43 | digest.update( buf ) 44 | end 45 | 46 | io.rewind 47 | return digest.hexdigest 48 | end 49 | 50 | end # class Thingfish::Processor::SHA256 51 | 52 | -------------------------------------------------------------------------------- /lib/thingfish/mixins.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # vim: set nosta noet ts=4 sw=4: 3 | # frozen_string_literal: true 4 | 5 | require 'securerandom' 6 | 7 | require 'thingfish' unless defined?( Thingfish ) 8 | 9 | module Thingfish 10 | 11 | # A collection of functions for dealing with object IDs. 12 | module Normalization 13 | 14 | ############### 15 | module_function 16 | ############### 17 | 18 | ### Generate a new object ID. 19 | def make_object_id 20 | return normalize_oid( SecureRandom.uuid ) 21 | end 22 | 23 | 24 | ### Normalize the given +oid+. 25 | def normalize_oid( oid ) 26 | return oid.to_s.downcase 27 | end 28 | 29 | 30 | ### Return a copy of the given +collection+ after being normalized. 31 | def normalize_keys( collection ) 32 | if collection.respond_to?( :keys ) 33 | return collection.each_with_object({}) do |(key,val),new_hash| 34 | n_key = normalize_key( key ) 35 | new_hash[ n_key ] = val 36 | end 37 | 38 | elsif collection.respond_to?( :map ) 39 | return collection.map {|key| normalize_key(key) } 40 | end 41 | 42 | return nil 43 | end 44 | 45 | 46 | ### Return a normalized copy of +key+. 47 | def normalize_key( key ) 48 | return key.to_s.downcase.gsub( /[^\w:]+/, '_' ) 49 | end 50 | 51 | end # module Normalization 52 | 53 | 54 | end # module Thingfish 55 | 56 | # vim: set nosta noet ts=4 sw=4: 57 | 58 | -------------------------------------------------------------------------------- /spec/thingfish/processor/sha256_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../../helpers' 4 | 5 | require 'rspec' 6 | require 'thingfish/processor' 7 | 8 | require 'strelka/httprequest/metadata' 9 | 10 | 11 | RSpec.describe Thingfish::Processor, "SHA256" do 12 | 13 | before( :all ) do 14 | Strelka::HTTPRequest.class_eval { include Strelka::HTTPRequest::Metadata } 15 | end 16 | 17 | 18 | let( :processor ) { described_class.create(:sha256) } 19 | 20 | let( :factory ) do 21 | Mongrel2::RequestFactory.new( 22 | :route => '/', 23 | :headers => {:accept => '*/*'}) 24 | end 25 | 26 | 27 | it "generates a sha256 checksum from an uploaded file" do 28 | req = factory.post( '/tf', fixture_data('APIC-1-image.mp3'), 'Content-type' => 'audio/mp3' ) 29 | processor.process_request( req ) 30 | 31 | expect( req.metadata['checksum'] ).to eq( 'e6b7070cbec90cdc2d8206819d86d100f076f480c9ae19d3eb8f878b3b86f2d6' ) 32 | end 33 | 34 | 35 | it "generates a sha256 checksum for related resources" do 36 | content = "data data data data data" 37 | req = factory.post( '/tf', fixture_data('APIC-1-image.mp3'), 'Content-type' => 'audio/mp3' ) 38 | req.add_related_resource( StringIO.new( content ), {} ) 39 | 40 | processor.process_request( req ) 41 | 42 | related = req.related_resources 43 | related_metadata = related.first.last 44 | 45 | expect( related_metadata[:checksum] ).to eq( Digest::SHA256.hexdigest(content) ) 46 | end 47 | end 48 | 49 | # vim: set nosta noet ts=4 sw=4 ft=rspec: 50 | -------------------------------------------------------------------------------- /certs/ged.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID+DCCAmCgAwIBAgIBAzANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBdnZWQv 3 | REM9RmFlcmllTVVEL0RDPW9yZzAeFw0yMDEyMjQyMDU1MjlaFw0yMTEyMjQyMDU1 4 | MjlaMCIxIDAeBgNVBAMMF2dlZC9EQz1GYWVyaWVNVUQvREM9b3JnMIIBojANBgkq 5 | hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAvyVhkRzvlEs0fe7145BYLfN6njX9ih5H 6 | L60U0p0euIurpv84op9CNKF9tx+1WKwyQvQP7qFGuZxkSUuWcP/sFhDXL1lWUuIl 7 | M4uHbGCRmOshDrF4dgnBeOvkHr1fIhPlJm5FO+Vew8tSQmlDsosxLUx+VB7DrVFO 8 | 5PU2AEbf04GGSrmqADGWXeaslaoRdb1fu/0M5qfPTRn5V39sWD9umuDAF9qqil/x 9 | Sl6phTvgBrG8GExHbNZpLARd3xrBYLEFsX7RvBn2UPfgsrtvpdXjsHGfpT3IPN+B 10 | vQ66lts4alKC69TE5cuKasWBm+16A4aEe3XdZBRNmtOu/g81gvwA7fkJHKllJuaI 11 | dXzdHqq+zbGZVSQ7pRYHYomD0IiDe1DbIouFnPWmagaBnGHwXkDT2bKKP+s2v21m 12 | ozilJg4aar2okb/RA6VS87o+d7g6LpDDMMQjH4G9OPnJENLdhu8KnPw/ivSVvQw7 13 | N2I4L/ZOIe2DIVuYH7aLHfjZDQv/mNgpAgMBAAGjOTA3MAkGA1UdEwQCMAAwCwYD 14 | VR0PBAQDAgSwMB0GA1UdDgQWBBRyjf55EbrHagiRLqt5YAd3yb8k4DANBgkqhkiG 15 | 9w0BAQsFAAOCAYEAMYegZanJi8zq7QKPT7wqXefX4C88I5JWeBHR3PvvWK0CwyMV 16 | peyiu5I13w/lYX+HUZjE4qsSpJMJFXWl4WZCOo+AMprOcf0PxfuJpxCej5D4tavf 17 | vRfhahSw7XJrcZih/3J+/UgoH7R05MJ+8LTcy3HGrB3a0vTafjm8OY7Xpa0LJDoN 18 | JDqxK321VIHyTibbKeA1hWSE6ljlQDvFbTqiCj3Ulp1jTv3TOlvRl8fqcfhxUJI0 19 | +5Q82jJODjEN+GaWs0V+NlrbU94cXwS2PH5dXogftB5YYA5Ex8A0ikZ73xns4Hdo 20 | XxdLdd92F5ovxA23j/rKe/IDwqr6FpDkU3nPXH/Qp0TVGv9zZnVJc/Z6ChkuWj8z 21 | pW7JAyyiiHZgKKDReDrA2LA7Zs3o/7KA6UtUH0FHf8LYhcK+pfHk6RtjRe65ffw+ 22 | MCh97sQ/Z/MOusb5+QddBmB+k8EicXyGNl4b5L4XpL7fIQu+Y96TB3JEJlShxFD9 23 | k9FjI4d9EP54gS/4 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /lib/thingfish/datastore.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'securerandom' 5 | require 'pluggability' 6 | require 'stringio' 7 | require 'strelka' 8 | 9 | require 'thingfish' unless defined?( Thingfish ) 10 | require 'thingfish/mixins' 11 | 12 | # The base class for storage mechanisms used by Thingfish to store its data 13 | # blobs. 14 | class Thingfish::Datastore 15 | extend Pluggability, 16 | Strelka::AbstractClass 17 | include Enumerable, 18 | Thingfish::Normalization 19 | 20 | 21 | # Pluggability API -- set the prefix for implementations of Datastore 22 | plugin_prefixes 'thingfish/datastore' 23 | 24 | # AbstractClass API -- register some virtual methods that must be implemented 25 | # in subclasses 26 | pure_virtual :save, 27 | :replace, 28 | :fetch, 29 | :each, 30 | :include?, 31 | :each_oid, 32 | :remove 33 | 34 | 35 | # :TODO: Make a utility method that provides normalization for IO handling 36 | # (restore .pos, etc.) 37 | # def with_io( io ) ... end 38 | 39 | ### Return a representation of the object as a String suitable for debugging. 40 | def inspect 41 | return "#<%p:%#016x>" % [ 42 | self.class, 43 | self.object_id * 2 44 | ] 45 | end 46 | 47 | 48 | ### Provide transactional consistency to the provided block. Concrete datastores should 49 | ### override this if they can implement it. By default it's a no-op. 50 | def transaction 51 | yield 52 | end 53 | 54 | end # class Thingfish::Datastore 55 | 56 | -------------------------------------------------------------------------------- /lib/thingfish/metastore.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'pluggability' 5 | require 'strelka' 6 | require 'strelka/mixins' 7 | 8 | require 'thingfish' unless defined?( Thingfish ) 9 | require 'thingfish/mixins' 10 | 11 | # The base class for storage mechanisms used by Thingfish to store its data 12 | # blobs. 13 | class Thingfish::Metastore 14 | extend Pluggability, 15 | Strelka::AbstractClass 16 | include Thingfish::Normalization 17 | 18 | 19 | # Pluggability API -- set the prefix for implementations of Metastore 20 | plugin_prefixes 'thingfish/metastore' 21 | 22 | # AbstractClass API -- register some virtual methods that must be implemented 23 | # in subclasses 24 | pure_virtual :oids, 25 | :each_oid, 26 | :save, 27 | :search, 28 | :fetch, 29 | :fetch_value, 30 | :fetch_related_oids, 31 | :merge, 32 | :include?, 33 | :remove, 34 | :remove_except, 35 | :size 36 | 37 | ### Return a representation of the object as a String suitable for debugging. 38 | def inspect 39 | return "#<%p:%#016x %d objects>" % [ 40 | self.class, 41 | self.object_id * 2, 42 | self.size 43 | ] 44 | end 45 | 46 | 47 | ### Provide transactional consistency to the provided block. Concrete metastores should 48 | ### override this if they can implement it. By default it's a no-op. 49 | def transaction 50 | yield 51 | end 52 | 53 | 54 | end # class Thingfish::Metastore 55 | 56 | -------------------------------------------------------------------------------- /etc/mongrel2-config.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # vim: set nosta noet ts=4 sw=4: 3 | # frozen_string_literal: true 4 | 5 | # 6 | # This script generates a Mongrel2 configuration database suitable for 7 | # getting a Thingfish handler running. 8 | # 9 | # Load it with: 10 | # 11 | # $ m2sh.rb -c mongrel2.sqlite load examples/mongrel2-config.rb 12 | # 13 | # Afterwards, ensure the path to the mongrel2.sqlite file is in your 14 | # thingfish.conf: 15 | # 16 | # mongrel2: 17 | # configdb: /path/to/mongrel2.sqlite 18 | # 19 | # ... and start the mongrel2 daemon: 20 | # 21 | # $ mongrel2 /path/to/mongrel2.sqlite thingfish 22 | # 23 | # In production use, you'll likely want to mount the Thingfish handler 24 | # within the URI space of an existing Mongrel2 environment. 25 | # 26 | 27 | require 'mongrel2' 28 | require 'mongrel2/config/dsl' 29 | 30 | server 'thingfish' do 31 | name 'Thingfish' 32 | default_host 'localhost' 33 | 34 | access_log 'logs/access.log' 35 | error_log 'logs/error.log' 36 | chroot '' 37 | pid_file 'run/mongrel2.pid' 38 | 39 | bind_addr '0.0.0.0' 40 | port 3474 41 | 42 | xrequest '/usr/local/lib/mongrel2/filters/sendfile.so' 43 | 44 | host 'localhost' do 45 | route '/', handler( 'tcp://127.0.0.1:9900', 'thingfish' ) 46 | end 47 | end 48 | 49 | setting 'zeromq.threads', 1 50 | setting 'limits.content_length', 250_000 51 | setting 'server.daemonize', false 52 | setting 'upload.temp_store', 'var/uploads/mongrel2.upload.XXXXXX' 53 | 54 | mkdir_p 'var/uploads' 55 | mkdir_p 'run' 56 | mkdir_p 'logs' 57 | 58 | -------------------------------------------------------------------------------- /spec/thingfish/processor_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../helpers' 4 | 5 | require 'rspec' 6 | require 'thingfish/processor' 7 | 8 | 9 | RSpec.describe Thingfish::Processor do 10 | 11 | before( :all ) do 12 | setup_logging() 13 | end 14 | 15 | 16 | it "has pluggability" do 17 | expect( described_class.plugin_type ).to eq( 'Processor' ) 18 | end 19 | 20 | 21 | it "defines a (no-op) method for handling requests" do 22 | expect { 23 | described_class.new.on_request( nil ) 24 | }.to_not raise_error 25 | end 26 | 27 | 28 | it "defines a (no-op) method for handling responses" do 29 | expect { 30 | described_class.new.on_response( nil ) 31 | }.to_not raise_error 32 | end 33 | 34 | 35 | describe "a subclass" do 36 | 37 | let!( :subclass ) { Class.new(described_class) } 38 | 39 | it "can declare a list of media types it handles" do 40 | subclass.handled_types( 'image/*', 'video/*' ) 41 | expect( subclass.handled_types.size ).to be( 2 ) 42 | expect( subclass.handled_types[0] ).to be_a( Strelka::HTTPRequest::MediaType ) 43 | end 44 | 45 | describe "instance" do 46 | 47 | let!( :instance ) do 48 | subclass.handled_types( 'audio/mpeg', 'audio/mpg', 'audio/mp3' ) 49 | subclass.new 50 | end 51 | 52 | 53 | it "knows that it doesn't handle a type it hasn't registered" do 54 | expect( instance ).to_not be_handled_type( 'image/png' ) 55 | expect( instance ).to be_handled_type( 'audio/mp3' ) 56 | end 57 | 58 | 59 | end 60 | 61 | end 62 | 63 | end 64 | 65 | # vim: set nosta noet ts=4 sw=4 ft=rspec: 66 | -------------------------------------------------------------------------------- /certs/mahlon.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIENDCCApygAwIBAgIBATANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBdtYWhs 3 | b24vREM9bWFydGluaS9EQz1udTAeFw0yMDAyMTkyMTU4NDBaFw0yMTAyMTgyMTU4 4 | NDBaMCIxIDAeBgNVBAMMF21haGxvbi9EQz1tYXJ0aW5pL0RDPW51MIIBojANBgkq 5 | hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA3cz7ILM8n+Y5nvz7mVRVqE8LusWdT8NX 6 | nlnETynDndenI+a2S3j22DR+U4ooGUjuCHE3iR1CVmTDGbxFfNRfmnC1AN9Hybat 7 | ewW+onvMBye7yfO0bJB5vkqaW5vd35rzquOffgBtJMo7rPRu6pX8RkL34Wnew4J7 8 | POooUcYbWSAO934HSCUC8wVm6b4v/ejVF1Lk44Dz45jtMqtR7KTAtpipdbTXAarO 9 | HQy3eVes/0oTqhk4CP50r1KP09nUHTn2lzVaCN9vmNE/Jwe0AuQ9ImvZXPpCsMMl 10 | V03/tuJ++48sVmOIusJkASPupXcdI6zqsjYw2vLMFtuYNskRSvwbn6Wm6x9hLWWj 11 | IRp5FvHPORLRCHFizXRmXZ3PyFHqbv6m4yG0SyfMzOXPk3Hn5dqqmK+BFCihTZIN 12 | fqpBmuxyNEE21fSO9ALLlWeW9ffg9Ye5Sc1n3yEyv8rPb9VDvi1B5N6xIcDFMNVs 13 | RiCamNbET4Sq9VIYwYtcB1f6EataqFEhAgMBAAGjdTBzMAkGA1UdEwQCMAAwCwYD 14 | VR0PBAQDAgSwMB0GA1UdDgQWBBR8KtAhZIhe2uPQHCgU5HurIG7crTAcBgNVHREE 15 | FTATgRFtYWhsb25AbWFydGluaS5udTAcBgNVHRIEFTATgRFtYWhsb25AbWFydGlu 16 | aS5udTANBgkqhkiG9w0BAQsFAAOCAYEAHXlLXIKQUjd0VYj2mPgMheMjLEtmhHu+ 17 | 7NdIv8Bz4rpKAdhypy30xjukGLTOKBp1C0TjfHXowW/icK0bv9CO9Chbc09/+Ed2 18 | K5IsyENen+YLeLfE8dguq5tHlfocbFilRGHt8BHHO9BpPpAYoPt/76SCC2NaU5vN 19 | 33YTCpaVP0raS6E4i+xtx5PNdKoeTdrgwCQtUBhGf3L9YbZy1UaSeAyng5keuOzV 20 | Mu2osihEB0GE0pOZJNpI6ow+0emwN/XvBKHpN9D2bjbvKetyQSrm0OniaZBXIGzW 21 | Bg0JmajxUaGYWnz+QFADT+HLPmekxF3mB4+0ymZCHKPC+04h6RDjvkEOji6Jm+VB 22 | JHjnceUEejSXTkZAKAmiOAtnX4j1MM1DLiFMlZ5Wbt7hbiOiO5HoN9p9snZhYqSU 23 | JyAQQloqY/KyzQqxPlKdMNmBxRU+Cdarp05lEI6Sfj9MdrndoIL6MT/f6PgoCWZQ 24 | xEarK1Fn47yS4UZqRi6VgKc3JHscX9x4 25 | -----END CERTIFICATE----- 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2015, Michael Granger and Mahlon E. Smith. 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are 6 | permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, this 12 | list of conditions and the following disclaimer in the documentation and/or 13 | other materials provided with the distribution. 14 | 15 | * Neither the name of the authors, nor the names of its contributors may be used to 16 | endorse or promote products derived from this software without specific prior 17 | written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 26 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 27 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 28 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /experiments/NEW.md: -------------------------------------------------------------------------------- 1 | Thingfish API -- Desired 2 | ================================================= 3 | 4 | # introspection 5 | OPTIONS /v1 6 | 7 | # Search (via params), fetch all assets 8 | GET /v1 9 | 10 | GET /v1? 11 | filter=title:Mahl*%20Smith,(format:(image/jpeg|image/png),extent:<100|format:image/icns)& 12 | limit=5& 13 | offset=100& 14 | order=title& 15 | casefold=true& 16 | direction=desc 17 | 18 | DEFAULT_FLAGS = { 19 | :casefold => false, 20 | :order => nil, 21 | :limit => nil, 22 | :offset => 0, 23 | :direction => :asc 24 | } 25 | 26 | metastore.search( sexp, flags={} ) 27 | [ 28 | ['title', 'Mahl* Smith' ], 29 | [:or, 30 | ['format', [:or, [ 31 | 'image/jpeg', 32 | 'image/png' 33 | ]], 34 | 'extent', '<100' 35 | ], 36 | ['format', 'image/icns'] 37 | ] 38 | ] 39 | 40 | # fetch an asset body 41 | GET /v1/«uuid» * 42 | 43 | # create a new asset 44 | POST /v1 * 45 | 46 | # update (replace) an asset body 47 | PUT /v1/«uuid» * 48 | 49 | # remove an asset and its metadata 50 | DELETE /v1/«uuid» * 51 | 52 | # retrieve all metadata associated with an asset 53 | GET /v1/«uuid»/metadata * 54 | 55 | # retrieve values for an asset's metadata key 56 | GET /v1/«uuid»/metadata/«key» * 57 | 58 | # append additional metadata for an asset 59 | POST /v1/«uuid»/metadata * 60 | 61 | # add a value for an asset's specific metadata key 62 | POST /v1/«uuid»/metadata/«key» * 63 | 64 | # replace metadata for an asset 65 | PUT /v1/«uuid»/metadata 66 | 67 | # update an asset's specific metadata key 68 | PUT /v1/«uuid»/metadata/«key» * 69 | 70 | # remove all user metadata for an asset 71 | DELETE /v1/«uuid»/metadata * 72 | 73 | # delete an asset's specific metadata key 74 | DELETE /v1/«uuid»/metadata/«key» * 75 | 76 | 77 | -------------------------------------------------------------------------------- /lib/strelka/httprequest/metadata.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # vim: set nosta noet ts=4 sw=4: 3 | # frozen_string_literal: true 4 | 5 | require 'strelka/constants' 6 | require 'strelka/httprequest' unless defined?( Strelka::HTTPRequest ) 7 | 8 | require 'thingfish' 9 | require 'thingfish/mixins' 10 | 11 | 12 | # The mixin that adds methods to Strelka::HTTPRequest for Thingfish metadata. 13 | # 14 | # request.metadata 15 | # request.add_metadata 16 | # 17 | module Strelka::HTTPRequest::Metadata 18 | include Strelka::Constants, 19 | Thingfish::Normalization 20 | 21 | 22 | ### Set up some data structures for metadata. 23 | def initialize( * ) 24 | super 25 | 26 | @metadata = {} 27 | @related_resources = {} 28 | end 29 | 30 | 31 | ###### 32 | public 33 | ###### 34 | 35 | # The Hash of Thingfish metadata associated with the request 36 | attr_reader :metadata 37 | 38 | # The Hash of related resources 39 | attr_reader :related_resources 40 | 41 | 42 | ### Merge the metadata in the given +metadata+ hash into the request's current 43 | ### metadata. 44 | def add_metadata( metadata ) 45 | self.log.debug "Adding metadata to the request: %p" % [ metadata ] 46 | metadata = normalize_keys( metadata ) 47 | self.metadata.merge!( metadata ) 48 | end 49 | 50 | 51 | ### Add a resource that's related to the one in the request. 52 | def add_related_resource( io, metadata ) 53 | metadata = normalize_keys( metadata ) 54 | metadata.merge!( self.extract_related_metadata(io) ) 55 | self.log.debug "Adding related resource: %p %p" % [ io, metadata ] 56 | self.related_resources[ io ] = metadata 57 | end 58 | 59 | 60 | ### Extract some default metadata from related resources. 61 | def extract_related_metadata( io ) 62 | metadata = {} 63 | 64 | metadata['extent'] = io.size 65 | 66 | return metadata 67 | end 68 | 69 | end 70 | 71 | -------------------------------------------------------------------------------- /spec/thingfish/mixins_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rspec -cfd -b 2 | # vim: set noet nosta sw=4 ts=4 : 3 | 4 | require_relative '../helpers' 5 | 6 | require 'rspec' 7 | 8 | require 'thingfish/mixins' 9 | 10 | 11 | RSpec.describe Thingfish, 'mixins' do 12 | 13 | # A collection of functions for dealing with object IDs. 14 | describe 'Normalization' do 15 | 16 | it 'can generate a new object ID' do 17 | expect( Thingfish::Normalization.make_object_id ).to match( UUID_PATTERN ) 18 | end 19 | 20 | it 'can normalize an object ID' do 21 | expect( 22 | Thingfish::Normalization.normalize_oid( TEST_UUID.upcase ) 23 | ).to_not match( /[A-Z]/ ) 24 | end 25 | 26 | it 'can normalize Hash metadata keys' do 27 | metadata = { :pork => 1, :sausaged => 2 } 28 | expect( Thingfish::Normalization.normalize_keys(metadata) ). 29 | to eq({ 'pork' => 1, 'sausaged' => 2 }) 30 | end 31 | 32 | it 'can normalize an Array of metadata keys' do 33 | values = [ :pork, :sausaged ] 34 | expect( Thingfish::Normalization.normalize_keys(values) ). 35 | to eq([ 'pork', 'sausaged' ]) 36 | expect( values.first ).to be( :pork ) 37 | end 38 | 39 | it "won't modify the original array of metadata keys" do 40 | values = [ :pork, :sausaged ] 41 | normalized = Thingfish::Normalization.normalize_keys( values ) 42 | 43 | expect( values.first ).to be( :pork ) 44 | expect( normalized ).to_not be( values ) 45 | end 46 | 47 | it "replaces non metadata key characters with underscores" do 48 | expect( Thingfish::Normalization::normalize_key('Sausaged!') ).to eq( 'sausaged_' ) 49 | expect( Thingfish::Normalization::normalize_key('SO sausaged') ).to eq( 'so_sausaged' ) 50 | expect( Thingfish::Normalization::normalize_key('*/porky+-') ).to eq( '_porky_' ) 51 | end 52 | 53 | it "preserves colons in metadata keys" do 54 | expect( Thingfish::Normalization::normalize_key('pork:sausaged') ). 55 | to eq( 'pork:sausaged' ) 56 | end 57 | 58 | 59 | end # module Normalization 60 | 61 | 62 | end 63 | 64 | -------------------------------------------------------------------------------- /spec/thingfish/datastore_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../helpers' 4 | 5 | require 'rspec' 6 | require 'thingfish/datastore' 7 | 8 | class TestingDatastore < Thingfish::Datastore 9 | end 10 | 11 | 12 | RSpec.describe Thingfish::Datastore do 13 | 14 | it "is abstract" do 15 | expect { described_class.new }.to raise_error( NoMethodError, /private/i ) 16 | end 17 | 18 | 19 | it "acts as a factory for its concrete derivatives" do 20 | expect( described_class.create('testing') ).to be_a( TestingDatastore ) 21 | end 22 | 23 | 24 | describe "an instance of a concrete derivative" do 25 | 26 | let( :store ) { described_class.create('testing') } 27 | 28 | it "raises an error if it doesn't implement #save" do 29 | expect { store.save(TEST_PNG_DATA) }.to raise_error( NotImplementedError, /save/ ) 30 | end 31 | 32 | it "raises an error if it doesn't implement #replace" do 33 | expect { store.replace(TEST_UUID) }.to raise_error( NotImplementedError, /replace/ ) 34 | end 35 | 36 | it "raises an error if it doesn't implement #fetch" do 37 | expect { store.fetch(TEST_UUID) }.to raise_error( NotImplementedError, /fetch/ ) 38 | end 39 | 40 | it "raises an error if it doesn't implement #each" do 41 | expect { store.each }.to raise_error( NotImplementedError, /each/ ) 42 | end 43 | 44 | it "raises an error if it doesn't implement #include?" do 45 | expect { store.include?(TEST_UUID) }.to raise_error( NotImplementedError, /include\?/ ) 46 | end 47 | 48 | it "raises an error if it doesn't implement #each_oid" do 49 | expect { store.each_oid }.to raise_error( NotImplementedError, /each_oid/ ) 50 | end 51 | 52 | it "raises an error if it doesn't implement #remove" do 53 | expect { store.remove(TEST_UUID) }.to raise_error( NotImplementedError, /remove/ ) 54 | end 55 | 56 | it "provides a transactional block method" do 57 | expect {|block| store.transaction(&block) }.to yield_with_no_args 58 | end 59 | 60 | end 61 | 62 | end 63 | 64 | # vim: set nosta noet ts=4 sw=4 ft=rspec: 65 | -------------------------------------------------------------------------------- /spec/helpers.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | # coding: utf-8 3 | 4 | BEGIN { 5 | require 'pathname' 6 | 7 | basedir = Pathname.new( __FILE__ ).dirname.parent 8 | strelkadir = basedir.parent + 'Strelka' 9 | strelkalibdir = strelkadir + 'lib' 10 | mongrel2dir = basedir.parent + 'Mongrel2' 11 | mongrel2libdir = mongrel2dir + 'lib' 12 | 13 | $LOAD_PATH.unshift( strelkalibdir.to_s ) unless $LOAD_PATH.include?( strelkalibdir.to_s ) 14 | $LOAD_PATH.unshift( mongrel2libdir.to_s ) unless $LOAD_PATH.include?( mongrel2libdir.to_s ) 15 | } 16 | 17 | # SimpleCov test coverage reporting; enable this using the :coverage rake task 18 | require 'simplecov' if ENV['COVERAGE'] 19 | 20 | require 'stringio' 21 | require 'time' 22 | 23 | 24 | require 'loggability' 25 | require 'loggability/spechelpers' 26 | require 'configurability' 27 | require 'configurability/behavior' 28 | 29 | require 'rspec' 30 | require 'mongrel2' 31 | require 'mongrel2/testing' 32 | 33 | require 'strelka' 34 | require 'strelka/testing' 35 | require 'strelka/authprovider' 36 | 37 | require 'thingfish' 38 | require 'thingfish/spechelpers' 39 | 40 | 41 | Loggability.format_with( :color ) if $stdout.tty? 42 | 43 | 44 | ### Mock with RSpec 45 | RSpec.configure do |config| 46 | include Strelka::Constants 47 | include Thingfish::SpecHelpers 48 | include Thingfish::SpecHelpers::Constants 49 | 50 | config.mock_with( :rspec ) do |mock| 51 | mock.syntax = :expect 52 | end 53 | 54 | config.disable_monkey_patching! 55 | config.example_status_persistence_file_path = "spec/.status" 56 | config.filter_run :focus 57 | config.filter_run_when_matching :focus 58 | config.order = :random 59 | config.profile_examples = 5 60 | config.run_all_when_everything_filtered = true 61 | config.shared_context_metadata_behavior = :apply_to_host_groups 62 | # config.warnings = true 63 | 64 | config.include( Loggability::SpecHelpers ) 65 | config.include( Mongrel2::SpecHelpers ) 66 | config.include( Mongrel2::Constants ) 67 | config.include( Mongrel2::Config::DSL ) 68 | config.include( Strelka::Constants ) 69 | config.include( Strelka::Testing ) 70 | config.include( Thingfish::SpecHelpers ) 71 | end 72 | 73 | # vim: set nosta noet ts=4 sw=4: 74 | 75 | -------------------------------------------------------------------------------- /lib/thingfish/processor.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'pluggability' 5 | require 'strelka/httprequest/acceptparams' 6 | 7 | require 'thingfish' unless defined?( Thingfish ) 8 | 9 | 10 | # Thingfish asset processor base class. 11 | class Thingfish::Processor 12 | extend Pluggability 13 | 14 | 15 | plugin_prefixes 'thingfish/processor' 16 | 17 | 18 | ### Get/set the list of media types this processor can handle. 19 | def self::handled_types( *mediatypes ) 20 | if mediatypes.empty? 21 | @handled_types ||= [] 22 | else 23 | @handled_types = mediatypes.collect do |type| 24 | Strelka::HTTPRequest::MediaType.parse(type) 25 | end 26 | end 27 | 28 | return @handled_types 29 | end 30 | 31 | 32 | ### Filter hook for request, pass to processor if it is able to 33 | ### handle the +request+ content type. 34 | def process_request( request ) 35 | return unless self.handled_path?( request ) 36 | if self.handled_type?( request.content_type ) 37 | on_request( request ) 38 | end 39 | end 40 | 41 | 42 | ### Process the data and/or metadata in the +request+. 43 | def on_request( request ) 44 | # No-op by default 45 | end 46 | 47 | 48 | ### Filter hook for response, pass to processor if it is able to 49 | ### handle the +response+ content type. 50 | def process_response( response ) 51 | return unless self.handled_path?( response.request ) 52 | if self.handled_type?( response.content_type ) 53 | on_response( response ) 54 | end 55 | end 56 | 57 | 58 | ### Process the data and/or metadata in the +response+. 59 | def on_response( response ) 60 | # No-op by default 61 | end 62 | 63 | 64 | ### Returns +true+ if the given media +type+ is one the processor handles. 65 | def handled_type?( type ) 66 | return true if self.class.handled_types.empty? 67 | self.class.handled_types.find {|handled_type| type =~ handled_type } 68 | end 69 | alias_method :is_handled_type?, :handled_type? 70 | 71 | 72 | ### Returns +true+ if the given +request+'s path is one that should 73 | ### be processed. 74 | def handled_path?( request ) 75 | return ! request.path.match( %r|^/?[\w\-]+/metadata| ) 76 | end 77 | 78 | end # class Thingfish::Processor 79 | 80 | -------------------------------------------------------------------------------- /lib/thingfish/datastore/memory.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'thingfish' unless defined?( Thingfish ) 5 | require 'thingfish/datastore' unless defined?( Thingfish::Datastore ) 6 | 7 | 8 | 9 | # An in-memory datastore for testing and tryout purposes. 10 | class Thingfish::Datastore::Memory < Thingfish::Datastore 11 | extend Loggability 12 | 13 | # Loggability API -- log to the :thingfish logger 14 | log_to :thingfish 15 | 16 | 17 | ### Create a new MemoryDatastore, using the given +storage+ object to store 18 | ### data in. The +storage+ should quack like a Hash. 19 | def initialize( storage={} ) 20 | @storage = storage 21 | end 22 | 23 | 24 | ### Save the +data+ read from the specified +io+ and return an ID that can be 25 | ### used to fetch it later. 26 | def save( io ) 27 | oid = make_object_id() 28 | offset = io.pos 29 | data = io.read.dup 30 | 31 | self.log.debug "Saving %d bytes of data under OID %s" % [ data.bytesize, oid ] 32 | @storage[ oid ] = data 33 | 34 | io.pos = offset 35 | return oid 36 | end 37 | 38 | 39 | ### Replace the existing object associated with +oid+ with the data read from the 40 | ### given +io+. 41 | def replace( oid, io ) 42 | offset = io.pos 43 | data = io.read.dup 44 | oid = normalize_oid( oid ) 45 | 46 | self.log.debug "Replacing data under OID %s with %d bytes" % [ oid, data.bytesize ] 47 | @storage[ oid ] = data 48 | 49 | io.pos = offset 50 | return true 51 | end 52 | 53 | 54 | ### Fetch the data corresponding to the given +oid+ as an IOish object. 55 | def fetch( oid ) 56 | oid = normalize_oid( oid ) 57 | self.log.debug "Fetching data for OID %s" % [ oid ] 58 | data = @storage[ oid ] or return nil 59 | return StringIO.new( data ) 60 | end 61 | 62 | 63 | ### Remove the data associated with +oid+ from the Datastore. 64 | def remove( oid ) 65 | oid = normalize_oid( oid ) 66 | @storage.delete( oid ) 67 | end 68 | 69 | 70 | ### Return +true+ if the datastore has data associated with the specified +oid+. 71 | def include?( oid ) 72 | oid = normalize_oid( oid ) 73 | return @storage.include?( oid ) 74 | end 75 | 76 | 77 | ### Iterator -- yield the UUID of each object in the datastore to the block, or 78 | ### return an Enumerator for each UUID if called without a block. 79 | def each_oid( &block ) 80 | return @storage.each_key( &block ) 81 | end 82 | 83 | 84 | ### Iterator -- yield a pair: 85 | ### UUID => datablob 86 | ### of each object in the datastore to the block, or return an Enumerator 87 | ### for each UUID if called without a block. 88 | def each( &block ) 89 | return @storage.each( &block ) 90 | end 91 | 92 | end # class Thingfish::Datastore::Memory 93 | 94 | -------------------------------------------------------------------------------- /.hgsigs: -------------------------------------------------------------------------------- 1 | bf8dfe2584415f58a4966647dcc96dc493cbeec3 0 iQIcBAABAgAGBQJYKfK/AAoJEGE7GvLhImG9nfwP/jKGm2SqJDRcxLqTKFM0yc57ASJdoQQPa88yL2/87vwxHaqL6YOFOiOXF+BKJ6estoBmLj4mKI5AUqiRa4X9tXfbYeV3yAU87ORpXIbzTqTGP5w0Pozhftosj9p4O+cIdJppR5s41ZJmy5RqFBT/YAtAsiqkSXhADGSGXkeOdFbTYXusCl+aOS8AH5d0ye8ZjMHqVo+07L0//J7U+krGj4HfvRlr0fmY9QlWizr4RkkXroRzsT8GlfKpjTEENyInJMrpjMicSQgq1nBOSOPVvxbWMigg7JqGcuZQ4xGoNt0DLM9cpKf4ttHpjUtuwzINam2JYaXzNCJ898uejDdZvxRuoi034cWDbK2MEDCyZ086NLv5RctiWIF7Q5Npf9iyogCKDTFXxeVdMLLGP8ows06k9MgUapuCWmxDxHB2Y1v9Cs7nJRKNpzrzKZW2wIqzFotULHIcE7iKSkNw4fRll7r4M7Z9M5zvujLt4U7uPHRKkUGZwMAqISYpX6Hbi3J1nL07tj6FYNn4nsA2Q3YC5amyTPtEz7st5speWGfBX25H/xEH1gtPA3IE+pKRI6IXW0ZqZp3ahWoHr/Aw3auHbpHyeFFkVW/uEodEr3e2IPyDBqH8u4z9fn9mK7ruMn7iu81pkP/EVW5rsLittJUi6O8kDYac0/I/CyjVhcwCrPQ9 2 | 23c489bc96188a5ba675d7d490df88b26eceec35 0 iQIcBAABAgAGBQJYKgRbAAoJEGE7GvLhImG9oO0P/0BLcxCWvNXQwCljCtI71W1G7JB5C1Ia7ERqAOcs8pSYEYHGpjaW4DVF4lBgUpK8WXJ6glFYX6D10R+STu++qng4+htgD+FYpLkAapdcd2NCo4QudjH0+YEZs7tfx73K9qe/wL1/jxaFGmAYviEdn+EQHFGiJQXqX6o6ejP8ugNA8OpcFW857tsClkZVkX2fOAF3c/lrMr+fmupurQ5NHG7XaeB+zY85yHV/oCpNSAKHjbwfT01ZRwAmlQedYph0RFpGSc7ppGwC2ggQWpNISpT0UHHpVF92GGmk3PdKHrB5iPNwv3XfFc73WkxdPdvYvFKlehYRes45OPghtsigf9gd+AGOosjczB0Lcw4c2S0Tlxfrx8OGyFhfMJnf5xqHE9lf9rJl+Bmpiasx7esW/DjbLq7lhdSCIjle0YivZYF5wrR7+WuUx+/To5Xbs3/MVsiLlAkitN1PiYd714qB0Y/IJsKFsd0/8y4vb2/sAGXz61BaDwCYs/wzHhasdmNWRWMxcszuHVIpQ+R7pMhCDTpuv2XRJExKkDhN5wwnEmor1w4dWsUboLitX5am6y3C8GrH4t75/FcBCBMBW+f9WDrUH28/44MXTUM2jL+8Ndwh7vq4ITD2/E5b8aqOOgyzLVkQYpl5NbynOa7UKanE6o9hXHBMcLhgqjoWJuOVpm9S 3 | 2bc8257c7537a3ef6c9dbd3a4165847e09435bdc 0 iQIVAwUAWH0aZQdBeh7mMHXSAQrDhg//VIDoZN40+aKG8wi9UqGNOkh1FbVSFt4CoYk/+gxignHtWmAm6J16mDOqupP9C9p1CUK5dEdAYSBDj90r/sJO0jtOwNXvr0/OnorYyvk7BKPkZWxL892xmkLLtBSkgdnex826e5iGr59KenlzgcL7RnDlVK09bGI1sM2imcPlEHgMC2kfgjkjMI/JkiEcIU5Prp4oDlTQgHuTWY6xkKXcLiW+aBhstaFUCgYudlnUCYOlefHYjxesy4U2ow8XQycY7pgfNIzInDYG3NoF+z8cWO0fMiPB8tx+P+VafbwUi2X+gwEa3oLI/GlY3OCxVC1nmG1tVd2fEMY9msPTwTfkBh/xTbjW+sOc/myw1+H/aAlW6nwXkVYtE6Jj4qUlbDLCtynuSS3JAWeIZkU1OuTDBD+rCcL5SmJaXKeYPnlA/ETCOJWucC3qTNzEgLgaWd2+pnq+gAgYdkw1kpVGL+MrnzETH5fsjr9j0MOkDyAtzjGoQFhd8c6J5bFWKh4GyVa9XCyoY1jRcxQPmGA5dKzpIsejSYE46k9BeXvnB6xz6zNWhKogHlTO/TsNpCRJPyHYGl6x3DEmi/HojfuoF3bgU9q7TXVh9+aKnHBU+7QsPb22TxBlZ4OcjnjuGiVnkXD43Z/VDGogllaogUypzy8i9oatS5yTgGCM22d6NFRN6a0= 4 | 9b28c599413a1ac17e8d725793400af584176f5c 0 iQIzBAABCAAdFiEEoYl52o0gA4yRUmHQ4yEXIpU5F6MFAl/xcqUACgkQ4yEXIpU5F6MnYQ/8DgxhRlu8txZ8vTLpilPhjDpV9JfseeHtYrf5kDM3oRh55SCsuxCPwk/Ta0k9e1gzqunM1SlkGuKiAHlO7Y6RQnJl+bsi1Fk6QP7+2PH+kZMx90XpoDII1ZdJd1MfLnA/o6p0eTrDRMo0yR1v5QxybMts97+yJwIQKkk24VfkoRHmAYaUfFSy12Q8HzYTwd+goglPrXT4nGOxvuiImtZQ08S45YmiHK7oahTs/r0IQ9kwFniJKB+EHezr/lU25TTVzYIn/lkXiVpEoIZ8VCXr2xLXpjPa9ZCqkoIpNCWQl23yJa36++A3tWhC7xCEHLgZbWLpOojTMtXslSMczTOrCrG7NzGA7GRx0+hAvL7CQ5OyHp2xiGxnFPu6cMDpZJ4Bog12+BDKBu/P/OaK55jydG0HLf0x8QpzXhxZqeT8sBTDm+QY+PUFdRVNHPIoUsGjXWHUOA7AxcESp/1Z+RAPlzti9d015BNk/WYnkoOJu0hwmPO+vzbVcBuj4Yo+UVwO/sOd8z01i6s3C8VPUqEyZVp8JlWXz3JWdnkzlwzI9m/YAkh9w4zUaTpte0uTp/vq4q4m7ArqIwTrHPgl9QBSEiFMO9gwMgmZctrABBjV7XBsCtWcLSrgh8Z4eCBlp1Egff0OI1M9KMfgcSvnypG3kDuVTkqTgbWgMPlHSlje9Gs= 5 | -------------------------------------------------------------------------------- /spec/thingfish/metastore_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../helpers' 4 | 5 | require 'rspec' 6 | require 'thingfish/metastore' 7 | 8 | class TestingMetastore < Thingfish::Metastore 9 | end 10 | 11 | 12 | RSpec.describe Thingfish::Metastore do 13 | 14 | before( :all ) do 15 | setup_logging() 16 | end 17 | 18 | 19 | it "is abstract" do 20 | expect { described_class.new }.to raise_error( NoMethodError, /private/i ) 21 | end 22 | 23 | 24 | it "acts as a factory for its concrete derivatives" do 25 | expect( described_class.create('testing') ).to be_a( TestingMetastore ) 26 | end 27 | 28 | 29 | describe "an instance of a concrete derivative" do 30 | 31 | let( :store ) { described_class.create('testing') } 32 | 33 | it "raises an error if it doesn't implement #oids" do 34 | expect { store.oids }.to raise_error( NotImplementedError, /oids/ ) 35 | end 36 | 37 | it "raises an error if it doesn't implement #each_oid" do 38 | expect { store.each_oid }.to raise_error( NotImplementedError, /each_oid/ ) 39 | end 40 | 41 | it "raises an error if it doesn't implement #fetch" do 42 | expect { store.fetch(TEST_UUID) }.to raise_error( NotImplementedError, /fetch/ ) 43 | end 44 | 45 | it "raises an error if it doesn't implement #fetch_value" do 46 | expect { store.fetch_value(TEST_UUID, :format) }. 47 | to raise_error( NotImplementedError, /fetch_value/ ) 48 | end 49 | 50 | it "raises an error if it doesn't implement #search" do 51 | expect { store.search(limit: 100) }.to raise_error( NotImplementedError, /search/ ) 52 | end 53 | 54 | it "raises an error if it doesn't implement #save" do 55 | expect { 56 | store.save( TEST_UUID, {name: 'foo'} ) 57 | }.to raise_error( NotImplementedError, /save/ ) 58 | end 59 | 60 | it "raises an error if it doesn't implement #merge" do 61 | expect { 62 | store.merge( TEST_UUID, {name: 'foo'} ) 63 | }.to raise_error( NotImplementedError, /merge/ ) 64 | end 65 | 66 | it "raises an error if it doesn't implement #include?" do 67 | expect { store.include?(TEST_UUID) }.to raise_error( NotImplementedError, /include\?/ ) 68 | end 69 | 70 | it "raises an error if it doesn't implement #remove" do 71 | expect { store.remove(TEST_UUID) }.to raise_error( NotImplementedError, /remove/ ) 72 | end 73 | 74 | it "raises an error if it doesn't implement #remove_except" do 75 | expect { store.remove_except(TEST_UUID, :format) }. 76 | to raise_error( NotImplementedError, /remove_except/ ) 77 | end 78 | 79 | it "raises an error if it doesn't implement #size" do 80 | expect { store.size }.to raise_error( NotImplementedError, /size/ ) 81 | end 82 | 83 | it "raises an error if it doesn't implement #fetch_related_oids" do 84 | expect { 85 | store.fetch_related_oids( TEST_UUID ) 86 | }.to raise_error( NotImplementedError, /fetch_related_oids/i ) 87 | end 88 | 89 | it "provides a transactional block method" do 90 | expect {|block| store.transaction(&block) }.to yield_with_no_args 91 | end 92 | end 93 | 94 | end 95 | 96 | # vim: set nosta noet ts=4 sw=4 ft=rspec: 97 | -------------------------------------------------------------------------------- /thingfish.gemspec: -------------------------------------------------------------------------------- 1 | # -*- frozen_string_literal: true -*- 2 | # stub: thingfish 0.8.0.pre.20200304144036 ruby lib 3 | 4 | Gem::Specification.new do |s| 5 | s.name = "thingfish".freeze 6 | s.version = "0.8.0.pre.20200304144036" 7 | 8 | s.required_rubygems_version = Gem::Requirement.new("> 1.3.1".freeze) if s.respond_to? :required_rubygems_version= 9 | s.metadata = { "bug_tracker_uri" => "https://todo.sr.ht/~ged/thingfish", "changelog_uri" => "https://thing.fish/docs/History_md.html", "documentation_uri" => "https://thing.fish/docs/", "homepage_uri" => "https://thing.fish", "source_uri" => "https://hg.sr.ht/~ged/thingfish" } if s.respond_to? :metadata= 10 | s.require_paths = ["lib".freeze] 11 | s.authors = ["Michael Granger".freeze, "Mahlon E. Smith".freeze] 12 | s.date = "2020-03-04" 13 | s.description = "Thingfish is a extensible, web-based digital asset manager. It can be used to store chunks of data on the network in an application-independent way, link the chunks together with metadata, and then search for the chunk you need later and fetch it, all through a REST API.".freeze 14 | s.email = ["ged@FaerieMUD.org".freeze, "mahlon@martini.nu".freeze] 15 | s.executables = ["thingfish".freeze] 16 | s.files = [".simplecov".freeze, "History.md".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "bin/thingfish".freeze, "lib/strelka/app/metadata.rb".freeze, "lib/strelka/apps.rb".freeze, "lib/strelka/httprequest/metadata.rb".freeze, "lib/thingfish.rb".freeze, "lib/thingfish/behaviors.rb".freeze, "lib/thingfish/datastore.rb".freeze, "lib/thingfish/datastore/memory.rb".freeze, "lib/thingfish/handler.rb".freeze, "lib/thingfish/metastore.rb".freeze, "lib/thingfish/metastore/memory.rb".freeze, "lib/thingfish/mixins.rb".freeze, "lib/thingfish/processor.rb".freeze, "lib/thingfish/processor/sha256.rb".freeze, "lib/thingfish/spechelpers.rb".freeze, "spec/data/APIC-1-image.mp3".freeze, "spec/data/APIC-2-images.mp3".freeze, "spec/data/PIC-1-image.mp3".freeze, "spec/data/PIC-2-images.mp3".freeze, "spec/helpers.rb".freeze, "spec/spec.opts".freeze, "spec/thingfish/datastore/memory_spec.rb".freeze, "spec/thingfish/datastore_spec.rb".freeze, "spec/thingfish/handler_spec.rb".freeze, "spec/thingfish/metastore/memory_spec.rb".freeze, "spec/thingfish/metastore_spec.rb".freeze, "spec/thingfish/mixins_spec.rb".freeze, "spec/thingfish/processor/sha256_spec.rb".freeze, "spec/thingfish/processor_spec.rb".freeze, "spec/thingfish_spec.rb".freeze] 17 | s.homepage = "https://thing.fish".freeze 18 | s.licenses = ["BSD-3-Clause".freeze] 19 | s.rubygems_version = "3.1.2".freeze 20 | s.summary = "Thingfish is a extensible, web-based digital asset manager.".freeze 21 | 22 | if s.respond_to? :specification_version then 23 | s.specification_version = 4 24 | end 25 | 26 | if s.respond_to? :add_runtime_dependency then 27 | s.add_runtime_dependency(%q.freeze, ["~> 0.14"]) 28 | s.add_development_dependency(%q.freeze, ["~> 0.10"]) 29 | s.add_development_dependency(%q.freeze, ["~> 0.4"]) 30 | s.add_development_dependency(%q.freeze, ["~> 0.18"]) 31 | else 32 | s.add_dependency(%q.freeze, ["~> 0.14"]) 33 | s.add_dependency(%q.freeze, ["~> 0.10"]) 34 | s.add_dependency(%q.freeze, ["~> 0.4"]) 35 | s.add_dependency(%q.freeze, ["~> 0.18"]) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /experiments/Processors.rdoc: -------------------------------------------------------------------------------- 1 | = ThingFish Processors 2 | 3 | == Processor Objects 4 | 5 | Processors are programs that modify or enhance uploaded assets in one or more of three ways: 6 | 7 | 1. Modify uploaded assets synchronously before they're saved 8 | 2. Modify assets synchronously on the fly when they're downloaded 9 | 3. Modify uploaded assets asynchronously after they're saved 10 | 11 | 12 | The basic Processor interface is: 13 | 14 | class Thingfish::Processor 15 | 16 | def process_request( request ) 17 | # No-op by default 18 | end 19 | 20 | def process_response( response ) 21 | # No-op by default 22 | end 23 | 24 | def process( io ) 25 | # No-op by default 26 | end 27 | 28 | end 29 | 30 | 31 | The first two cases run inside the Thingfish daemon, and the third runs inside an asynchronous 32 | processor daemon. Thingfish writes the UUID of every newly-uploaded asset to a PUB zeromq endpoint, 33 | so asynchronous processing systems need only SUBscribe to that endpoint, and dispatch processing 34 | jobs to one or more daemons when a new uuid is read. 35 | 36 | 37 | === Request (Synchronous) Processor 38 | 39 | An example of a synchronous upload processor adds metadata extracted from the ID3 tags of uploaded 40 | MP3 files: 41 | 42 | class Thingfish::ID3Processor < Thingfish::Processor 43 | 44 | def handle_request( request ) 45 | return unless request.content_type == 'audio/mp3' 46 | 47 | mp3 = request.body.read 48 | metadata = extract_some_id3_shit( mp3 ) 49 | request.add_metadata( metadata ) 50 | end 51 | 52 | end 53 | 54 | 55 | === Request/Response (Synchronous) Processor 56 | 57 | An example of a processor that adds two watermarks to images: 58 | 59 | * when an image is uploaded, it adds an invisible watermark to the image data, which then becomes a 60 | permanent part of the asset. 61 | * when an image is downloaded, it adds a visible watermark to the image if it's being downloaded 62 | from an external site. 63 | 64 | class Thingfish::WaterMarker < Thingfish::Processor 65 | 66 | def handle_request( request ) 67 | return unless request.content_type =~ /^image/ 68 | 69 | image = request.body.read 70 | watermarked_image = add_invisible_watermark( image ) 71 | request.body.rewind 72 | request.body.write( watermarked_image ) 73 | end 74 | 75 | def handle_response( response ) 76 | return unless response.content_type =~ /^image/ && 77 | !from_internal_network( response ) 78 | 79 | image = response.body.read 80 | watermarked_image = add_visible_watermark( image ) 81 | response.body.rewind 82 | response.body.write( watermarked_image ) 83 | end 84 | 85 | end 86 | 87 | 88 | === Asynchronous-only Processor 89 | 90 | This is an example of a processor that adds a thumbnail to videos that have been uploaded 91 | based on the video's keyframes. It doesn't touch uploads or downloads, but since it's 92 | time-consuming, it can be run asynchronously at any point after the upload. 93 | 94 | class Thingfish::VideoProcessor < Thingfish::Processor 95 | 96 | def process_async( asset ) 97 | asset.metadata[:thumbnail] = extract_keyframe( asset.data ) 98 | end 99 | 100 | end 101 | 102 | 103 | 104 | request.add_related_resource( ) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Thingfish 2 | 3 | home 4 | : https://thing.fish 5 | 6 | code 7 | : https://hg.sr.ht/~ged/thingfish 8 | 9 | docs 10 | : https://thing.fish/docs/ 11 | 12 | 13 | ## Description 14 | 15 | Thingfish is a extensible, web-based digital asset manager. It can be used to 16 | store chunks of data on the network in an application-independent way, link the 17 | chunks together with metadata, and then search for the chunk you need later and 18 | fetch it, all through a REST API. 19 | 20 | 21 | ## Requirements 22 | 23 | Thingfish is written in ruby, and is tested using [version 2.7](http://www.ruby-lang.org/en/downloads/). Other versions may work, 24 | but are not tested. 25 | 26 | 27 | ## Installation 28 | 29 | You can install Thingfish via Rubygems: 30 | 31 | $ gem install thingfish 32 | 33 | This will install the basic server and its dependencies. Additional functionality is available via separate gems in the following namespaces: 34 | 35 | `thingfish-metastore-*` 36 | : Storage backends for resource metadata 37 | 38 | `thingfish-filestore-*` 39 | : Storage backends for resources themselves 40 | 41 | `thingfish-processor-*` 42 | : Filters and extractors for resources 43 | 44 | 45 | ## Contributing 46 | 47 | You can check out the current development source 48 | {with Mercurial}[http://bitbucket.org/ged/thingfish], or 49 | if you prefer Git, via the project's 50 | {Github mirror}[https://github.com/ged/thingfish]. 51 | 52 | After checking out the source, run: 53 | 54 | $ rake newb 55 | 56 | This task will install any missing dependencies, run the tests/specs, and 57 | generate the API documentation. 58 | 59 | You can submit bug reports, suggestions, and read more about future plans at 60 | {the project page}[http://bitbucket.org/ged/thingfish]. 61 | 62 | 63 | ## Authors 64 | 65 | * Michael Granger 66 | * Mahlon E. Smith 67 | 68 | 69 | ## Contributors 70 | 71 | * Jeremiah Jordan 72 | * Ben Bleything 73 | * Jeff Davis 74 | 75 | 76 | ## License 77 | 78 | Copyright (c) 2007-2020, Michael Granger and Mahlon E. Smith 79 | All rights reserved. 80 | 81 | Redistribution and use in source and binary forms, with or without 82 | modification, are permitted provided that the following conditions are met: 83 | 84 | * Redistributions of source code must retain the above copyright notice, 85 | this list of conditions and the following disclaimer. 86 | 87 | * Redistributions in binary form must reproduce the above copyright notice, 88 | this list of conditions and the following disclaimer in the documentation 89 | and/or other materials provided with the distribution. 90 | 91 | * Neither the name of the author/s, nor the names of the project's 92 | contributors may be used to endorse or promote products derived from this 93 | software without specific prior written permission. 94 | 95 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 96 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 97 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 98 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 99 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 100 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 101 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 102 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 103 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 104 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 105 | 106 | 107 | -------------------------------------------------------------------------------- /experiments/ragel_search_parsing.pl: -------------------------------------------------------------------------------- 1 | # vim: set noet nosta sw=4 ts=4 ft=ragel : 2 | # 3 | # title:Mahl*%20Smith,(format:(image/jpeg|image/png),extent:<100|format:image/icns) 4 | # 5 | # [ 6 | # ['title', 'Mahl* Smith' ], 7 | # [:or, 8 | # ['format', [:or, [ 9 | # 'image/jpeg', 10 | # 'image/png' 11 | # ]], 12 | # [:lt, 'extent', '100'], 13 | # ], 14 | # ['format', 'image/icns'] 15 | # ] 16 | # ] 17 | 18 | 19 | %%{ 20 | machine filter_to_sexp; 21 | 22 | action set_mark { mark = p; debug "set mark to %d" % [ p ] } 23 | action key_end { key = filter.extract( mark, p ); debug( key ); } 24 | action val_end { debug("k:%p v:%p" % [ key, filter.extract(mark, p) ]) } 25 | 26 | action and { debug 'AND' } 27 | action or { debug 'OR' } 28 | 29 | action popen { pdepth = pdepth + 1 } 30 | action pclose { pdepth = pdepth - 1 } 31 | 32 | uri_valid_chars = ( [a-zA-z0-9\-*%\/\.] ); # anything else requires URI encoding 33 | 34 | popen = '(' >popen; 35 | pclose = ')' >pclose; 36 | parens = ( popen | pclose ); 37 | 38 | or = '|' >or; 39 | and = ',' >and; 40 | gt = '>'; 41 | lt = '<'; 42 | oper = ( gt | lt ); 43 | bool = ( and | or ); 44 | 45 | key = ( [a-zA-Z0-9\-]+ . ':' ) >set_mark %key_end; 46 | value = ( uri_valid_chars )+ >set_mark %val_end; 47 | 48 | # TODO: friggen everything of consequence 49 | main:= key value ( bool key value )* %{ debug "filter complete"; filter.valid = true }; 50 | 51 | # main:= |* 52 | # key value => { filter.value = true }; 53 | # *|; 54 | }%% 55 | 56 | 57 | require 'pp' 58 | 59 | ### 60 | ### 61 | class QueryFilter 62 | 63 | # FIXME: what to do on parse errors? raise? or just set invalid? 64 | class Error < RuntimeError; end 65 | class ParseError < Error; end 66 | 67 | # Ragel accessors are injected into the class. 68 | %% write data; 69 | 70 | ######################################################################## 71 | ### C L A S S M E T H O D S 72 | ######################################################################## 73 | 74 | ### Parse a filter string into an S-Expression. 75 | ### 76 | def self::parse( filter_str ) 77 | #ts = te = act = 0 78 | key = '' 79 | 80 | filter = new( filter_str ) 81 | data = filter.data 82 | 83 | mark = 0 84 | pdepth = 0 85 | sexp = [] 86 | 87 | %% write init; 88 | eof = pe 89 | %% write exec; 90 | 91 | filter.valid = false if filter.valid && ! pdepth.zero? 92 | # raise ParseError, "Unmatched parenthesis" unless pdepth.zero? 93 | 94 | self.debug "%p" % [ filter ] 95 | filter.extract( 0, 5 ) 96 | 97 | filter.instance_variable_set( :@sexp, sexp ) 98 | return filter 99 | end 100 | 101 | def self::debug( msg ) 102 | $stderr.puts " #{msg}" if $DEBUG 103 | end 104 | 105 | 106 | ######################################################################## 107 | ### I N S T A N C E M E T H O D S 108 | ######################################################################## 109 | 110 | ### Instantiate a new QueryFilter, provided a +filter+ string. 111 | ### 112 | private_class_method :new 113 | def initialize( filter ) # :nodoc: 114 | @str = filter 115 | @data = filter.to_s.unpack( 'c*' ) 116 | @valid = false 117 | end 118 | 119 | 120 | # The array of character values, as signed 8-bit integers. 121 | attr_reader :data 122 | 123 | # Is the filter string parsable? 124 | attr_accessor :valid 125 | 126 | ### Stringify the filter (returning the original argument.) 127 | ### 128 | def to_s 129 | return @str 130 | end 131 | 132 | 133 | ### Return the S-Expression of the filter. 134 | ### 135 | def to_sexp 136 | return @s_exp 137 | end 138 | 139 | 140 | ### Inspection string. 141 | ### 142 | def inspect 143 | return "<%s:0x%08x filter:%p valid:%p>" % [ 144 | self.class.name, 145 | self.object_id * 2, 146 | self.to_s, 147 | self.valid 148 | ] 149 | end 150 | 151 | 152 | ### Given a start and ending scanner position, 153 | ### return an ascii representation of the data slice. 154 | ### 155 | def extract( start, fin ) 156 | slice = @data[ start, fin ] 157 | return '' unless slice 158 | return slice.pack( 'c*' ) 159 | end 160 | end 161 | 162 | 163 | 164 | 165 | 166 | while str = gets 167 | str.chomp! 168 | begin 169 | qf = QueryFilter.parse( str ) 170 | 171 | rescue => err 172 | puts "%s -> %s" % [ err.class.name, err.message ] 173 | end 174 | end 175 | 176 | -------------------------------------------------------------------------------- /lib/thingfish/metastore/memory.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'thingfish' unless defined?( Thingfish ) 5 | require 'thingfish/metastore' unless defined?( Thingfish::Metastore ) 6 | 7 | 8 | 9 | # An in-memory metastore for testing and tryout purposes. 10 | class Thingfish::Metastore::Memory < Thingfish::Metastore 11 | extend Loggability 12 | include Thingfish::Normalization 13 | 14 | # Loggability API -- log to the :thingfish logger 15 | log_to :thingfish 16 | 17 | 18 | ### Create a new MemoryMetastore, using the given +storage+ object to store 19 | ### data in. The +storage+ should quack like a Hash. 20 | def initialize( storage={} ) 21 | @storage = storage 22 | end 23 | 24 | 25 | ## 26 | # The raw Hash of metadata 27 | attr_reader :storage 28 | 29 | 30 | ### Return an Array of all stored OIDs. 31 | def oids 32 | return @storage.keys 33 | end 34 | 35 | 36 | ### Iterate over the OID of each entry in the store, yielding to the block if one is given 37 | ### or returning an Enumerator if one is not. 38 | def each_oid( &block ) 39 | return @storage.each_key( &block ) 40 | end 41 | 42 | 43 | ### Save the +metadata+ Hash for the specified +oid+. 44 | def save( oid, metadata ) 45 | oid = normalize_oid( oid ) 46 | @storage[ oid ] = metadata.dup 47 | end 48 | 49 | 50 | ### Fetch the data corresponding to the given +oid+ as a Hash-ish object. 51 | def fetch( oid, *keys ) 52 | oid = normalize_oid( oid ) 53 | metadata = @storage[ oid ] or return nil 54 | 55 | if keys.empty? 56 | self.log.debug "Fetching metadata for OID %s" % [ oid ] 57 | return metadata.dup 58 | else 59 | self.log.debug "Fetching metadata for %p for OID %s" % [ keys, oid ] 60 | keys = normalize_keys( keys ) 61 | values = metadata.values_at( *keys ) 62 | return Hash[ [keys, values].transpose ] 63 | end 64 | end 65 | 66 | 67 | ### Fetch the value of the metadata associated with the given +key+ for the 68 | ### specified +oid+. 69 | def fetch_value( oid, key ) 70 | oid = normalize_oid( oid ) 71 | key = normalize_key( key ) 72 | data = @storage[ oid ] or return nil 73 | 74 | return data[ key ] 75 | end 76 | 77 | 78 | ### Fetch OIDs related to the given +oid+. 79 | def fetch_related_oids( oid ) 80 | oid = normalize_oid( oid ) 81 | self.log.debug "Fetching OIDs of resources related to %s" % [ oid ] 82 | return self.search( :criteria => {:relation => oid}, :include_related => true ) 83 | end 84 | 85 | 86 | ### Search the metastore for UUIDs which match the specified +criteria+ and 87 | ### return them as an iterator. 88 | def search( options={} ) 89 | ds = @storage.each_key 90 | self.log.debug "Starting search with %p" % [ ds ] 91 | 92 | ds = self.omit_related_resources( ds, options ) 93 | ds = self.apply_search_criteria( ds, options ) 94 | ds = self.apply_search_order( ds, options ) 95 | ds = self.apply_search_direction( ds, options ) 96 | ds = self.apply_search_limit( ds, options ) 97 | 98 | return ds.to_a 99 | end 100 | 101 | 102 | ### Omit related resources from the search dataset +ds+ unless the given 103 | ### +options+ specify otherwise. 104 | def omit_related_resources( ds, options ) 105 | unless options[:include_related] 106 | ds = ds.reject {|uuid| @storage[uuid]['relationship'] } 107 | end 108 | return ds 109 | end 110 | 111 | 112 | ### Apply the search :criteria from the specified +options+ to the collection 113 | ### in +ds+ and return the modified dataset. 114 | def apply_search_criteria( ds, options ) 115 | if (( criteria = options[:criteria] )) 116 | criteria.each do |field, value| 117 | self.log.debug " applying criteria: %p => %p" % [ field.to_s, value ] 118 | ds = ds.select {|uuid| @storage[uuid][field.to_s] == value } 119 | end 120 | end 121 | 122 | return ds 123 | end 124 | 125 | 126 | ### Apply the search :order from the specified +options+ to the collection in 127 | ### +ds+ and return the modified dataset. 128 | def apply_search_order( ds, options ) 129 | if (( fields = options[:order] )) 130 | ds = ds.to_a.sort_by do |uuid| 131 | @storage[ uuid ].values_at( *fields.compact ).map {|val| val || ''} 132 | end 133 | end 134 | 135 | return ds 136 | end 137 | 138 | 139 | ### Apply the search :direction from the specified +options+ to the collection 140 | ### in +ds+ and return the modified dataset. 141 | def apply_search_direction( ds, options ) 142 | ds.reverse! if options[:direction] && options[:direction] == 'desc' 143 | return ds 144 | end 145 | 146 | 147 | ### Apply the search :limit from the specified +options+ to the collection in 148 | ### +ds+ and return the modified dataset. 149 | def apply_search_limit( ds, options ) 150 | if (( limit = options[:limit] )) 151 | self.log.debug " limiting to %s results" % [ limit ] 152 | offset = options[:offset] || 0 153 | ds = ds.to_a.slice( offset, limit ) 154 | end 155 | 156 | return ds 157 | end 158 | 159 | 160 | ### Update the metadata for the given +oid+ with the specified +values+ hash. 161 | def merge( oid, values ) 162 | oid = normalize_oid( oid ) 163 | values = normalize_keys( values ) 164 | @storage[ oid ].merge!( values ) 165 | end 166 | 167 | 168 | ### Remove all metadata associated with +oid+ from the Metastore. 169 | def remove( oid, *keys ) 170 | oid = normalize_oid( oid ) 171 | if keys.empty? 172 | @storage.delete( oid ) 173 | else 174 | keys = normalize_keys( keys ) 175 | @storage[ oid ].delete_if {|key, _| keys.include?(key) } 176 | end 177 | end 178 | 179 | 180 | ### Remove all metadata associated with +oid+ except for the specified +keys+. 181 | def remove_except( oid, *keys ) 182 | oid = normalize_oid( oid ) 183 | keys = normalize_keys( keys ) 184 | @storage[ oid ].keep_if {|key,_| keys.include?(key) } 185 | end 186 | 187 | 188 | ### Returns +true+ if the metastore has metadata associated with the specified +oid+. 189 | def include?( oid ) 190 | oid = normalize_oid( oid ) 191 | return @storage.include?( oid ) 192 | end 193 | 194 | 195 | ### Returns the number of objects the store contains. 196 | def size 197 | return @storage.size 198 | end 199 | 200 | end # class Thingfish::Metastore::Memory 201 | 202 | -------------------------------------------------------------------------------- /lib/thingfish/spechelpers.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | # vim: set nosta noet ts=4 sw=4 ft=ruby: 4 | 5 | require 'time' 6 | require 'thingfish' 7 | require 'rspec' 8 | 9 | 10 | ### RSpec helper functions. 11 | module Thingfish::SpecHelpers 12 | 13 | module Constants 14 | TEST_APPID = 'thingfish-test' 15 | TEST_SEND_SPEC = 'tcp://127.0.0.1:9999' 16 | TEST_RECV_SPEC = 'tcp://127.0.0.1:9998' 17 | 18 | UUID_PATTERN = /[[:xdigit:]]{8}(?:-[[:xdigit:]]{4}){3}-[[:xdigit:]]{12}/i 19 | 20 | TEST_UUID = 'E5DFEEAB-3525-4F14-B4DB-2772D0B9987F' 21 | 22 | TEST_TEXT_DATA = "Pork sausage. Pork! Sausage!".b 23 | TEST_TEXT_DATA_IO = StringIO.new( TEST_TEXT_DATA ) 24 | TEST_PNG_DATA = ("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMA" + 25 | "AQAABQABDQottAAAAABJRU5ErkJggg==").unpack('m').first 26 | TEST_PNG_DATA_IO = StringIO.new( TEST_PNG_DATA ) 27 | 28 | TEST_METADATA = [ 29 | {"useragent" => "ChunkersTheClown v2.0", 30 | "extent" => 1072, 31 | "uploadaddress" => "127.0.0.1", 32 | "format" => "application/rtf", 33 | "created" => Time.parse('2010-10-14 00:08:21 UTC'), 34 | "title" => "How to use the Public folder.rtf"}, 35 | {"useragent" => "ChunkersTheClown v2.0", 36 | "extent" => 832604, 37 | "uploadaddress" => "127.0.0.1", 38 | "format" => "image/jpeg", 39 | "created" => Time.parse('2011-09-06 20:10:54 UTC'), 40 | "title" => "IMG_0316.JPG"}, 41 | {"useragent" => "ChunkersTheClown v2.0", 42 | "extent" => 2253642, 43 | "uploadaddress" => "127.0.0.1", 44 | "format" => "image/jpeg", 45 | "created" => Time.parse('2011-09-06 20:10:49 UTC'), 46 | "title" => "IMG_0544.JPG"}, 47 | {"useragent" => "ChunkersTheClown v2.0", 48 | "extent" => 694785, 49 | "uploadaddress" => "127.0.0.1", 50 | "format" => "image/jpeg", 51 | "created" => Time.parse('2011-09-06 20:10:52 UTC'), 52 | "title" => "IMG_0552.JPG"}, 53 | {"useragent" => "ChunkersTheClown v2.0", 54 | "extent" => 1579773, 55 | "uploadaddress" => "127.0.0.1", 56 | "format" => "image/jpeg", 57 | "created" => Time.parse('2011-09-06 20:10:56 UTC'), 58 | "title" => "IMG_0748.JPG"}, 59 | {"useragent" => "ChunkersTheClown v2.0", 60 | "extent" => 6464493, 61 | "uploadaddress" => "127.0.0.1", 62 | "format" => "image/jpeg", 63 | "created" => Time.parse('2011-10-14 05:05:23 UTC'), 64 | "title" => "IMG_1700.JPG"}, 65 | {"useragent" => "ChunkersTheClown v2.0", 66 | "extent" => 388727, 67 | "uploadaddress" => "127.0.0.1", 68 | "format" => "image/jpeg", 69 | "created" => Time.parse('2011-12-28 01:23:27 UTC'), 70 | "title" => "IMG_3553.jpg"}, 71 | {"useragent" => "ChunkersTheClown v2.0", 72 | "extent" => 1354, 73 | "uploadaddress" => "127.0.0.1", 74 | "format" => "text/plain", 75 | "created" => Time.parse('2013-09-09 15:43:31 UTC'), 76 | "title" => "agilemanifesto.txt"}, 77 | {"useragent" => "ChunkersTheClown v2.0", 78 | "extent" => 3059035, 79 | "uploadaddress" => "127.0.0.1", 80 | "format" => "image/jpeg", 81 | "created" => Time.parse('2013-04-18 00:25:56 UTC'), 82 | "title" => "bacon.jpg"}, 83 | {"useragent" => "ChunkersTheClown v2.0", 84 | "extent" => 71860, 85 | "uploadaddress" => "127.0.0.1", 86 | "format" => "image/jpeg", 87 | "created" => Time.parse('2011-09-06 20:10:57 UTC'), 88 | "title" => "boom.jpg"}, 89 | {"useragent" => "ChunkersTheClown v2.0", 90 | "extent" => 2115410, 91 | "uploadaddress" => "127.0.0.1", 92 | "format" => "audio/mp3", 93 | "created" => Time.parse('2013-09-09 15:42:49 UTC'), 94 | "title" => "craigslist_erotica.mp3"}, 95 | {"useragent" => "ChunkersTheClown v2.0", 96 | "extent" => 377445, 97 | "uploadaddress" => "127.0.0.1", 98 | "format" => "image/jpeg", 99 | "created" => Time.parse('2012-02-09 17:06:44 UTC'), 100 | "title" => "cubes.jpg"}, 101 | {"useragent" => "ChunkersTheClown v2.0", 102 | "extent" => 240960, 103 | "uploadaddress" => "127.0.0.1", 104 | "format" => "audio/mp3", 105 | "created" => Time.parse('2013-09-09 15:42:58 UTC'), 106 | "title" => "gay_clowns.mp3"}, 107 | {"useragent" => "ChunkersTheClown v2.0", 108 | "extent" => 561792, 109 | "uploadaddress" => "127.0.0.1", 110 | "format" => "image/jpeg", 111 | "created" => Time.parse('2011-09-06 20:10:57 UTC'), 112 | "title" => "aaaaaaaa"}, 113 | {"useragent" => "ChunkersTheClown v2.0", 114 | "extent" => 1104950, 115 | "uploadaddress" => "127.0.0.1", 116 | "format" => "image/jpeg", 117 | "created" => Time.parse('2013-09-09 15:37:25 UTC'), 118 | "title" => "joss.jpg"}, 119 | {"useragent" => "ChunkersTheClown v2.0", 120 | "extent" => 163, 121 | "uploadaddress" => "127.0.0.1", 122 | "format" => "text/plain", 123 | "created" => Time.parse('2013-01-23 07:52:44 UTC'), 124 | "title" => "macbook.txt"}, 125 | {"useragent" => "ChunkersTheClown v2.0", 126 | "extent" => 2130567, 127 | "uploadaddress" => "127.0.0.1", 128 | "format" => "image/png", 129 | "created" => Time.parse('2012-03-15 05:15:07 UTC'), 130 | "title" => "marbles.png"}, 131 | {"useragent" => "ChunkersTheClown v2.0", 132 | "extent" => 8971, 133 | "uploadaddress" => "127.0.0.1", 134 | "format" => "image/gif", 135 | "created" => Time.parse('2013-01-15 19:15:35 UTC'), 136 | "title" => "trusttom.GIF"} 137 | ].freeze 138 | TEST_METADATA.each {|hash| hash.freeze } 139 | 140 | 141 | end # module Constants 142 | 143 | include Constants 144 | 145 | 146 | # Load fixture data from the ThingFish spec data directory 147 | FIXTURE_DIR = Pathname( __FILE__ ).dirname.parent.parent + 'spec/data' 148 | 149 | 150 | RSpec::Matchers.define :be_a_uuid do |expected| 151 | match do |actual| 152 | actual =~ UUID_PATTERN 153 | end 154 | end 155 | 156 | 157 | ### Load and return the data from the fixture with the specified +filename+. 158 | def fixture_data( filename ) 159 | fixture = FIXTURE_DIR + filename 160 | return fixture.open( 'r', encoding: 'binary' ) 161 | end 162 | 163 | end # Thingfish::SpecHelpers 164 | 165 | 166 | -------------------------------------------------------------------------------- /lib/thingfish/behaviors.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'rspec' 5 | 6 | require 'thingfish/handler' 7 | 8 | 9 | RSpec.shared_examples "a Thingfish metastore" do 10 | 11 | let( :metastore ) do 12 | Thingfish::Metastore.create( described_class ) 13 | end 14 | 15 | 16 | it "can save and fetch data" do 17 | metastore.save( TEST_UUID, TEST_METADATA.first ) 18 | expect( metastore.fetch(TEST_UUID) ).to eq( TEST_METADATA.first ) 19 | expect( metastore.fetch(TEST_UUID) ).to_not be( TEST_METADATA.first ) 20 | end 21 | 22 | 23 | it "returns nil when fetching metadata for an object that doesn't exist" do 24 | expect( metastore.fetch(TEST_UUID) ).to be_nil 25 | end 26 | 27 | 28 | it "doesn't care about the case of the UUID when saving and fetching data" do 29 | metastore.save( TEST_UUID.downcase, TEST_METADATA.first.freeze ) 30 | expect( metastore.fetch(TEST_UUID) ).to eq( TEST_METADATA.first ) 31 | end 32 | 33 | 34 | it "can fetch a single metadata value for a given oid" do 35 | metastore.save( TEST_UUID, TEST_METADATA.first ) 36 | expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( TEST_METADATA.first['format'] ) 37 | expect( metastore.fetch_value(TEST_UUID, :extent) ).to eq( TEST_METADATA.first['extent'] ) 38 | end 39 | 40 | 41 | it "can fetch a slice of data for a given oid" do 42 | metastore.save( TEST_UUID, TEST_METADATA.first ) 43 | expect( metastore.fetch(TEST_UUID, :format, :extent) ).to eq({ 44 | 'format' => TEST_METADATA.first['format'], 45 | 'extent' => TEST_METADATA.first['extent'], 46 | }) 47 | end 48 | 49 | 50 | it "returns nil when fetching a slice of data for an object that doesn't exist" do 51 | expect( metastore.fetch_value(TEST_UUID, :format) ).to be_nil 52 | end 53 | 54 | 55 | it "doesn't care about the case of the UUID when fetching data" do 56 | metastore.save( TEST_UUID, TEST_METADATA.first ) 57 | expect( metastore.fetch_value(TEST_UUID.downcase, :format) ).to eq( TEST_METADATA.first['format'] ) 58 | end 59 | 60 | 61 | it "can update data" do 62 | metastore.save( TEST_UUID, TEST_METADATA.first ) 63 | metastore.merge( TEST_UUID, format: 'image/jpeg' ) 64 | 65 | expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( 'image/jpeg' ) 66 | end 67 | 68 | 69 | it "doesn't care about the case of the UUID when updating data" do 70 | metastore.save( TEST_UUID, TEST_METADATA.first ) 71 | metastore.merge( TEST_UUID.downcase, format: 'image/jpeg' ) 72 | 73 | expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( 'image/jpeg' ) 74 | end 75 | 76 | 77 | it "can remove metadata for a UUID" do 78 | metastore.save( TEST_UUID, TEST_METADATA.first ) 79 | metastore.remove( TEST_UUID ) 80 | 81 | expect( metastore.fetch(TEST_UUID) ).to be_nil 82 | end 83 | 84 | 85 | it "can remove a single key/value pair from the metadata for a UUID" do 86 | metastore.save( TEST_UUID, TEST_METADATA.first ) 87 | metastore.remove( TEST_UUID, :useragent ) 88 | 89 | expect( metastore.fetch_value(TEST_UUID, :useragent) ).to be_nil 90 | end 91 | 92 | 93 | it "can truncate metadata not in a list of OIDs for a UUID" do 94 | keys = Thingfish::Handler::OPERATIONAL_METADATA_KEYS 95 | metastore.save( TEST_UUID, TEST_METADATA.first ) 96 | metastore.remove_except( TEST_UUID, *keys ) 97 | 98 | metadata = metastore.fetch( TEST_UUID ) 99 | expect( metadata.size ).to eq( keys.size ) 100 | expect( metadata.keys ).to include( *keys.map(&:to_s) ) 101 | end 102 | 103 | 104 | it "knows if it has data for a given OID" do 105 | metastore.save( TEST_UUID, TEST_METADATA.first ) 106 | expect( metastore ).to include( TEST_UUID ) 107 | end 108 | 109 | 110 | it "knows how many objects it contains" do 111 | expect( metastore.size ).to eq( 0 ) 112 | metastore.save( TEST_UUID, TEST_METADATA.first ) 113 | expect( metastore.size ).to eq( 1 ) 114 | end 115 | 116 | 117 | it "knows how to fetch UUIDs for related resources" do 118 | rel_uuid1 = SecureRandom.uuid 119 | rel_uuid2 = SecureRandom.uuid 120 | unrel_uuid = SecureRandom.uuid 121 | 122 | metastore.save( TEST_UUID, TEST_METADATA.first ) 123 | metastore.save( rel_uuid1, TEST_METADATA[1].merge('relation' => TEST_UUID.downcase) ) 124 | metastore.save( rel_uuid2, TEST_METADATA[2].merge('relation' => TEST_UUID.downcase) ) 125 | metastore.save( unrel_uuid, TEST_METADATA[3] ) 126 | 127 | uuids = metastore.fetch_related_oids( TEST_UUID ) 128 | 129 | expect( uuids ).to include( rel_uuid1, rel_uuid2 ) 130 | expect( uuids ).to_not include( unrel_uuid ) 131 | end 132 | 133 | 134 | context "with some uploaded metadata" do 135 | 136 | before( :each ) do 137 | @uuids = [] 138 | TEST_METADATA.each do |file| 139 | uuid = SecureRandom.uuid 140 | @uuids << uuid 141 | metastore.save( uuid, file ) 142 | end 143 | end 144 | 145 | 146 | it "can fetch an array of all of its OIDs" do 147 | expect( metastore.oids ).to eq( @uuids ) 148 | end 149 | 150 | it "can iterate over each of the store's oids" do 151 | uuids = [] 152 | metastore.each_oid {|u| uuids << u } 153 | 154 | expect( uuids ).to eq( @uuids ) 155 | end 156 | 157 | it "can provide an enumerator over each of the store's oids" do 158 | expect( metastore.each_oid.to_a ).to eq( @uuids ) 159 | end 160 | 161 | it "can search for uuids" do 162 | expect( metastore.search.to_a ).to eq( metastore.oids ) 163 | end 164 | 165 | it "can apply criteria to searches" do 166 | results = metastore.search( criteria: {format: 'audio/mp3'} ) 167 | expect( results.size ).to eq( 2 ) 168 | results.each do |uuid| 169 | expect( metastore.fetch_value(uuid, 'format') ).to eq( 'audio/mp3' ) 170 | end 171 | end 172 | 173 | it "can limit the number of results returned from a search" do 174 | expect( metastore.search( limit: 2 ).to_a ).to eq( metastore.oids[0,2] ) 175 | end 176 | 177 | it "can order the results returned from a search" do 178 | results = metastore.search( order: %w[title created] ).to_a 179 | sorted_uuids = metastore.each_oid. 180 | map {|oid| metastore.fetch(oid, :title, :created).merge(oid: oid) }. 181 | sort_by {|tuple| tuple.values_at('title', 'created') }. 182 | map {|tuple| tuple[:oid] } 183 | 184 | expect( results ).to eq( sorted_uuids ) 185 | end 186 | 187 | end 188 | 189 | 190 | end 191 | 192 | 193 | RSpec.shared_examples "a Thingfish datastore" do 194 | 195 | let( :png_io ) { StringIO.new(TEST_PNG_DATA.dup) } 196 | let( :text_io ) { StringIO.new(TEST_TEXT_DATA.dup) } 197 | 198 | 199 | it "returns a UUID when saving" do 200 | expect( store.save(png_io) ).to be_a_uuid() 201 | end 202 | 203 | 204 | it "restores the position of the IO after saving" do 205 | png_io.pos = 11 206 | store.save( png_io ) 207 | expect( png_io.pos ).to eq( 11 ) 208 | end 209 | 210 | 211 | it "can replace existing data" do 212 | new_uuid = store.save( text_io ) 213 | store.replace( new_uuid, png_io ) 214 | 215 | rval = store.fetch( new_uuid ) 216 | expect( rval ).to respond_to( :read ) 217 | expect( rval.read ).to eq( TEST_PNG_DATA ) 218 | end 219 | 220 | 221 | it "doesn't care about the case of the uuid when replacing" do 222 | new_uuid = store.save( text_io ) 223 | store.replace( new_uuid.upcase, png_io ) 224 | 225 | rval = store.fetch( new_uuid ) 226 | expect( rval ).to respond_to( :read ) 227 | expect( rval.read ).to eq( TEST_PNG_DATA ) 228 | end 229 | 230 | 231 | it "can fetch saved data" do 232 | oid = store.save( text_io ) 233 | rval = store.fetch( oid ) 234 | 235 | expect( rval ).to respond_to( :read ) 236 | expect( rval.external_encoding ).to eq( Encoding::ASCII_8BIT ) 237 | expect( rval.read ).to eq( TEST_TEXT_DATA ) 238 | end 239 | 240 | 241 | it "doesn't care about the case of the uuid when fetching" do 242 | oid = store.save( text_io ) 243 | rval = store.fetch( oid.upcase ) 244 | 245 | expect( rval ).to respond_to( :read ) 246 | expect( rval.read ).to eq( TEST_TEXT_DATA ) 247 | end 248 | 249 | 250 | it "can remove data" do 251 | oid = store.save( text_io ) 252 | store.remove( oid ) 253 | 254 | expect( store.fetch(oid) ).to be_nil 255 | end 256 | 257 | 258 | it "knows if it has data for a given OID" do 259 | oid = store.save( text_io ) 260 | expect( store ).to include( oid ) 261 | end 262 | 263 | end 264 | -------------------------------------------------------------------------------- /lib/thingfish/handler.rb: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | # frozen_string_literal: true 3 | 4 | require 'strelka' 5 | require 'strelka/app' 6 | 7 | require 'configurability' 8 | require 'loggability' 9 | 10 | require 'thingfish' unless defined?( Thingfish ) 11 | require 'thingfish/processor' 12 | 13 | # 14 | # Network-accessable datastore service 15 | # 16 | class Thingfish::Handler < Strelka::App 17 | extend Loggability, 18 | Configurability 19 | 20 | 21 | # Strelka App ID 22 | ID = 'thingfish' 23 | 24 | 25 | # Loggability API -- log to the :thingfish logger 26 | log_to :thingfish 27 | 28 | 29 | # Configurability API -- set config defaults 30 | CONFIG_DEFAULTS = { 31 | datastore: 'memory', 32 | metastore: 'memory', 33 | processors: [], 34 | event_socket_uri: 'tcp://127.0.0.1:3475', 35 | } 36 | 37 | # Metadata keys which aren't directly modifiable via the REST API 38 | # :TODO: Consider making either all of these or a subset of them 39 | # be immutable. 40 | OPERATIONAL_METADATA_KEYS = %w[ 41 | format 42 | extent 43 | created 44 | uploadaddress 45 | ] 46 | 47 | # Metadata keys that must be provided by plugins for related resources 48 | REQUIRED_RELATED_METADATA_KEYS = %w[ 49 | relationship 50 | format 51 | ] 52 | 53 | 54 | require 'thingfish/mixins' 55 | require 'thingfish/datastore' 56 | require 'thingfish/metastore' 57 | extend Strelka::MethodUtilities 58 | 59 | 60 | ## 61 | # Configurability API 62 | config_key :thingfish 63 | 64 | # The configured datastore type 65 | singleton_attr_accessor :datastore 66 | 67 | # The configured metastore type 68 | singleton_attr_accessor :metastore 69 | 70 | # The ZMQ socket for publishing various resource events. 71 | singleton_attr_accessor :event_socket_uri 72 | 73 | # The list of configured processors. 74 | singleton_attr_accessor :processors 75 | 76 | 77 | ### Load the Thingfish::Processors in the given +processor_list+ and return an instance 78 | ### of each one. 79 | def self::load_processors( processor_list ) 80 | self.log.info "Loading processors" 81 | processors = [] 82 | 83 | processor_list.each do |processor_type| 84 | begin 85 | processors << Thingfish::Processor.create( processor_type ) 86 | self.log.debug " loaded %s: %p" % [ processor_type, processors.last ] 87 | rescue LoadError => err 88 | self.log.error "%p: %s while loading the %s processor" % 89 | [ err.class, err.message, processor_type ] 90 | end 91 | end 92 | 93 | return processors 94 | end 95 | 96 | 97 | ### Configurability API -- install the configuration 98 | def self::configure( config=nil ) 99 | config = self.defaults.merge( config || {} ) 100 | 101 | self.datastore = config[:datastore] 102 | self.metastore = config[:metastore] 103 | self.event_socket_uri = config[:event_socket_uri] 104 | 105 | self.plugin( :filters ) # pre-load the filters plugin for deferred config 106 | 107 | self.processors = self.load_processors( config[:processors] ) 108 | self.processors.each do |processor| 109 | self.filter( :request, &processor.method(:process_request) ) 110 | self.filter( :response, &processor.method(:process_response) ) 111 | end 112 | end 113 | 114 | 115 | ### Set up the metastore, datastore, and event socket when the handler is 116 | ### created. 117 | def initialize( * ) # :notnew: 118 | super 119 | 120 | @datastore = Thingfish::Datastore.create( self.class.datastore ) 121 | @metastore = Thingfish::Metastore.create( self.class.metastore ) 122 | @event_socket = nil 123 | end 124 | 125 | 126 | ###### 127 | public 128 | ###### 129 | 130 | # The datastore 131 | attr_reader :datastore 132 | 133 | # The metastore 134 | attr_reader :metastore 135 | 136 | # The PUB socket on which resource events are published 137 | attr_reader :event_socket 138 | 139 | 140 | ### Run the handler -- overridden to set up the event socket on startup. 141 | def run 142 | self.setup_event_socket 143 | super 144 | end 145 | 146 | 147 | ### Set up the event socket. 148 | def setup_event_socket 149 | if self.class.event_socket_uri && ! @event_socket 150 | @event_socket = CZTop::Socket::PUB.new 151 | @event_socket.options.linger = 0 152 | @event_socket.bind( self.class.event_socket_uri ) 153 | end 154 | end 155 | 156 | 157 | ### Shutdown handler hook. 158 | def shutdown 159 | self.event_socket.close if self.event_socket 160 | super 161 | end 162 | 163 | 164 | ### Restart handler hook. 165 | def restart 166 | if self.event_socket 167 | oldsock = @event_socket 168 | @event_socket = @event_socket.dup 169 | oldsock.close 170 | end 171 | 172 | super 173 | end 174 | 175 | 176 | ######################################################################## 177 | ### P L U G I N S 178 | ######################################################################## 179 | 180 | # 181 | # Strelka plugin for Thingfish metadata 182 | # 183 | plugin :metadata 184 | 185 | 186 | # 187 | # Global params 188 | # 189 | plugin :parameters 190 | param :uuid 191 | param :key, :word 192 | param :limit, :integer, "The maximum number of records to return." 193 | param :offset, :integer, "The offset into the result set to use as the first result." 194 | param :order, /^(?[[:word:]]+(?:,[[:word:]]+)*)/, 195 | "The name(s) of the fields to order results by." 196 | param :direction, /^(asc|desc)$/i, "The order direction (ascending or descending)" 197 | param :casefold, :boolean, "Whether or not to convert to lowercase before matching" 198 | param :relationship, /^[\w\-]+$/, "The name of the relationship between two resources" 199 | 200 | 201 | # 202 | # Content negotiation 203 | # 204 | plugin :negotiation 205 | 206 | 207 | # 208 | # Filters 209 | # 210 | plugin :filters 211 | 212 | ### Modify outgoing headers on all responses to include version info. 213 | ### 214 | filter :response do |res| 215 | res.headers.x_thingfish = Thingfish.version_string( true ) 216 | end 217 | 218 | 219 | # 220 | # Routing 221 | # 222 | plugin :routing 223 | router :exclusive 224 | 225 | # GET /serverinfo 226 | # Return various information about the handler configuration. 227 | get '/serverinfo' do |req| 228 | res = req.response 229 | info = { 230 | :version => Thingfish.version_string( true ), 231 | :metastore => self.metastore.class.name, 232 | :datastore => self.datastore.class.name 233 | } 234 | 235 | self.check_resource_permissions( req ) 236 | res.for( :text, :json, :yaml ) { info } 237 | return res 238 | end 239 | 240 | 241 | # 242 | # Datastore routes 243 | # 244 | 245 | # GET / 246 | # Fetch a list of all objects 247 | get do |req| 248 | finish_with HTTP::BAD_REQUEST, req.params.error_messages.join(', ') unless req.params.okay? 249 | 250 | self.check_resource_permissions( req ) 251 | 252 | uuids = self.metastore.search( req.params.valid ) 253 | self.log.debug "UUIDs are: %p" % [ uuids ] 254 | 255 | base_uri = req.base_uri 256 | list = uuids.collect do |uuid| 257 | uri = base_uri.dup 258 | uri.path += '/' unless uri.path[-1] == '/' 259 | uri.path += uuid 260 | 261 | metadata = self.metastore.fetch( uuid ) 262 | metadata['uri'] = uri.to_s 263 | metadata['uuid'] = uuid 264 | 265 | metadata 266 | end 267 | 268 | res = req.response 269 | res.for( :json, :yaml ) { list } 270 | res.for( :text ) do 271 | list.collect {|entry| "%s [%s, %0.2fB]" % entry.values_at(:url, :format, :extent) } 272 | end 273 | 274 | return res 275 | end 276 | 277 | 278 | # GET /«uuid»/related 279 | # Fetch a list of all objects related to «uuid» 280 | get ':uuid/related' do |req| 281 | finish_with HTTP::BAD_REQUEST, req.params.error_messages.join(', ') unless req.params.okay? 282 | 283 | uuid = req.params[ :uuid ] 284 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 285 | 286 | uuids = self.metastore.fetch_related_oids( uuid ) 287 | self.log.debug "Related UUIDs are: %p" % [ uuids ] 288 | 289 | base_uri = req.base_uri 290 | list = uuids.collect do |uuid| 291 | uri = base_uri.dup 292 | uri.path += '/' unless uri.path[-1] == '/' 293 | uri.path += uuid 294 | 295 | metadata = self.metastore.fetch( uuid ) 296 | metadata['uri'] = uri.to_s 297 | metadata['uuid'] = uuid 298 | 299 | metadata 300 | end 301 | 302 | res = req.response 303 | res.for( :json, :yaml ) { list } 304 | res.for( :text ) do 305 | list.collect {|entry| "%s [%s, %0.2fB]" % entry.values_at(:url, :format, :extent) } 306 | end 307 | 308 | return res 309 | end 310 | 311 | 312 | # GET /«uuid»/related/«relationship» 313 | # Get the data for the resource related to the one to the given +uuid+ via the 314 | # specified +relationship+. 315 | get ':uuid/related/:relationship' do |req| 316 | uuid = req.params[:uuid] 317 | rel = req.params[:relationship] 318 | 319 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 320 | 321 | primary_metadata = self.metastore.fetch( uuid ) 322 | self.check_resource_permissions( req, uuid, primary_metadata ) 323 | 324 | criteria = { 325 | 'relation' => uuid, 326 | 'relationship' => rel, 327 | } 328 | uuid = self.metastore.search( criteria: criteria, include_related: true ).first or 329 | finish_with( HTTP::NOT_FOUND, "No such related resource." ) 330 | 331 | object = self.datastore.fetch( uuid ) or 332 | raise "Metadata for non-existant resource %p" % [ uuid ] 333 | metadata = self.metastore.fetch( uuid ) 334 | 335 | res = req.response 336 | self.add_cache_headers( req, metadata ) 337 | self.add_content_disposition( res, metadata ) 338 | 339 | res.body = object 340 | res.content_type = metadata['format'] 341 | res.status = HTTP::OK 342 | 343 | return res 344 | end 345 | 346 | 347 | # GET /«uuid» 348 | # Fetch an object by ID 349 | get ':uuid' do |req| 350 | uuid = req.params[:uuid] 351 | object = self.datastore.fetch( uuid ) 352 | metadata = self.metastore.fetch( uuid ) 353 | 354 | finish_with HTTP::NOT_FOUND, "No such object." unless object && metadata 355 | self.check_resource_permissions( req, nil, metadata ) 356 | 357 | res = req.response 358 | res.content_type = metadata['format'] 359 | 360 | self.add_cache_headers( req, metadata ) 361 | self.add_content_disposition( res, metadata ) 362 | 363 | if object.respond_to?( :path ) 364 | path = Pathname( object.path ) 365 | chroot_path = path.relative_path_from( req.server_chroot ) 366 | res.extend_reply_with( :sendfile ) 367 | res.headers.content_length = object.size 368 | res.extended_reply_data << File::SEPARATOR + chroot_path.to_s 369 | else 370 | res.body = object 371 | end 372 | 373 | res.status = HTTP::OK 374 | return res 375 | end 376 | 377 | 378 | # POST / 379 | # Upload a new object. 380 | post do |req| 381 | uuid, metadata = self.save_resource( req ) 382 | self.send_event( :created, :uuid => uuid ) 383 | 384 | uri = req.base_uri.dup 385 | uri.path += '/' unless uri.path[-1] == '/' 386 | uri.path += uuid 387 | 388 | res = req.response 389 | res.headers.location = uri 390 | res.headers.x_thingfish_uuid = uuid 391 | res.status = HTTP::CREATED 392 | 393 | res.for( :text, :json, :yaml ) { metadata } 394 | 395 | return res 396 | end 397 | 398 | 399 | # PUT /«uuid» 400 | # Replace the data associated with +uuid+. 401 | put ':uuid' do |req| 402 | uuid = req.params[:uuid] 403 | self.datastore.include?( uuid ) or 404 | finish_with HTTP::NOT_FOUND, "No such object." 405 | 406 | self.check_resource_permissions( req, uuid ) 407 | self.remove_related_resources( uuid ) 408 | self.save_resource( req, uuid ) 409 | self.send_event( :replaced, :uuid => uuid ) 410 | 411 | res = req.response 412 | res.status = HTTP::NO_CONTENT 413 | 414 | return res 415 | end 416 | 417 | 418 | # DELETE /«uuid» 419 | # Remove the object associated with +uuid+. 420 | delete ':uuid' do |req| 421 | uuid = req.params[:uuid] 422 | 423 | self.check_resource_permissions( req, uuid ) 424 | 425 | self.remove_related_resources( uuid ) 426 | metadata = self.metastore.remove( uuid ) 427 | self.datastore.remove( uuid ) or finish_with( HTTP::NOT_FOUND, "No such object." ) 428 | self.send_event( :deleted, :uuid => uuid ) 429 | 430 | res = req.response 431 | res.status = HTTP::OK 432 | 433 | # TODO: Remove in favor of default metadata when the metastore 434 | # knows what that is. 435 | res.for( :text ) do 436 | "%d bytes for %s deleted." % [ metadata['extent'], uuid ] 437 | end 438 | res.for( :json, :yaml ) {{ uuid: uuid, extent: metadata['extent'] }} 439 | 440 | return res 441 | end 442 | 443 | 444 | # 445 | # Metastore routes 446 | # 447 | 448 | # GET /«uuid»/metadata 449 | # Fetch all metadata for «uuid». 450 | get ':uuid/metadata' do |req| 451 | uuid = req.params[:uuid] 452 | 453 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 454 | 455 | metadata = self.normalized_metadata_for( uuid ) 456 | self.check_resource_permissions( req, uuid, metadata ) 457 | 458 | res = req.response 459 | res.status = HTTP::OK 460 | res.for( :json, :yaml ) { metadata } 461 | 462 | return res 463 | end 464 | 465 | 466 | # GET /«uuid»/metadata/«key» 467 | # Fetch metadata value associated with «key» for «uuid». 468 | get ':uuid/metadata/:key' do |req| 469 | uuid = req.params[:uuid] 470 | key = req.params[:key] 471 | 472 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 473 | 474 | metadata = self.metastore.fetch( uuid ) 475 | self.check_resource_permissions( req, uuid, metadata ) 476 | 477 | res = req.response 478 | res.status = HTTP::OK 479 | res.for( :json, :yaml ) { metadata[key] } 480 | 481 | return res 482 | end 483 | 484 | 485 | # POST /«uuid»/metadata/«key» 486 | # Create a metadata value associated with «key» for «uuid». 487 | post ':uuid/metadata/:key' do |req| 488 | uuid = req.params[:uuid] 489 | key = req.params[:key] 490 | 491 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 492 | finish_with( HTTP::CONFLICT, "Key already exists." ) unless 493 | self.metastore.fetch_value( uuid, key ).nil? 494 | 495 | metadata = self.metastore.fetch( uuid ) 496 | self.check_resource_permissions( req, uuid, metadata ) 497 | 498 | self.metastore.merge( uuid, key => req.body.read ) 499 | self.send_event( :metadata_updated, :uuid => uuid, :key => key ) 500 | 501 | res = req.response 502 | res.headers.location = req.uri.to_s 503 | res.body = nil 504 | res.status = HTTP::CREATED 505 | 506 | return res 507 | end 508 | 509 | 510 | # PUT /«uuid»/metadata/«key» 511 | # Replace or create a metadata value associated with «key» for «uuid». 512 | put ':uuid/metadata/:key' do |req| 513 | uuid = req.params[:uuid] 514 | key = req.params[:key] 515 | 516 | finish_with( HTTP::FORBIDDEN, "Protected metadata." ) if 517 | OPERATIONAL_METADATA_KEYS.include?( key ) 518 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 519 | previous_value = self.metastore.fetch( uuid, key ) 520 | 521 | metadata = self.metastore.fetch( uuid ) 522 | self.check_resource_permissions( req, uuid, metadata ) 523 | 524 | self.metastore.merge( uuid, key => req.body.read ) 525 | self.send_event( :metadata_replaced, :uuid => uuid, :key => key ) 526 | 527 | res = req.response 528 | res.body = nil 529 | 530 | if previous_value 531 | res.status = HTTP::NO_CONTENT 532 | else 533 | res.headers.location = req.uri.to_s 534 | res.status = HTTP::CREATED 535 | end 536 | 537 | return res 538 | end 539 | 540 | 541 | # PUT /«uuid»/metadata 542 | # Replace user metadata for «uuid». 543 | put ':uuid/metadata' do |req| 544 | uuid = req.params[:uuid] 545 | 546 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 547 | 548 | metadata = self.metastore.fetch( uuid ) 549 | self.check_resource_permissions( req, uuid, metadata ) 550 | 551 | op_metadata = self.metastore.fetch( uuid, *OPERATIONAL_METADATA_KEYS ) 552 | new_metadata = self.extract_metadata( req ) 553 | self.metastore.transaction do 554 | self.metastore.remove_except( uuid, *OPERATIONAL_METADATA_KEYS ) 555 | self.metastore.merge( uuid, new_metadata.merge(op_metadata) ) 556 | end 557 | self.send_event( :metadata_replaced, :uuid => uuid ) 558 | 559 | res = req.response 560 | res.status = HTTP::OK 561 | res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) } 562 | 563 | return res 564 | end 565 | 566 | 567 | # POST /«uuid»/metadata 568 | # Merge new metadata into the existing metadata for «uuid». 569 | post ':uuid/metadata' do |req| 570 | uuid = req.params[:uuid] 571 | 572 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 573 | 574 | metadata = self.metastore.fetch( uuid ) 575 | self.check_resource_permissions( req, uuid, metadata ) 576 | 577 | new_metadata = self.extract_metadata( req ) 578 | self.metastore.merge( uuid, new_metadata ) 579 | self.send_event( :metadata_updated, :uuid => uuid ) 580 | 581 | res = req.response 582 | res.status = HTTP::OK 583 | res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) } 584 | 585 | return res 586 | end 587 | 588 | 589 | # DELETE /«uuid»/metadata 590 | # Remove all (but operational) metadata associated with «uuid». 591 | delete ':uuid/metadata' do |req| 592 | uuid = req.params[:uuid] 593 | 594 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 595 | 596 | metadata = self.metastore.fetch( uuid ) 597 | self.check_resource_permissions( req, uuid, metadata ) 598 | 599 | self.metastore.remove_except( uuid, *OPERATIONAL_METADATA_KEYS ) 600 | self.send_event( :metadata_deleted, :uuid => uuid ) 601 | 602 | res = req.response 603 | res.status = HTTP::OK 604 | res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) } 605 | 606 | return res 607 | end 608 | 609 | 610 | # DELETE /«uuid»/metadata/«key» 611 | # Remove the metadata associated with «key» for the given «uuid». 612 | delete ':uuid/metadata/:key' do |req| 613 | uuid = req.params[:uuid] 614 | key = req.params[:key] 615 | 616 | finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid ) 617 | finish_with( HTTP::FORBIDDEN, "Protected metadata." ) if 618 | OPERATIONAL_METADATA_KEYS.include?( key ) 619 | 620 | metadata = self.metastore.fetch( uuid ) 621 | self.check_resource_permissions( req, uuid, metadata ) 622 | 623 | self.metastore.remove( uuid, key ) 624 | self.send_event( :metadata_deleted, :uuid => uuid, :key => key ) 625 | 626 | res = req.response 627 | res.status = HTTP::NO_CONTENT 628 | 629 | return res 630 | end 631 | 632 | 633 | ######### 634 | protected 635 | ######### 636 | 637 | 638 | ### Save the resource in the given +request+'s body and any associated metadata 639 | ### or additional resources. 640 | def save_resource( request, uuid=nil ) 641 | metadata = request.metadata 642 | metadata.merge!( self.extract_header_metadata(request) ) 643 | metadata.merge!( self.extract_default_metadata(request) ) 644 | 645 | self.check_resource_permissions( request, uuid, metadata ) 646 | self.verify_operational_metadata( metadata ) 647 | 648 | if uuid 649 | self.log.info "Replacing resource %s (encoding: %p)" % 650 | [ uuid, request.headers.content_encoding ] 651 | self.datastore.replace( uuid, request.body ) 652 | self.metastore.merge( uuid, metadata ) 653 | else 654 | self.log.info "Saving new resource (encoding: %p)." % 655 | [ request.headers.content_encoding ] 656 | uuid = self.datastore.save( request.body ) 657 | self.metastore.save( uuid, metadata ) 658 | end 659 | 660 | self.save_related_resources( request, uuid ) 661 | 662 | return uuid, metadata 663 | end 664 | 665 | 666 | ### Save any related resources in the given +request+ with a relationship to the 667 | ### resource with the given +uuid+. 668 | def save_related_resources( request, uuid ) 669 | request.related_resources.each do |io, metadata| 670 | self.log.debug "Saving a resource related to %s: %p" % [ uuid, metadata ] 671 | next unless self.check_related_metadata( metadata ) 672 | 673 | self.log.debug " related metadata checks passed; storing it." 674 | metadata = self.extract_connection_metadata( request ).merge( metadata ) 675 | self.log.debug " metadata is: %p" % [ metadata ] 676 | r_uuid = self.datastore.save( io ) 677 | metadata['created'] = Time.now.getgm 678 | metadata['relation'] = uuid 679 | 680 | self.metastore.save( r_uuid, metadata ) 681 | self.log.debug " %s for %s saved as %s" % 682 | [ metadata['relationship'], uuid, r_uuid ] 683 | end 684 | end 685 | 686 | 687 | ### Remove any resources that are related to the one with the specified +uuid+. 688 | def remove_related_resources( uuid ) 689 | self.metastore.fetch_related_oids( uuid ).each do |r_uuid| 690 | self.datastore.remove( r_uuid ) 691 | self.metastore.remove( r_uuid ) 692 | self.log.info "Removed related resource %s for %s." % [ r_uuid, uuid ] 693 | end 694 | end 695 | 696 | 697 | ### Do some consistency checks on the given +metadata+ for a related resource, 698 | ### returning +true+ if it meets the requirements. 699 | def check_related_metadata( metadata ) 700 | REQUIRED_RELATED_METADATA_KEYS.each do |attribute| 701 | unless metadata[ attribute ] 702 | self.log.error "Metadata for required resource must include '#{attribute}' attribute!" 703 | return false 704 | end 705 | end 706 | return true 707 | end 708 | 709 | 710 | ### Overridden from the base handler class to allow spooled uploads. 711 | def handle_async_upload_start( request ) 712 | self.log.info "Starting asynchronous upload: %s" % 713 | [ request.headers.x_mongrel2_upload_start ] 714 | return nil 715 | end 716 | 717 | 718 | ### Return a Hash of default metadata extracted from the given +request+. 719 | def extract_default_metadata( request ) 720 | return self.extract_connection_metadata( request ).merge( 721 | 'extent' => request.headers.content_length, 722 | 'format' => request.content_type, 723 | 'created' => Time.now.getgm 724 | ) 725 | end 726 | 727 | 728 | ### Return a Hash of metadata extracted from the connection information 729 | ### of the given +request+. 730 | def extract_connection_metadata( request ) 731 | return { 732 | 'useragent' => request.headers.user_agent, 733 | 'uploadaddress' => request.remote_ip, 734 | } 735 | end 736 | 737 | 738 | ### Extract and validate supplied metadata from the +request+. 739 | def extract_metadata( req ) 740 | new_metadata = req.params.fields.dup 741 | new_metadata = self.remove_operational_metadata( new_metadata ) 742 | return new_metadata 743 | end 744 | 745 | 746 | ### Extract metadata from X-ThingFish-* headers from the given +request+ and return 747 | ### them as a Hash. 748 | def extract_header_metadata( request ) 749 | self.log.debug "Extracting metadata from headers: %p" % [ request.headers ] 750 | metadata = {} 751 | request.headers.each do |header, value| 752 | name = header.downcase[ /^x_thingfish_(?[[:alnum:]\-]+)$/i, :name ] or next 753 | self.log.debug "Found metadata header %p" % [ header ] 754 | metadata[ name ] = value 755 | end 756 | 757 | return metadata 758 | end 759 | 760 | 761 | ### Fetch the current metadata for +uuid+, altering it for easier 762 | ### round trips with REST. 763 | def normalized_metadata_for( uuid ) 764 | return self.metastore.fetch(uuid).merge( 'uuid' => uuid ) 765 | end 766 | 767 | 768 | ### Check that the metadata provided contains valid values for 769 | ### the operational keys, before saving a resource to disk. 770 | def verify_operational_metadata( metadata ) 771 | if metadata.values_at( *OPERATIONAL_METADATA_KEYS ).any?( &:nil? ) 772 | finish_with( HTTP::BAD_REQUEST, "Missing operational attribute." ) 773 | end 774 | end 775 | 776 | 777 | ### Prune operational +metadata+ from the provided hash. 778 | def remove_operational_metadata( metadata ) 779 | operationals = OPERATIONAL_METADATA_KEYS + [ 'uuid' ] 780 | return metadata.reject{|key, _| operationals.include?(key) } 781 | end 782 | 783 | 784 | ### Send an event of +type+ with the given +msg+ over the zmq event socket. 785 | def send_event( type, msg ) 786 | esock = self.event_socket or return 787 | self.log.debug "Publishing %p event: %p" % [ type, msg ] 788 | esock << CZTop::Message.new([ type.to_s, Yajl.dump(msg) ]) 789 | end 790 | 791 | 792 | ### Add browser cache headers for resources. 793 | ### Last-Modified is always added. ETag support requires the sha256 794 | ### processor plugin to be enabled for stored resources. 795 | ### 796 | def add_cache_headers( request, metadata ) 797 | response = request.response 798 | 799 | # ETag takes precedence if available. 800 | # 801 | if (( checksum = metadata['checksum'] )) 802 | if (( match = request.headers[ :if_none_match ] )) 803 | match = match.gsub( '"', '' ).split( /,\s*/ ) 804 | finish_with( HTTP::NOT_MODIFIED ) if match.include?( checksum ) 805 | end 806 | response.headers[ :etag ] = checksum 807 | end 808 | 809 | return unless metadata[ 'created' ] 810 | 811 | if (( modified = request.headers[ :if_modified_since ] )) 812 | finish_with( HTTP::NOT_MODIFIED ) if Time.parse( modified ) <= metadata['created'].round 813 | end 814 | response.headers[ :last_modified ] = metadata[ 'created' ].httpdate 815 | 816 | return 817 | end 818 | 819 | 820 | ### Add a filename "hint" for browsers, if the resource being fetched 821 | ### has a 'title' attribute. 822 | def add_content_disposition( res, metadata ) 823 | return unless metadata[ 'title' ] 824 | title = metadata[ 'title' ].encode( 'us-ascii', :undef => :replace ) 825 | res.headers[ :content_disposition ] = "filename=%p" % [ title ] 826 | end 827 | 828 | 829 | ### Supply a method that child handlers can override. The regular auth 830 | ### plugin runs too early (but can also be used), this hook allows 831 | ### child handlers to make access decisions based on the +request+ 832 | ### object, +uuid+ of the resource, or +metadata+ of the fetched resource. 833 | def check_resource_permissions( request, uuid=nil, metadata=nil ); end 834 | 835 | 836 | end # class Thingfish::Handler 837 | 838 | # vim: set nosta noet ts=4 sw=4: 839 | -------------------------------------------------------------------------------- /spec/thingfish/handler_spec.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../helpers' 4 | 5 | require 'rspec' 6 | require 'thingfish/handler' 7 | require 'thingfish/processor' 8 | 9 | 10 | RSpec.describe Thingfish::Handler do 11 | 12 | EVENT_SOCKET_URI = 'tcp://127.0.0.1:*' 13 | 14 | before( :all ) do 15 | Thingfish::Handler.configure( :event_socket_uri => EVENT_SOCKET_URI ) 16 | Thingfish::Handler.install_plugins 17 | end 18 | 19 | let( :png_io ) { StringIO.new( TEST_PNG_DATA.dup ) } 20 | let( :text_io ) { StringIO.new( TEST_TEXT_DATA.dup ) } 21 | let( :handler ) { described_class.new(TEST_APPID, TEST_SEND_SPEC, TEST_RECV_SPEC) } 22 | 23 | 24 | # 25 | # Shared behaviors 26 | # 27 | 28 | it_should_behave_like "an object with Configurability" 29 | 30 | 31 | # 32 | # Examples 33 | # 34 | 35 | context "misc api" do 36 | 37 | let( :factory ) do 38 | Mongrel2::RequestFactory.new( 39 | :route => '/', 40 | :headers => {:accept => '*/*'}) 41 | end 42 | 43 | it 'returns interesting configuration info' do 44 | req = factory.get( '/serverinfo', content_type: 'text/plain' ) 45 | res = handler.handle( req ) 46 | 47 | expect( res.status_line ).to match( /200 ok/i ) 48 | expect( res.headers ).to include( 'x-thingfish' ) 49 | end 50 | 51 | end 52 | 53 | 54 | context "datastore api" do 55 | 56 | let( :factory ) do 57 | Mongrel2::RequestFactory.new( 58 | :route => '/', 59 | :headers => {:accept => '*/*'}) 60 | end 61 | 62 | 63 | it 'accepts a POSTed upload' do 64 | req = factory.post( '/', TEST_TEXT_DATA ) 65 | req.content_type = 'text/plain' 66 | req.headers.content_length = TEST_TEXT_DATA.bytesize 67 | res = handler.handle( req ) 68 | 69 | expect( res.status_line ).to match( /201 created/i ) 70 | expect( res.headers.location.to_s ).to match( %r:/#{UUID_PATTERN}$: ) 71 | end 72 | 73 | 74 | it "accepts an upload POSTED via Mongrel's async API'" do 75 | # Need the config to look up the async upload path relative to the server's chroot 76 | Mongrel2::Config.db = Mongrel2::Config.in_memory_db 77 | Mongrel2::Config.init_database 78 | server( 'thingfish' ) do 79 | chroot '' 80 | host 'localhost' do 81 | route '/', handler( 'tcp://127.0.0.1:9900', 'thingfish' ) 82 | end 83 | end 84 | 85 | spool_path = '/var/spool/uploadfile.672' 86 | upload_size = 645_000 87 | fh = instance_double( "File", size: upload_size, rewind: 0, pos: 0, :pos= => nil, 88 | read: TEST_TEXT_DATA ) 89 | expect( FileTest ).to receive( :exist? ).with( spool_path ).and_return( true ) 90 | expect( File ).to receive( :open ). 91 | with( spool_path, 'r', encoding: Encoding::ASCII_8BIT ). 92 | and_return( fh ) 93 | 94 | start_req = factory.post( '/', '', 95 | x_mongrel2_upload_start: spool_path, 96 | content_length: upload_size, 97 | content_type: 'text/plain' ) 98 | upload_req = factory.post( '/', '', 99 | x_mongrel2_upload_start: spool_path, 100 | x_mongrel2_upload_done: spool_path, 101 | content_length: upload_size, 102 | content_type: 'text/plain' ) 103 | 104 | start_res = handler.dispatch_request( start_req ) 105 | upload_res = handler.dispatch_request( upload_req ) 106 | 107 | expect( start_res ).to be_nil 108 | 109 | expect( upload_res.status_line ).to match( /201 created/i ) 110 | expect( upload_res.headers.location.to_s ).to match( %r:/#{UUID_PATTERN}$: ) 111 | end 112 | 113 | 114 | it "accepts resources added to a POSTed resource by processors" do 115 | imageio = StringIO.new( TEST_PNG_DATA ) 116 | 117 | subclass = Class.new( described_class ) 118 | subclass.filter( :request ) do |req| 119 | req.add_related_resource( imageio, 120 | relationship: 'thumbnail', 121 | format: 'image/png', 122 | extent: TEST_PNG_DATA.bytesize ) 123 | end 124 | subclass.metastore = 'memory' 125 | subclass.datastore = 'memory' 126 | handler = subclass.new( TEST_APPID, TEST_SEND_SPEC, TEST_RECV_SPEC ) 127 | 128 | req = factory.post( '/', TEST_TEXT_DATA, content_type: 'text/plain' ) 129 | res = handler.handle( req ) 130 | 131 | metastore = handler.metastore 132 | oid = res.headers.x_thingfish_uuid 133 | related_oid = metastore.fetch_related_oids( oid ).first 134 | 135 | expect( 136 | metastore.fetch_value(related_oid, 'uploadaddress') 137 | ).to eq( metastore.fetch_value(oid, 'uploadaddress') ) 138 | end 139 | 140 | 141 | it 'returns a BAD_REQUEST if unable to extract operational metadata' do 142 | req = factory.post( '/', TEST_TEXT_DATA ) 143 | req.content_type = 'text/plain' 144 | 145 | res = handler.handle( req ) 146 | res.body.rewind 147 | expect( res.status ).to be( HTTP::BAD_REQUEST ) 148 | expect( res.body.read ).to match( /missing operational attribute/i ) 149 | end 150 | 151 | 152 | it "allows additional metadata to be attached to uploads via X-Thingfish-* headers" do 153 | headers = { 154 | content_type: 'text/plain', 155 | x_thingfish_title: 'Muffin the Panda Goes To School', 156 | x_thingfish_tags: 'rapper,ukraine,potap', 157 | } 158 | req = factory.post( '/', TEST_TEXT_DATA, headers ) 159 | req.headers.content_length = TEST_TEXT_DATA.bytesize 160 | res = handler.handle( req ) 161 | 162 | expect( res.status_line ).to match( /201 created/i ) 163 | expect( res.headers.location.to_s ).to match( %r:/#{UUID_PATTERN}$: ) 164 | 165 | uuid = res.headers.x_thingfish_uuid 166 | expect( handler.metastore.fetch_value(uuid, 'title') ). 167 | to eq( 'Muffin the Panda Goes To School' ) 168 | expect( handler.metastore.fetch_value(uuid, 'tags') ).to eq( 'rapper,ukraine,potap' ) 169 | end 170 | 171 | 172 | it 'replaces content via PUT' do 173 | uuid = handler.datastore.save( text_io ) 174 | handler.metastore.save( uuid, {'format' => 'text/plain'} ) 175 | 176 | req = factory.put( "/#{uuid}", png_io, content_type: 'image/png' ) 177 | req.headers.content_length = png_io.read.bytesize 178 | res = handler.handle( req ) 179 | 180 | expect( res.status ).to eq( HTTP::NO_CONTENT ) 181 | expect( handler.datastore.fetch(uuid).read ).to eq( TEST_PNG_DATA ) 182 | expect( handler.metastore.fetch(uuid) ).to include( 'format' => 'image/png' ) 183 | end 184 | 185 | 186 | it "doesn't care about the case of the UUID when replacing content via PUT" do 187 | uuid = handler.datastore.save( text_io ) 188 | handler.metastore.save( uuid, {'format' => 'text/plain'} ) 189 | 190 | req = factory.put( "/#{uuid.upcase}", png_io, content_type: 'image/png' ) 191 | req.headers.content_length = png_io.read.bytesize 192 | res = handler.handle( req ) 193 | 194 | expect( res.status ).to eq( HTTP::NO_CONTENT ) 195 | expect( handler.datastore.fetch(uuid).read ).to eq( TEST_PNG_DATA ) 196 | expect( handler.metastore.fetch(uuid) ).to include( 'format' => 'image/png' ) 197 | end 198 | 199 | 200 | it 'can fetch all uploaded data' do 201 | text_uuid = handler.datastore.save( text_io ) 202 | handler.metastore.save( text_uuid, { 203 | 'format' => 'text/plain', 204 | 'extent' => text_io.string.bytesize 205 | }) 206 | png_uuid = handler.datastore.save( png_io ) 207 | handler.metastore.save( png_uuid, { 208 | 'format' => 'image/png', 209 | 'extent' => png_io.string.bytesize 210 | }) 211 | 212 | req = factory.get( '/' ) 213 | res = handler.handle( req ) 214 | content = Yajl::Parser.parse( res.body.read ) 215 | 216 | expect( res.status_line ).to match( /200 ok/i ) 217 | expect( res.headers.content_type ).to eq( 'application/json' ) 218 | expect( content ).to be_a( Array ) 219 | expect( content[0] ).to be_a( Hash ) 220 | expect( content[0]['uri'] ).to eq( "#{req.base_uri}#{text_uuid}" ) 221 | expect( content[0]['format'] ).to eq( "text/plain" ) 222 | expect( content[0]['extent'] ).to eq( text_io.string.bytesize ) 223 | expect( content[1] ).to be_a( Hash ) 224 | expect( content[1]['uri'] ).to eq( "#{req.base_uri}#{png_uuid}" ) 225 | expect( content[1]['format'] ).to eq( 'image/png' ) 226 | expect( content[1]['extent'] ).to eq( png_io.string.bytesize ) 227 | end 228 | 229 | 230 | it 'can fetch all related data for a single resource' do 231 | main_uuid = handler.datastore.save( png_io ) 232 | handler.metastore.save( main_uuid, { 233 | 'format' => 'image/png', 234 | 'extent' => png_io.string.bytesize 235 | }) 236 | related_uuid = handler.datastore.save( png_io ) 237 | handler.metastore.save( related_uuid, { 238 | 'format' => 'image/png', 239 | 'extent' => png_io.string.bytesize, 240 | 'relation' => main_uuid, 241 | 'relationship' => "twinsies" 242 | }) 243 | 244 | req = factory.get( "/#{main_uuid}/related" ) 245 | res = handler.handle( req ) 246 | content = Yajl::Parser.parse( res.body.read ) 247 | 248 | expect( res.status_line ).to match( /200 ok/i ) 249 | expect( res.headers.content_type ).to eq( 'application/json' ) 250 | expect( content ).to be_a( Array ) 251 | expect( content[0] ).to be_a( Hash ) 252 | expect( content[0]['uri'] ).to eq( "#{req.base_uri}#{related_uuid}" ) 253 | expect( content[0]['format'] ).to eq( "image/png" ) 254 | expect( content[0]['extent'] ).to eq( png_io.string.bytesize ) 255 | expect( content[0]['uuid'] ).to eq( related_uuid ) 256 | expect( content[0]['relation'] ).to eq( main_uuid ) 257 | end 258 | 259 | 260 | it 'can fetch a related resource by name' do 261 | main_uuid = handler.datastore.save( png_io ) 262 | handler.metastore.save( main_uuid, { 263 | 'format' => 'image/png', 264 | 'extent' => png_io.string.bytesize 265 | }) 266 | related_uuid = handler.datastore.save( png_io ) 267 | handler.metastore.save( related_uuid, { 268 | 'format' => 'image/png', 269 | 'extent' => png_io.string.bytesize, 270 | 'relation' => main_uuid, 271 | 'relationship' => "twinsies", 272 | 'title' => 'Make America Smart Again.png', 273 | 'checksum' => '123456' 274 | }) 275 | 276 | req = factory.get( "/#{main_uuid}/related/twinsies" ) 277 | res = handler.handle( req ) 278 | 279 | expect( res.status_line ).to match( /200 ok/i ) 280 | expect( res.headers.content_type ).to eq( 'image/png' ) 281 | expect( res.headers.etag ).to eq( '123456' ) 282 | expect( res.headers.content_disposition ).to eq( 'filename="Make America Smart Again.png"' ) 283 | expect( res.body.read ).to eq( TEST_PNG_DATA ) 284 | end 285 | 286 | 287 | it "404s when attempting to fetch a resource related to a non-existant resource" do 288 | req = factory.get( "/#{TEST_UUID}/related/twinsies" ) 289 | res = handler.handle( req ) 290 | 291 | expect( res.status_line ).to match( /404 not found/i ) 292 | end 293 | 294 | 295 | it "404s when attempting to fetch a related resource that doesn't exist" do 296 | uuid = handler.datastore.save( png_io ) 297 | handler.metastore.save( uuid, { 298 | 'format' => 'image/png', 299 | 'extent' => png_io.string.bytesize 300 | }) 301 | 302 | req = factory.get( "/#{uuid}/related/twinsies" ) 303 | res = handler.handle( req ) 304 | 305 | expect( res.status_line ).to match( /404 not found/i ) 306 | end 307 | 308 | 309 | it "can fetch an uploaded chunk of data" do 310 | uuid = handler.datastore.save( png_io ) 311 | handler.metastore.save( uuid, {'format' => 'image/png'} ) 312 | 313 | req = factory.get( "/#{uuid}" ) 314 | result = handler.handle( req ) 315 | 316 | expect( result.status_line ).to match( /200 ok/i ) 317 | expect( result.body.read ).to eq( png_io.string ) 318 | expect( result.headers.content_type ).to eq( 'image/png' ) 319 | end 320 | 321 | 322 | it "returns a 404 Not Found when asked to fetch an object that doesn't exist" do 323 | req = factory.get( "/#{TEST_UUID}" ) 324 | result = handler.handle( req ) 325 | 326 | expect( result.status_line ).to match( /404 not found/i ) 327 | end 328 | 329 | 330 | it "returns a 404 Not Found when asked to fetch an object that doesn't exist in the metastore" do 331 | uuid = handler.datastore.save( png_io ) 332 | 333 | req = factory.get( "/#{uuid}" ) 334 | result = handler.handle( req ) 335 | 336 | expect( result.status_line ).to match( /404 not found/i ) 337 | end 338 | 339 | 340 | it "doesn't care about the case of the UUID when fetching uploaded data" do 341 | uuid = handler.datastore.save( png_io ) 342 | handler.metastore.save( uuid, {'format' => 'image/png'} ) 343 | 344 | req = factory.get( "/#{uuid.upcase}" ) 345 | result = handler.handle( req ) 346 | 347 | expect( result.status_line ).to match( /200 ok/i ) 348 | expect( result.body.read ).to eq( png_io.string ) 349 | expect( result.headers.content_type ).to eq( 'image/png' ) 350 | end 351 | 352 | 353 | it "adds date cache headers to resources" do 354 | created = Time.now 355 | uuid = handler.datastore.save( png_io ) 356 | handler.metastore.save( uuid, 'format' => 'image/png', 'created' => created ) 357 | 358 | req = factory.get( "/#{uuid}" ) 359 | result = handler.handle( req ) 360 | 361 | expect( result.status_line ).to match( /200 ok/i ) 362 | expect( result.headers.last_modified ).to eq( created.httpdate ) 363 | end 364 | 365 | 366 | it "adds content cache headers to resources with a checksum attribute" do 367 | uuid = handler.datastore.save( png_io ) 368 | handler.metastore.save( uuid, 'format' => 'image/png', 'checksum' => '123456' ) 369 | 370 | req = factory.get( "/#{uuid}" ) 371 | result = handler.handle( req ) 372 | 373 | expect( result.status_line ).to match( /200 ok/i ) 374 | expect( result.headers.etag ).to eq( '123456' ) 375 | end 376 | 377 | 378 | it "adds content disposition filename, if the resource has a title" do 379 | uuid = handler.datastore.save( png_io ) 380 | handler.metastore.save( uuid, {'format' => 'image/png', 'title' => 'spょler"py.txt'} ) 381 | 382 | req = factory.get( "/#{uuid}" ) 383 | result = handler.handle( req ) 384 | 385 | expect( result.status_line ).to match( /200 ok/i ) 386 | expect( result.body.read ).to eq( png_io.string ) 387 | expect( result.headers.content_type ).to eq( 'image/png' ) 388 | expect( result.headers.content_disposition ).to eq( 'filename="sp?ler\"py.txt"' ) 389 | end 390 | 391 | 392 | it "returns a 304 not modified for unchanged date cache requests" do 393 | created = Time.now 394 | uuid = handler.datastore.save( png_io ) 395 | handler.metastore.save( uuid, 'format' => 'image/png', 'created' => created ) 396 | 397 | req = factory.get( "/#{uuid}" ) 398 | req.headers[ :if_modified_since ] = ( Time.now - 300 ).httpdate 399 | result = handler.handle( req ) 400 | 401 | expect( result.status_line ).to match( /304 not modified/i ) 402 | expect( result.body.read ).to be_empty 403 | end 404 | 405 | 406 | it "returns a 304 not modified for unchanged content cache requests" do 407 | uuid = handler.datastore.save( png_io ) 408 | handler.metastore.save( uuid, 'format' => 'image/png', 'checksum' => '123456' ) 409 | 410 | req = factory.get( "/#{uuid}" ) 411 | req.headers[ :if_none_match ] = '123456' 412 | result = handler.handle( req ) 413 | 414 | expect( result.status_line ).to match( /304 not modified/i ) 415 | expect( result.body.read ).to be_empty 416 | end 417 | 418 | 419 | it "can remove everything associated with an object id" do 420 | uuid = handler.datastore.save( png_io ) 421 | handler.metastore.save( uuid, { 422 | 'format' => 'image/png', 423 | 'extent' => 288, 424 | }) 425 | 426 | req = factory.delete( "/#{uuid}" ) 427 | result = handler.handle( req ) 428 | 429 | expect( result.status_line ).to match( /200 ok/i ) 430 | expect( handler.metastore.include?(uuid) ).to be_falsey 431 | expect( handler.datastore.include?(uuid) ).to be_falsey 432 | end 433 | 434 | 435 | it "returns a 404 Not Found when asked to remove an object that doesn't exist" do 436 | req = factory.delete( "/#{TEST_UUID}" ) 437 | result = handler.handle( req ) 438 | 439 | expect( result.status_line ).to match( /404 not found/i ) 440 | end 441 | 442 | 443 | end 444 | 445 | 446 | context "metastore api" do 447 | 448 | let( :factory ) do 449 | Mongrel2::RequestFactory.new( 450 | :route => '/', 451 | :headers => {:accept => 'application/json'}) 452 | end 453 | 454 | it "can fetch the metadata associated with uploaded data" do 455 | uuid = handler.datastore.save( png_io ) 456 | handler.metastore.save( uuid, { 457 | 'format' => 'image/png', 458 | 'extent' => 288, 459 | 'created' => Time.at(1378313840), 460 | }) 461 | 462 | req = factory.get( "/#{uuid}/metadata" ) 463 | result = handler.handle( req ) 464 | content = result.body.read 465 | 466 | content_hash = Yajl::Parser.parse( content ) 467 | 468 | expect( result.status ).to eq( 200 ) 469 | expect( result.headers.content_type ).to eq( 'application/json' ) 470 | expect( content_hash ).to be_a( Hash ) 471 | expect( content_hash['uuid'] ).to eq( uuid ) 472 | expect( content_hash['extent'] ).to eq( 288 ) 473 | expect( content_hash['created'] ).to eq( Time.at(1378313840).to_s ) 474 | end 475 | 476 | 477 | it "returns a 404 Not Found when fetching metadata for an object that doesn't exist" do 478 | req = factory.get( "/#{TEST_UUID}/metadata" ) 479 | result = handler.handle( req ) 480 | 481 | expect( result.status_line ).to match( /404 not found/i ) 482 | end 483 | 484 | 485 | it "can fetch a value for a single metadata key" do 486 | uuid = handler.datastore.save( png_io ) 487 | handler.metastore.save( uuid, { 488 | 'format' => 'image/png', 489 | 'extent' => 288, 490 | }) 491 | 492 | req = factory.get( "/#{uuid}/metadata/extent" ) 493 | result = handler.handle( req ) 494 | result.body.rewind 495 | content = result.body.read 496 | 497 | expect( result.status ).to eq( 200 ) 498 | expect( result.headers.content_type ).to eq( 'application/json' ) 499 | expect( content ).to eq( "288" ) 500 | end 501 | 502 | 503 | it "returns a 404 Not Found when fetching a single metadata value for a uuid that doesn't exist" do 504 | req = factory.get( "/#{TEST_UUID}/metadata/extent" ) 505 | result = handler.handle( req ) 506 | 507 | expect( result.status_line ).to match( /404 not found/i ) 508 | end 509 | 510 | 511 | it "doesn't error when fetching a non-existent metadata value" do 512 | uuid = handler.datastore.save( png_io ) 513 | handler.metastore.save( uuid, { 514 | 'format' => 'image/png', 515 | 'extent' => 288, 516 | }) 517 | 518 | req = factory.get( "/#{uuid}/metadata/hururrgghh" ) 519 | result = handler.handle( req ) 520 | 521 | content = Yajl::Parser.parse( result.body.read ) 522 | 523 | expect( result.status ).to eq( 200 ) 524 | expect( result.headers.content_type ).to eq( 'application/json' ) 525 | 526 | expect( content ).to be_nil 527 | end 528 | 529 | 530 | it "can merge in new metadata for an existing resource with a POST" do 531 | uuid = handler.datastore.save( png_io ) 532 | handler.metastore.save( uuid, { 533 | 'format' => 'image/png', 534 | 'extent' => 288 535 | }) 536 | 537 | body_json = Yajl.dump({ 'comment' => 'Ignore me!', 'uuid' => 123 }) 538 | req = factory.post( "/#{uuid}/metadata", body_json, 'Content-type' => 'application/json' ) 539 | result = handler.handle( req ) 540 | 541 | expect( result.status ).to eq( HTTP::OK ) 542 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'Ignore me!' ) 543 | expect( handler.metastore.fetch_value(uuid, 'uuid') ).to be_nil 544 | end 545 | 546 | 547 | it "ignores attempts to alter operational metadata when merging" do 548 | uuid = handler.datastore.save( png_io ) 549 | handler.metastore.save( uuid, { 550 | 'format' => 'image/png', 551 | 'extent' => 288, 552 | }) 553 | 554 | body_json = Yajl.dump({ 'format' => 'text/plain', 'comment' => 'Ignore me!' }) 555 | req = factory.post( "/#{uuid}/metadata", body_json, 'Content-type' => 'application/json' ) 556 | result = handler.handle( req ) 557 | 558 | expect( result.status ).to eq( HTTP::OK ) 559 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'Ignore me!' ) 560 | expect( handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' ) 561 | end 562 | 563 | 564 | it "can create single metadata values with a POST" do 565 | uuid = handler.datastore.save( png_io ) 566 | handler.metastore.save( uuid, { 567 | 'format' => 'image/png', 568 | 'extent' => 288, 569 | }) 570 | 571 | req = factory.post( "/#{uuid}/metadata/comment", "urrrg", 'Content-type' => 'text/plain' ) 572 | result = handler.handle( req ) 573 | 574 | expect( result.status ).to eq( HTTP::CREATED ) 575 | expect( result.headers.location ).to match( %r|#{uuid}/metadata/comment$| ) 576 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'urrrg' ) 577 | end 578 | 579 | 580 | it "returns NOT_FOUND when attempting to create metadata for a non-existent object" do 581 | req = factory.post( "/#{TEST_UUID}/metadata/comment", "urrrg", 'Content-type' => 'text/plain' ) 582 | result = handler.handle( req ) 583 | 584 | expect( result.status ).to eq( HTTP::NOT_FOUND ) 585 | expect( result.body.string ).to match( /no such object/i ) 586 | end 587 | 588 | 589 | it "returns CONFLICT when attempting to create a single metadata value if it already exists" do 590 | uuid = handler.datastore.save( png_io ) 591 | handler.metastore.save( uuid, { 592 | 'format' => 'image/png', 593 | 'extent' => 288, 594 | 'comment' => 'nill bill' 595 | }) 596 | 597 | req = factory.post( "/#{uuid}/metadata/comment", "urrrg", 'Content-type' => 'text/plain' ) 598 | result = handler.handle( req ) 599 | 600 | expect( result.status ).to eq( HTTP::CONFLICT ) 601 | expect( result.body.string ).to match( /already exists/i ) 602 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'nill bill' ) 603 | end 604 | 605 | 606 | it "can create single metadata values with a PUT" do 607 | uuid = handler.datastore.save( png_io ) 608 | handler.metastore.save( uuid, { 609 | 'format' => 'image/png', 610 | 'extent' => 288, 611 | }) 612 | 613 | req = factory.put( "/#{uuid}/metadata/comment", "urrrg", 'Content-type' => 'text/plain' ) 614 | result = handler.handle( req ) 615 | 616 | expect( result.status ).to eq( HTTP::NO_CONTENT ) 617 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'urrrg' ) 618 | end 619 | 620 | 621 | it "can replace a single metadata value with a PUT" do 622 | uuid = handler.datastore.save( png_io ) 623 | handler.metastore.save( uuid, { 624 | 'format' => 'image/png', 625 | 'extent' => 288, 626 | 'comment' => 'nill bill' 627 | }) 628 | 629 | req = factory.put( "/#{uuid}/metadata/comment", "urrrg", 'Content-type' => 'text/plain' ) 630 | result = handler.handle( req ) 631 | 632 | expect( result.status ).to eq( HTTP::NO_CONTENT ) 633 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'urrrg' ) 634 | end 635 | 636 | 637 | it "returns FORBIDDEN when attempting to replace a operational metadata value with a PUT" do 638 | uuid = handler.datastore.save( png_io ) 639 | handler.metastore.save( uuid, { 640 | 'format' => 'image/png', 641 | 'extent' => 288, 642 | 'comment' => 'nill bill' 643 | }) 644 | 645 | req = factory.put( "/#{uuid}/metadata/format", "image/gif", 'Content-type' => 'text/plain' ) 646 | result = handler.handle( req ) 647 | 648 | expect( result.status ).to eq( HTTP::FORBIDDEN ) 649 | expect( result.body.string ).to match( /protected metadata/i ) 650 | expect( handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' ) 651 | end 652 | 653 | 654 | it "can replace all metadata with a PUT" do 655 | uuid = handler.datastore.save( png_io ) 656 | handler.metastore.save( uuid, { 657 | 'format' => 'image/png', 658 | 'extent' => 288, 659 | 'comment' => 'nill bill', 660 | 'ephemeral' => 'butterflies', 661 | }) 662 | 663 | req = factory.put( "/#{uuid}/metadata", %[{"comment":"Yeah."}], 664 | 'Content-type' => 'application/json' ) 665 | result = handler.handle( req ) 666 | 667 | expect( result.status ).to eq( HTTP::OK ) 668 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'Yeah.' ) 669 | expect( handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' ) 670 | expect( handler.metastore ).to_not include( 'ephemeral' ) 671 | end 672 | 673 | 674 | it "can remove all non-default metadata with a DELETE" do 675 | timestamp = Time.now.getgm 676 | uuid = handler.datastore.save( png_io ) 677 | handler.metastore.save( uuid, { 678 | 'format' => 'image/png', 679 | 'extent' => 288, 680 | 'comment' => 'nill bill', 681 | 'useragent' => 'Inky/2.0', 682 | 'uploadaddress' => '127.0.0.1', 683 | 'created' => timestamp, 684 | }) 685 | 686 | req = factory.delete( "/#{uuid}/metadata" ) 687 | result = handler.handle( req ) 688 | 689 | expect( result.status ).to eq( HTTP::OK ) 690 | expect( result.body.string ).to_not be_empty 691 | expect( handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' ) 692 | expect( handler.metastore.fetch_value(uuid, 'extent') ).to eq( 288 ) 693 | expect( handler.metastore.fetch_value(uuid, 'uploadaddress') ).to eq( '127.0.0.1' ) 694 | expect( handler.metastore.fetch_value(uuid, 'created') ).to eq( timestamp ) 695 | 696 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to be_nil 697 | expect( handler.metastore.fetch_value(uuid, 'useragent') ).to be_nil 698 | end 699 | 700 | 701 | it "can remove a single metadata value with DELETE" do 702 | uuid = handler.datastore.save( png_io ) 703 | handler.metastore.save( uuid, { 704 | 'format' => 'image/png', 705 | 'comment' => 'nill bill' 706 | }) 707 | 708 | req = factory.delete( "/#{uuid}/metadata/comment" ) 709 | result = handler.handle( req ) 710 | 711 | expect( result.status ).to eq( HTTP::NO_CONTENT ) 712 | expect( result.body.string ).to be_empty 713 | expect( handler.metastore.fetch_value(uuid, 'comment') ).to be_nil 714 | expect( handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' ) 715 | end 716 | 717 | 718 | it "returns FORBIDDEN when attempting to remove a operational metadata value with a DELETE" do 719 | uuid = handler.datastore.save( png_io ) 720 | handler.metastore.save( uuid, { 721 | 'format' => 'image/png' 722 | }) 723 | 724 | req = factory.delete( "/#{uuid}/metadata/format" ) 725 | result = handler.handle( req ) 726 | 727 | expect( result.status ).to eq( HTTP::FORBIDDEN ) 728 | expect( result.body.string ).to match( /protected metadata/i ) 729 | expect( handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' ) 730 | end 731 | end 732 | 733 | 734 | context "processors" do 735 | 736 | before( :all ) do 737 | @original_filters = described_class.filters.dup 738 | described_class.filters.replace({ :request => [], :response => [], :both => [] }) 739 | end 740 | 741 | after( :all ) do 742 | described_class.filters.replace( @original_filters ) 743 | end 744 | 745 | before( :each ) do 746 | described_class.processors.clear 747 | described_class.filters.values.each( &:clear ) 748 | end 749 | 750 | 751 | let( :factory ) do 752 | Mongrel2::RequestFactory.new( 753 | :route => '/', 754 | :headers => {:accept => '*/*'}) 755 | end 756 | 757 | let!( :test_processor ) do 758 | klass = Class.new( Thingfish::Processor ) do 759 | extend Loggability 760 | log_to :thingfish 761 | 762 | handled_types 'text/plain' 763 | 764 | def initialize( * ) 765 | super 766 | @was_called = false 767 | end 768 | attr_reader :was_called 769 | 770 | def self::name; 'Thingfish::Processor::Test'; end 771 | def on_request( request ) 772 | @was_called = true 773 | self.log.debug "Adding a comment to a request." 774 | request.add_metadata( 'test:comment' => "Yo, it totally worked." ) 775 | 776 | io = StringIO.new( "Chunkers!" ) 777 | io.rewind 778 | related_metadata = { 'format' => 'text/plain', 'relationship' => 'comment' } 779 | request.add_related_resource( io, related_metadata ) 780 | end 781 | def on_response( response ) 782 | @was_called = true 783 | content = response.body.read 784 | response.body.rewind 785 | response.body.print( content.reverse ) 786 | response.body.rewind 787 | end 788 | end 789 | # Re-call inherited so it associates the processor plugin with its name 790 | Thingfish::Processor.inherited( klass ) 791 | klass 792 | end 793 | 794 | 795 | it "loads configured processors when it is instantiated" do 796 | logger = Loggability[ described_class ] 797 | logger.debug( "*** %p" % described_class.filters ) 798 | logger.debug( "*** %p" % @original_filters ) 799 | 800 | described_class.configure( :processors => %w[test] ) 801 | 802 | expect( described_class.processors ).to be_an( Array ) 803 | 804 | processor = described_class.processors.first 805 | expect( processor ).to be_an_instance_of( test_processor ) 806 | end 807 | 808 | 809 | it "processes requests" do 810 | described_class.configure( :processors => %w[test] ) 811 | 812 | req = factory.post( '/', TEST_TEXT_DATA, content_type: 'text/plain' ) 813 | req.headers.content_length = TEST_TEXT_DATA.bytesize 814 | res = handler.handle( req ) 815 | uuid = res.headers.x_thingfish_uuid 816 | 817 | Thingfish.logger.debug "Metastore contains: %p" % [ handler.metastore.storage ] 818 | 819 | expect( handler.metastore.fetch(uuid) ). 820 | to include( 'test:comment' => 'Yo, it totally worked.') 821 | related_uuids = handler.metastore.fetch_related_oids( uuid ) 822 | expect( related_uuids.size ).to eq( 1 ) 823 | 824 | r_uuid = related_uuids.first.downcase 825 | expect( handler.metastore.fetch_value(r_uuid, 'relation') ).to eq( uuid ) 826 | expect( handler.metastore.fetch_value(r_uuid, 'format') ).to eq( 'text/plain' ) 827 | expect( handler.metastore.fetch_value(r_uuid, 'extent') ).to eq( 9 ) 828 | expect( handler.metastore.fetch_value(r_uuid, 'relationship') ).to eq( 'comment' ) 829 | 830 | expect( handler.datastore.fetch(r_uuid).read ).to eq( 'Chunkers!' ) 831 | end 832 | 833 | 834 | it "doesn't process requests for paths under the metadata uri-space" do 835 | described_class.configure( :processors => %w[test] ) 836 | processor = described_class.processors.first 837 | 838 | req = factory.post( "/#{TEST_UUID}/metadata", TEST_TEXT_DATA, content_type: 'text/plain' ) 839 | handler.handle( req ) 840 | 841 | expect( processor.was_called ).to be_falsey 842 | end 843 | 844 | 845 | it "processes responses" do 846 | described_class.configure( :processors => %w[test] ) 847 | 848 | uuid = handler.datastore.save( text_io ) 849 | handler.metastore.save( uuid, {'format' => 'text/plain'} ) 850 | 851 | req = factory.get( "/#{uuid}" ) 852 | res = handler.handle( req ) 853 | 854 | res.body.rewind 855 | expect( res.body.read ).to eq( TEST_TEXT_DATA.reverse ) 856 | end 857 | 858 | 859 | it "doesn't process responses for paths under the metadata uri-space" do 860 | described_class.configure( :processors => %w[test] ) 861 | processor = described_class.processors.first 862 | 863 | uuid = handler.datastore.save( text_io ) 864 | handler.metastore.save( uuid, {'format' => 'text/plain'} ) 865 | 866 | req = factory.get( "/#{uuid}/metadata" ) 867 | handler.handle( req ) 868 | 869 | expect( processor.was_called ).to be_falsey 870 | end 871 | end 872 | 873 | 874 | context "event hook" do 875 | 876 | let( :factory ) do 877 | Mongrel2::RequestFactory.new( 878 | :route => '/', 879 | :headers => {:accept => '*/*'}) 880 | end 881 | 882 | before( :each ) do 883 | handler.setup_event_socket 884 | 885 | @subsock = CZTop::Socket::SUB.new 886 | @subsock.options.linger = 0 887 | @subsock.subscribe( '' ) 888 | @subsock.connect( handler.event_socket.last_endpoint ) 889 | end 890 | 891 | after( :each ) do 892 | @subsock.close 893 | end 894 | 895 | it "publishes notifications about uploaded assets to a PUBSUB socket" do 896 | req = factory.post( '/', TEST_TEXT_DATA, content_type: 'text/plain' ) 897 | req.headers.content_length = TEST_TEXT_DATA.bytesize 898 | 899 | poller = CZTop::Poller.new 900 | poller.add_reader( @subsock ) 901 | 902 | handler.handle( req ) 903 | event = poller.wait( 500 ) 904 | 905 | expect( event ).to_not be_nil 906 | 907 | message = event.socket.receive 908 | expect( message.frames.count ).to be( 2 ) 909 | 910 | expect( message.frames.first.to_s ).to eq( 'created' ) 911 | expect( message.frames.last.to_s ).to match( /^\{"uuid":"#{UUID_PATTERN}"\}$/ ) 912 | end 913 | end 914 | 915 | end 916 | 917 | # vim: set nosta noet ts=4 sw=4 ft=rspec: 918 | --------------------------------------------------------------------------------