├── .gitignore ├── .travis.yml ├── Dockerfile ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin ├── rake └── rspec ├── docker-compose.yml ├── lib ├── generators │ └── refile │ │ └── postgres │ │ ├── initializer │ │ ├── USAGE │ │ ├── initializer_generator.rb │ │ └── templates │ │ │ └── refile.rb │ │ └── migration │ │ ├── USAGE │ │ ├── migration_generator.rb │ │ └── templates │ │ └── migration.rb.erb └── refile │ ├── postgres.rb │ └── postgres │ ├── backend.rb │ ├── backend │ └── reader.rb │ ├── smart_transaction.rb │ └── version.rb ├── migration_to_1_3_0.md ├── postgres-setup ├── refile-postgres.gemspec └── spec ├── refile └── postgres │ └── backend_spec.rb └── spec_helper.rb /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | *.bundle 11 | *.so 12 | *.o 13 | *.a 14 | mkmf.log 15 | .ruby-version 16 | .ruby-gemset 17 | .vagrant 18 | berks-cookbooks 19 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | cache: bundler 3 | rvm: 4 | - 2.4 5 | - 2.5 6 | - 2.6 7 | - 2.7.0-preview3 8 | addons: 9 | postgresql: "11" 10 | before_script: 11 | - cat $TRAVIS_BUILD_DIR/postgres-setup | psql -U postgres 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.6.3 2 | 3 | RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \ 4 | && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ 5 | && apt-get update -qq \ 6 | && apt-get install -y build-essential libpq-dev postgresql-client-11 postgresql-contrib-11 7 | 8 | 9 | ARG INSTALL_BUNDLER_VERSION=2.0.2 10 | 11 | RUN gem install bundler --version=${INSTALL_BUNDLER_VERSION} 12 | 13 | ENV BUNDLER_VERSION=${INSTALL_BUNDLER_VERSION} 14 | 15 | ENV APP_PATH=/app 16 | 17 | RUN mkdir ${APP_PATH} 18 | 19 | WORKDIR ${APP_PATH} 20 | 21 | ADD . ${APP_PATH} 22 | 23 | CMD bundle check || bundle install; bundle exec rspec spec 24 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | git_source(:github) { |repo| "https://github.com/#{repo}.git" } 3 | 4 | gem "refile", github: "refile/refile", branch: "master" 5 | 6 | gemspec 7 | 8 | gem "rspec" 9 | gem "webmock" 10 | gem "pry" 11 | gem "pry-byebug" 12 | gem "rails", "~> 6.0.0" 13 | gem "rake" 14 | gem "simplecov" 15 | gem "codeclimate-test-reporter" 16 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2019 Krists Ozols 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Refile::Postgres 2 | 3 | A PostgreSQL backend for [Refile](https://github.com/elabs/refile). 4 | 5 | [![Build Status](https://travis-ci.org/krists/refile-postgres.svg?branch=master)](https://travis-ci.org/krists/refile-postgres) 6 | [![Code Climate](https://codeclimate.com/github/krists/refile-postgres/badges/gpa.svg)](https://codeclimate.com/github/krists/refile-postgres) 7 | [![Test Coverage](https://codeclimate.com/github/krists/refile-postgres/badges/coverage.svg)](https://codeclimate.com/github/krists/refile-postgres/coverage) 8 | 9 | ## Why? 10 | 11 | * You want to store all your data in one place to simplify backups and replication 12 | * ACID 13 | 14 | ## Take into account 15 | 16 | * Gem is developed and tested using Postgresql 9.3, Ruby 2.1 and ActiveRecord 4.x. It might work with earlier versions. 17 | * Performance hit storing files in database 18 | * Higher memory requirements for database 19 | * Backups can take significantly longer 20 | 21 | ## Installation 22 | 23 | Add this line to your application's Gemfile: 24 | 25 | ```ruby 26 | gem 'refile-postgres', '~> 1.4.0' 27 | ``` 28 | 29 | And then execute: 30 | 31 | $ bundle 32 | 33 | Or install it yourself as: 34 | 35 | $ gem install refile-postgres 36 | 37 | ## Usage with Rails 38 | 39 | Application must have sql as schema format to properly dump OID type. 40 | 41 | ```ruby 42 | # config/application.rb 43 | config.active_record.schema_format = :sql 44 | ``` 45 | 46 | Generate migration for table were to store list of attachments. 47 | 48 | $ rails g refile:postgres:migration 49 | 50 | Run the migrations 51 | 52 | $ rake db:migrate 53 | 54 | Generate initializer and set Refile::Postgres as `store` backend. 55 | 56 | $ rails g refile:postgres:initializer 57 | 58 | ## Upgrade to 1.3.0 59 | 60 | If you have been using refile-postgres before 1.3.0 version then please follow [upgrade guide](https://github.com/krists/refile-postgres/blob/master/migration_to_1_3_0.md) 61 | 62 | 63 | ## Contributing 64 | 65 | 1. Fork it ( https://github.com/krists/refile-postgres/fork ) 66 | 2. Create your feature branch (`git checkout -b my-new-feature`) 67 | 3. Commit your changes (`git commit -am 'Add some feature'`) 68 | 4. Push to the branch (`git push origin my-new-feature`) 69 | 5. Create a new Pull Request 70 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | begin 4 | require 'rspec/core/rake_task' 5 | RSpec::Core::RakeTask.new(:spec) 6 | task :default => :spec 7 | rescue LoadError 8 | # no rspec available 9 | end 10 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rake' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "pathname" 12 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path("../bundle", __FILE__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require "rubygems" 27 | require "bundler/setup" 28 | 29 | load Gem.bin_path("rake", "rake") 30 | -------------------------------------------------------------------------------- /bin/rspec: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rspec' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "pathname" 12 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path("../bundle", __FILE__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require "rubygems" 27 | require "bundler/setup" 28 | 29 | load Gem.bin_path("rspec-core", "rspec") 30 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | app: 4 | build: . 5 | depends_on: 6 | - postgres 7 | volumes: 8 | - gems:/usr/local/bundle 9 | - .:/app 10 | environment: 11 | POSTGRES_HOST: postgres 12 | POSTGRES_PORT: 5432 13 | POSTGRES_DB: refile_test 14 | POSTGRES_USER: refile_postgres_test_user 15 | POSTGRES_PASSWORD: refilepostgres 16 | postgres: 17 | image: postgres:11 18 | environment: 19 | POSTGRES_USER: refile_postgres_test_user 20 | POSTGRES_PASSWORD: refilepostgres 21 | POSTGRES_DB: refile_test 22 | volumes: 23 | gems: 24 | driver: local -------------------------------------------------------------------------------- /lib/generators/refile/postgres/initializer/USAGE: -------------------------------------------------------------------------------- 1 | Description: 2 | Generates initializer and sets Postgres as store backend 3 | 4 | Example: 5 | rails generate initializer 6 | 7 | This will create: 8 | config/initializers/refile.rb 9 | -------------------------------------------------------------------------------- /lib/generators/refile/postgres/initializer/initializer_generator.rb: -------------------------------------------------------------------------------- 1 | class Refile::Postgres::InitializerGenerator < Rails::Generators::Base 2 | source_root File.expand_path('../templates', __FILE__) 3 | def copy_initializer_file 4 | copy_file "refile.rb", "config/initializers/refile.rb" 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /lib/generators/refile/postgres/initializer/templates/refile.rb: -------------------------------------------------------------------------------- 1 | require "refile" 2 | Refile.configure do |config| 3 | connection = lambda { |&blk| ActiveRecord::Base.connection_pool.with_connection { |con| blk.call(con.raw_connection) } } 4 | config.store = Refile::Postgres::Backend.new(connection) 5 | end 6 | -------------------------------------------------------------------------------- /lib/generators/refile/postgres/migration/USAGE: -------------------------------------------------------------------------------- 1 | Description: 2 | Generates database migration for table to story list of attachments 3 | 4 | Example: 5 | rails generate refile:postgres:migration 6 | 7 | This will create: 8 | db/migrate/_create_refile_attachments.rb 9 | -------------------------------------------------------------------------------- /lib/generators/refile/postgres/migration/migration_generator.rb: -------------------------------------------------------------------------------- 1 | require "refile" 2 | require 'rails/generators/active_record' 3 | class Refile::Postgres::MigrationGenerator < Rails::Generators::Base 4 | include Rails::Generators::Migration 5 | argument :table_name, type: :string, default: Refile::Postgres::Backend::DEFAULT_REGISTRY_TABLE 6 | source_root File.expand_path('../templates', __FILE__) 7 | 8 | def self.next_migration_number(path) 9 | Time.now.utc.strftime("%Y%m%d%H%M%S") 10 | end 11 | 12 | def copy_migration_file 13 | migration_template "migration.rb.erb", "db/migrate/create_#{table_name}.rb" 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/generators/refile/postgres/migration/templates/migration.rb.erb: -------------------------------------------------------------------------------- 1 | class Create<%= table_name.camelize %> < ActiveRecord::Migration[4.2] 2 | def up 3 | execute %Q{ 4 | CREATE TABLE <%= table_name %> ( 5 | id integer NOT NULL, 6 | oid oid NOT NULL, 7 | namespace character varying NOT NULL, 8 | created_at timestamp without time zone DEFAULT ('now'::text)::timestamp without time zone 9 | ); 10 | 11 | CREATE SEQUENCE <%= table_name %>_id_seq 12 | START WITH 1 13 | INCREMENT BY 1 14 | NO MINVALUE 15 | NO MAXVALUE 16 | CACHE 1; 17 | 18 | ALTER SEQUENCE <%= table_name %>_id_seq OWNED BY <%= table_name %>.id; 19 | 20 | ALTER TABLE ONLY <%= table_name %> ALTER COLUMN id SET DEFAULT nextval('<%= table_name %>_id_seq'::regclass); 21 | 22 | ALTER TABLE ONLY <%= table_name %> ADD CONSTRAINT <%= table_name %>_pkey PRIMARY KEY (id); 23 | 24 | CREATE INDEX index_<%= table_name %>_on_namespace ON <%= table_name %> USING btree (namespace); 25 | 26 | CREATE INDEX index_<%= table_name %>_on_oid ON <%= table_name %> USING btree (oid); 27 | } 28 | end 29 | 30 | def drop 31 | drop_table :<%= table_name %> 32 | end 33 | end 34 | 35 | -------------------------------------------------------------------------------- /lib/refile/postgres.rb: -------------------------------------------------------------------------------- 1 | require "refile/backend_macros" 2 | require "refile/postgres/version" 3 | require "refile/postgres/smart_transaction" 4 | require "refile/postgres/backend" 5 | require "refile/postgres/backend/reader" 6 | 7 | module Refile 8 | module Postgres 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/refile/postgres/backend.rb: -------------------------------------------------------------------------------- 1 | module Refile 2 | module Postgres 3 | class Backend 4 | include SmartTransaction 5 | extend Refile::BackendMacros 6 | RegistryTableDoesNotExistError = Class.new(StandardError) 7 | DEFAULT_REGISTRY_TABLE = "refile_attachments" 8 | DEFAULT_NAMESPACE = "default" 9 | PG_LARGE_OBJECT_METADATA_TABLE = "pg_largeobject_metadata" 10 | READ_CHUNK_SIZE = 16384 11 | 12 | def initialize(connection_or_proc, max_size: nil, namespace: DEFAULT_NAMESPACE, registry_table: DEFAULT_REGISTRY_TABLE) 13 | @connection_or_proc = connection_or_proc 14 | @namespace = namespace.to_s 15 | @registry_table = registry_table 16 | @registry_table_validated = false 17 | @max_size = max_size 18 | end 19 | 20 | attr_reader :namespace, :max_size 21 | 22 | def registry_table 23 | unless @registry_table_validated 24 | with_connection do |connection| 25 | connection.exec_params("SELECT * FROM pg_catalog.pg_tables WHERE tablename = $1::varchar;", [@registry_table]) do |result| 26 | if result.count == 0 27 | raise RegistryTableDoesNotExistError.new(%{Please create a table "#{@registry_table}" where backend could store list of attachments}) 28 | end 29 | end 30 | end 31 | @registry_table_validated = true 32 | end 33 | @registry_table 34 | end 35 | 36 | verify_uploadable def upload(uploadable) 37 | with_connection do |connection| 38 | oid = connection.lo_creat 39 | ensure_in_transaction(connection) do 40 | begin 41 | handle = connection.lo_open(oid, PG::INV_WRITE) 42 | connection.lo_truncate(handle, 0) 43 | buffer = "" # reuse the same buffer 44 | until uploadable.eof? 45 | uploadable.read(READ_CHUNK_SIZE, buffer) 46 | connection.lo_write(handle, buffer) 47 | end 48 | uploadable.close 49 | connection.exec_params("INSERT INTO #{registry_table} (oid, namespace) VALUES ($1::oid, $2::varchar);", [oid, namespace]) 50 | Refile::File.new(self, oid.to_s) 51 | ensure 52 | connection.lo_close(handle) 53 | end 54 | end 55 | end 56 | end 57 | 58 | verify_id def open(id) 59 | if exists?(id) 60 | Reader.new(@connection_or_proc, id) 61 | else 62 | raise ArgumentError.new("No such attachment with ID: #{id}") 63 | end 64 | end 65 | 66 | verify_id def read(id) 67 | if exists?(id) 68 | open(id).read 69 | else 70 | nil 71 | end 72 | end 73 | 74 | verify_id def get(id) 75 | Refile::File.new(self, id) 76 | end 77 | 78 | verify_id def exists?(id) 79 | with_connection do |connection| 80 | connection.exec_params(%{ 81 | SELECT count(*) FROM #{registry_table} 82 | INNER JOIN #{PG_LARGE_OBJECT_METADATA_TABLE} 83 | ON #{registry_table}.oid = #{PG_LARGE_OBJECT_METADATA_TABLE}.oid 84 | WHERE #{registry_table}.namespace = $1::varchar 85 | AND #{registry_table}.oid = $2::integer; 86 | }, [namespace, id.to_s.to_i]) do |result| 87 | result[0]["count"].to_i > 0 88 | end 89 | end 90 | end 91 | 92 | verify_id def size(id) 93 | if exists?(id) 94 | open(id).size 95 | else 96 | nil 97 | end 98 | end 99 | 100 | verify_id def delete(id) 101 | if exists?(id) 102 | with_connection do |connection| 103 | ensure_in_transaction(connection) do 104 | rez = connection.exec_params(%{ 105 | SELECT * FROM #{registry_table} 106 | WHERE #{registry_table}.oid = $1::integer 107 | LIMIT 1 108 | }, [id.to_s.to_i]) 109 | oid = rez[0]['oid'].to_i 110 | connection.lo_unlink(oid) 111 | connection.exec_params("DELETE FROM #{registry_table} WHERE oid = $1::oid;", [oid]) 112 | end 113 | end 114 | end 115 | end 116 | 117 | def clear!(confirm = nil) 118 | raise Refile::Confirm unless confirm == :confirm 119 | registry_table 120 | with_connection do |connection| 121 | ensure_in_transaction(connection) do 122 | connection.exec_params(%{ 123 | SELECT #{registry_table}.oid FROM #{registry_table} 124 | INNER JOIN #{PG_LARGE_OBJECT_METADATA_TABLE} ON #{registry_table}.oid = #{PG_LARGE_OBJECT_METADATA_TABLE}.oid 125 | WHERE #{registry_table}.namespace = $1::varchar; 126 | }, [namespace]) do |result| 127 | result.each_row do |row| 128 | connection.lo_unlink(row[0].to_i) 129 | end 130 | end 131 | connection.exec_params("DELETE FROM #{registry_table} WHERE namespace = $1::varchar;", [namespace]) 132 | end 133 | end 134 | end 135 | 136 | end 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /lib/refile/postgres/backend/reader.rb: -------------------------------------------------------------------------------- 1 | module Refile 2 | module Postgres 3 | class Backend 4 | class Reader 5 | STREAM_CHUNK_SIZE = 16384 6 | include SmartTransaction 7 | 8 | def initialize(connection_or_proc, oid) 9 | @connection_or_proc = connection_or_proc 10 | @oid = oid.to_s.to_i 11 | @closed = false 12 | @pos = 0 13 | end 14 | 15 | attr_reader :oid, :pos 16 | 17 | def read(length = nil, buffer = nil) 18 | result = if length 19 | raise "closed" if @closed 20 | with_connection do |connection| 21 | smart_transaction(connection) do |descriptor| 22 | connection.lo_lseek(descriptor, @pos, PG::SEEK_SET) 23 | data = connection.lo_read(descriptor, length) 24 | @pos = connection.lo_tell(descriptor) 25 | data 26 | end 27 | end 28 | else 29 | with_connection do |connection| 30 | smart_transaction(connection) do |descriptor| 31 | connection.lo_read(descriptor, size) 32 | end 33 | end 34 | end 35 | buffer.replace(result) if buffer and result 36 | result 37 | end 38 | 39 | def eof? 40 | with_connection do |connection| 41 | smart_transaction(connection) do |descriptor| 42 | @pos == size 43 | end 44 | end 45 | end 46 | 47 | def each 48 | if block_given? 49 | until eof? 50 | yield(read(STREAM_CHUNK_SIZE)) 51 | end 52 | else 53 | to_enum 54 | end 55 | end 56 | 57 | def size 58 | @size ||= fetch_size 59 | end 60 | 61 | def close 62 | @closed = true 63 | end 64 | 65 | private 66 | 67 | def fetch_size 68 | with_connection do |connection| 69 | smart_transaction(connection) do |descriptor| 70 | current_position = connection.lo_tell(descriptor) 71 | end_position = connection.lo_lseek(descriptor, 0, PG::SEEK_END) 72 | connection.lo_lseek(descriptor, current_position, PG::SEEK_SET) 73 | end_position 74 | end 75 | end 76 | end 77 | 78 | end 79 | end 80 | end 81 | end 82 | 83 | 84 | -------------------------------------------------------------------------------- /lib/refile/postgres/smart_transaction.rb: -------------------------------------------------------------------------------- 1 | module Refile 2 | module Postgres 3 | module SmartTransaction 4 | INIT_CONNECTION_ARG_ERROR_MSG = "When initializing new Refile::Postgres::Backend first argument should be an instance of PG::Connection or a lambda/proc that yields it." 5 | PQTRANS_INTRANS = 2 # (idle, within transaction block) 6 | 7 | def smart_transaction(connection) 8 | result = nil 9 | ensure_in_transaction(connection) do 10 | begin 11 | handle = connection.lo_open(oid) 12 | result = yield handle 13 | connection.lo_close(handle) 14 | end 15 | end 16 | result 17 | end 18 | 19 | def ensure_in_transaction(connection) 20 | if connection.transaction_status == PQTRANS_INTRANS 21 | yield 22 | else 23 | connection.transaction do 24 | yield 25 | end 26 | end 27 | end 28 | 29 | def with_connection 30 | if @connection_or_proc.is_a?(PG::Connection) 31 | yield @connection_or_proc 32 | else 33 | if @connection_or_proc.is_a?(Proc) 34 | block_has_been_executed = false 35 | value = nil 36 | @connection_or_proc.call do |connection| 37 | block_has_been_executed = true 38 | raise ArgumentError.new(INIT_CONNECTION_ARG_ERROR_MSG) unless connection.is_a?(PG::Connection) 39 | value = yield connection 40 | end 41 | raise ArgumentError.new(INIT_CONNECTION_ARG_ERROR_MSG) unless block_has_been_executed 42 | value 43 | else 44 | raise ArgumentError.new(INIT_CONNECTION_ARG_ERROR_MSG) 45 | end 46 | end 47 | end 48 | 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/refile/postgres/version.rb: -------------------------------------------------------------------------------- 1 | module Refile 2 | module Postgres 3 | VERSION = "1.4.1" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /migration_to_1_3_0.md: -------------------------------------------------------------------------------- 1 | # Migration to refile-postgres version 1.3.0 2 | 3 | Please check [issue](https://github.com/krists/refile-postgres/issues/9) for more details. 4 | 5 | 1) Change Rails schema dump format to SQL in your `config/application.rb` file 6 | ```ruby 7 | # Use structure.sql instead of schema.rb 8 | config.active_record.schema_format = :sql 9 | ``` 10 | 2) Change version number in Gemfile 11 | ```ruby 12 | gem 'refile-postgres', '~> 1.3.0' 13 | ``` 14 | 3) Create Rails migration 15 | ``` 16 | rails g migration refile_postgres_migration_to_1_3_0 17 | ``` 18 | 4) Add content to migration 19 | ```ruby 20 | class RefilePostgresMigrationTo130 < ActiveRecord::Migration 21 | def up 22 | execute <<-SQL 23 | DROP INDEX index_refile_attachments_on_namespace; 24 | ALTER TABLE refile_attachments RENAME TO old_refile_attachments; 25 | ALTER TABLE ONLY old_refile_attachments RENAME CONSTRAINT refile_attachments_pkey TO old_refile_attachments_pkey; 26 | CREATE TABLE refile_attachments ( 27 | id integer NOT NULL, 28 | oid oid NOT NULL, 29 | namespace character varying NOT NULL, 30 | created_at timestamp without time zone DEFAULT ('now'::text)::timestamp without time zone 31 | ); 32 | ALTER TABLE ONLY refile_attachments ADD CONSTRAINT refile_attachments_pkey PRIMARY KEY (id); 33 | ALTER SEQUENCE refile_attachments_id_seq RESTART OWNED BY refile_attachments.id; 34 | ALTER TABLE ONLY refile_attachments ALTER COLUMN id SET DEFAULT nextval('refile_attachments_id_seq'::regclass); 35 | INSERT INTO refile_attachments (oid, namespace) SELECT id, namespace FROM old_refile_attachments; 36 | CREATE INDEX index_refile_attachments_on_namespace ON refile_attachments USING btree (namespace); 37 | CREATE INDEX index_refile_attachments_on_oid ON refile_attachments USING btree (oid); 38 | DROP TABLE old_refile_attachments; 39 | SQL 40 | end 41 | 42 | def down 43 | raise ActiveRecord::IrreversibleMigration 44 | end 45 | end 46 | ``` 47 | 5) Now it is safe to run [vacuumlo](http://www.postgresql.org/docs/9.5/static/vacuumlo.html) 48 | -------------------------------------------------------------------------------- /postgres-setup: -------------------------------------------------------------------------------- 1 | DROP DATABASE IF EXISTS refile_test; 2 | DROP ROLE IF EXISTS refile_postgres_test_user; 3 | CREATE ROLE refile_postgres_test_user WITH NOSUPERUSER CREATEDB LOGIN PASSWORD 'refilepostgres'; 4 | CREATE DATABASE refile_test WITH OWNER refile_postgres_test_user; 5 | -------------------------------------------------------------------------------- /refile-postgres.gemspec: -------------------------------------------------------------------------------- 1 | require_relative "./lib/refile/postgres/version" 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "refile-postgres" 5 | spec.version = Refile::Postgres::VERSION 6 | spec.authors = ["Krists Ozols"] 7 | spec.email = ["krists.ozols@gmail.com"] 8 | spec.summary = %q{Postgres database as a backend for Refile} 9 | spec.description = %q{Postgres database as a backend for Refile. Uses "Large Objects". See https://www.postgresql.org/docs/current/largeobjects.html for more info.} 10 | spec.homepage = "https://github.com/krists/refile-postgres" 11 | spec.license = "MIT" 12 | spec.files = Dir["lib/**/*", "LICENSE.txt", "README.md"] 13 | spec.require_paths = ["lib"] 14 | spec.add_dependency "refile", [">= 0.6", "< 0.8"] 15 | spec.add_dependency "pg" 16 | end 17 | -------------------------------------------------------------------------------- /spec/refile/postgres/backend_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | require "tempfile" 3 | 4 | describe Refile::Postgres::Backend do 5 | let(:connection) { test_connection } 6 | let(:backend) { Refile::Postgres::Backend.new(connection_or_proc, max_size: 100) } 7 | 8 | context "Connection tests" do 9 | context "when not using procs and providing PG::Connection directly" do 10 | let(:connection_or_proc) { connection } 11 | it "reuses the same PG::Connection" do 12 | expect(backend.with_connection { |c| c.db }).to eq(TEST_DB_NAME) 13 | end 14 | end 15 | 16 | context "when using proc" do 17 | context "when lambda does not yield a block but returns connection" do 18 | let(:connection_or_proc) { lambda { connection } } 19 | it "raises argument error" do 20 | expect { 21 | backend.with_connection { |c| c.db } 22 | }.to raise_error(ArgumentError, "When initializing new Refile::Postgres::Backend first argument should be an instance of PG::Connection or a lambda/proc that yields it.") 23 | end 24 | end 25 | 26 | context "when lambda does yield a PG::Connection" do 27 | let(:connection_or_proc) { lambda { |&blk| blk.call(connection) } } 28 | it "is usable in queries" do 29 | expect(backend.with_connection { |c| c.db }).to eq(TEST_DB_NAME) 30 | end 31 | end 32 | end 33 | end 34 | 35 | describe "#registry_table" do 36 | context "when no registry table is present" do 37 | it "raises an exception" do 38 | drop_registry_table 39 | expect { 40 | Refile::Postgres::Backend.new(test_connection, max_size: 100).registry_table 41 | }.to raise_error Refile::Postgres::Backend::RegistryTableDoesNotExistError 42 | end 43 | end 44 | 45 | context "when registry tables exist in multiple schemas" do 46 | before do 47 | test_connection.exec %{ 48 | CREATE SCHEMA other_schema; 49 | CREATE TABLE IF NOT EXISTS other_schema.#{Refile::Postgres::Backend::DEFAULT_REGISTRY_TABLE} 50 | ( id serial NOT NULL ); 51 | } 52 | end 53 | 54 | after do 55 | test_connection.exec %{ 56 | DROP SCHEMA other_schema CASCADE; 57 | } 58 | end 59 | 60 | it "does not raise an exception" do 61 | expect { 62 | Refile::Postgres::Backend.new(test_connection, max_size: 100).registry_table 63 | }.not_to raise_error 64 | end 65 | end 66 | end 67 | 68 | describe "Orphaned large object cleaning" do 69 | let(:connection_or_proc) { test_connection } 70 | let(:backend) { Refile::Postgres::Backend.new(connection_or_proc, max_size: 10000 ) } 71 | it "does not garbage collect attachments after vacuumlo call" do 72 | uploadable = File.open(File.expand_path(__FILE__)) 73 | file = backend.upload(uploadable) 74 | expect(backend.exists?(file.id)).to eq(true) 75 | run_vacuumlo 76 | expect(backend.exists?(file.id)).to eq(true) 77 | end 78 | end 79 | 80 | context "Refile Provided tests" do 81 | let(:connection_or_proc) { connection } 82 | it_behaves_like :backend 83 | end 84 | 85 | describe "Content streaming" do 86 | let(:connection_or_proc) { test_connection } 87 | let(:backend) { Refile::Postgres::Backend.new(connection_or_proc, max_size: 1000000 ) } 88 | it "allows to steam large file" do 89 | expect(Refile::Postgres::Backend::Reader::STREAM_CHUNK_SIZE).to eq(16384) 90 | uploadable = Tempfile.new("test-file") 91 | uploadable.write "A" * Refile::Postgres::Backend::Reader::STREAM_CHUNK_SIZE 92 | uploadable.write "B" * Refile::Postgres::Backend::Reader::STREAM_CHUNK_SIZE 93 | uploadable.write "C" * Refile::Postgres::Backend::Reader::STREAM_CHUNK_SIZE 94 | uploadable.close 95 | uploadable.open 96 | file = backend.upload(uploadable) 97 | expect(backend.exists?(file.id)).to eq(true) 98 | reader = backend.open(file.id) 99 | enum = reader.each 100 | expect(enum.next).to eq("A" * Refile::Postgres::Backend::Reader::STREAM_CHUNK_SIZE) 101 | expect(enum.next).to eq("B" * Refile::Postgres::Backend::Reader::STREAM_CHUNK_SIZE) 102 | expect(enum.next).to eq("C" * Refile::Postgres::Backend::Reader::STREAM_CHUNK_SIZE) 103 | expect { enum.next }.to raise_error(StopIteration) 104 | end 105 | 106 | it "allows to steam small file" do 107 | uploadable = Tempfile.new("test-file") 108 | uploadable.write "QWERTY" 109 | uploadable.close 110 | uploadable.open 111 | file = backend.upload(uploadable) 112 | expect(backend.exists?(file.id)).to eq(true) 113 | reader = backend.open(file.id) 114 | enum = reader.each 115 | expect(enum.next).to eq("QWERTY") 116 | expect { enum.next }.to raise_error(StopIteration) 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require "simplecov" 2 | SimpleCov.start 3 | 4 | $LOAD_PATH.unshift(File.join(Gem::Specification.find_by_name("refile").gem_dir, "spec")) 5 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 6 | 7 | require "refile/spec_helper" 8 | require "pg" 9 | require "pry" 10 | require "refile/postgres" 11 | require "open3" 12 | 13 | WebMock.disable!(:except => [:codeclimate_test_reporter]) 14 | 15 | TEST_DB_NAME = ENV.fetch('POSTGRES_DB', 'refile_test') 16 | TEST_DB_HOST = ENV.fetch('POSTGRES_HOST','localhost') 17 | TEST_DB_USER = ENV.fetch('POSTGRES_USER','refile_postgres_test_user') 18 | TEST_DB_PASSWD = ENV.fetch('POSTGRES_PASSWORD','refilepostgres') 19 | module DatabaseHelpers 20 | def test_connection 21 | @@connection ||= PG.connect(host: TEST_DB_HOST, dbname: TEST_DB_NAME, user: TEST_DB_USER, password: TEST_DB_PASSWD) 22 | end 23 | 24 | def create_registy_table(name = Refile::Postgres::Backend::DEFAULT_REGISTRY_TABLE) 25 | test_connection.exec %Q{ 26 | DROP TABLE IF EXISTS #{name}; 27 | CREATE TABLE #{name} ( 28 | id integer NOT NULL, 29 | oid oid NOT NULL, 30 | namespace character varying NOT NULL, 31 | created_at timestamp without time zone DEFAULT ('now'::text)::timestamp without time zone 32 | ); 33 | 34 | CREATE SEQUENCE #{name}_id_seq 35 | START WITH 1 36 | INCREMENT BY 1 37 | NO MINVALUE 38 | NO MAXVALUE 39 | CACHE 1; 40 | 41 | ALTER SEQUENCE #{name}_id_seq OWNED BY #{name}.id; 42 | 43 | ALTER TABLE ONLY #{name} ALTER COLUMN id SET DEFAULT nextval('#{name}_id_seq'::regclass); 44 | 45 | ALTER TABLE ONLY #{name} ADD CONSTRAINT #{name}_pkey PRIMARY KEY (id); 46 | 47 | CREATE INDEX index_#{name}_on_namespace ON #{name} USING btree (namespace); 48 | 49 | CREATE INDEX index_#{name}_on_oid ON #{name} USING btree (oid); 50 | } 51 | end 52 | 53 | def drop_registry_table(name = Refile::Postgres::Backend::DEFAULT_REGISTRY_TABLE) 54 | test_connection.exec %{ DROP TABLE IF EXISTS #{name} CASCADE; } 55 | end 56 | 57 | def run_vacuumlo 58 | command = "export PGPASSWORD=#{TEST_DB_PASSWD}; vacuumlo -h #{TEST_DB_HOST} -U #{TEST_DB_USER} -w -v #{TEST_DB_NAME}" 59 | Open3.popen3(command) do |stdin, stdout, stderr, thread| 60 | stdin.close 61 | IO.copy_stream(stderr, $stderr) 62 | IO.copy_stream(stdout, $stdout) 63 | end 64 | end 65 | end 66 | 67 | RSpec.configure do |config| 68 | config.include DatabaseHelpers 69 | 70 | config.around(:each) do |example| 71 | create_registy_table 72 | example.run 73 | drop_registry_table 74 | end 75 | end 76 | --------------------------------------------------------------------------------