├── .gitignore ├── .rspec ├── .ruby-version ├── .vscode └── launch.json ├── Gemfile ├── Gemfile.lock ├── Procfile ├── README.md ├── Rakefile ├── app ├── assets │ ├── config │ │ └── manifest.js │ ├── images │ │ └── .keep │ ├── javascripts │ │ ├── application.js │ │ ├── cable.js │ │ ├── channels │ │ │ └── .keep │ │ └── home.js │ └── stylesheets │ │ ├── application.css │ │ ├── home.scss │ │ └── normalize.css ├── builder │ ├── doc_builder.rb │ ├── epub_builder.rb │ ├── html_builder.rb │ ├── mobi_builder.rb │ └── pdf_builder.rb ├── channels │ └── application_cable │ │ ├── channel.rb │ │ └── connection.rb ├── controllers │ ├── application_controller.rb │ ├── concerns │ │ └── .keep │ ├── home_controller.rb │ └── static_controller.rb ├── jobs │ ├── application_job.rb │ ├── delayed_builder.rb │ ├── document_cleanup_job.rb │ └── helpers │ │ ├── application_helper.rb │ │ └── home_helper.rb ├── mailers │ └── application_mailer.rb ├── models │ ├── application_record.rb │ ├── chapter.rb │ ├── concerns │ │ └── .keep │ ├── document.rb │ ├── image.rb │ ├── proxy.rb │ ├── request.rb │ ├── story.rb │ └── target.rb ├── scraper │ ├── ffn_scraper.rb │ ├── forum_scraper.rb │ ├── scraper.rb │ ├── scraper_error.rb │ └── toc_scraper.rb ├── templates │ ├── css │ │ ├── epub.css │ │ ├── ffn.css │ │ ├── fp.css │ │ ├── html.css │ │ ├── qq.css │ │ ├── sb.css │ │ └── sv.css │ ├── epub │ │ ├── META-INF │ │ │ └── container.xml │ │ ├── OPS │ │ │ ├── book │ │ │ │ ├── chapter.xhtml.erb │ │ │ │ ├── cover.xhtml.erb │ │ │ │ ├── frontmatter.xhtml.erb │ │ │ │ ├── table-of-contents.ncx.erb │ │ │ │ └── table-of-contents.xhtml.erb │ │ │ ├── css │ │ │ │ └── sv.css │ │ │ └── package.opf.erb │ │ └── mimetype │ ├── html │ │ └── story.html.erb │ ├── images │ │ ├── favicon.png │ │ ├── ffn.png │ │ ├── fp.png │ │ ├── qq.png │ │ ├── sb.png │ │ ├── sv.png │ │ └── xenforo-smilies-sprite.png │ └── pdf │ │ ├── content.html.erb │ │ ├── filler.html.erb │ │ └── frontmatter.html.erb └── views │ ├── home │ └── index.html.erb │ ├── layouts │ ├── application.html.erb │ ├── mailer.html.erb │ └── mailer.text.erb │ ├── release-notes │ └── 1.md │ ├── shared │ ├── _footer.html.erb │ ├── _google_analytics.html.erb │ └── _header.html.erb │ └── static │ ├── about.html.erb │ └── contact.html.erb ├── bin ├── bundle ├── rails ├── rake ├── setup ├── spring └── update ├── brakeman.html ├── config.ru ├── config ├── application.rb ├── boot.rb ├── cable.yml ├── database.yml ├── environment.rb ├── environments │ ├── development.rb │ ├── production.rb │ └── test.rb ├── initializers │ ├── application_controller_renderer.rb │ ├── assets.rb │ ├── aws.rb │ ├── backtrace_silencers.rb │ ├── cookies_serializer.rb │ ├── filter_parameter_logging.rb │ ├── inflections.rb │ ├── mime_types.rb │ ├── new_framework_defaults.rb │ ├── pdfkit.rb │ ├── redis.rb │ ├── resque-pool.rb │ ├── resque.rb │ ├── rollbar.rb │ ├── session_store.rb │ ├── targets.rb │ └── wrap_parameters.rb ├── locales │ └── en.yml ├── puma.rb ├── resque-pool.yml ├── routes.rb ├── secrets.yml ├── spring.rb └── targets.yml ├── db ├── migrate │ ├── 20160721160005_create_stories.rb │ ├── 20160721160220_create_chapters.rb │ ├── 20160721160529_create_requests.rb │ ├── 20160723172334_create_documents.rb │ ├── 20160807110240_add_meta_data_to_stories.rb │ ├── 20160903153614_add_progress_to_requests.rb │ ├── 20160903164837_remove_current_chapter_from_requests.rb │ ├── 20160903164949_add_current_chapters_to_requests.rb │ ├── 20160903194955_add_extension_to_requests.rb │ ├── 20160914112318_create_scraper_queues.rb │ ├── 20160919094109_remove_queue_from_scraper_queues.rb │ ├── 20160928190251_add_doc_idto_request.rb │ ├── 20160929173836_add_aws_infoto_documents.rb │ ├── 20160929174442_add_aws_urlto_requests.rb │ ├── 20170112142726_create_images.rb │ ├── 20170112152157_add_aws_key_to_images.rb │ ├── 20170227112542_add_domain_to_stories.rb │ ├── 20170302132440_add_dates_to_chapters.rb │ ├── 20170302153532_add_strategy_to_request.rb │ ├── 20170702114104_create_targets.rb │ ├── 20170702114530_drop_scraper_queues.rb │ ├── 20170702121451_add_target_to_requests.rb │ └── 20181110200218_create_proxies.rb ├── schema.rb └── seeds.rb ├── lib ├── assets │ └── .keep ├── gifsicle ├── kindlegen ├── tasks │ ├── .keep │ └── resque.rake └── wkhtmltopdf ├── log └── .keep ├── public ├── 404.html ├── 422.html ├── 500.html ├── apple-touch-icon-precomposed.png ├── apple-touch-icon.png ├── favicon.ico ├── images │ ├── books.jpg │ ├── error.png │ ├── favicon.png │ ├── github.png │ ├── header_logo.png │ ├── omnibuser.png │ └── twitter.png └── robots.txt ├── spec ├── factories │ ├── chapters.rb │ ├── documents.rb │ ├── images.rb │ ├── requests.rb │ └── stories.rb ├── features │ ├── fanfiction_spec.rb │ ├── forum_spec.rb │ └── toc_spec.rb ├── models │ ├── builders │ │ ├── doc_builder_spec.rb │ │ ├── epub_builder_spec.rb │ │ ├── html_builder_spec.rb │ │ ├── mobi_builder_spec.rb │ │ └── pdf_builder_spec.rb │ ├── chapter_spec.rb │ ├── document_spec.rb │ ├── image_spec.rb │ ├── request_spec.rb │ └── story_spec.rb ├── rails_helper.rb ├── spec_helper.rb └── support │ ├── document_macros.rb │ ├── form_macros.rb │ └── images │ ├── dancing_banana.gif │ ├── dice.png │ └── lake.jpg ├── tmp └── .keep └── vendor └── assets ├── javascripts └── .keep └── stylesheets └── .keep /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files for more about ignoring files. 2 | # 3 | # If you find yourself ignoring temporary files generated by your text editor 4 | # or operating system, you probably want to add a global ignore instead: 5 | # git config --global core.excludesfile '~/.gitignore_global' 6 | 7 | # Ignore bundler config. 8 | /.bundle 9 | 10 | # Ignore all logfiles and tempfiles. 11 | /log/* 12 | /tmp/* 13 | !/log/.keep 14 | !/tmp/.keep 15 | 16 | # Ignore Byebug command history file. 17 | .byebug_history 18 | 19 | # Ignore documents 20 | #/public/documents/* 21 | #/public/images/* 22 | 23 | # 3rd party tools 24 | #/lib/kindlegen 25 | #/lib/wkhtmltopdf 26 | #/lib/gifsicle 27 | 28 | /config/initializers/00_dev_env.rb 29 | dump.rdb 30 | 31 | /test/images 32 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --require spec_helper 3 | --format documentation 4 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.3.8 2 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "chrome", 9 | "request": "launch", 10 | "name": "Launch Chrome against localhost", 11 | "url": "http://localhost:8080", 12 | "webRoot": "${workspaceFolder}" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'rollbar' 4 | gem 'resque-rollbar' 5 | 6 | gem 'colorscore' 7 | gem 'combine_pdf' 8 | gem 'redcarpet', '~> 3.3.4' 9 | gem 'rubyzip' 10 | gem "mini_magick" 11 | gem 'image_optim' 12 | gem 'image_optim_pack' 13 | gem 'mechanize', '~> 2.7' 14 | gem 'pdfkit', '~> 0.8' 15 | gem 'wkhtmltopdf-binary-edge', '~> 0.12.2.1' 16 | gem "resque", "~> 1.24.1" 17 | gem 'aws-sdk-v1' 18 | gem 'resque-pool' 19 | gem 'resque-web', require: 'resque_web' 20 | # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' 21 | gem 'rails', '~> 5.0.0' 22 | # Use postgresql as the database for Active Record 23 | gem 'pg', '~> 0.18' 24 | # Use Puma as the app server 25 | gem 'puma', '~> 3.0' 26 | # Use SCSS for stylesheets 27 | gem 'sass-rails', '~> 5.0' 28 | # Use Uglifier as compressor for JavaScript assets 29 | gem 'uglifier', '>= 1.3.0' 30 | gem 'therubyracer' 31 | # Use CoffeeScript for .coffee assets and views 32 | gem 'coffee-rails', '~> 4.2' 33 | # See https://github.com/rails/execjs#readme for more supported runtimes 34 | # gem 'therubyracer', platforms: :ruby 35 | 36 | # Use jquery as the JavaScript library 37 | gem 'jquery-rails' 38 | gem 'jquery-ui-rails', '~> 5.0.5' 39 | # Turbolinks makes navigating your web application faster. Read more: https://github.com/turbolinks/turbolinks 40 | # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder 41 | gem 'jbuilder', '~> 2.5' 42 | # Use Redis adapter to run Action Cable in production 43 | gem 'redis' 44 | # Use ActiveModel has_secure_password 45 | # gem 'bcrypt', '~> 3.1.7' 46 | 47 | # Use Capistrano for deployment 48 | # gem 'capistrano-rails', group: :development 49 | 50 | group :development, :test do 51 | # Call 'byebug' anywhere in the code to stop execution and get a debugger console 52 | gem 'rack-timeout' 53 | gem 'byebug', platform: :mri 54 | gem 'rspec-rails', '~> 3.5' 55 | gem 'factory_girl_rails' 56 | gem 'byebug' 57 | gem 'pry' 58 | gem 'pry-byebug' 59 | end 60 | 61 | group :test do 62 | gem 'faker' 63 | gem 'capybara' 64 | gem 'launchy' 65 | gem 'selenium-webdriver', '~> 2.53' 66 | gem 'shoulda-matchers', '~> 3.1' 67 | 68 | end 69 | 70 | group :development do 71 | # Access an IRB console on exception pages or by using <%= console %> anywhere in the code. 72 | gem 'web-console' 73 | gem 'listen', '~> 3.0.5' 74 | # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring 75 | gem 'spring' 76 | gem 'spring-watcher-listen', '~> 2.0.0' 77 | end 78 | 79 | # Windows does not include zoneinfo files, so bundle the tzinfo-data gem 80 | gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] 81 | 82 | #ruby "2.3.8" 83 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | worker: bundle exec resque-pool 2 | resque: env TERM_CHILD=1 bundle exec rake resque:work 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # Add your own tasks in files placed in lib/tasks ending in .rake, 2 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 3 | 4 | require_relative 'config/application' 5 | require 'resque/pool/tasks' 6 | 7 | Rails.application.load_tasks 8 | 9 | # this task will get called before resque:pool:setup 10 | # and preload the rails environment in the pool manager 11 | task "resque:pool:setup" do 12 | # close any sockets or files in pool manager 13 | ActiveRecord::Base.connection.disconnect! 14 | Resque::Pool.after_prefork do 15 | ActiveRecord::Base.establish_connection 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /app/assets/config/manifest.js: -------------------------------------------------------------------------------- 1 | //= link_tree ../images 2 | //= link_directory ../javascripts .js 3 | //= link_directory ../stylesheets .css 4 | -------------------------------------------------------------------------------- /app/assets/images/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/assets/images/.keep -------------------------------------------------------------------------------- /app/assets/javascripts/application.js: -------------------------------------------------------------------------------- 1 | // This is a manifest file that'll be compiled into application.js, which will include all the files 2 | // listed below. 3 | // 4 | // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts, 5 | // or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path. 6 | // 7 | // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the 8 | // compiled file. JavaScript code in this file should be added after the last require_* statement. 9 | // 10 | // Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details 11 | // about supported directives. 12 | // 13 | //= require jquery 14 | //= require jquery_ujs 15 | //= require jquery-ui/progressbar 16 | //= require_tree . 17 | -------------------------------------------------------------------------------- /app/assets/javascripts/cable.js: -------------------------------------------------------------------------------- 1 | // Action Cable provides the framework to deal with WebSockets in Rails. 2 | // You can generate new channels where WebSocket features live using the rails generate channel command. 3 | // 4 | //= require action_cable 5 | //= require_self 6 | //= require_tree ./channels 7 | 8 | (function() { 9 | this.App || (this.App = {}); 10 | 11 | App.cable = ActionCable.createConsumer(); 12 | 13 | }).call(this); 14 | -------------------------------------------------------------------------------- /app/assets/javascripts/channels/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/assets/javascripts/channels/.keep -------------------------------------------------------------------------------- /app/assets/javascripts/home.js: -------------------------------------------------------------------------------- 1 | // # Place all the behaviors and hooks related to the matching controller here. 2 | // # All this logic will automatically be available in application.js. 3 | // # You can use CoffeeScript in this file: http://coffeescript.org/ 4 | 5 | $(document).ready(function() { 6 | clearFeedback(); 7 | enableInput(); 8 | $("#q").val(""); 9 | $("#request_form").on("ajax:success", function(e, data, status, xhr){ 10 | clearFeedback(); 11 | disableInput(); 12 | startQueries(xhr); 13 | }).on("ajax:error", function(e, xhr, status, error){ 14 | updateErrors(xhr.responseJSON.status); 15 | }) 16 | 17 | }); 18 | 19 | 20 | function startQueries(request) { 21 | var request = jQuery.parseJSON(request.responseText); 22 | initializeProgress(); 23 | $.ajax({ 24 | url: '/scrape/' + request.id, 25 | type: "POST", 26 | dataType: "json" 27 | }).done(function(id){ 28 | 29 | }).fail(function(jqXHR, textStatus, errorThrown){ 30 | }); 31 | var interval = 2000; 32 | function checkStatus() { 33 | $.ajax({ 34 | type: 'GET', 35 | url: '/requests/' + request.id, 36 | dataType: 'json' 37 | }).always(function(request, status){ 38 | if (request.complete !== true) { 39 | if (request.current_chapters === null || request.total_chapters === null) { 40 | } else { 41 | updateProgress(request.current_chapters, request.total_chapters); 42 | } 43 | setTimeout(checkStatus, interval); 44 | } else{ 45 | if (request.status == "Success") { 46 | enableInput(); 47 | updateDownload(request.aws_url); 48 | } else{ 49 | enableInput(); 50 | updateErrors(request.status); 51 | } 52 | } 53 | }); 54 | } 55 | setTimeout(checkStatus, interval); 56 | } 57 | 58 | function disableInput(){ 59 | $("#submit").hide(); 60 | $("input").prop("disabled", true); 61 | } 62 | 63 | function enableInput(){ 64 | $("input").prop("disabled", false); 65 | $("#submit").show(); 66 | } 67 | 68 | function clearFeedback(){ 69 | $("#download").hide(); 70 | $("#error-explanation").hide(); 71 | $("#progressbar").hide(); 72 | $("#error-container").empty(); 73 | $("#download").empty(); 74 | 75 | } 76 | function updateErrors(html){ 77 | $("#download").hide(); 78 | $("#progressbar").hide(); 79 | $("#error-container").empty(); 80 | $("#error-container").append("error:

" + html + "

"); 81 | $("#error-explanation").show(); 82 | } 83 | function updateDownload(aws_url){ 84 | $("#error-explanation").hide(); 85 | $("#progressbar").hide(); 86 | $("#download").empty(); 87 | $("#download").append(" Download Ebook "); 88 | $("#download").show(); 89 | } 90 | 91 | function initializeProgress(){ 92 | $("#error-explanation").hide(); 93 | $("#download").hide(); 94 | $(".progress-label").text( "Loading..." ); 95 | $("#progressbar").show(); 96 | 97 | 98 | $( "#progressbar" ).progressbar({ 99 | value: false, 100 | change: function() { 101 | if ($( "#progressbar" ).progressbar( "value" ) !== false) { 102 | $( ".progress-label" ).text( $( "#progressbar" ).progressbar( "value" ) + "/" + $( "#progressbar" ).progressbar( "option", "max" ) ); 103 | } 104 | 105 | }, 106 | complete: function() { 107 | $( ".progress-label" ).text( "Creating Ebook..." ); 108 | } 109 | }); 110 | } 111 | 112 | function updateProgress(current, total){ 113 | $( "#progressbar" ).progressbar( "option", "max", total ); 114 | $( "#progressbar" ).progressbar( "value", current ); 115 | 116 | } 117 | -------------------------------------------------------------------------------- /app/assets/stylesheets/application.css: -------------------------------------------------------------------------------- 1 | /* 2 | * This is a manifest file that'll be compiled into application.css, which will include all the files 3 | * listed below. 4 | * 5 | * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets, 6 | * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path. 7 | * 8 | * You're free to add application-wide styles to this file and they'll appear at the bottom of the 9 | * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS 10 | * files in this directory. Styles in this file should be added after the last require_* statement. 11 | * It is generally better to create a new file per style scope. 12 | * 13 | *= require normalize 14 | *= require jquery-ui/progressbar 15 | *= require_tree . 16 | *= require_self 17 | */ 18 | -------------------------------------------------------------------------------- /app/builder/doc_builder.rb: -------------------------------------------------------------------------------- 1 | class DocBuilder 2 | include ActiveModel::Model 3 | attr_accessor :doc 4 | 5 | def build 6 | #this is never used, as all subclasses override it 7 | #consider deleting 8 | @file = File.new(@doc.path, 'w+') 9 | add_file_header 10 | add_file_frontmatter 11 | @doc.story.chapters.order(:number).each do |chapter| 12 | add_chapter(chapter) 13 | end 14 | add_file_footer 15 | @file.close 16 | @file.path 17 | end 18 | 19 | def render_template(template_path, output_path) 20 | template = File.read("#{@template_dir}/#{template_path}") 21 | f = File.new("#{@directory}/#{output_path}", 'w+') 22 | f << ERB.new(template).result(binding) 23 | f.close 24 | end 25 | 26 | end 27 | -------------------------------------------------------------------------------- /app/builder/epub_builder.rb: -------------------------------------------------------------------------------- 1 | require 'zip' 2 | include ERB::Util 3 | 4 | class EPUBBuilder < DocBuilder 5 | attr_accessor :template_dir, :input, :domain 6 | def build 7 | @story = @doc.story 8 | @template_dir = Rails.root.join("app", "templates", 'epub') 9 | @input = ["mimetype", "META-INF/container.xml", "OPS/package.opf", "OPS/book/table-of-contents.xhtml", 10 | "OPS/book/cover.xhtml", "OPS/book/frontmatter.xhtml", "OPS/book/table-of-contents.ncx"] 11 | @domain = @story.domain 12 | create_directory_structure 13 | create_mimetype 14 | create_meta_inf_container 15 | add_images 16 | add_styles 17 | create_cover 18 | create_frontmatter 19 | create_content 20 | create_toc 21 | create_toc_ncx 22 | create_package_opf 23 | zip_directory 24 | end 25 | 26 | def create_directory_structure 27 | @directory = "/tmp/#{@doc.filename}" 28 | FileUtils.remove_dir(@directory) if Dir.exist?(@directory) 29 | Dir.mkdir(@directory) 30 | Dir.mkdir("#{@directory}/META-INF") 31 | Dir.mkdir("#{@directory}/OPS") 32 | Dir.mkdir("#{@directory}/OPS/book") 33 | Dir.mkdir("#{@directory}/OPS/css") 34 | Dir.mkdir("#{@directory}/OPS/images") 35 | end 36 | 37 | def create_mimetype 38 | render_template("mimetype", "mimetype") 39 | end 40 | 41 | def create_meta_inf_container 42 | render_template("META-INF/container.xml", "META-INF/container.xml") 43 | end 44 | 45 | def add_images 46 | @story.images.where(cover: false).each do |image| 47 | image.download("#{@directory}/OPS/images") 48 | @input << "OPS/images/#{image.name}" 49 | end 50 | FileUtils.cp("#{@template_dir}/../images/xenforo-smilies-sprite.png", 51 | "#{@directory}/OPS/images/xenforo-smilies-sprite.png") if @domain == 'sv' || @domain == 'sb' 52 | end 53 | 54 | def add_styles 55 | render_template("../css/#{@domain}.css", "OPS/css/#{@domain}.css") if @domain 56 | render_template("../css/epub.css", "OPS/css/main.css") 57 | @input << "OPS/css/#{@domain}.css" if @domain 58 | @input << "OPS/css/main.css" 59 | end 60 | 61 | def copy_from_template(path) 62 | FileUtils.cp("#{@template_dir}/#{path}", "#{@directory}/#{path}") 63 | end 64 | 65 | def create_cover 66 | @cover = @story.cover_image 67 | if @cover 68 | @cover.download("#{@directory}/OPS/images") 69 | @cover_path = "OPS/images/#{@cover.name}" 70 | @cover_name = @cover.name 71 | else 72 | if @domain 73 | name = @domain 74 | else 75 | name = 'favicon' 76 | end 77 | @cover_path = "OPS/images/#{name}.png" 78 | @cover_name = name 79 | FileUtils.cp("#{@template_dir}/../images/#{name}.png", "#{@directory}/OPS/images/#{name}.png") 80 | end 81 | render_template("OPS/book/cover.xhtml.erb", "OPS/book/cover.xhtml") 82 | @input << @cover_path 83 | end 84 | 85 | def create_frontmatter 86 | render_template("OPS/book/frontmatter.xhtml.erb", "OPS/book/frontmatter.xhtml") 87 | end 88 | 89 | def create_content 90 | @story.chapters.order(:number).each do |chapter| 91 | @chapter = chapter 92 | template = File.read("#{@template_dir}/OPS/book/chapter.xhtml.erb") 93 | filename = "Chapter_#{chapter.number.to_s.rjust(3, '0')}.xhtml" 94 | @input << "OPS/book/#{filename}" 95 | f = File.new("#{@directory}/OPS/book/#{filename}", 'w+') 96 | f << ERB.new(template).result(binding) 97 | f.close 98 | end 99 | end 100 | 101 | def create_toc 102 | render_template("OPS/book/table-of-contents.xhtml.erb", "OPS/book/table-of-contents.xhtml") 103 | end 104 | 105 | def create_toc_ncx 106 | render_template("OPS/book/table-of-contents.ncx.erb", "OPS/book/table-of-contents.ncx") 107 | end 108 | 109 | def create_package_opf 110 | render_template("OPS/package.opf.erb", "OPS/package.opf") 111 | end 112 | 113 | def zip_directory 114 | #refactor to not use input array, and just recursively zip dir 115 | zip_name = "/tmp/#{@doc.filename}.epub" 116 | File.delete(zip_name) if File.exist?(zip_name) 117 | Zip::File.open(zip_name, Zip::File::CREATE) do |zipfile| 118 | @input.each do |filename| 119 | # Two arguments: 120 | # - The name of the file as it will appear in the archive 121 | # - The original file, including the path to find it 122 | zipfile.add(filename, @directory + '/' + filename) 123 | end 124 | #zipfile.get_output_stream("myFile") { |os| os.write "myFile contains just this" } 125 | end 126 | end 127 | end 128 | -------------------------------------------------------------------------------- /app/builder/html_builder.rb: -------------------------------------------------------------------------------- 1 | require 'zip' 2 | include ERB::Util 3 | 4 | class HTMLBuilder < DocBuilder 5 | attr_accessor :template_dir, :input, :domain 6 | 7 | def build(nozip: false) 8 | @story = @doc.story 9 | @template_dir = Rails.root.join("app", "templates", 'html') 10 | @input = ["story.html"] 11 | @domain = @story.domain 12 | create_directory_structure 13 | add_styles 14 | create_cover 15 | add_images 16 | create_story 17 | if nozip 18 | "#{@directory}/story.html" 19 | else 20 | zip_directory 21 | end 22 | end 23 | 24 | def create_directory_structure 25 | @directory = "/tmp/#{@doc.filename}" 26 | FileUtils.remove_dir(@directory) if Dir.exist?(@directory) 27 | Dir.mkdir(@directory) 28 | Dir.mkdir("#{@directory}/files") 29 | Dir.mkdir("#{@directory}/files/css") 30 | Dir.mkdir("#{@directory}/files/images") 31 | end 32 | 33 | def add_styles 34 | render_template("../css/#{@domain}.css", "files/css/#{@domain}.css") if @domain 35 | render_template("../css/html.css", "files/css/main.css") 36 | @input << "files/css/#{@domain}.css" if @domain 37 | @input << "files/css/main.css" 38 | end 39 | 40 | def create_cover 41 | @cover = @story.cover_image 42 | if @cover 43 | @cover.download("#{@directory}/files/images") 44 | @cover_path = "files/images/#{@cover.name}" 45 | @cover_name = @cover.name 46 | else 47 | if @domain 48 | name = @domain 49 | else 50 | name = 'favicon' 51 | end 52 | @cover_path = "files/images/#{name}.png" 53 | @cover_name = name 54 | FileUtils.cp("#{@template_dir}/../images/#{name}.png", "#{@directory}/files/images/#{name}.png") 55 | end 56 | @input << @cover_path 57 | end 58 | 59 | def add_images 60 | @story.images.where(cover: false).each do |image| 61 | image.download("#{@directory}/files/images") 62 | @input << "files/images/#{image.name}" 63 | end 64 | if @domain == 'sv' || @domain == 'sb' 65 | FileUtils.cp("#{@template_dir}/../images/xenforo-smilies-sprite.png", 66 | "#{@directory}/files/images/xenforo-smilies-sprite.png") 67 | @input << "files/images/xenforo-smilies-sprite.png" 68 | end 69 | 70 | end 71 | 72 | def create_story 73 | render_template("story.html.erb", "story.html") 74 | end 75 | 76 | def zip_directory 77 | zip_name = "/tmp/#{@doc.filename}.zip" 78 | File.delete(zip_name) if File.exist?(zip_name) 79 | Zip::File.open(zip_name, Zip::File::CREATE) do |zipfile| 80 | @input.each do |filename| 81 | # Two arguments: 82 | # - The name of the file as it will appear in the archive 83 | # - The original file, including the path to find it 84 | zipfile.add(filename, @directory + '/' + filename) 85 | end 86 | #zipfile.get_output_stream("myFile") { |os| os.write "myFile contains just this" } 87 | end 88 | end 89 | 90 | 91 | end 92 | -------------------------------------------------------------------------------- /app/builder/mobi_builder.rb: -------------------------------------------------------------------------------- 1 | class MOBIBuilder < DocBuilder 2 | def build 3 | epub_doc_id = @doc.story.build('epub') 4 | epub_doc_path = Document.find(epub_doc_id).path 5 | log = %x<#{Rails.root.join("lib", "kindlegen")} #{epub_doc_path} -verbose -dont_append_source > 6 | Rails.logger.info(log) 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /app/builder/pdf_builder.rb: -------------------------------------------------------------------------------- 1 | include ERB::Util 2 | 3 | class PDFBuilder < HTMLBuilder 4 | attr_accessor :template_dir, :input, :domain, :files 5 | 6 | def build 7 | @story = @doc.story 8 | @template_dir = Rails.root.join("app", "templates", 'html') 9 | @files = ["frontmatter"] 10 | @input = ["story.html"] 11 | @domain = @story.domain 12 | create_directory_structure 13 | add_styles 14 | create_cover 15 | add_images 16 | create_frontmatter 17 | create_content 18 | render_template('../pdf/filler.html.erb', 'filler.html') 19 | convert_to_pdf 20 | combine_pdfs 21 | end 22 | 23 | def create_frontmatter 24 | render_template("../pdf/frontmatter.html.erb", "frontmatter.html") 25 | end 26 | 27 | def create_content 28 | @chapters = @story.chapters.order(:number) 29 | index = 0 30 | @chapters.each_slice(10) do |chapter_chunk| 31 | @chapter_chunk = chapter_chunk 32 | index += 1 33 | filename = "content_#{index}" 34 | @files << filename 35 | render_template("../pdf/content.html.erb", "#{filename}.html") 36 | end 37 | end 38 | 39 | def convert_to_pdf 40 | @files.each do |filename| 41 | pdf = PDFKit.new(File.open("#{@directory}/#{filename}.html"), 42 | margin_top: 10, margin_bottom: 10, 43 | margin_left: 0, margin_right: 0, quiet: true, 44 | header_html: "#{@directory}/filler.html", 45 | footer_html: "#{@directory}/filler.html", 46 | load_error_handling: 'ignore') 47 | pdf.to_file("#{@directory}/#{filename}.pdf") 48 | end 49 | end 50 | 51 | def combine_pdfs 52 | pdf = CombinePDF.new 53 | @files.each do |filename| 54 | pdf << CombinePDF.load("#{@directory}/#{filename}.pdf") 55 | end 56 | pdf.save @doc.path 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /app/channels/application_cable/channel.rb: -------------------------------------------------------------------------------- 1 | module ApplicationCable 2 | class Channel < ActionCable::Channel::Base 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /app/channels/application_cable/connection.rb: -------------------------------------------------------------------------------- 1 | module ApplicationCable 2 | class Connection < ActionCable::Connection::Base 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | class ApplicationController < ActionController::Base 2 | protect_from_forgery with: :exception 3 | end 4 | -------------------------------------------------------------------------------- /app/controllers/concerns/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/controllers/concerns/.keep -------------------------------------------------------------------------------- /app/controllers/home_controller.rb: -------------------------------------------------------------------------------- 1 | 2 | class HomeController < ApplicationController 3 | require 'open-uri' 4 | 5 | def index 6 | @canonical = "http://omnibuser.com" 7 | end 8 | 9 | def status 10 | request = Request.find(params[:id]) 11 | respond_to do |format| 12 | format.json {render json: request.to_json, status: :ok} 13 | end 14 | end 15 | 16 | def scrape 17 | puts "begin scrape" 18 | respond_to do |format| 19 | begin 20 | @request = Request.find(params[:id]) 21 | @request.update(complete: false, status: "In Progress") 22 | @request.scrape 23 | format.json {render json: @request, status: :ok} 24 | rescue ScraperError => e 25 | @request.update(complete: true, status: e) 26 | Rollbar.error(e) 27 | format.json {render json: @request, status: 422} 28 | rescue Exception => e 29 | @request.update(complete: true, status: "Sorry, something went wrong.") 30 | Rollbar.error(e) 31 | format.json {render json: @request, status: 422} 32 | end 33 | end 34 | end 35 | 36 | def new 37 | respond_to do |format| 38 | begin 39 | @request = Request.create(url: params[:q], extension: params[:ext], strategy: params[:strategy], recent_number: params[:recent_number], status: "Initializing") 40 | format.json {render json: @request.to_json, status: :created} 41 | rescue Exception => e 42 | @request = Request.new 43 | @request.status = e 44 | @request.complete = true 45 | Rollbar.error(e) 46 | format.json {render json: @request, status: 422} 47 | end 48 | end 49 | end 50 | 51 | private 52 | def request_params 53 | params.require(:q, :ext, :strategy, :recent_number) 54 | end 55 | 56 | end 57 | -------------------------------------------------------------------------------- /app/controllers/static_controller.rb: -------------------------------------------------------------------------------- 1 | class StaticController < ApplicationController 2 | 3 | def about 4 | @title = "Omnibuser | About" 5 | @canonical = "http://omnibuser.com/about" 6 | end 7 | 8 | def contact 9 | @title = "Omnibuser | Contact" 10 | @canonical = "http://omnibuser.com/contact" 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /app/jobs/application_job.rb: -------------------------------------------------------------------------------- 1 | class ApplicationJob < ActiveJob::Base 2 | end 3 | -------------------------------------------------------------------------------- /app/jobs/delayed_builder.rb: -------------------------------------------------------------------------------- 1 | class DelayedBuilder 2 | @queue = :build 3 | 4 | def self.perform(request_id) 5 | request = Request.find(request_id) 6 | story = request.story 7 | begin 8 | doc_id = story.build(request.extension) 9 | request.update(doc_id: doc_id, complete: true, status: "Success") 10 | rescue Exception => e 11 | request.update(complete: true, status: e) 12 | raise e 13 | end 14 | end 15 | 16 | 17 | end 18 | -------------------------------------------------------------------------------- /app/jobs/document_cleanup_job.rb: -------------------------------------------------------------------------------- 1 | class DocumentCleanupJob < ApplicationJob 2 | queue_as :default 3 | #need to setup a job backend for production 4 | def perform(doc) 5 | doc.destroy 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /app/jobs/helpers/application_helper.rb: -------------------------------------------------------------------------------- 1 | module ApplicationHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/jobs/helpers/home_helper.rb: -------------------------------------------------------------------------------- 1 | module HomeHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/mailers/application_mailer.rb: -------------------------------------------------------------------------------- 1 | class ApplicationMailer < ActionMailer::Base 2 | default from: 'from@example.com' 3 | layout 'mailer' 4 | end 5 | -------------------------------------------------------------------------------- /app/models/application_record.rb: -------------------------------------------------------------------------------- 1 | class ApplicationRecord < ActiveRecord::Base 2 | self.abstract_class = true 3 | end 4 | -------------------------------------------------------------------------------- /app/models/chapter.rb: -------------------------------------------------------------------------------- 1 | class Chapter < ApplicationRecord 2 | belongs_to :story 3 | validates :number, uniqueness: {scope: :story_id} 4 | after_create :ensure_title 5 | 6 | def epub 7 | nodeset = self.xhtml 8 | nodeset.search('img').each{|i| i['src'] = "../images/#{i['src']}"} 9 | content = nodeset.root.to_s 10 | content.gsub!('
', '
') 11 | content.gsub!('
', '
') 12 | content 13 | end 14 | 15 | def html 16 | nodeset = self.xhtml 17 | nodeset.search('img').each do |i| 18 | i['src'] = "files/images/#{i['src']}" unless i['src'].blank? 19 | end 20 | content = nodeset.root.to_s 21 | content.gsub!('
', '
') 22 | content.gsub!('
', '
') 23 | content 24 | end 25 | 26 | def xhtml 27 | nodeset = Nokogiri::XML(self.content) 28 | nodeset.search('.bbCodeSpoilerButton').each do |button| 29 | button.keys.each{|att| button.delete att} 30 | button['class'] = 'omni-spoilerTextContainer' 31 | button.name = 'div' 32 | end 33 | nodeset.search('.JsOnly').each{|js| js.remove} 34 | nodeset.search('noscript').each{|n| n.replace Nokogiri::XML.fragment(n.children)} 35 | nodeset.css('iframe').each do |iframe| 36 | url = iframe['src']&.sub(/^\/\//, '') 37 | if url 38 | url = "http://#{url}" unless url.start_with?('http') 39 | new_node = nodeset.create_element "a", "#{url}", href: "#{url}" 40 | iframe.replace new_node 41 | end 42 | end 43 | nodeset.search('img').each{|i| i.remove if i['src'].nil?} 44 | nodeset.search('span').each do |span| 45 | if span.children.search('div').count > 0 46 | span.name = 'div' 47 | end 48 | end 49 | nodeset.search('.AttributionLink').each{|n| n.remove} 50 | nodeset.search('.quoteExpand').each{|n| n.remove} 51 | nodeset.search('.quoteExpand .shrinker').each{|n| n.remove} 52 | nodeset.search('.bbCodeQuote').each{|n| n['data-page-break'] = 'avoid'} 53 | nodeset.search('.adv_tabs_wrapper').each{|n| n['style'] = ""} 54 | nodeset.search('.adv_tabs_noscript_content').each{|n| n['style'] = ""} 55 | nodeset.search('script').each{|s| s.remove} 56 | nodeset.search('table').each do |table| 57 | if table['border'] == '0' 58 | table['border'] = '' 59 | elsif !table['border'].blank? && table['border'] != '1' 60 | table['border'] = '1' 61 | end 62 | table.remove_attribute('cellspacing') 63 | table.remove_attribute('cellpadding') 64 | end 65 | nodeset.xpath("//*[@style]").each{|n| n['style'] = n['style'].sub('color: transparent', 'opacity: 0.5')} 66 | nodeset 67 | end 68 | 69 | def ensure_title 70 | if self.title.blank? 71 | self.update(title: "Chapter #{self.number}") 72 | end 73 | end 74 | 75 | end 76 | -------------------------------------------------------------------------------- /app/models/concerns/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/models/concerns/.keep -------------------------------------------------------------------------------- /app/models/document.rb: -------------------------------------------------------------------------------- 1 | class Document < ApplicationRecord 2 | belongs_to :story 3 | before_create :sanitize_filename 4 | after_create :build 5 | before_destroy :delete_file 6 | 7 | def sanitize_filename 8 | self.filename.gsub!(/[^[:alpha:]]/, '_') 9 | self.filename.gsub!(/_{2,}/, '_') 10 | self.filename.gsub!(/^_/, '') 11 | self.filename.gsub!(/_$/, '') 12 | self.filename = self.filename.slice(0, 230) 13 | add_chapter_numbers if self.story.request.strategy == 'recent' 14 | end 15 | 16 | def add_chapter_numbers 17 | chapters = self.story.chapters.order(:number) 18 | if chapters.count == 1 19 | self.filename = "#{self.filename}_#{chapters.first.number}" 20 | else 21 | self.filename = "#{self.filename}_#{chapters.first.number}-#{chapters.last.number}" 22 | end 23 | end 24 | 25 | def path 26 | "/tmp/#{filename}.#{extension}" 27 | end 28 | 29 | def delete_file 30 | File.delete(self.path) if File.exist?(self.path) 31 | end 32 | 33 | def build 34 | builder = case extension 35 | when 'html' 36 | self.extension = 'zip' 37 | HTMLBuilder 38 | when 'mobi' 39 | MOBIBuilder 40 | when 'epub' 41 | EPUBBuilder 42 | when 'pdf' 43 | PDFBuilder 44 | end 45 | builder.new(doc: self).build 46 | upload 47 | end 48 | 49 | def upload 50 | obj = S3_BUCKET.objects["documents/#{self.id}/#{self.filename}.#{self.extension}"] 51 | obj.write( 52 | file: path, 53 | acl: :public_read 54 | ) 55 | self.update(aws_url: obj.public_url, aws_key: obj.key) 56 | self.story.request.update(aws_url: obj.public_url) 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /app/models/image.rb: -------------------------------------------------------------------------------- 1 | class Image < ApplicationRecord 2 | belongs_to :story 3 | before_create :generate_name 4 | 5 | def generate_name 6 | self.filename = SecureRandom.hex 7 | end 8 | 9 | def name 10 | "#{self.filename}.#{self.extension}" 11 | end 12 | 13 | 14 | def path 15 | "/tmp/#{self.name}" 16 | end 17 | 18 | def self.run_compression_test 19 | #this method is not used in production and is purely for testing 20 | #different compression strategies during development 21 | input_dir = Rails.root.join("test", "images", "input") 22 | output_dir = Rails.root.join("test", "images", "output") 23 | 24 | Dir.foreach(input_dir) do |file| 25 | next if file == '.' || file == '..' 26 | input = Rails.root.join("#{input_dir}", file).to_s 27 | output = Rails.root.join("#{output_dir}", file).to_s 28 | image = Image.create(story_id: 202, 29 | extension: input.split('.')[-1], 30 | source_url: 'test', 31 | cover: false, 32 | filename: "#{file.split('.')[0]}_50compressed") 33 | if image.extension == 'gif' 34 | output = Rails.root.join("#{output_dir}", image.name).to_s 35 | %x<#{Rails.root.join("lib", "gifsicle")} -O2 --lossy=80 #{input} -o #{output} > 36 | else 37 | i = MiniMagick::Image.open(input) 38 | if image.extension == 'png' 39 | i.background '#FFFFFF' 40 | i.alpha 'remove' 41 | end 42 | i.format 'jpg' 43 | i.quality 50 44 | image.update(extension: 'jpg') 45 | output = Rails.root.join("#{output_dir}", image.name).to_s 46 | i.write output 47 | end 48 | end 49 | 50 | end 51 | 52 | def compress(background_color="#FFFFFF") 53 | input = "#{self.path}.temp" 54 | if self.extension == 'gif' 55 | output = self.path 56 | %x<#{Rails.root.join("lib", "gifsicle")} -O2 --lossy=80 #{input} -o #{output} > 57 | else 58 | image = MiniMagick::Image.open(input) 59 | if self.extension == 'png' 60 | image.background background_color 61 | image.alpha 'remove' 62 | end 63 | image.format 'jpg' 64 | if image.width > 1000 65 | new_width = 1000 66 | new_height = (image.height * new_width)/image.width 67 | image.resize "#{new_width}x#{new_height}" 68 | end 69 | image.quality 50 70 | self.update(extension: 'jpg') 71 | output = self.path 72 | image.write output 73 | end 74 | end 75 | 76 | def download(dir="/tmp") 77 | open("#{dir}/#{self.name}", 'wb') do |file| 78 | file << open(self.aws_url).read 79 | end 80 | path 81 | end 82 | 83 | def upload 84 | obj = S3_BUCKET.objects["images/#{self.name}"] 85 | obj.write( 86 | file: path, 87 | acl: :public_read 88 | ) 89 | self.update(aws_url: obj.public_url, aws_key: obj.key) 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /app/models/proxy.rb: -------------------------------------------------------------------------------- 1 | class Proxy < ApplicationRecord 2 | end -------------------------------------------------------------------------------- /app/models/request.rb: -------------------------------------------------------------------------------- 1 | class Request < ApplicationRecord 2 | belongs_to :story, required: false 3 | belongs_to :target, required: false 4 | before_create :normalize_url 5 | after_create :set_target 6 | 7 | def scrape 8 | raise ScraperError, "Please enter a URL" if url.blank? 9 | begin 10 | Resque.enqueue(target.scraper_class, self.id) 11 | rescue Exception => e 12 | self.update!(complete: true, status: e) 13 | raise e 14 | end 15 | end 16 | 17 | def normalize_url 18 | self.url += '/' unless self.url.blank? || self.url.split('').last == '/' 19 | end 20 | 21 | 22 | def set_target 23 | target = Target.find{|t| url.include?(t.domain)} 24 | raise ScraperError, "The website you entered is not currently supported. See the About page for a list of supported sites, or the Contact page to request support for a new site" unless target 25 | self.update!(target_id: target.id) 26 | end 27 | 28 | end 29 | -------------------------------------------------------------------------------- /app/models/story.rb: -------------------------------------------------------------------------------- 1 | class Story < ApplicationRecord 2 | has_many :chapters, dependent: :destroy 3 | has_many :documents, dependent: :destroy 4 | has_many :images, dependent: :destroy 5 | has_one :request 6 | 7 | after_create :add_domain 8 | 9 | def build(ext) 10 | @doc = Document.create(story_id: self.id, filename: self.title, 11 | extension: ext) 12 | @doc.id 13 | end 14 | 15 | def cover_image 16 | self.images.find_by(cover: true) 17 | end 18 | 19 | def has_image(url) 20 | self.images.where(cover:false).find_by(source_url: url) 21 | end 22 | 23 | def list_images 24 | #unused, untested, consider deleting 25 | images = [] 26 | self.chapters.each {|chapter| chapter.xhtml.search('img').each {|i| images << i['src']} } 27 | images.uniq! 28 | images.select{|i| !i.include?('styles/sv_smiles')} 29 | end 30 | 31 | def add_domain 32 | if self.url.include?('fanfiction.net') 33 | self.update(domain: 'ffn') 34 | elsif self.url.include?('fictionpress.com') 35 | self.update(domain: 'fp') 36 | elsif self.url.include?('forums.sufficientvelocity.com') 37 | self.update(domain: 'sv') 38 | elsif self.url.include?('forums.spacebattles.com') 39 | self.update(domain: 'sb') 40 | elsif self.url.include?('forum.questionablequesting.com') 41 | self.update(domain: 'qq') 42 | else 43 | nil 44 | end 45 | end 46 | 47 | end 48 | -------------------------------------------------------------------------------- /app/models/target.rb: -------------------------------------------------------------------------------- 1 | class Target < ApplicationRecord 2 | has_many :requests, dependent: :nullify 3 | validates :domain, presence: true, uniqueness: true 4 | 5 | after_create :touch 6 | after_create :load_target_data 7 | 8 | def touch 9 | self.update!(last_access: Time.now) 10 | end 11 | 12 | def scraper_class 13 | scraper.constantize 14 | end 15 | 16 | def load_target_data 17 | return if target_data 18 | yml = YAML.load(File.read(Rails.root.join('config', 'targets.yml'))) 19 | self.update!(target_data: yml[domain]) 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /app/scraper/ffn_scraper.rb: -------------------------------------------------------------------------------- 1 | class FFNScraper < Scraper 2 | @queue = :scrape 3 | 4 | def get_base_url 5 | @url.match(/(fictionpress\.com|fanfiction\.net)\/s\/\d+\//) 6 | end 7 | 8 | def get_story 9 | @target_data = @target.target_data 10 | @page = get_metadata_page 11 | @story = Story.create(url: @base_url, 12 | title: get_story_title, 13 | author: get_author, 14 | meta_data: get_metadata) 15 | get_cover_image 16 | chapter_urls = get_chapter_urls 17 | if @request.strategy == 'recent' 18 | offset = chapter_urls.length - @request.recent_number 19 | chapter_urls = chapter_urls.last(@request.recent_number) 20 | end 21 | offset = 0 unless offset && offset > 0 22 | @request.update(total_chapters: chapter_urls.length, current_chapters: 0) 23 | get_chapters(chapter_urls, offset: offset) 24 | end 25 | 26 | def get_metadata 27 | summary = @page.at_css(@target_data['summary']).text 28 | meta = @page.at_css(@target_data['meta']).text 29 | {summary: summary, info: meta}.to_json 30 | end 31 | 32 | def get_story_title 33 | if @page.at_css(@target_data['title']) 34 | return @page.at_css(@target_data['title']).text.strip 35 | else 36 | raise ScraperError, "Cannot find a story at url provided. Please recheck the url." 37 | end 38 | end 39 | 40 | def get_author 41 | @page.xpath(@target_data['author']).first.text.strip 42 | end 43 | 44 | def get_cover_image 45 | image = @page.search('.cimage') 46 | return unless image[1] 47 | parts = image[1]['src'].split('/') 48 | parts[-1] = '180/' 49 | url = parts.join('/') 50 | scrape_image(url, cover: true) 51 | end 52 | 53 | def get_chapter_urls 54 | unless @page.css(@target_data['chapter_urls']).empty? 55 | @page.at_css(@target_data['chapter_urls']).css("option").map do |option| 56 | "https://www.#{@base_url}#{option['value']}/" 57 | end 58 | else 59 | [@page.uri] 60 | end 61 | end 62 | 63 | def get_chapter_title 64 | options = @page.css(@target_data['chapter_titles']) 65 | title = "" 66 | options.each do |option| 67 | if option['selected'] 68 | title = option.text.sub(/^\d+\./, '').strip 69 | break 70 | end 71 | end 72 | title 73 | end 74 | 75 | def get_chapters(chapter_urls, offset: 0) 76 | chapter_urls.each_with_index do |chapter, index| 77 | @page = queue_page(chapter) unless chapter == @page.uri 78 | Chapter.create(title: get_chapter_title, 79 | content: get_chapter_content, 80 | number: index + offset + 1, 81 | story_id: @story.id) 82 | @request.increment!(:current_chapters) 83 | end 84 | end 85 | 86 | def get_chapter_content 87 | @page.at_css(@target_data['chapter_content']).to_s 88 | end 89 | 90 | def get_metadata_page 91 | queue_page("https://www.#{@base_url}1/") 92 | end 93 | 94 | 95 | end 96 | -------------------------------------------------------------------------------- /app/scraper/forum_scraper.rb: -------------------------------------------------------------------------------- 1 | class ForumScraper < Scraper 2 | @queue = :scrape 3 | 4 | def get_story 5 | @target_data = @target.target_data 6 | @page = get_metadata_page 7 | title = get_story_title 8 | 9 | chapter_urls_with_dates = get_chapter_urls_with_dates(@page.css(@target_data['overlay_threadmark'])) 10 | chapter_urls = chapter_urls_with_dates.map(&:first).map{|url| url.split('#')[0]} 11 | @page = queue_page("https://#{@base_url}") 12 | @story = Story.create(url: @base_url, 13 | title: title, 14 | author: get_author, 15 | meta_data: get_metadata) 16 | get_cover_image 17 | if @request.strategy == 'all' 18 | @request.update(total_chapters: chapter_urls.length, current_chapters: 0) 19 | if reader_mode 20 | puts "reader mode true" 21 | first_post = @page.at_css(@target_data['post']) 22 | if first_post['class'].include?('hasThreadmark') 23 | @page = queue_page("https://#{@base_url}/reader") 24 | get_reader_chapters 25 | else 26 | create_chapter(first_post, 1, title: "Intro") 27 | @page = queue_page("https://#{@base_url}/reader") 28 | get_reader_chapters(2) 29 | end 30 | else 31 | puts "reader mode false" 32 | get_chapters(chapter_urls) 33 | end 34 | elsif @request.strategy == 'recent' 35 | chapter_urls, offset = recent_urls(chapter_urls_with_dates, @request.recent_number) 36 | @request.update(total_chapters: chapter_urls.count, current_chapters: 0) 37 | offset = 1 if offset < 1 38 | get_recent_chapters(chapter_urls, offset) 39 | end 40 | end 41 | 42 | def get_recent_chapters(chapter_urls, offset) 43 | chapter_urls.each do |url| 44 | chunks = url.split('#') 45 | post_id = chunks[1] if chunks[1] 46 | @page = queue_page(url) 47 | if post_id 48 | node = (@page.at_css("#js-#{post_id}") || @page.at_css("##{post_id}")) 49 | else 50 | node = threadmark_nodes.first 51 | end 52 | create_chapter(node, offset) 53 | offset += 1 54 | @request.increment!(:current_chapters) 55 | end 56 | end 57 | 58 | def recent_urls(urls, number) 59 | return urls.sort{|a, b| a[1] <=> b[1]}.last(number).map{|x| x[0]}, urls.length - @request.recent_number + 1 60 | end 61 | 62 | def get_chapter_urls_with_dates(threadmark_targets, urls=[]) 63 | threadmark_targets.each do |t| 64 | if t.attributes['class'].value.include?('ThreadmarkFetcher') || 65 | t.attributes['class'].value.include?('structItem--threadmark-filler') 66 | urls.concat(get_chapter_urls_with_dates(get_new_threadmarks(t).css(@target_data['threadmark_list_item']), urls)) 67 | else 68 | urls << chapter_url_and_date(t) 69 | end 70 | end 71 | urls 72 | end 73 | 74 | def get_new_threadmarks(threadmark_fetcher) 75 | if target.domain.include?("sufficientvelocity") || target.domain.include?("spacebattles") 76 | @agent.get("https://#{@target.domain}#{threadmark_fetcher.at_css('div').attributes['data-fetchurl'].value}") 77 | else 78 | @agent.post("https://#{@target.domain}/index.php?threads/threadmarks/load-range", 79 | 'min' => threadmark_fetcher.attributes['data-range-min'].value, 80 | 'max' => threadmark_fetcher.attributes['data-range-max'].value, 81 | 'thread_id' => threadmark_fetcher.attributes['data-thread-id'].value 82 | ) 83 | end 84 | end 85 | 86 | def chapter_url_and_date(threadmark) 87 | begin 88 | date = threadmark.at_css(@target_data['threadmark_date']).text.to_date 89 | rescue StandardError 90 | date = Time.now.to_date 91 | end 92 | 93 | [absolute_url(threadmark.at_css(@target_data['threadmark_url'])['href'], @page.uri), date] 94 | end 95 | 96 | def threadmark_nodes 97 | @page.css(@target_data['threadmark']).map{|node| ["sv", "sb"].include?(story.domain) ? node.parent : node} 98 | end 99 | 100 | def get_reader_chapters(index=1) 101 | threadmark_nodes.each do |chapter| 102 | create_chapter(chapter, index) 103 | index += 1 104 | @request.increment!(:current_chapters) 105 | end 106 | if next_page 107 | @page = queue_page(next_page) 108 | get_reader_chapters(index) 109 | end 110 | end 111 | 112 | def next_page 113 | return false unless @page.at_css('.PageNav nav') 114 | @page.at_css('.PageNav nav').css('a').each do |a| 115 | return absolute_url(a['href'], @page.uri) if a.text == "Next >" 116 | end 117 | false 118 | end 119 | 120 | def get_publish_date(node) 121 | date = node&.at_css(@target_data['chapter_pub_date']) 122 | if date 123 | date.text&.to_date 124 | else 125 | nil 126 | end 127 | end 128 | 129 | def get_edit_date(node) 130 | date = node&.at_css(@target_data['chapter_edit_date']) 131 | if date 132 | date['data-datestring']&.to_date 133 | else 134 | nil 135 | end 136 | end 137 | 138 | def create_chapter(node, number, title: nil) 139 | Chapter.create(title: title ||= get_chapter_title(node), 140 | content: get_chapter_content(node), 141 | number: number, 142 | story_id: @story.id, 143 | publish_date: get_publish_date(node), 144 | edit_date: get_edit_date(node)) 145 | end 146 | 147 | def get_chapters(chapter_urls) 148 | chapter_urls.uniq! 149 | @index = 1 150 | chapter_urls.each do |url| 151 | @page = queue_page(url) 152 | threadmark_nodes.each do |chapter| 153 | create_chapter(chapter, @index) 154 | @index += 1 155 | @request.increment!(:current_chapters) 156 | end 157 | end 158 | end 159 | 160 | def reader_mode 161 | @page.at_css('.readerToggle') 162 | end 163 | 164 | def get_base_url 165 | @url.match(/(forums|forum)\.(sufficientvelocity|spacebattles|questionablequesting)\.com\/threads\/.+\.\d+/) 166 | end 167 | 168 | def get_metadata 169 | if @page.uri.to_s == "https://#{@base_url}/threadmarks" 170 | "" 171 | else 172 | pub_date = @page.at_css(@target_data['story_pub_date']).text 173 | {published: pub_date}.to_json 174 | end 175 | 176 | end 177 | 178 | def get_cover_image 179 | unless @page.uri.to_s == "https://#{@base_url}/threadmarks" 180 | return unless @page.at_css(@target_data['avatar']) 181 | parts = @page.at_css(@target_data['avatar'])['src'].split('/') 182 | return unless parts[-3] 183 | parts[-3] = 'l' 184 | url = "https://#{@page.uri.host}/#{parts.join('/')}" 185 | scrape_image(url, cover: true) 186 | end 187 | end 188 | 189 | def get_metadata_page 190 | begin 191 | queue_page("https://#{@base_url}/threadmarks") 192 | rescue StandardError => e 193 | if e.to_s.start_with?('404') 194 | raise ScraperError, "No threadmarks were found for this post. At this time only threads with threadmarks can be converted." 195 | else raise e 196 | end 197 | end 198 | end 199 | 200 | def get_story_title 201 | (@page.at_css(".titleBar h1") || @page.at_css(".p-title-value")).text.gsub(/( - Threadmarks|Threadmarks for: )/, '').strip 202 | end 203 | 204 | def get_author 205 | if @page.uri.to_s == "https://#{@base_url}/threadmarks" 206 | "" 207 | else 208 | @page.css(@target_data['post']).first.attr("data-author") 209 | end 210 | end 211 | 212 | def get_chapter_urls 213 | @page.css("#{@target_data['threadmark_list_item']} a").map do |t| 214 | "https://#{@base_url.to_s.split('threads/')[0]}#{t.attr('href')}".sub(/#post-\d+/, '') 215 | end 216 | end 217 | 218 | def get_chapter_title(chapter) 219 | chapter.at_css(@target_data['chapter_threadmark_text']).text.gsub(@target_data['chapter_threadmark_fluff'], '').strip 220 | end 221 | 222 | def get_chapter_content(chapter) 223 | content = chapter.at_css(@target_data['chapter_content']) 224 | content = absolutify_urls(content) 225 | extract_images(content).to_xml 226 | end 227 | end 228 | -------------------------------------------------------------------------------- /app/scraper/scraper.rb: -------------------------------------------------------------------------------- 1 | class ProxiesExhausted < StandardError; end 2 | 3 | class Scraper 4 | include ActiveModel::Model 5 | @queue = :scrape 6 | attr_accessor :url, :doc_id, :request, :squeue, :agent, :story, :target, :offset 7 | 8 | USER_AGENTS = [ 9 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", 10 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36", 11 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", 12 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0", 13 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", 14 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0", 15 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15", 16 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" 17 | ] 18 | 19 | def self.perform(request_id) 20 | begin 21 | request = Request.find(request_id) 22 | story = self.new(url: request.url, request: request, target: request.target).scrape 23 | request.update!(story_id: story.id) 24 | Resque.enqueue(DelayedBuilder, request.id) 25 | rescue Exception => e 26 | request.update(complete: true, status: e) 27 | raise e 28 | end 29 | end 30 | 31 | def scrape 32 | @base_url = get_base_url 33 | raise ScraperError, "Cannot find a story at url provided. Please recheck the url." unless @base_url 34 | @proxies = Proxy.all.to_a 35 | @agent = Mechanize.new 36 | @agent.user_agent = "Omnibuser 1.2 www.omnibuser.com" 37 | get_story 38 | @story 39 | end 40 | 41 | def get_page(url) 42 | puts "Retrieving page #{url}" 43 | tries = 5 44 | @proxies.shuffle! 45 | proxy_index = 0 46 | begin 47 | if true || self.class == FFNScraper && @proxies.length > 0 48 | proxy = @proxies[proxy_index] 49 | @agent.set_proxy(proxy.ip, proxy.port, proxy.username, proxy.password) 50 | @agent.user_agent = USER_AGENTS.sample 51 | response = @agent.get(url) 52 | proxy.update!(successful_request_count: proxy.successful_request_count + 1, last_successful_request: Time.now) 53 | response 54 | else 55 | @agent.get(url) 56 | end 57 | rescue Exception => e 58 | if true || self.class == FFNScraper && @proxies.length > 0 59 | proxy = @proxies[proxy_index] 60 | proxy.increment!(:failed_request_count) 61 | proxy_index += 1 62 | end 63 | tries -= 1 64 | if tries > 0 65 | if true || self.class == FFNScraper && @proxies.length > 0 66 | Rollbar.warning(e, "FFN connection failure for proxy #{proxy.ip}") 67 | end 68 | retry 69 | else 70 | if true || self.class == FFNScraper && @proxies.length > 0 71 | raise ProxiesExhausted.new("Could not connect to #{request.url}") 72 | else 73 | raise e 74 | end 75 | end 76 | end 77 | end 78 | 79 | def queue_page(url) 80 | #make this dynamic+generic and use it for images too 81 | delay = 1.0 82 | @target.reload 83 | if Time.now - @target.last_access > delay 84 | @target.update!(last_access: Time.now) 85 | get_page(url) 86 | else 87 | sleep(delay - (Time.now - @target.last_access) + rand) 88 | queue_page(url) 89 | end 90 | end 91 | 92 | def extract_images(content) 93 | content.search('img').each do |img| 94 | next if img['src'].blank? 95 | duplicate = @story.has_image(absolute_url(img['src'], @page.uri)) 96 | if duplicate 97 | img['src'] = "#{duplicate.name}" 98 | else 99 | image = scrape_image(absolute_url(img['src'], @page.uri)) 100 | img['src'] = "#{image}" 101 | end 102 | end 103 | content 104 | end 105 | 106 | def scrape_image(url, cover: false) 107 | begin 108 | puts "Retrieving image at #{url}" 109 | sleep(1) 110 | src = queue_page(url) 111 | rescue Exception => e 112 | puts e 113 | end 114 | if src&.class == Mechanize::Image 115 | image = Image.create(story_id: @story.id, 116 | extension: src['content-type'].split('/')[-1], 117 | source_url: url, 118 | cover: cover) 119 | background_color = @target_data['image_background'] 120 | unless url.include?('clear.png') 121 | begin 122 | src.save("#{image.path}.temp") 123 | image.compress(background_color) 124 | rescue Exception => e 125 | puts e 126 | end 127 | end 128 | src.save(image.path) unless File.exist?(image.path) 129 | image.upload 130 | end 131 | image&.name 132 | end 133 | 134 | def absolute_url(url, reference) 135 | url = url.split('#') 136 | unless url[0]&.start_with?('http') || url[0].blank? 137 | url[0] = "/#{url[0]}" unless url[0].start_with?('/') 138 | new_uri = URI::Generic.build({scheme: reference.scheme, 139 | host: reference.host, 140 | path: url[0].split('?')[0], 141 | query: url[0].split('?')[1], 142 | fragment: url[1]}) 143 | end 144 | if new_uri 145 | new_uri.to_s 146 | else 147 | url.join('#') 148 | end 149 | end 150 | 151 | def absolutify_urls(content) 152 | content.search('a').each do |a| 153 | next if a['href'].blank? 154 | a['href'] = absolute_url(a['href'], @page.uri) 155 | end 156 | content 157 | end 158 | 159 | def filter_nodes(nodeset, filters) 160 | filters&.each do |filter| 161 | nodeset.search(filter).each{|n| n.remove} 162 | end 163 | nodeset 164 | end 165 | 166 | end 167 | -------------------------------------------------------------------------------- /app/scraper/scraper_error.rb: -------------------------------------------------------------------------------- 1 | class ScraperError < StandardError 2 | 3 | end 4 | -------------------------------------------------------------------------------- /app/scraper/toc_scraper.rb: -------------------------------------------------------------------------------- 1 | class TOCScraper < Scraper 2 | @queue = :scrape 3 | 4 | def get_base_url 5 | @target.target_data['toc_url'] 6 | end 7 | 8 | def get_story 9 | @target_data = @target.target_data 10 | @page = queue_page(@target_data['toc_url']) 11 | chapter_urls = get_chapter_urls 12 | @story = Story.create(url: @target.domain, 13 | title: @target_data['title'], 14 | author: @target_data['author']) 15 | get_cover_image 16 | @offset = 0 17 | if @request.strategy == 'recent' 18 | @offset = chapter_urls.length - @request.recent_number 19 | chapter_urls = chapter_urls.last(@request.recent_number) 20 | end 21 | @request.update(total_chapters: chapter_urls.length, current_chapters: 0) 22 | get_chapters(chapter_urls) 23 | end 24 | 25 | def get_cover_image 26 | return unless @target_data['cover_image_url'] 27 | scrape_image(@target_data['cover_image_url'], cover: true) 28 | end 29 | 30 | def get_chapter_urls 31 | filter_nodes(@page, @target_data['toc_filters']) 32 | @page.css(@target_data['chapter_urls']).map do |url| 33 | if url['href'].include?(@target.domain) 34 | if url['href'].start_with?('http') 35 | url['href'] 36 | else 37 | "http://#{url['href']}" 38 | end 39 | else 40 | next 41 | end 42 | end.compact 43 | end 44 | 45 | def get_chapters(chapter_urls) 46 | chapter_urls.each_with_index do |chapter, index| 47 | @page = queue_page(chapter) 48 | Chapter.create(title: get_chapter_title, 49 | content: get_chapter_content, 50 | number: index + offset + 1, 51 | story_id: @story.id) 52 | @request.increment!(:current_chapters) 53 | end 54 | end 55 | 56 | def get_chapter_content 57 | content = @page.at_css(@target_data['chapter_content']) 58 | filter_nodes(content, @target_data['content_filters']) 59 | extract_images(content).to_xml 60 | end 61 | 62 | def get_chapter_title 63 | @page.at_css(@target_data['chapter_title']).text 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /app/templates/css/epub.css: -------------------------------------------------------------------------------- 1 | @charset "utf-8"; 2 | @namespace epub 'http://www.idpf.org/2007/ops'; 3 | 4 | /* Remove margin for WebKit-based reading systems. 5 | */ 6 | body { 7 | margin: 0; 8 | } 9 | 10 | /* Disable hyphenation for headings to avoid single-syllable-lines. 11 | */ 12 | h1, 13 | h2 { 14 | -epub-hyphens: none; 15 | -webkit-hyphens: none; 16 | -moz-hyphens: none; 17 | hyphens: none; 18 | } 19 | 20 | /* Set the minimum amount of lines to show up on a seperate page. (There is not much support for this at the moment.) 21 | */ 22 | p, 23 | blockquote { 24 | orphans: 2; 25 | widows: 2; 26 | } 27 | 28 | /* Turn on hyphenation for paragraphs and captions only. 29 | */ 30 | p, 31 | figcaption { 32 | -epub-hyphens: auto; 33 | -webkit-hyphens: auto; 34 | -moz-hyphens: auto; 35 | hyphens: auto; 36 | } 37 | 38 | /* Shortcodes for page-break rules. 39 | Use data attributes to designate if and how the page should be broken before, inside or after an element. 40 | */ 41 | h1, h2, h3, h4, h5, h6, 42 | table, img, figure, video, 43 | [data-page-break~=inside][data-page-break~=avoid] { page-break-inside: avoid; } 44 | [data-page-break~=after] { page-break-after: always; } 45 | h1, h2, h3, h4, h5, h6, 46 | [data-page-break~=after][data-page-break~=avoid] { page-break-after: avoid; } 47 | [data-page-break~=before] { page-break-before: always; } 48 | [data-page-break~=before][data-page-break~=avoid] { page-break-before: avoid; } 49 | img[data-page-break~=before] { page-break-before: left; } 50 | 51 | /* Custom Styles 52 | */ 53 | -------------------------------------------------------------------------------- /app/templates/css/ffn.css: -------------------------------------------------------------------------------- 1 | body{ 2 | font-family: Verdana,'Lucida Grande','Lucida Sans Unicode','Open Sans',Arial,sans-serif; 3 | font-size: 1em; 4 | line-height: 1.25; 5 | } 6 | 7 | .filler{ 8 | height: 30mm; 9 | } 10 | 11 | .omni-chapter-title{ 12 | text-align: center; 13 | font-size: 200%; 14 | font-weight: bold; 15 | } 16 | .omni-title{ 17 | font-size: 200%; 18 | text-align: center; 19 | } 20 | -------------------------------------------------------------------------------- /app/templates/css/fp.css: -------------------------------------------------------------------------------- 1 | body{ 2 | font-family: Verdana,'Lucida Grande','Lucida Sans Unicode','Open Sans',Arial,sans-serif; 3 | font-size: 1em; 4 | line-height: 1.25; 5 | } 6 | 7 | .filler{ 8 | height: 30mm; 9 | } 10 | 11 | .omni-chapter-title{ 12 | text-align: center; 13 | font-size: 200%; 14 | font-weight: bold; 15 | } 16 | .omni-title{ 17 | font-size: 200%; 18 | text-align: center; 19 | } 20 | -------------------------------------------------------------------------------- /app/templates/css/html.css: -------------------------------------------------------------------------------- 1 | body{ 2 | font-size: 16pt; 3 | padding: 20px 40px; 4 | } 5 | 6 | #omni-frontmatter-container{ 7 | page-break-after: always; 8 | } 9 | 10 | #omni-story-container{ 11 | page-break-after: always; 12 | } 13 | 14 | .omni-chapter{ 15 | page-break-before: always; 16 | } 17 | 18 | .omni-spacer{ 19 | page-break-before: always; 20 | margin: 0; 21 | padding: 0; 22 | height: 338.5mm; 23 | width: 210mm; 24 | } 25 | 26 | img{ 27 | margin: 20px; 28 | } 29 | -------------------------------------------------------------------------------- /app/templates/css/qq.css: -------------------------------------------------------------------------------- 1 | body{ 2 | color: rgb(86, 86, 86); 3 | background-color: rgb(234,235,235); 4 | font-family: 'Trebuchet MS',Helvetica,Arial,sans-serif; 5 | } 6 | 7 | .filler{ 8 | height: 30mm; 9 | background-color: rgb(234,235,235); 10 | } 11 | 12 | a{ 13 | color: rgb(25,105,196); 14 | text-decoration: none; 15 | } 16 | 17 | .contents-list{ 18 | 19 | } 20 | 21 | .adv_accordion dt, .adv_tabs_noscript_title{ 22 | border-top: 1px solid rgb(177, 177, 177); 23 | border-left: 1px solid rgb(177, 177, 177); 24 | border-right: 1px solid rgb(177, 177, 177); 25 | margin-bottom: 0; 26 | margin-top: 10px; 27 | margin-left: 0; 28 | page-break-after: avoid; 29 | } 30 | 31 | .adv_accordion dd, .adv_tabs_noscript_content { 32 | margin-left: 0; 33 | margin-top: 0; 34 | border: 1px solid rgb(177, 177, 177); 35 | padding: 10px; 36 | page-break-before: avoid; 37 | 38 | } 39 | 40 | 41 | blockquote{ 42 | margin: none; 43 | } 44 | 45 | .omni-chapter-title{ 46 | text-align: center; 47 | font-size: 200%; 48 | font-weight: bold; 49 | } 50 | .omni-title{ 51 | font-size: 200%; 52 | text-align: center; 53 | } 54 | .omni-spoilerTextContainer{ 55 | margin-top: 10px; 56 | font-weight: bold; 57 | } 58 | 59 | .SpoilerTitle{ 60 | } 61 | 62 | .bbCodeSpoilerText{ 63 | border: 1px solid rgb(177, 177, 177); 64 | padding: 10px; 65 | margin-bottom :10px; 66 | 67 | } 68 | 69 | .bbCodeQuote{ 70 | padding: 20px; 71 | margin-left: 20px; 72 | } 73 | .quoteContainer{ 74 | border: 1px dashed rgb(177, 177, 177); 75 | padding: 10px; 76 | 77 | } 78 | 79 | img.mceSmilie, 80 | img.mceSmilieSprite { 81 | vertical-align: text-bottom; 82 | margin: 0 1px 83 | } 84 | img.mceSmilieSprite.mceSmilie1 { 85 | width: 18px; 86 | height: 18px; 87 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px 0px 88 | } 89 | img.mceSmilieSprite.mceSmilie2 { 90 | width: 18px; 91 | height: 18px; 92 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -21px 93 | } 94 | img.mceSmilieSprite.mceSmilie3 { 95 | width: 18px; 96 | height: 18px; 97 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -42px 98 | } 99 | img.mceSmilieSprite.mceSmilie4 { 100 | width: 18px; 101 | height: 18px; 102 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px 0px 103 | } 104 | img.mceSmilieSprite.mceSmilie5 { 105 | width: 18px; 106 | height: 18px; 107 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -21px 108 | } 109 | img.mceSmilieSprite.mceSmilie6 { 110 | width: 18px; 111 | height: 18px; 112 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px 0px 113 | } 114 | img.mceSmilieSprite.mceSmilie7 { 115 | width: 18px; 116 | height: 18px; 117 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px -21px 118 | } 119 | img.mceSmilieSprite.mceSmilie8 { 120 | width: 18px; 121 | height: 18px; 122 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px 0px 123 | } 124 | img.mceSmilieSprite.mceSmilie9 { 125 | width: 18px; 126 | height: 18px; 127 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px -42px 128 | } 129 | img.mceSmilieSprite.mceSmilie10 { 130 | width: 18px; 131 | height: 18px; 132 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px -42px 133 | } 134 | img.mceSmilieSprite.mceSmilie11 { 135 | width: 18px; 136 | height: 18px; 137 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px -21px 138 | } 139 | img.mceSmilieSprite.mceSmilie12 { 140 | width: 18px; 141 | height: 18px; 142 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -80px -42px 143 | } 144 | img.mceSmilieSprite.mceSmilie58 { 145 | width: 18px; 146 | height: 18px; 147 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -84px 148 | } 149 | img.mceSmilieSprite.mceSmilie59 { 150 | width: 18px; 151 | height: 18px; 152 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -126px 153 | } 154 | img.mceSmilieSprite.mceSmilie60 { 155 | width: 18px; 156 | height: 18px; 157 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -80px -126px 158 | } 159 | img.mceSmilieSprite.mceSmilie61 { 160 | width: 18px; 161 | height: 18px; 162 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -63px 163 | } 164 | img.mceSmilieSprite.mceSmilie62 { 165 | width: 18px; 166 | height: 18px; 167 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -63px 168 | } 169 | img.mceSmilieSprite.mceSmilie63 { 170 | width: 18px; 171 | height: 18px; 172 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -42px 173 | } 174 | -------------------------------------------------------------------------------- /app/templates/css/sb.css: -------------------------------------------------------------------------------- 1 | body{ 2 | color: rgb(204,204,204); 3 | background-color: rgb(25, 31, 45); 4 | font-family: tahoma,arial,helvetica,sans-serif; 5 | } 6 | .filler{ 7 | height: 30mm; 8 | background-color: rgb(25, 31, 45); 9 | } 10 | 11 | a{ 12 | color: rgb(0,255,0); 13 | text-decoration: none; 14 | } 15 | 16 | .contents-list{ 17 | 18 | } 19 | 20 | .adv_accordion dt, .adv_tabs_noscript_title{ 21 | border-top: 1px solid rgb(65, 92, 135); 22 | border-left: 1px solid rgb(65, 92, 135); 23 | border-right: 1px solid rgb(65, 92, 135); 24 | margin-bottom: 0; 25 | margin-top: 10px; 26 | margin-left: 0; 27 | page-break-after: avoid; 28 | } 29 | 30 | .adv_accordion dd, .adv_tabs_noscript_content { 31 | margin-left: 0; 32 | margin-top: 0; 33 | border: 1px solid rgb(65, 92, 135); 34 | padding: 10px; 35 | page-break-before: avoid; 36 | 37 | } 38 | 39 | 40 | blockquote{ 41 | margin: none; 42 | } 43 | 44 | .omni-chapter-title{ 45 | text-align: center; 46 | font-size: 200%; 47 | font-weight: bold; 48 | } 49 | .omni-title{ 50 | font-size: 200%; 51 | text-align: center; 52 | } 53 | .omni-spoilerTextContainer{ 54 | margin-top: 10px; 55 | font-weight: bold; 56 | } 57 | 58 | .SpoilerTitle{ 59 | } 60 | 61 | .bbCodeSpoilerText{ 62 | border: 1px solid rgb(65, 92, 135); 63 | padding: 10px; 64 | margin-bottom :10px; 65 | 66 | } 67 | 68 | .bbCodeQuote{ 69 | padding: 20px; 70 | margin-left: 20px; 71 | } 72 | .quoteContainer{ 73 | border: 1px dashed rgb(65, 92, 135); 74 | padding: 10px; 75 | 76 | 77 | } 78 | 79 | img.mceSmilie, 80 | img.mceSmilieSprite { 81 | vertical-align: text-bottom; 82 | margin: 0 1px 83 | } 84 | img.mceSmilieSprite.mceSmilie1 { 85 | width: 18px; 86 | height: 18px; 87 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px 0px 88 | } 89 | img.mceSmilieSprite.mceSmilie2 { 90 | width: 18px; 91 | height: 18px; 92 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -21px 93 | } 94 | img.mceSmilieSprite.mceSmilie3 { 95 | width: 18px; 96 | height: 18px; 97 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -42px 98 | } 99 | img.mceSmilieSprite.mceSmilie4 { 100 | width: 18px; 101 | height: 18px; 102 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px 0px 103 | } 104 | img.mceSmilieSprite.mceSmilie5 { 105 | width: 18px; 106 | height: 18px; 107 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -21px 108 | } 109 | img.mceSmilieSprite.mceSmilie6 { 110 | width: 18px; 111 | height: 18px; 112 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px 0px 113 | } 114 | img.mceSmilieSprite.mceSmilie7 { 115 | width: 18px; 116 | height: 18px; 117 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px -21px 118 | } 119 | img.mceSmilieSprite.mceSmilie8 { 120 | width: 18px; 121 | height: 18px; 122 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px 0px 123 | } 124 | img.mceSmilieSprite.mceSmilie9 { 125 | width: 18px; 126 | height: 18px; 127 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px -42px 128 | } 129 | img.mceSmilieSprite.mceSmilie10 { 130 | width: 18px; 131 | height: 18px; 132 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px -42px 133 | } 134 | img.mceSmilieSprite.mceSmilie11 { 135 | width: 18px; 136 | height: 18px; 137 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px -21px 138 | } 139 | img.mceSmilieSprite.mceSmilie12 { 140 | width: 18px; 141 | height: 18px; 142 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -80px -42px 143 | } 144 | img.mceSmilieSprite.mceSmilie58 { 145 | width: 18px; 146 | height: 18px; 147 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -84px 148 | } 149 | img.mceSmilieSprite.mceSmilie59 { 150 | width: 18px; 151 | height: 18px; 152 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -126px 153 | } 154 | img.mceSmilieSprite.mceSmilie60 { 155 | width: 18px; 156 | height: 18px; 157 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -80px -126px 158 | } 159 | img.mceSmilieSprite.mceSmilie61 { 160 | width: 18px; 161 | height: 18px; 162 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -63px 163 | } 164 | img.mceSmilieSprite.mceSmilie62 { 165 | width: 18px; 166 | height: 18px; 167 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -63px 168 | } 169 | img.mceSmilieSprite.mceSmilie63 { 170 | width: 18px; 171 | height: 18px; 172 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -42px 173 | } 174 | -------------------------------------------------------------------------------- /app/templates/css/sv.css: -------------------------------------------------------------------------------- 1 | body{ 2 | color: rgb(230,230,230); 3 | background-color: rgb(40,40,40); 4 | font-family: 'Trebuchet MS',Helvetica,Arial,sans-serif; 5 | } 6 | 7 | .filler{ 8 | height: 30mm; 9 | background-color: rgb(40,40,40); 10 | } 11 | 12 | a{ 13 | color: rgb(40,161,221); 14 | text-decoration: none; 15 | } 16 | 17 | .contents-list{ 18 | 19 | } 20 | 21 | .adv_accordion dt, .adv_tabs_noscript_title{ 22 | border-top: 1px solid rgb(48,75,101); 23 | border-left: 1px solid rgb(48,75,101); 24 | border-right: 1px solid rgb(48,75,101); 25 | margin-bottom: 0; 26 | margin-top: 10px; 27 | margin-left: 0; 28 | page-break-after: avoid; 29 | } 30 | 31 | .adv_accordion dd, .adv_tabs_noscript_content { 32 | margin-left: 0; 33 | margin-top: 0; 34 | border: 1px solid rgb(48,75,101); 35 | padding: 10px; 36 | page-break-before: avoid; 37 | 38 | } 39 | 40 | 41 | blockquote{ 42 | margin: none; 43 | } 44 | 45 | .omni-chapter-title{ 46 | text-align: center; 47 | font-size: 200%; 48 | font-weight: bold; 49 | } 50 | .omni-title{ 51 | font-size: 200%; 52 | text-align: center; 53 | } 54 | .omni-spoilerTextContainer{ 55 | margin-top: 10px; 56 | font-weight: bold; 57 | } 58 | 59 | .SpoilerTitle{ 60 | } 61 | 62 | .bbCodeSpoilerText{ 63 | border: 1px solid rgb(48,75,101); 64 | padding: 10px; 65 | margin-bottom :10px; 66 | 67 | } 68 | 69 | .bbCodeQuote{ 70 | padding: 20px; 71 | margin-left: 20px; 72 | } 73 | .quoteContainer{ 74 | border: 1px dashed rgb(48,75,101); 75 | padding: 10px; 76 | 77 | } 78 | 79 | img.mceSmilie, 80 | img.mceSmilieSprite { 81 | vertical-align: text-bottom; 82 | margin: 0 1px 83 | } 84 | img.mceSmilieSprite.mceSmilie1 { 85 | width: 18px; 86 | height: 18px; 87 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px 0px 88 | } 89 | img.mceSmilieSprite.mceSmilie2 { 90 | width: 18px; 91 | height: 18px; 92 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -21px 93 | } 94 | img.mceSmilieSprite.mceSmilie3 { 95 | width: 18px; 96 | height: 18px; 97 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -42px 98 | } 99 | img.mceSmilieSprite.mceSmilie4 { 100 | width: 18px; 101 | height: 18px; 102 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px 0px 103 | } 104 | img.mceSmilieSprite.mceSmilie5 { 105 | width: 18px; 106 | height: 18px; 107 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -21px 108 | } 109 | img.mceSmilieSprite.mceSmilie6 { 110 | width: 18px; 111 | height: 18px; 112 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px 0px 113 | } 114 | img.mceSmilieSprite.mceSmilie7 { 115 | width: 18px; 116 | height: 18px; 117 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px -21px 118 | } 119 | img.mceSmilieSprite.mceSmilie8 { 120 | width: 18px; 121 | height: 18px; 122 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px 0px 123 | } 124 | img.mceSmilieSprite.mceSmilie9 { 125 | width: 18px; 126 | height: 18px; 127 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px -42px 128 | } 129 | img.mceSmilieSprite.mceSmilie10 { 130 | width: 18px; 131 | height: 18px; 132 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px -42px 133 | } 134 | img.mceSmilieSprite.mceSmilie11 { 135 | width: 18px; 136 | height: 18px; 137 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px -21px 138 | } 139 | img.mceSmilieSprite.mceSmilie12 { 140 | width: 18px; 141 | height: 18px; 142 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -80px -42px 143 | } 144 | img.mceSmilieSprite.mceSmilie58 { 145 | width: 18px; 146 | height: 18px; 147 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -84px 148 | } 149 | img.mceSmilieSprite.mceSmilie59 { 150 | width: 18px; 151 | height: 18px; 152 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -126px 153 | } 154 | img.mceSmilieSprite.mceSmilie60 { 155 | width: 18px; 156 | height: 18px; 157 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -80px -126px 158 | } 159 | img.mceSmilieSprite.mceSmilie61 { 160 | width: 18px; 161 | height: 18px; 162 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -63px 163 | } 164 | img.mceSmilieSprite.mceSmilie62 { 165 | width: 18px; 166 | height: 18px; 167 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -63px 168 | } 169 | img.mceSmilieSprite.mceSmilie63 { 170 | width: 18px; 171 | height: 18px; 172 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -42px 173 | } 174 | -------------------------------------------------------------------------------- /app/templates/epub/META-INF/container.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /app/templates/epub/OPS/book/chapter.xhtml.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= html_escape(@story.title) %> 6 | 7 | 8 | 9 | 10 |
11 |

<%= html_escape(@chapter.title) %>

12 |
13 | <%= @chapter.epub %> 14 |
15 | 16 | 17 | -------------------------------------------------------------------------------- /app/templates/epub/OPS/book/cover.xhtml.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= html_escape(@story.title) %> 6 | 7 | 8 | 9 | 10 | <% if @cover %> 11 | <%= "Book Cover" %> 12 | <% else %> 13 | <%= "Book Cover" %> 14 | <% end %> 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /app/templates/epub/OPS/book/frontmatter.xhtml.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= html_escape(@story.title) %> 6 | 7 | 8 | 9 | 10 | 11 |

<%= @story.title %>

12 |

by <%= @story.author %>

13 |
14 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /app/templates/epub/OPS/book/table-of-contents.ncx.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Contents 12 | 13 | 14 | <%= @story.author %> 15 | 16 | 17 | <% @story.chapters.order(:number).each do |chapter|%> 18 | 19 | <%= html_escape(chapter.title) %> 20 | 21 | 22 | <% end %> 23 | 24 | 25 | -------------------------------------------------------------------------------- /app/templates/epub/OPS/book/table-of-contents.xhtml.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= html_escape(@story.title) %> 6 | 7 | 8 | 9 | 10 | 11 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /app/templates/epub/OPS/css/sv.css: -------------------------------------------------------------------------------- 1 | body{ 2 | color: rgb(230,230,230); 3 | background-color: rgb(40,40,40); 4 | font-family: 'Trebuchet MS',Helvetica,Arial,sans-serif; 5 | } 6 | 7 | a{ 8 | color: rgb(40,161,221); 9 | text-decoration: none; 10 | } 11 | 12 | .contents-list{ 13 | 14 | } 15 | 16 | .adv_accordion dt, .adv_tabs_noscript_title{ 17 | border-top: 1px solid rgb(48,75,101); 18 | border-left: 1px solid rgb(48,75,101); 19 | border-right: 1px solid rgb(48,75,101); 20 | margin-bottom: 0; 21 | margin-top: 10px; 22 | margin-left: 0; 23 | page-break-after: avoid; 24 | } 25 | 26 | .adv_accordion dd, .adv_tabs_noscript_content { 27 | margin-left: 0; 28 | margin-top: 0; 29 | border: 1px solid rgb(48,75,101); 30 | padding: 10px; 31 | page-break-before: avoid; 32 | 33 | } 34 | 35 | 36 | blockquote{ 37 | margin: none; 38 | } 39 | 40 | .omni-chapter-title{ 41 | text-align: center; 42 | font-size: 200%; 43 | font-weight: bold; 44 | } 45 | .omni-title{ 46 | font-size: 200%; 47 | text-align: center; 48 | } 49 | .omni-spoilerTextContainer{ 50 | margin-top: 10px; 51 | font-weight: bold; 52 | } 53 | 54 | .SpoilerTitle{ 55 | } 56 | 57 | .bbCodeSpoilerText{ 58 | border: 1px solid rgb(48,75,101); 59 | padding: 10px; 60 | margin-bottom :10px; 61 | 62 | } 63 | 64 | .bbCodeQuote{ 65 | padding: 20px; 66 | margin-left: 20px; 67 | } 68 | .quoteContainer{ 69 | border: 1px dashed rgb(48,75,101); 70 | padding: 10px; 71 | 72 | } 73 | 74 | img.mceSmilie, 75 | img.mceSmilieSprite { 76 | vertical-align: text-bottom; 77 | margin: 0 1px 78 | } 79 | img.mceSmilieSprite.mceSmilie1 { 80 | width: 18px; 81 | height: 18px; 82 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px 0px 83 | } 84 | img.mceSmilieSprite.mceSmilie2 { 85 | width: 18px; 86 | height: 18px; 87 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -21px 88 | } 89 | img.mceSmilieSprite.mceSmilie3 { 90 | width: 18px; 91 | height: 18px; 92 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -42px 93 | } 94 | img.mceSmilieSprite.mceSmilie4 { 95 | width: 18px; 96 | height: 18px; 97 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px 0px 98 | } 99 | img.mceSmilieSprite.mceSmilie5 { 100 | width: 18px; 101 | height: 18px; 102 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -21px 103 | } 104 | img.mceSmilieSprite.mceSmilie6 { 105 | width: 18px; 106 | height: 18px; 107 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px 0px 108 | } 109 | img.mceSmilieSprite.mceSmilie7 { 110 | width: 18px; 111 | height: 18px; 112 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px -21px 113 | } 114 | img.mceSmilieSprite.mceSmilie8 { 115 | width: 18px; 116 | height: 18px; 117 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px 0px 118 | } 119 | img.mceSmilieSprite.mceSmilie9 { 120 | width: 18px; 121 | height: 18px; 122 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -20px -42px 123 | } 124 | img.mceSmilieSprite.mceSmilie10 { 125 | width: 18px; 126 | height: 18px; 127 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px -42px 128 | } 129 | img.mceSmilieSprite.mceSmilie11 { 130 | width: 18px; 131 | height: 18px; 132 | background: url('../images/xenforo-smilies-sprite.png') no-repeat 0px -21px 133 | } 134 | img.mceSmilieSprite.mceSmilie12 { 135 | width: 18px; 136 | height: 18px; 137 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -80px -42px 138 | } 139 | img.mceSmilieSprite.mceSmilie58 { 140 | width: 18px; 141 | height: 18px; 142 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -84px 143 | } 144 | img.mceSmilieSprite.mceSmilie59 { 145 | width: 18px; 146 | height: 18px; 147 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -126px 148 | } 149 | img.mceSmilieSprite.mceSmilie60 { 150 | width: 18px; 151 | height: 18px; 152 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -80px -126px 153 | } 154 | img.mceSmilieSprite.mceSmilie61 { 155 | width: 18px; 156 | height: 18px; 157 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -40px -63px 158 | } 159 | img.mceSmilieSprite.mceSmilie62 { 160 | width: 18px; 161 | height: 18px; 162 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -63px 163 | } 164 | img.mceSmilieSprite.mceSmilie63 { 165 | width: 18px; 166 | height: 18px; 167 | background: url('../images/xenforo-smilies-sprite.png') no-repeat -60px -42px 168 | } 169 | -------------------------------------------------------------------------------- /app/templates/epub/OPS/package.opf.erb: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | <%= @story.url %> 7 | <%= @story.url %> 8 | <%= html_escape(@story.title) %> 9 | Downloaded via omnibuser.com 10 | Web-Fic 11 | en 12 | <%= Time.now.utc.iso8601 %> 13 | <%= Time.now.utc.iso8601 %> 14 | <%= html_escape(@story.author) %> 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | <% @story.chapters.order(:number).each do |chapter| %> 27 | <% num = "#{chapter.number.to_s.rjust(3, '0')}" %> 28 | <%= "" %> 29 | <% end %> 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | <% if @cover %> 38 | <%= "" %> 39 | <% else %> 40 | <%= "" %> 41 | <% end %> 42 | 43 | <% @story.images.where(cover: false).each_with_index do |image, index| %> 44 | <%= "" %> 45 | <% end %> 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | <% @story.chapters.order(:number).each do |chapter| %> 58 | <% num = "#{chapter.number.to_s.rjust(3, '0')}" %> 59 | <%= "" %> 60 | <% end %> 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /app/templates/epub/mimetype: -------------------------------------------------------------------------------- 1 | application/epub+zip -------------------------------------------------------------------------------- /app/templates/html/story.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= html_escape(@story.title) %> 6 | 7 | 8 | 9 | 10 |
11 | <% if @cover %> 12 | <%= "Book Cover" %> 13 | <% else %> 14 | <%= "Book Cover" %> 15 | <% end %> 16 |
17 | 18 |
19 |

<%= @story.title %>

20 |

by <%= @story.author %>

21 |
22 | 31 |
32 | 33 |
34 | 43 |
44 | 45 |
46 | <% @story.chapters.order(:number).each do |chapter|%> 47 |
48 |

<%= html_escape(chapter.title) %>

49 |
50 | <%= chapter.html %> 51 |
52 | <% end %> 53 | 54 |
55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /app/templates/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/templates/images/favicon.png -------------------------------------------------------------------------------- /app/templates/images/ffn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/templates/images/ffn.png -------------------------------------------------------------------------------- /app/templates/images/fp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/templates/images/fp.png -------------------------------------------------------------------------------- /app/templates/images/qq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/templates/images/qq.png -------------------------------------------------------------------------------- /app/templates/images/sb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/templates/images/sb.png -------------------------------------------------------------------------------- /app/templates/images/sv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/templates/images/sv.png -------------------------------------------------------------------------------- /app/templates/images/xenforo-smilies-sprite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/app/templates/images/xenforo-smilies-sprite.png -------------------------------------------------------------------------------- /app/templates/pdf/content.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= html_escape(@story.title) %> 6 | 7 | 8 | 9 | 10 |
11 | <% @chapter_chunk.each do |chapter|%> 12 |
13 |

<%= html_escape(chapter.title) %>

14 |
15 | <%= chapter.html %> 16 |
17 | <% unless chapter == @chapter_chunk.last %> 18 |
 
19 | <% end %> 20 | <% end %> 21 |
22 |
 
23 | 24 | 25 | -------------------------------------------------------------------------------- /app/templates/pdf/filler.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |
8 |   9 |
10 | 11 | 12 | -------------------------------------------------------------------------------- /app/templates/pdf/frontmatter.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= html_escape(@story.title) %> 6 | 7 | 8 | 9 | 10 |
11 | <% if @cover %> 12 | <%= "Book Cover" %> 13 | <% else %> 14 | <%= "Book Cover" %> 15 | <% end %> 16 |
17 | 18 |
19 |

<%= @story.title %>

20 |

by <%= @story.author %>

21 |
22 | 31 |
32 | 33 |
34 | 43 |
44 |
 
45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /app/views/home/index.html.erb: -------------------------------------------------------------------------------- 1 | 6 |
7 |

Convert your favourite online fiction into a convenient ebook!

8 | 9 |
10 | <%= form_tag('/', remote: true, id: "request_form") do %> 11 | <%= text_field_tag(:q) %> 12 | <%= submit_tag("Get", class: 'button', id: 'submit', data: {disable_with: "Get"}) %> 13 |
14 |
15 | <%= label_tag(:ext, "Select ebook file type:")%> 16 | <%= select_tag(:ext, options_for_select(['epub', 'html', 'mobi', 'pdf']), class:'dropdown') %> 17 |
18 |
19 |
20 |
21 | <%= radio_button_tag(:strategy, "all", true) %> 22 | <%= label_tag(:strategy_all, "Get all chapters") %> 23 |
24 |
25 | <%= radio_button_tag(:strategy, "recent") %> 26 | <%= label_tag :strategy_recent do %> 27 | Get most recent <%= select_tag(:recent_number, options_for_select([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])) %> chapter(s) 28 | <% end %> 29 |
30 |
31 | <% end %> 32 |
33 | 34 | <% if @doc_id %> 35 | <%= link_to "Download file", "/documents/#{@doc_id}", method: :post %> 36 | <% end %> 37 | <% flash.each do |name, msg| -%> 38 | <%= content_tag :div, msg, class: name %> 39 | <% end -%> 40 | 43 | 44 |
45 |
46 | 47 | 50 | 51 |
52 |

Enter a link to the story in the box above, select the file type you want for the ebook, and hit Get. See the About page for a list of supported sites.

53 |
54 | 55 |
56 | 57 |
58 | <% markdown = Redcarpet::Markdown.new(Redcarpet::Render::HTML) %> 59 | <% notes = Dir.glob("#{Rails.root.join("app", "views", "release-notes").to_s}/*") %> 60 | <% notes.map!{|n| File.new(n)} %> 61 | <% notes.sort!{|a, b| b.path.split('/')[-1].split('.')[0].to_i <=> a.path.split('/')[-1].split('.')[0].to_i} %> 62 | <% notes.each do |note| %> 63 |
64 | <%= markdown.render(note.read).html_safe %> 65 |
66 |
67 | <% end %> 68 |
69 | -------------------------------------------------------------------------------- /app/views/layouts/application.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | <%= @title || "Omnibuser"%> 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | <%= csrf_meta_tags %> 15 | 16 | 17 | 18 | 19 | 20 | 22 | <%= render 'shared/header' %> 23 |
24 |
25 | <%= yield %> 26 | <%= render 'shared/footer' %> 27 | 28 | 29 | <%= stylesheet_link_tag 'application', media: 'all'%> 30 | <%= javascript_include_tag 'application'%> 31 | <%= render partial: "shared/google_analytics"%> 32 | 33 | -------------------------------------------------------------------------------- /app/views/layouts/mailer.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 8 | 9 | 10 | 11 | <%= yield %> 12 | 13 | 14 | -------------------------------------------------------------------------------- /app/views/layouts/mailer.text.erb: -------------------------------------------------------------------------------- 1 | <%= yield %> 2 | -------------------------------------------------------------------------------- /app/views/release-notes/1.md: -------------------------------------------------------------------------------- 1 | **November 28th, 2019 - Omnibuser is no longer being actively maintained** 2 | 3 | I will continue to fix critical bugs, but I will not be doing any new feature development or addressing minor problems. 4 | 5 | I originally built Omnibuser as a way to learn Ruby on Rails. It was mostly for personal use, and I didn't expect many people to be interested in it. I ended up being wrong about that - a slow trickle of people started using it after I posted it on reddit and since then the numbers have steadily grown. I've been amazed and thankful for how many people found my little fanfic app useful - we've had over 300,000 downloads since release! 6 | 7 | I had a lot more ideas for Omnibuser, new features and ways to improve the user experience. I use it quite a bit myself, and I know how clunky and annoying it can be, and how many cool things it can't do. But shortly after releasing V1.1, I got an awesome job thanks to my new Rails skills. As a very junior developer, writing code for 8 hours a day was pretty draining, and I found I wasn't really able to work on personal projects on top of that. So all the ideas I've had, and all the cool ideas people have emailed me about, have been gathering dust on a Trello board somewhere for the past couple of years. 8 | 9 | I'm a bit more experienced now, and I'm finally able to put consistent time into personal projects on top of my day job. I thought about coming back to Omnibuser and adding all those cool features. But it's been a long time since I've worked on it, and looking at it now is like reading someone else's code. The code quality is pretty terrible - it looks like it was written by someone just starting to learn Rails! I wouldn't be happy slapping new stuff on top of such an amateurish foundation, I'd have to rewrite the whole thing. 10 | 11 | So instead of doing that, I'm working on a new app, which takes a different approach to the "reading fanfic as an ebook" idea. It's still in early stages, but I'm very excited about it, and when it's released I think it will make Omnibuser obsolete. So if you've been waiting patiently to see if your suggestion ever makes it into Omnibuser, all hope is not lost. I'll be taking on board all the feedback I've received, and I'll aim to make the new app the best fanfic reading tool possible! 12 | 13 | No release date for the new app yet, it won't be ready for at least a few months. If you'd be interested in helping me beta test closer to release, please do get in touch! 14 | -------------------------------------------------------------------------------- /app/views/shared/_footer.html.erb: -------------------------------------------------------------------------------- 1 | 12 | -------------------------------------------------------------------------------- /app/views/shared/_google_analytics.html.erb: -------------------------------------------------------------------------------- 1 | <% if Rails.env == "production" %> 2 | 12 | <% end %> 13 | -------------------------------------------------------------------------------- /app/views/shared/_header.html.erb: -------------------------------------------------------------------------------- 1 | 11 | -------------------------------------------------------------------------------- /app/views/static/about.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 |

4 | Omnibuser is a simple tool for converting online fiction into ebooks. I made this in my spare time because I was sick of my Kindle gathering dust while I read web serials. 5 |

6 |

7 | Just enter a link to the story you want, and Omnibuser will grab it and deliver it to you in whatever format you want. 8 |

9 |
10 |
11 |

Supported sites

12 |

13 | Right now I'm just supporting the sites that I personally use, that don't already have an ebook download feature (AO3 already has one, for example). If there are any sites I've missed that you'd like to see, get in contact with me and I'll look into it. 14 |

15 | 22 |

Currently the forums are only supported if the story thread uses threadmarks.

23 |
24 | 25 |
26 |

Why is it taking so long to fetch the story?

27 |

28 | In order to be considerate to the hosting sites, Omnibuser will only fetch a new chapter from a particular site once every couple of seconds. This means that if several users are trying to get stories from the same site at the same time, each request will take longer to fulfill. 29 |

30 |
31 | 32 |
33 |

"Omnibuser"? What?

34 |

35 | An omnibus is a bunch of short works by a particular author collected into a single book. This tool collects a bunch of serially-published chapters into a single ebook. It makes omnibuses. It's an omnibuser. Look, domain names are hard, okay? 36 |

37 |
38 |
39 | -------------------------------------------------------------------------------- /app/views/static/contact.html.erb: -------------------------------------------------------------------------------- 1 |
2 |

Feel free to get in touch with me for suggestions, bug reports, or really anything at all.

3 | 4 |

Email: contact@omnibuser.com

5 |

Twitter

6 |

Github

7 |
8 | -------------------------------------------------------------------------------- /bin/bundle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 3 | load Gem.bin_path('bundler', 'bundle') 4 | -------------------------------------------------------------------------------- /bin/rails: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | begin 3 | load File.expand_path('../spring', __FILE__) 4 | rescue LoadError => e 5 | raise unless e.message.include?('spring') 6 | end 7 | APP_PATH = File.expand_path('../config/application', __dir__) 8 | require_relative '../config/boot' 9 | require 'rails/commands' 10 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | begin 3 | load File.expand_path('../spring', __FILE__) 4 | rescue LoadError => e 5 | raise unless e.message.include?('spring') 6 | end 7 | require_relative '../config/boot' 8 | require 'rake' 9 | Rake.application.run 10 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'pathname' 3 | require 'fileutils' 4 | include FileUtils 5 | 6 | # path to your application root. 7 | APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) 8 | 9 | def system!(*args) 10 | system(*args) || abort("\n== Command #{args} failed ==") 11 | end 12 | 13 | chdir APP_ROOT do 14 | # This script is a starting point to setup your application. 15 | # Add necessary setup steps to this file. 16 | 17 | puts '== Installing dependencies ==' 18 | system! 'gem install bundler --conservative' 19 | system('bundle check') || system!('bundle install') 20 | 21 | # puts "\n== Copying sample files ==" 22 | # unless File.exist?('config/database.yml') 23 | # cp 'config/database.yml.sample', 'config/database.yml' 24 | # end 25 | 26 | puts "\n== Preparing database ==" 27 | system! 'bin/rails db:setup' 28 | 29 | puts "\n== Removing old logs and tempfiles ==" 30 | system! 'bin/rails log:clear tmp:clear' 31 | 32 | puts "\n== Restarting application server ==" 33 | system! 'bin/rails restart' 34 | end 35 | -------------------------------------------------------------------------------- /bin/spring: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # This file loads spring without using Bundler, in order to be fast. 4 | # It gets overwritten when you run the `spring binstub` command. 5 | 6 | unless defined?(Spring) 7 | require 'rubygems' 8 | require 'bundler' 9 | 10 | if (match = Bundler.default_lockfile.read.match(/^GEM$.*?^ (?: )*spring \((.*?)\)$.*?^$/m)) 11 | Gem.paths = { 'GEM_PATH' => [Bundler.bundle_path.to_s, *Gem.path].uniq.join(Gem.path_separator) } 12 | gem 'spring', match[1] 13 | require 'spring/binstub' 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /bin/update: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'pathname' 3 | require 'fileutils' 4 | include FileUtils 5 | 6 | # path to your application root. 7 | APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) 8 | 9 | def system!(*args) 10 | system(*args) || abort("\n== Command #{args} failed ==") 11 | end 12 | 13 | chdir APP_ROOT do 14 | # This script is a way to update your development environment automatically. 15 | # Add necessary update steps to this file. 16 | 17 | puts '== Installing dependencies ==' 18 | system! 'gem install bundler --conservative' 19 | system('bundle check') || system!('bundle install') 20 | 21 | puts "\n== Updating database ==" 22 | system! 'bin/rails db:migrate' 23 | 24 | puts "\n== Removing old logs and tempfiles ==" 25 | system! 'bin/rails log:clear tmp:clear' 26 | 27 | puts "\n== Restarting application server ==" 28 | system! 'bin/rails restart' 29 | end 30 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | # This file is used by Rack-based servers to start the application. 2 | 3 | require_relative 'config/environment' 4 | 5 | run Rails.application 6 | -------------------------------------------------------------------------------- /config/application.rb: -------------------------------------------------------------------------------- 1 | require_relative 'boot' 2 | 3 | require 'rails/all' 4 | 5 | # Require the gems listed in Gemfile, including any gems 6 | # you've limited to :test, :development, or :production. 7 | Bundler.require(*Rails.groups) 8 | 9 | module Omnibuser 10 | class Application < Rails::Application 11 | config.autoload_paths += Dir[Rails.root.join('app', 'builder')] 12 | 13 | config.generators do |g| 14 | g.test_framework :rspec, 15 | fixtures: true, 16 | view_specs: false, 17 | helper_specs: false, 18 | routing_specs: false, 19 | controller_specs: true, 20 | request_specs: false 21 | g.fixture_replacement :factory_girl, dir: "spec/factories" 22 | end 23 | 24 | 25 | # Settings in config/environments/* take precedence over those specified here. 26 | # Application configuration should go into files in config/initializers 27 | # -- all .rb files in that directory are automatically loaded. 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /config/boot.rb: -------------------------------------------------------------------------------- 1 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) 2 | 3 | require 'bundler/setup' # Set up gems listed in the Gemfile. 4 | -------------------------------------------------------------------------------- /config/cable.yml: -------------------------------------------------------------------------------- 1 | development: 2 | adapter: async 3 | 4 | test: 5 | adapter: async 6 | 7 | production: 8 | adapter: redis 9 | url: redis://localhost:6379/1 10 | -------------------------------------------------------------------------------- /config/database.yml: -------------------------------------------------------------------------------- 1 | # PostgreSQL. Versions 9.1 and up are supported. 2 | # 3 | # Install the pg driver: 4 | # gem install pg 5 | # On OS X with Homebrew: 6 | # gem install pg -- --with-pg-config=/usr/local/bin/pg_config 7 | # On OS X with MacPorts: 8 | # gem install pg -- --with-pg-config=/opt/local/lib/postgresql84/bin/pg_config 9 | # On Windows: 10 | # gem install pg 11 | # Choose the win32 build. 12 | # Install PostgreSQL and put its /bin directory on your path. 13 | # 14 | # Configure Using Gemfile 15 | # gem 'pg' 16 | # 17 | default: &default 18 | adapter: postgresql 19 | encoding: unicode 20 | # For details on connection pooling, see rails configuration guide 21 | # http://guides.rubyonrails.org/configuring.html#database-pooling 22 | pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> 23 | 24 | development: 25 | <<: *default 26 | database: omnibuser_development 27 | 28 | # The specified database role being used to connect to postgres. 29 | # To create additional roles in postgres see `$ createuser --help`. 30 | # When left blank, postgres will use the default role. This is 31 | # the same name as the operating system user that initialized the database. 32 | #username: omnibuser 33 | 34 | # The password associated with the postgres role (username). 35 | #password: 36 | 37 | # Connect on a TCP socket. Omitted by default since the client uses a 38 | # domain socket that doesn't need configuration. Windows does not have 39 | # domain sockets, so uncomment these lines. 40 | #host: localhost 41 | 42 | # The TCP port the server listens on. Defaults to 5432. 43 | # If your server runs on a different port number, change accordingly. 44 | #port: 5432 45 | 46 | # Schema search path. The server defaults to $user,public 47 | #schema_search_path: myapp,sharedapp,public 48 | 49 | # Minimum log levels, in increasing order: 50 | # debug5, debug4, debug3, debug2, debug1, 51 | # log, notice, warning, error, fatal, and panic 52 | # Defaults to warning. 53 | #min_messages: notice 54 | 55 | # Warning: The database defined as "test" will be erased and 56 | # re-generated from your development database when you run "rake". 57 | # Do not set this db to the same as development or production. 58 | test: 59 | <<: *default 60 | database: omnibuser_test 61 | 62 | # As with config/secrets.yml, you never want to store sensitive information, 63 | # like your database password, in your source code. If your source code is 64 | # ever seen by anyone, they now have access to your database. 65 | # 66 | # Instead, provide the password as a unix environment variable when you boot 67 | # the app. Read http://guides.rubyonrails.org/configuring.html#configuring-a-database 68 | # for a full rundown on how to provide these environment variables in a 69 | # production deployment. 70 | # 71 | # On Heroku and other platform providers, you may have a full connection URL 72 | # available as an environment variable. For example: 73 | # 74 | # DATABASE_URL="postgres://myuser:mypass@localhost/somedatabase" 75 | # 76 | # You can use this database configuration with: 77 | # 78 | # production: 79 | # url: <%= ENV['DATABASE_URL'] %> 80 | # 81 | production: 82 | <<: *default 83 | database: omnibuser_production 84 | username: omnibuser 85 | password: <%= ENV['OMNIBUSER_DATABASE_PASSWORD'] %> 86 | -------------------------------------------------------------------------------- /config/environment.rb: -------------------------------------------------------------------------------- 1 | # Load the Rails application. 2 | require_relative 'application' 3 | 4 | # Initialize the Rails application. 5 | Rails.application.initialize! 6 | -------------------------------------------------------------------------------- /config/environments/development.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # In the development environment your application's code is reloaded on 5 | # every request. This slows down response time but is perfect for development 6 | # since you don't have to restart the web server when you make code changes. 7 | config.cache_classes = false 8 | 9 | # Do not eager load code on boot. 10 | config.eager_load = false 11 | 12 | # Show full error reports. 13 | config.consider_all_requests_local = true 14 | 15 | # Enable/disable caching. By default caching is disabled. 16 | if Rails.root.join('tmp/caching-dev.txt').exist? 17 | config.action_controller.perform_caching = true 18 | 19 | config.cache_store = :memory_store 20 | config.public_file_server.headers = { 21 | 'Cache-Control' => 'public, max-age=172800' 22 | } 23 | else 24 | config.action_controller.perform_caching = false 25 | 26 | config.cache_store = :null_store 27 | end 28 | 29 | # Don't care if the mailer can't send. 30 | config.action_mailer.raise_delivery_errors = false 31 | 32 | config.action_mailer.perform_caching = false 33 | 34 | # Print deprecation notices to the Rails logger. 35 | config.active_support.deprecation = :log 36 | 37 | # Raise an error on page load if there are pending migrations. 38 | config.active_record.migration_error = :page_load 39 | 40 | # Debug mode disables concatenation and preprocessing of assets. 41 | # This option may cause significant delays in view rendering with a large 42 | # number of complex assets. 43 | config.assets.debug = true 44 | 45 | # Suppress logger output for asset requests. 46 | config.assets.quiet = true 47 | 48 | # Raises error for missing translations 49 | # config.action_view.raise_on_missing_translations = true 50 | 51 | # Use an evented file watcher to asynchronously detect changes in source code, 52 | # routes, locales, etc. This feature depends on the listen gem. 53 | config.file_watcher = ActiveSupport::EventedFileUpdateChecker 54 | end 55 | -------------------------------------------------------------------------------- /config/environments/production.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # Code is not reloaded between requests. 5 | config.cache_classes = true 6 | 7 | # Eager load code on boot. This eager loads most of Rails and 8 | # your application in memory, allowing both threaded web servers 9 | # and those relying on copy on write to perform better. 10 | # Rake tasks automatically ignore this option for performance. 11 | config.eager_load = true 12 | 13 | # Full error reports are disabled and caching is turned on. 14 | config.consider_all_requests_local = false 15 | config.action_controller.perform_caching = true 16 | 17 | # Disable serving static files from the `/public` folder by default since 18 | # Apache or NGINX already handles this. 19 | config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? 20 | 21 | # Compress JavaScripts and CSS. 22 | config.assets.js_compressor = :uglifier 23 | # config.assets.css_compressor = :sass 24 | 25 | # Do not fallback to assets pipeline if a precompiled asset is missed. 26 | config.assets.compile = false 27 | 28 | # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb 29 | 30 | # Enable serving of images, stylesheets, and JavaScripts from an asset server. 31 | # config.action_controller.asset_host = 'http://assets.example.com' 32 | 33 | # Specifies the header that your server uses for sending files. 34 | # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache 35 | # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX 36 | 37 | # Mount Action Cable outside main process or domain 38 | # config.action_cable.mount_path = nil 39 | # config.action_cable.url = 'wss://example.com/cable' 40 | # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] 41 | 42 | # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. 43 | # config.force_ssl = true 44 | 45 | # Use the lowest log level to ensure availability of diagnostic information 46 | # when problems arise. 47 | config.log_level = :debug 48 | 49 | # Prepend all log lines with the following tags. 50 | config.log_tags = [ :request_id ] 51 | 52 | # Use a different cache store in production. 53 | # config.cache_store = :mem_cache_store 54 | 55 | # Use a real queuing backend for Active Job (and separate queues per environment) 56 | # config.active_job.queue_adapter = :resque 57 | # config.active_job.queue_name_prefix = "omnibuser_#{Rails.env}" 58 | config.action_mailer.perform_caching = false 59 | 60 | # Ignore bad email addresses and do not raise email delivery errors. 61 | # Set this to true and configure the email server for immediate delivery to raise delivery errors. 62 | # config.action_mailer.raise_delivery_errors = false 63 | 64 | # Enable locale fallbacks for I18n (makes lookups for any locale fall back to 65 | # the I18n.default_locale when a translation cannot be found). 66 | config.i18n.fallbacks = true 67 | 68 | # Send deprecation notices to registered listeners. 69 | config.active_support.deprecation = :notify 70 | 71 | # Use default logging formatter so that PID and timestamp are not suppressed. 72 | config.log_formatter = ::Logger::Formatter.new 73 | 74 | # Use a different logger for distributed setups. 75 | # require 'syslog/logger' 76 | # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') 77 | 78 | if ENV["RAILS_LOG_TO_STDOUT"].present? 79 | logger = ActiveSupport::Logger.new(STDOUT) 80 | logger.formatter = config.log_formatter 81 | config.logger = ActiveSupport::TaggedLogging.new(logger) 82 | end 83 | 84 | # Do not dump schema after migrations. 85 | config.active_record.dump_schema_after_migration = false 86 | end 87 | -------------------------------------------------------------------------------- /config/environments/test.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # The test environment is used exclusively to run your application's 5 | # test suite. You never need to work with it otherwise. Remember that 6 | # your test database is "scratch space" for the test suite and is wiped 7 | # and recreated between test runs. Don't rely on the data there! 8 | config.cache_classes = true 9 | 10 | # Do not eager load code on boot. This avoids loading your whole application 11 | # just for the purpose of running a single test. If you are using a tool that 12 | # preloads Rails for running tests, you may have to set it to true. 13 | config.eager_load = false 14 | 15 | # Configure public file server for tests with Cache-Control for performance. 16 | config.public_file_server.enabled = true 17 | config.public_file_server.headers = { 18 | 'Cache-Control' => 'public, max-age=3600' 19 | } 20 | 21 | # Show full error reports and disable caching. 22 | config.consider_all_requests_local = true 23 | config.action_controller.perform_caching = false 24 | 25 | # Raise exceptions instead of rendering exception templates. 26 | config.action_dispatch.show_exceptions = false 27 | 28 | # Disable request forgery protection in test environment. 29 | config.action_controller.allow_forgery_protection = false 30 | config.action_mailer.perform_caching = false 31 | 32 | # Tell Action Mailer not to deliver emails to the real world. 33 | # The :test delivery method accumulates sent emails in the 34 | # ActionMailer::Base.deliveries array. 35 | config.action_mailer.delivery_method = :test 36 | 37 | # Print deprecation notices to the stderr. 38 | config.active_support.deprecation = :stderr 39 | 40 | # Raises error for missing translations 41 | # config.action_view.raise_on_missing_translations = true 42 | end 43 | -------------------------------------------------------------------------------- /config/initializers/application_controller_renderer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # ApplicationController.renderer.defaults.merge!( 4 | # http_host: 'example.org', 5 | # https: false 6 | # ) 7 | -------------------------------------------------------------------------------- /config/initializers/assets.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Version of your assets, change this if you want to expire all your assets. 4 | Rails.application.config.assets.version = '1.0' 5 | 6 | # Add additional assets to the asset load path 7 | # Rails.application.config.assets.paths << Emoji.images_path 8 | 9 | # Precompile additional assets. 10 | # application.js, application.css, and all non-JS/CSS in app/assets folder are already added. 11 | # Rails.application.config.assets.precompile += %w( search.js ) 12 | -------------------------------------------------------------------------------- /config/initializers/aws.rb: -------------------------------------------------------------------------------- 1 | AWS.config( 2 | :access_key_id => ENV['AWS_ACCESS_KEY_ID'], 3 | :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'] 4 | ) 5 | # Aws.config.update({ 6 | # credentials: Aws::Credentials.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY']) 7 | # }) 8 | 9 | S3_BUCKET = AWS::S3.new.buckets[ENV['S3_BUCKET_NAME']] 10 | -------------------------------------------------------------------------------- /config/initializers/backtrace_silencers.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. 4 | # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } 5 | 6 | # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. 7 | # Rails.backtrace_cleaner.remove_silencers! 8 | -------------------------------------------------------------------------------- /config/initializers/cookies_serializer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Specify a serializer for the signed and encrypted cookie jars. 4 | # Valid options are :json, :marshal, and :hybrid. 5 | Rails.application.config.action_dispatch.cookies_serializer = :json 6 | -------------------------------------------------------------------------------- /config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Configure sensitive parameters which will be filtered from the log file. 4 | Rails.application.config.filter_parameters += [:password] 5 | -------------------------------------------------------------------------------- /config/initializers/inflections.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new inflection rules using the following format. Inflections 4 | # are locale specific, and you may define rules for as many different 5 | # locales as you wish. All of these examples are active by default: 6 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 7 | # inflect.plural /^(ox)$/i, '\1en' 8 | # inflect.singular /^(ox)en/i, '\1' 9 | # inflect.irregular 'person', 'people' 10 | # inflect.uncountable %w( fish sheep ) 11 | # end 12 | 13 | # These inflection rules are supported but not enabled by default: 14 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 15 | # inflect.acronym 'RESTful' 16 | # end 17 | -------------------------------------------------------------------------------- /config/initializers/mime_types.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new mime types for use in respond_to blocks: 4 | # Mime::Type.register "text/richtext", :rtf 5 | -------------------------------------------------------------------------------- /config/initializers/new_framework_defaults.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | # 3 | # This file contains migration options to ease your Rails 5.0 upgrade. 4 | # 5 | # Read the Rails 5.0 release notes for more info on each option. 6 | 7 | # Enable per-form CSRF tokens. Previous versions had false. 8 | Rails.application.config.action_controller.per_form_csrf_tokens = true 9 | 10 | # Enable origin-checking CSRF mitigation. Previous versions had false. 11 | Rails.application.config.action_controller.forgery_protection_origin_check = true 12 | 13 | # Make Ruby 2.4 preserve the timezone of the receiver when calling `to_time`. 14 | # Previous versions had false. 15 | ActiveSupport.to_time_preserves_timezone = true 16 | 17 | # Require `belongs_to` associations by default. Previous versions had false. 18 | Rails.application.config.active_record.belongs_to_required_by_default = true 19 | 20 | # Do not halt callback chains when a callback returns false. Previous versions had true. 21 | ActiveSupport.halt_callback_chains_on_return_false = false 22 | 23 | # Configure SSL options to enable HSTS with subdomains. Previous versions had false. 24 | Rails.application.config.ssl_options = { hsts: { subdomains: true } } 25 | -------------------------------------------------------------------------------- /config/initializers/pdfkit.rb: -------------------------------------------------------------------------------- 1 | PDFKit.configure do |config| 2 | config.wkhtmltopdf = Rails.root.join("lib", "wkhtmltopdf").to_s 3 | config.default_options = { 4 | :page_size => 'A4' 5 | } 6 | 7 | end 8 | -------------------------------------------------------------------------------- /config/initializers/redis.rb: -------------------------------------------------------------------------------- 1 | if ENV["REDISCLOUD_URL"] 2 | uri = URI.parse(ENV["REDISCLOUD_URL"]) 3 | $redis = Redis.new(:host => uri.host, :port => uri.port, :password => uri.password) 4 | Resque.redis = $redis 5 | end 6 | -------------------------------------------------------------------------------- /config/initializers/resque-pool.rb: -------------------------------------------------------------------------------- 1 | WORKER_CONCURRENCY = Integer(ENV["WORKER_CONCURRENCY"] || 5) 2 | -------------------------------------------------------------------------------- /config/initializers/resque.rb: -------------------------------------------------------------------------------- 1 | Resque.logger = Logger.new(STDOUT) 2 | 3 | require 'resque/failure/multiple' 4 | require 'resque/failure/redis' 5 | require 'resque/rollbar' 6 | 7 | Resque::Failure::Multiple.classes = [ Resque::Failure::Redis, Resque::Failure::Rollbar ] 8 | Resque::Failure.backend = Resque::Failure::Multiple -------------------------------------------------------------------------------- /config/initializers/rollbar.rb: -------------------------------------------------------------------------------- 1 | Rollbar.configure do |config| 2 | # Without configuration, Rollbar is enabled in all environments. 3 | # To disable in specific environments, set config.enabled=false. 4 | 5 | config.access_token = ENV['ROLLBAR_ACCESS_TOKEN'] 6 | config.use_resque 7 | 8 | 9 | # Here we'll disable in 'test': 10 | if Rails.env.test? 11 | config.enabled = false 12 | end 13 | 14 | # By default, Rollbar will try to call the `current_user` controller method 15 | # to fetch the logged-in user object, and then call that object's `id` 16 | # method to fetch this property. To customize: 17 | # config.person_method = "my_current_user" 18 | # config.person_id_method = "my_id" 19 | 20 | # Additionally, you may specify the following: 21 | # config.person_username_method = "username" 22 | # config.person_email_method = "email" 23 | 24 | # If you want to attach custom data to all exception and message reports, 25 | # provide a lambda like the following. It should return a hash. 26 | # config.custom_data_method = lambda { {:some_key => "some_value" } } 27 | 28 | # Add exception class names to the exception_level_filters hash to 29 | # change the level that exception is reported at. Note that if an exception 30 | # has already been reported and logged the level will need to be changed 31 | # via the rollbar interface. 32 | # Valid levels: 'critical', 'error', 'warning', 'info', 'debug', 'ignore' 33 | # 'ignore' will cause the exception to not be reported at all. 34 | # config.exception_level_filters.merge!('MyCriticalException' => 'critical') 35 | # 36 | # You can also specify a callable, which will be called with the exception instance. 37 | # config.exception_level_filters.merge!('MyCriticalException' => lambda { |e| 'critical' }) 38 | 39 | # Enable asynchronous reporting (uses girl_friday or Threading if girl_friday 40 | # is not installed) 41 | # config.use_async = true 42 | # Supply your own async handler: 43 | # config.async_handler = Proc.new { |payload| 44 | # Thread.new { Rollbar.process_from_async_handler(payload) } 45 | # } 46 | 47 | # Enable asynchronous reporting (using sucker_punch) 48 | # config.use_sucker_punch 49 | 50 | # Enable delayed reporting (using Sidekiq) 51 | # config.use_sidekiq 52 | # You can supply custom Sidekiq options: 53 | # config.use_sidekiq 'queue' => 'default' 54 | 55 | # If your application runs behind a proxy server, you can set proxy parameters here. 56 | # If https_proxy is set in your environment, that will be used. Settings here have precedence. 57 | # The :host key is mandatory and must include the URL scheme (e.g. 'http://'), all other fields 58 | # are optional. 59 | # 60 | # config.proxy = { 61 | # host: 'http://some.proxy.server', 62 | # port: 80, 63 | # user: 'username_if_auth_required', 64 | # password: 'password_if_auth_required' 65 | # } 66 | 67 | # If you run your staging application instance in production environment then 68 | # you'll want to override the environment reported by `Rails.env` with an 69 | # environment variable like this: `ROLLBAR_ENV=staging`. This is a recommended 70 | # setup for Heroku. See: 71 | # https://devcenter.heroku.com/articles/deploying-to-a-custom-rails-environment 72 | config.environment = ENV['ROLLBAR_ENV'].presence || Rails.env 73 | end 74 | -------------------------------------------------------------------------------- /config/initializers/session_store.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.session_store :cookie_store, key: '_omnibuser_session' 4 | -------------------------------------------------------------------------------- /config/initializers/targets.rb: -------------------------------------------------------------------------------- 1 | yml = YAML.load(File.read(Rails.root.join('config', 'targets.yml'))) 2 | yml.each do |site| 3 | target = Target.find_by(domain: site[0]) 4 | if target 5 | target.update!(scraper: site[1]['scraper'], target_data: site[1]['data']) 6 | else 7 | Target.create!(domain: site[0], scraper: site[1]['scraper'], target_data: site[1]['data']) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /config/initializers/wrap_parameters.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # This file contains settings for ActionController::ParamsWrapper which 4 | # is enabled by default. 5 | 6 | # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array. 7 | ActiveSupport.on_load(:action_controller) do 8 | wrap_parameters format: [:json] 9 | end 10 | 11 | # To enable root element in JSON for ActiveRecord objects. 12 | # ActiveSupport.on_load(:active_record) do 13 | # self.include_root_in_json = true 14 | # end 15 | -------------------------------------------------------------------------------- /config/locales/en.yml: -------------------------------------------------------------------------------- 1 | # Files in the config/locales directory are used for internationalization 2 | # and are automatically loaded by Rails. If you want to use locales other 3 | # than English, add the necessary files in this directory. 4 | # 5 | # To use the locales, use `I18n.t`: 6 | # 7 | # I18n.t 'hello' 8 | # 9 | # In views, this is aliased to just `t`: 10 | # 11 | # <%= t('hello') %> 12 | # 13 | # To use a different locale, set it with `I18n.locale`: 14 | # 15 | # I18n.locale = :es 16 | # 17 | # This would use the information in config/locales/es.yml. 18 | # 19 | # To learn more, please read the Rails Internationalization guide 20 | # available at http://guides.rubyonrails.org/i18n.html. 21 | 22 | en: 23 | hello: "Hello world" 24 | -------------------------------------------------------------------------------- /config/puma.rb: -------------------------------------------------------------------------------- 1 | # Puma can serve each request in a thread from an internal thread pool. 2 | # The `threads` method setting takes two numbers a minimum and maximum. 3 | # Any libraries that use thread pools should be configured to match 4 | # the maximum value specified for Puma. Default is set to 5 threads for minimum 5 | # and maximum, this matches the default thread size of Active Record. 6 | # 7 | threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 }.to_i 8 | threads threads_count, threads_count 9 | 10 | # Specifies the `port` that Puma will listen on to receive requests, default is 3000. 11 | # 12 | port ENV.fetch("PORT") { 3000 } 13 | 14 | # Specifies the `environment` that Puma will run in. 15 | # 16 | environment ENV.fetch("RAILS_ENV") { "development" } 17 | 18 | # Specifies the number of `workers` to boot in clustered mode. 19 | # Workers are forked webserver processes. If using threads and workers together 20 | # the concurrency of the application would be max `threads` * `workers`. 21 | # Workers do not work on JRuby or Windows (both of which do not support 22 | # processes). 23 | # 24 | # workers ENV.fetch("WEB_CONCURRENCY") { 2 } 25 | 26 | # Use the `preload_app!` method when specifying a `workers` number. 27 | # This directive tells Puma to first boot the application and load code 28 | # before forking the application. This takes advantage of Copy On Write 29 | # process behavior so workers use less memory. If you use this option 30 | # you need to make sure to reconnect any threads in the `on_worker_boot` 31 | # block. 32 | # 33 | # preload_app! 34 | 35 | # The code in the `on_worker_boot` will be called if you are using 36 | # clustered mode by specifying a number of `workers`. After each worker 37 | # process is booted this block will be run, if you are using `preload_app!` 38 | # option you will want to use this block to reconnect to any threads 39 | # or connections that may have been created at application boot, Ruby 40 | # cannot share connections between processes. 41 | # 42 | # on_worker_boot do 43 | # ActiveRecord::Base.establish_connection if defined?(ActiveRecord) 44 | # end 45 | 46 | # Allow puma to be restarted by `rails restart` command. 47 | plugin :tmp_restart 48 | -------------------------------------------------------------------------------- /config/resque-pool.yml: -------------------------------------------------------------------------------- 1 | --- 2 | scrape: <%= WORKER_CONCURRENCY %> 3 | build: 1 4 | -------------------------------------------------------------------------------- /config/routes.rb: -------------------------------------------------------------------------------- 1 | require "resque_web" 2 | 3 | Rails.application.routes.draw do 4 | root 'home#index' 5 | post '/', to: 'home#new' 6 | get '/requests/:id', to: 'home#status' 7 | post '/scrape/:id', to: 'home#scrape' 8 | 9 | get 'about', to: 'static#about' 10 | get 'contact', to: 'static#contact' 11 | 12 | mount ResqueWeb::Engine => "/resque_web" 13 | end 14 | -------------------------------------------------------------------------------- /config/secrets.yml: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Your secret key is used for verifying the integrity of signed cookies. 4 | # If you change this key, all old signed cookies will become invalid! 5 | 6 | # Make sure the secret is at least 30 characters and all random, 7 | # no regular words or you'll be exposed to dictionary attacks. 8 | # You can use `rails secret` to generate a secure secret key. 9 | 10 | # Make sure the secrets in this file are kept private 11 | # if you're sharing your code publicly. 12 | 13 | development: 14 | secret_key_base: ce398d9a721580538d5fc8becfe227e6109d032de2d9b62e5864932aaa418983fecf76b1b4680558f21fe366bca583b8a1833ec74b3d8942ef8e2b3307bca289 15 | 16 | test: 17 | secret_key_base: 822a5a1e2c80238a6fec9d25d47e3da10e26e55c78bb9038cdf7b537f67b09402039258a995729cd690f10b41d96f10c0e7bd3ea03dfad6f7942b39132f23e9a 18 | 19 | # Do not keep production secrets in the repository, 20 | # instead read values from the environment. 21 | production: 22 | secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> 23 | -------------------------------------------------------------------------------- /config/spring.rb: -------------------------------------------------------------------------------- 1 | %w( 2 | .ruby-version 3 | .rbenv-vars 4 | tmp/restart.txt 5 | tmp/caching-dev.txt 6 | ).each { |path| Spring.watch(path) } 7 | -------------------------------------------------------------------------------- /config/targets.yml: -------------------------------------------------------------------------------- 1 | fictionpress.com: 2 | scraper: 'FFNScraper' 3 | data: 4 | image_background: '#fff' 5 | summary: "#profile_top div.xcontrast_txt" 6 | meta: "#profile_top span.xgray.xcontrast_txt" 7 | title: "#profile_top .xcontrast_txt" 8 | author: "//a[starts-with(@href, '/u/')]" 9 | chapter_urls: "#chap_select" 10 | chapter_titles: '#chap_select option' 11 | chapter_content: "#storytext" 12 | 13 | fanfiction.net: 14 | scraper: 'FFNScraper' 15 | data: 16 | image_background: '#fff' 17 | summary: "#profile_top div.xcontrast_txt" 18 | meta: "#profile_top span.xgray.xcontrast_txt" 19 | title: "#profile_top .xcontrast_txt" 20 | author: "//a[starts-with(@href, '/u/')]" 21 | chapter_urls: "#chap_select" 22 | chapter_titles: '#chap_select option' 23 | chapter_content: "#storytext" 24 | 25 | forum.questionablequesting.com: 26 | scraper: 'ForumScraper' 27 | data: 28 | image_background: '#EAEBEB' 29 | post: '#messageList .message' 30 | threadmark: '.message.hasThreadmark' 31 | overlay_threadmark: '.ThreadmarkIndex .threadmarkList li' 32 | chapter_pub_date: '.messageInfo .messageMeta .datePermalink .DateTime' 33 | chapter_edit_date: '.messageInfo .editDate .DateTime' 34 | story_pub_date: '.message .primaryContent .messageMeta .datePermalink' 35 | avatar: '#messageList .message .avatar img' 36 | threadmark_list_item: '.threadmarkListItem' 37 | threadmark_url: '.PreviewTooltip' 38 | threadmark_date: '.DateTime' 39 | chapter_threadmark_text: '.threadmarker .label' 40 | chapter_threadmark_fluff: 'Threadmarks:' 41 | chapter_content: '.messageContent .messageText' 42 | 43 | forums.sufficientvelocity.com: 44 | scraper: 'ForumScraper' 45 | data: 46 | image_background: '#282828' 47 | post: '.message--post' 48 | threadmark: '.message--post .message-cell--threadmark-header' 49 | overlay_threadmark: '.structItem--threadmark' 50 | chapter_pub_date: '.messageInfo .messageMeta .datePermalink .DateTime' 51 | chapter_edit_date: '.messageInfo .editDate .DateTime' 52 | story_pub_date: '.message--post .message-main time' 53 | avatar: '.message--post .avatar img' 54 | threadmark_list_item: '.structItem--threadmark' 55 | threadmark_url: '.structItem-title a' 56 | threadmark_date: 'time' 57 | chapter_threadmark_text: '.threadmarkLabel' 58 | chapter_threadmark_fluff: 'Threadmarks:' 59 | chapter_content: '.message-body' 60 | 61 | forums.spacebattles.com: 62 | scraper: 'ForumScraper' 63 | data: 64 | image_background: '#191F2D' 65 | post: '.message--post' 66 | threadmark: '.message--post .message-cell--threadmark-header' 67 | overlay_threadmark: '.structItem--threadmark' 68 | chapter_pub_date: '.messageInfo .messageMeta .datePermalink .DateTime' 69 | chapter_edit_date: '.messageInfo .editDate .DateTime' 70 | story_pub_date: '.message--post .message-main time' 71 | avatar: '.message--post .avatar img' 72 | threadmark_list_item: '.structItem--threadmark' 73 | threadmark_url: '.structItem-title a' 74 | threadmark_date: 'time' 75 | chapter_threadmark_text: '.threadmarkLabel' 76 | chapter_threadmark_fluff: 'Threadmarks:' 77 | chapter_content: '.message-body' 78 | 79 | parahumans.wordpress.com: 80 | scraper: 'TOCScraper' 81 | data: 82 | image_background: '#fff' 83 | toc_url: https://parahumans.wordpress.com/table-of-contents/ 84 | title: Worm 85 | author: John Charles McCrae 86 | cover_image_url: https://parahumans.files.wordpress.com/2012/08/skitter-cover-2-a.jpg?w=1000 87 | chapter_urls: '#content .entry-content a:not(.sd-button)' 88 | chapter_content: '.entry-content' 89 | chapter_title: '.entry-title' 90 | content_filters: 91 | - 'a:contains("Next Chapter")' 92 | - 'a:contains("Last Chapter")' 93 | - '.sharedaddy' 94 | 95 | unsongbook.com: 96 | scraper: 'TOCScraper' 97 | data: 98 | image_background: '#fff' 99 | toc_url: http://unsongbook.com/ 100 | title: Unsong 101 | author: 'Scott Alexander' 102 | cover_image_url: http://unsongbook.com/wp-content/uploads/2016/01/icon2.png 103 | chapter_urls: '.pjgm-postcontent a:not(.share-icon)' 104 | chapter_content: '.pjgm-postcontent' 105 | chapter_title: '.pjgm-posttitle' 106 | content_filters: 107 | - '.sharedaddy' 108 | 109 | practicalguidetoevil.wordpress.com: 110 | scraper: 'TOCScraper' 111 | data: 112 | image_background: '#fff' 113 | toc_url: https://practicalguidetoevil.wordpress.com/table-of-contents/ 114 | title: A Practical Guide to Evil 115 | author: 'ErraticErrata' 116 | cover_image_url: http://orig02.deviantart.net/58b9/f/2016/131/4/b/catherine_foundling_by_sandara-da26nur.jpg 117 | chapter_urls: '.entry-content ul li a' 118 | chapter_content: '.entry-content' 119 | chapter_title: '.entry-header .entry-title' 120 | content_filters: 121 | - '.wpcnt' 122 | - '.sharedaddy' 123 | toc_filters: 124 | - '.sharedaddy' 125 | 126 | parahumans.net: 127 | scraper: 'TOCScraper' 128 | data: 129 | image_background: '#fff' 130 | toc_url: https://www.parahumans.net/table-of-contents/ 131 | title: Ward 132 | author: John Charles McCrae 133 | cover_image_url: https://i0.wp.com/www.parahumans.net/wp-content/uploads/2017/10/cropped-Ward-Banner-Proper-1.jpg?resize=1024%2C206&ssl=1 134 | chapter_urls: '#main .entry-content a:not(.sd-button)' 135 | chapter_content: '.entry-content' 136 | chapter_title: '.entry-title' 137 | content_filters: 138 | - 'a:contains("Next Chapter")' 139 | - 'a:contains("Previous Chapter")' 140 | - '.sharedaddy' 141 | -------------------------------------------------------------------------------- /db/migrate/20160721160005_create_stories.rb: -------------------------------------------------------------------------------- 1 | class CreateStories < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :stories do |t| 4 | t.string :url 5 | t.string :title 6 | t.string :author 7 | 8 | t.timestamps 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20160721160220_create_chapters.rb: -------------------------------------------------------------------------------- 1 | class CreateChapters < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :chapters do |t| 4 | t.belongs_to :story, index: true 5 | t.integer :number 6 | t.string :title 7 | t.string :content 8 | 9 | t.timestamps 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20160721160529_create_requests.rb: -------------------------------------------------------------------------------- 1 | class CreateRequests < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :requests do |t| 4 | t.belongs_to :story, index: true 5 | t.string :url 6 | 7 | t.timestamps 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20160723172334_create_documents.rb: -------------------------------------------------------------------------------- 1 | class CreateDocuments < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :documents do |t| 4 | t.belongs_to :story, index:true 5 | t.string :filename 6 | t.string :extension 7 | 8 | t.timestamps 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20160807110240_add_meta_data_to_stories.rb: -------------------------------------------------------------------------------- 1 | class AddMetaDataToStories < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :stories, :meta_data, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160903153614_add_progress_to_requests.rb: -------------------------------------------------------------------------------- 1 | class AddProgressToRequests < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :requests, :status, :string 4 | add_column :requests, :current_chapter, :integer 5 | add_column :requests, :total_chapters, :integer 6 | add_column :requests, :complete, :boolean 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /db/migrate/20160903164837_remove_current_chapter_from_requests.rb: -------------------------------------------------------------------------------- 1 | class RemoveCurrentChapterFromRequests < ActiveRecord::Migration[5.0] 2 | def change 3 | remove_column :requests, :current_chapter, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160903164949_add_current_chapters_to_requests.rb: -------------------------------------------------------------------------------- 1 | class AddCurrentChaptersToRequests < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :requests, :current_chapters, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160903194955_add_extension_to_requests.rb: -------------------------------------------------------------------------------- 1 | class AddExtensionToRequests < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :requests, :extension, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160914112318_create_scraper_queues.rb: -------------------------------------------------------------------------------- 1 | class CreateScraperQueues < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :scraper_queues do |t| 4 | t.string :domain 5 | t.datetime :last_access 6 | t.string :queue, array: true, default: [] 7 | t.timestamps 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20160919094109_remove_queue_from_scraper_queues.rb: -------------------------------------------------------------------------------- 1 | class RemoveQueueFromScraperQueues < ActiveRecord::Migration[5.0] 2 | def change 3 | remove_column :scraper_queues, :queue, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160928190251_add_doc_idto_request.rb: -------------------------------------------------------------------------------- 1 | class AddDocIdtoRequest < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :requests, :doc_id, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20160929173836_add_aws_infoto_documents.rb: -------------------------------------------------------------------------------- 1 | class AddAwsInfotoDocuments < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :documents, :aws_url, :string 4 | add_column :documents, :aws_key, :string 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20160929174442_add_aws_urlto_requests.rb: -------------------------------------------------------------------------------- 1 | class AddAwsUrltoRequests < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :requests, :aws_url, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170112142726_create_images.rb: -------------------------------------------------------------------------------- 1 | class CreateImages < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :images do |t| 4 | t.belongs_to :story, index:true 5 | t.string :source_url 6 | t.string :aws_url 7 | t.string :filename 8 | t.string :extension 9 | t.integer :size 10 | t.boolean :cover 11 | 12 | t.timestamps 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20170112152157_add_aws_key_to_images.rb: -------------------------------------------------------------------------------- 1 | class AddAwsKeyToImages < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :images, :aws_key, :string 4 | 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20170227112542_add_domain_to_stories.rb: -------------------------------------------------------------------------------- 1 | class AddDomainToStories < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :stories, :domain, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170302132440_add_dates_to_chapters.rb: -------------------------------------------------------------------------------- 1 | class AddDatesToChapters < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :chapters, :publish_date, :date 4 | add_column :chapters, :edit_date, :date 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20170302153532_add_strategy_to_request.rb: -------------------------------------------------------------------------------- 1 | class AddStrategyToRequest < ActiveRecord::Migration[5.0] 2 | def change 3 | add_column :requests, :strategy, :string 4 | add_column :requests, :recent_number, :integer 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20170702114104_create_targets.rb: -------------------------------------------------------------------------------- 1 | class CreateTargets < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :targets do |t| 4 | t.string :domain 5 | t.datetime :last_access 6 | t.string :scraper 7 | t.json :target_data 8 | 9 | t.timestamps 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20170702114530_drop_scraper_queues.rb: -------------------------------------------------------------------------------- 1 | class DropScraperQueues < ActiveRecord::Migration[5.0] 2 | def change 3 | drop_table :scraper_queues 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20170702121451_add_target_to_requests.rb: -------------------------------------------------------------------------------- 1 | class AddTargetToRequests < ActiveRecord::Migration[5.0] 2 | def change 3 | add_reference :requests, :target, foreign_key: true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20181110200218_create_proxies.rb: -------------------------------------------------------------------------------- 1 | class CreateProxies < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :proxies do |t| 4 | t.string :ip 5 | t.string :port 6 | t.string :username 7 | t.string :password 8 | t.integer :successful_request_count, default: 0 9 | t.integer :failed_request_count, default: 0 10 | t.datetime :last_successful_request 11 | t.timestamps 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /db/schema.rb: -------------------------------------------------------------------------------- 1 | # This file is auto-generated from the current state of the database. Instead 2 | # of editing this file, please use the migrations feature of Active Record to 3 | # incrementally modify your database, and then regenerate this schema definition. 4 | # 5 | # Note that this schema.rb definition is the authoritative source for your 6 | # database schema. If you need to create the application database on another 7 | # system, you should be using db:schema:load, not running all the migrations 8 | # from scratch. The latter is a flawed and unsustainable approach (the more migrations 9 | # you'll amass, the slower it'll run and the greater likelihood for issues). 10 | # 11 | # It's strongly recommended that you check this file into your version control system. 12 | 13 | ActiveRecord::Schema.define(version: 20181110200218) do 14 | 15 | # These are extensions that must be enabled in order to support this database 16 | enable_extension "plpgsql" 17 | 18 | create_table "chapters", force: :cascade do |t| 19 | t.integer "story_id" 20 | t.integer "number" 21 | t.string "title" 22 | t.string "content" 23 | t.datetime "created_at", null: false 24 | t.datetime "updated_at", null: false 25 | t.date "publish_date" 26 | t.date "edit_date" 27 | t.index ["story_id"], name: "index_chapters_on_story_id", using: :btree 28 | end 29 | 30 | create_table "documents", force: :cascade do |t| 31 | t.integer "story_id" 32 | t.string "filename" 33 | t.string "extension" 34 | t.datetime "created_at", null: false 35 | t.datetime "updated_at", null: false 36 | t.string "aws_url" 37 | t.string "aws_key" 38 | t.index ["story_id"], name: "index_documents_on_story_id", using: :btree 39 | end 40 | 41 | create_table "images", force: :cascade do |t| 42 | t.integer "story_id" 43 | t.string "source_url" 44 | t.string "aws_url" 45 | t.string "filename" 46 | t.string "extension" 47 | t.integer "size" 48 | t.boolean "cover" 49 | t.datetime "created_at", null: false 50 | t.datetime "updated_at", null: false 51 | t.string "aws_key" 52 | t.index ["story_id"], name: "index_images_on_story_id", using: :btree 53 | end 54 | 55 | create_table "proxies", force: :cascade do |t| 56 | t.string "ip" 57 | t.string "port" 58 | t.string "username" 59 | t.string "password" 60 | t.integer "successful_request_count", default: 0 61 | t.integer "failed_request_count", default: 0 62 | t.datetime "last_successful_request" 63 | t.datetime "created_at", null: false 64 | t.datetime "updated_at", null: false 65 | end 66 | 67 | create_table "requests", force: :cascade do |t| 68 | t.integer "story_id" 69 | t.string "url" 70 | t.datetime "created_at", null: false 71 | t.datetime "updated_at", null: false 72 | t.string "status" 73 | t.integer "total_chapters" 74 | t.boolean "complete" 75 | t.integer "current_chapters" 76 | t.string "extension" 77 | t.integer "doc_id" 78 | t.string "aws_url" 79 | t.string "strategy" 80 | t.integer "recent_number" 81 | t.integer "target_id" 82 | t.index ["story_id"], name: "index_requests_on_story_id", using: :btree 83 | t.index ["target_id"], name: "index_requests_on_target_id", using: :btree 84 | end 85 | 86 | create_table "stories", force: :cascade do |t| 87 | t.string "url" 88 | t.string "title" 89 | t.string "author" 90 | t.datetime "created_at", null: false 91 | t.datetime "updated_at", null: false 92 | t.string "meta_data" 93 | t.string "domain" 94 | end 95 | 96 | create_table "targets", force: :cascade do |t| 97 | t.string "domain" 98 | t.datetime "last_access" 99 | t.string "scraper" 100 | t.json "target_data" 101 | t.datetime "created_at", null: false 102 | t.datetime "updated_at", null: false 103 | end 104 | 105 | add_foreign_key "requests", "targets" 106 | end 107 | -------------------------------------------------------------------------------- /db/seeds.rb: -------------------------------------------------------------------------------- 1 | # This file should contain all the record creation needed to seed the database with its default values. 2 | # The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup). 3 | # 4 | # Examples: 5 | # 6 | # movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }]) 7 | # Character.create(name: 'Luke', movie: movies.first) 8 | -------------------------------------------------------------------------------- /lib/assets/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/lib/assets/.keep -------------------------------------------------------------------------------- /lib/gifsicle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/lib/gifsicle -------------------------------------------------------------------------------- /lib/kindlegen: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/lib/kindlegen -------------------------------------------------------------------------------- /lib/tasks/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/lib/tasks/.keep -------------------------------------------------------------------------------- /lib/tasks/resque.rake: -------------------------------------------------------------------------------- 1 | require 'resque/tasks' 2 | require 'resque/pool/tasks' 3 | 4 | # this task will get called before resque:pool:setup 5 | # and preload the rails environment in the pool manager 6 | if Rails.env.development? 7 | task "resque:setup" => :environment do 8 | ENV['TERM_CHILD'] = '1' 9 | ENV['RESQUE_TERM_TIMEOUT'] = '10' 10 | # generic worker setup, e.g. Hoptoad for failed jobs 11 | end 12 | end 13 | 14 | if Rails.env.production? 15 | task "resque:setup" => :environment do 16 | ENV['TERM_CHILD'] = '1' 17 | ENV['RESQUE_TERM_TIMEOUT'] = '10' 18 | end 19 | 20 | desc "Alias for resque:work (To run workers on Heroku)" 21 | task "jobs:work" => "resque:work" 22 | else 23 | task 'resque:setup' => :environment 24 | end 25 | 26 | task "resque:pool:setup" do 27 | # close any sockets or files in pool manager 28 | ActiveRecord::Base.connection.disconnect! 29 | # and re-open them in the resque worker parent 30 | Resque::Pool.after_prefork do |job| 31 | ActiveRecord::Base.establish_connection 32 | end 33 | end 34 | 35 | task "resque:pool:setup" do 36 | Resque::Pool.after_prefork do |job| 37 | Resque.redis.client.reconnect 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/wkhtmltopdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/lib/wkhtmltopdf -------------------------------------------------------------------------------- /log/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/log/.keep -------------------------------------------------------------------------------- /public/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The page you were looking for doesn't exist (404) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The page you were looking for doesn't exist.

62 |

You may have mistyped the address or the page may have moved.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /public/422.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The change you wanted was rejected (422) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The change you wanted was rejected.

62 |

Maybe you tried to change something you didn't have access to.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /public/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | We're sorry, but something went wrong (500) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

We're sorry, but something went wrong.

62 |
63 |

If you are the application owner check the logs for more information.

64 |
65 | 66 | 67 | -------------------------------------------------------------------------------- /public/apple-touch-icon-precomposed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/apple-touch-icon-precomposed.png -------------------------------------------------------------------------------- /public/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/apple-touch-icon.png -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/favicon.ico -------------------------------------------------------------------------------- /public/images/books.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/images/books.jpg -------------------------------------------------------------------------------- /public/images/error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/images/error.png -------------------------------------------------------------------------------- /public/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/images/favicon.png -------------------------------------------------------------------------------- /public/images/github.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/images/github.png -------------------------------------------------------------------------------- /public/images/header_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/images/header_logo.png -------------------------------------------------------------------------------- /public/images/omnibuser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/images/omnibuser.png -------------------------------------------------------------------------------- /public/images/twitter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/public/images/twitter.png -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | # 3 | # To ban all spiders from the entire site uncomment the next two lines: 4 | User-agent: * 5 | Disallow: /documents 6 | -------------------------------------------------------------------------------- /spec/factories/chapters.rb: -------------------------------------------------------------------------------- 1 | FactoryGirl.define do 2 | factory :chapter do 3 | association :story 4 | sequence(:number) {|n| n} 5 | title {Faker::Superhero.name} 6 | 7 | factory :untitled_chapter do 8 | title nil 9 | end 10 | 11 | factory :chapter_with_images do 12 | content "
\n
" 13 | end 14 | factory :chapter_with_srcless_images do 15 | content "
\n
" 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/factories/documents.rb: -------------------------------------------------------------------------------- 1 | FactoryGirl.define do 2 | factory :document do 3 | association :story 4 | filename 'generic_document' 5 | extension 'html' 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/factories/images.rb: -------------------------------------------------------------------------------- 1 | FactoryGirl.define do 2 | factory :image do 3 | cover false 4 | extension 'jpg' 5 | 6 | factory :cover_image do 7 | cover true 8 | end 9 | 10 | factory :gif do 11 | extension 'gif' 12 | end 13 | factory :png do 14 | extension 'png' 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/factories/requests.rb: -------------------------------------------------------------------------------- 1 | FactoryGirl.define do 2 | factory :request do 3 | url "https://www.fanfiction.net/s/5782108/" 4 | extension 'epub' 5 | strategy 'all' 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/factories/stories.rb: -------------------------------------------------------------------------------- 1 | FactoryGirl.define do 2 | factory :story do 3 | association :request 4 | url "fanfiction.net/s/5782108/" 5 | title "Harry Potter and the Methods of Rationality" 6 | author "Less Wrong" 7 | meta_data "{\"summary\":\"Petunia married a biochemist, and Harry grew up reading science and science fiction. Then came the Hogwarts letter, and a world of intriguing new possibilities to exploit. And new friends, like Hermione Granger, and Professor McGonagall, and Professor Quirrell... COMPLETE.\",\"info\":\"Rated: Fiction T - English - Drama/Humor - Harry P., Hermione G. - Chapters: 122 - Words: 661,619 - Reviews: 32,985 - Favs: 20,340 - Follows: 16,184 - Updated: 3/14/2015 - Published: 2/28/2010 - Status: Complete - id: 5782108 \"}" 8 | 9 | factory :ffn_story do 10 | url "https://www.fanfiction.net" 11 | end 12 | 13 | factory :fp_story do 14 | url "https://www.fictionpress.com" 15 | end 16 | 17 | factory :sv_story do 18 | url "https://forums.sufficientvelocity.com" 19 | end 20 | 21 | factory :sb_story do 22 | url "https://forums.spacebattles.com" 23 | end 24 | 25 | factory :qq_story do 26 | url "https://forum.questionablequesting.com" 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /spec/features/fanfiction_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | feature "FanFiction", js: true do 4 | let(:single_chapter) {'https://www.fanfiction.net/s/3853/1/Lily-Fly-Away'} 5 | let(:multi_chapter) {'https://www.fanfiction.net/s/195487/1/Bill-and-Ted-s-Adventure'} 6 | 7 | before :each do 8 | visit root_path 9 | end 10 | 11 | scenario "Scraping a single-chapter story" do 12 | get_story single_chapter, extension: 'epub' 13 | expect(page).to have_content('Download') 14 | end 15 | scenario "Scraping all chapters from a multi-chapter story" do 16 | get_story multi_chapter, extension: 'html' 17 | expect(page).to have_content('Download') 18 | end 19 | scenario "Scraping one chapter from a multi-chapter story" do 20 | get_story multi_chapter, extension: 'mobi', recent: true, count: 1 21 | expect(page).to have_content('Download') 22 | end 23 | scenario "Scraping 10 chapters from a story with fewer than 10 chapters" do 24 | get_story multi_chapter, extension: 'pdf', recent: true, count: 10 25 | expect(page).to have_content('Download') 26 | end 27 | end 28 | 29 | feature "FictionPress", js: true do 30 | let(:single_chapter) {'https://www.fictionpress.com/s/3593/1/The-Secret-Valley'} 31 | let(:multi_chapter) {'https://www.fictionpress.com/s/2737883/1/Incapaz-de-querer'} 32 | 33 | before :each do 34 | visit root_path 35 | end 36 | 37 | scenario "Scraping a single-chapter story" do 38 | get_story single_chapter, extension: 'epub' 39 | expect(page).to have_content('Download') 40 | end 41 | scenario "Scraping all chapters from a multi-chapter story" do 42 | get_story multi_chapter, extension: 'html' 43 | expect(page).to have_content('Download') 44 | end 45 | scenario "Scraping one chapter from a multi-chapter story" do 46 | get_story multi_chapter, extension: 'mobi', recent: true, count: 1 47 | expect(page).to have_content('Download') 48 | end 49 | scenario "Scraping 10 chapters from a story with fewer than 10 chapters" do 50 | get_story multi_chapter, extension: 'pdf', recent: true, count: 10 51 | expect(page).to have_content('Download') 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /spec/features/forum_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | feature "SpaceBattles", js: true do 4 | let(:unthreadmarked_op) {'https://forums.spacebattles.com/threads/under-your-mask-gundam-iron-blooded-orphans.463801/'} 5 | let(:large_images) {'https://forums.spacebattles.com/threads/ydia-ii-wild-wasteland-a-fallout-ranma-sm-cross.388980/'} 6 | 7 | before :each do 8 | visit root_path 9 | end 10 | 11 | scenario "Scraping all chapters from a multi-chapter story" do 12 | get_story large_images, extension: 'html' 13 | expect(page).to have_content('Download') 14 | end 15 | scenario "Scraping one chapter from a multi-chapter story" do 16 | get_story large_images, extension: 'mobi', recent: true, count: 1 17 | expect(page).to have_content('Download') 18 | end 19 | scenario "Scraping 10 chapters from a story with fewer than 10 chapters" do 20 | get_story large_images, extension: 'pdf', recent: true, count: 10 21 | expect(page).to have_content('Download') 22 | end 23 | scenario "Scraping a story with no threadmark for first post" do 24 | get_story unthreadmarked_op, extension: 'epub' 25 | expect(page).to have_content('Download') 26 | end 27 | end 28 | 29 | feature "Sufficient Velocity", js: true do 30 | let(:single_chapter) {'https://forums.sufficientvelocity.com/threads/archive-of-random-snips.32557/'} 31 | let(:unthreadmarked_op) {'https://forums.sufficientvelocity.com/threads/erlk%C3%B6nig-worm-au.33429/'} 32 | let(:large_images) {'https://forums.sufficientvelocity.com/threads/a-daughters-dedication-kancolle.33211/'} 33 | let(:many_threadmarks) {'https://forums.sufficientvelocity.com/threads/mauling-snarks-worm.41471/'} 34 | 35 | before :each do 36 | visit root_path 37 | end 38 | 39 | # scenario "Scraping a single-chapter story" do 40 | # get_story single_chapter, extension: 'epub' 41 | # expect(page).to have_content('Download') 42 | # end 43 | scenario "Scraping all chapters from a multi-chapter story" do 44 | get_story large_images, extension: 'html' 45 | expect(page).to have_content('Download') 46 | end 47 | scenario "Scraping one chapter from a multi-chapter story" do 48 | get_story large_images, extension: 'mobi', recent: true, count: 1 49 | expect(page).to have_content('Download') 50 | end 51 | scenario "Scraping 10 chapters from a story with fewer than 10 chapters" do 52 | get_story large_images, extension: 'pdf', recent: true, count: 10 53 | expect(page).to have_content('Download', wait: 100) 54 | end 55 | scenario "Scraping a story with no threadmark for first post" do 56 | get_story unthreadmarked_op, extension: 'epub' 57 | expect(page).to have_content('Download') 58 | end 59 | # scenario "Scraping a story with many threadmarks" do 60 | # get_story many_threadmarks, extension: 'html' 61 | # expect(page).to have_content('Download') 62 | # end 63 | end 64 | 65 | feature "Questionable Questing", js: true do 66 | let(:unthreadmarked_op) {'https://forum.questionablequesting.com/threads/a-prophets-portents-misc-original.2904/'} 67 | let(:large_images) {'https://forum.questionablequesting.com/threads/spitfire-quest-worm-x-d-d-quest-thread-1.1464/'} 68 | 69 | before :each do 70 | visit root_path 71 | end 72 | 73 | scenario "Scraping all chapters from a multi-chapter story" do 74 | get_story large_images, extension: 'html' 75 | expect(page).to have_content('Download') 76 | end 77 | scenario "Scraping one chapter from a multi-chapter story" do 78 | get_story large_images, extension: 'mobi', recent: true, count: 1 79 | expect(page).to have_content('Download') 80 | end 81 | scenario "Scraping 10 chapters from a story with fewer than 10 chapters" do 82 | get_story large_images, extension: 'pdf', recent: true, count: 10 83 | expect(page).to have_content('Download', wait: 100) 84 | end 85 | scenario "Scraping a story with no threadmark for first post" do 86 | get_story unthreadmarked_op, extension: 'epub' 87 | expect(page).to have_content('Download') 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /spec/features/toc_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | feature "TOCScraper", js: true do 4 | 5 | before :each do 6 | visit root_path 7 | end 8 | 9 | scenario "Scraping last 3 chapters of Worm" do 10 | get_story 'https://parahumans.wordpress.com', extension: 'epub', recent: true, count: 3 11 | expect(page).to have_content('Download') 12 | end 13 | 14 | scenario "Scraping last 7 chapters of Unsong" do 15 | get_story 'https://unsongbook.com', extension: 'pdf', recent: true, count: 7 16 | expect(page).to have_content('Download') 17 | end 18 | 19 | scenario "Scraping last 3 chapters of Practical Guide To Evil" do 20 | get_story 'https://practicalguidetoevil.wordpress.com', extension: 'mobi', recent: true, count: 3 21 | expect(page).to have_content('Download') 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/models/builders/doc_builder_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | require 'fileutils' 3 | 4 | describe DocBuilder do 5 | class TestBuilder < DocBuilder 6 | attr_accessor :template_dir, :directory 7 | end 8 | 9 | describe '#render_template' do 10 | 11 | before :all do 12 | @template_dir = '/tmp/test/template' 13 | @template = 'buildertest.erb' 14 | @directory = '/tmp/test/directory' 15 | @output = 'output' 16 | @template_path = "#{@template_dir}/#{@template}" 17 | @output_path = "#{@directory}/#{@output}" 18 | @doc = build(:document) 19 | builder = TestBuilder.new(doc: @doc, 20 | template_dir: @template_dir, directory: @directory) 21 | FileUtils.rm_r('/tmp/test') if Dir.exist?('/tmp/test') 22 | FileUtils.mkdir(['/tmp/test', @template_dir, @directory]) 23 | File.open(@template_path, 'w+') do |f| 24 | f << "<%= @doc.story.title %>" 25 | end 26 | builder.render_template(@template, @output) 27 | end 28 | 29 | it "creates a new file at the output path" do 30 | expect(File.exist?(@output_path)).to be true 31 | end 32 | it "renders the template into the file as ERB" do 33 | expect(File.read(@output_path)).to eq(@doc.story.title) 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/models/builders/html_builder_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe HTMLBuilder do 4 | before :all do 5 | @story = create(:story) 6 | @doc = Document.new(story_id: @story.id, filename: @story.title, 7 | extension: 'html') 8 | @doc.sanitize_filename 9 | 3.times {create(:chapter, story_id: @story.id)} 10 | FileUtils.rm_r("/tmp/#{@doc.filename}") if Dir.exist?("/tmp/#{@doc.filename}") 11 | @builder = HTMLBuilder.new(doc: @doc) 12 | @builder.template_dir = Rails.root.join("app", "templates", 'html') 13 | @builder.domain = @story.domain 14 | @builder.create_directory_structure 15 | end 16 | 17 | after :all do 18 | FileUtils.rm_r("/tmp/#{@doc.filename}") if Dir.exist?("/tmp/#{@doc.filename}") 19 | 20 | File.delete("/tmp/#{@doc.filename}.zip") if File.exist?("/tmp/#{@doc.filename}.zip") 21 | end 22 | 23 | describe '#build' do 24 | after :each do 25 | allow_any_instance_of(HTMLBuilder).to receive :zip_directory 26 | @builder.build 27 | end 28 | it "sets @domain" do 29 | allow_any_instance_of(HTMLBuilder).to receive :zip_directory 30 | @builder.domain = nil 31 | @builder.build 32 | expect(@builder.domain).to eq @story.domain 33 | end 34 | it "creates directory structure" do 35 | expect_any_instance_of(HTMLBuilder).to receive :create_directory_structure 36 | end 37 | it "adds style files" do 38 | expect_any_instance_of(HTMLBuilder).to receive :add_styles 39 | end 40 | it "adds cover image" do 41 | expect_any_instance_of(HTMLBuilder).to receive :create_cover 42 | end 43 | it "adds other images" do 44 | expect_any_instance_of(HTMLBuilder).to receive :add_images 45 | end 46 | it "adds story file" do 47 | expect_any_instance_of(HTMLBuilder).to receive :create_story 48 | end 49 | context "nozip is true" do 50 | it "returns the path to main story file" do 51 | expect(@builder.build(nozip: true)).to eq("/tmp/#{@doc.filename}/story.html") 52 | end 53 | end 54 | context "nozip is false" do 55 | it "zips the directory" do 56 | expect_any_instance_of(HTMLBuilder).to receive :zip_directory 57 | end 58 | end 59 | end 60 | describe '#create_directory_structure' do 61 | it "removes previous directory if it exists" do 62 | expect{@builder.create_directory_structure}.not_to raise_error 63 | end 64 | it "creates directory in /tmp named with doc filename" do 65 | expect(Dir.exist?("/tmp/#{@doc.filename}")).to be true 66 | end 67 | it "creates files subdir" do 68 | expect(Dir.exist?("/tmp/#{@doc.filename}/files")).to be true 69 | end 70 | it "creates styles subdir" do 71 | expect(Dir.exist?("/tmp/#{@doc.filename}/files/css")).to be true 72 | end 73 | it "creates images subdir" do 74 | expect(Dir.exist?("/tmp/#{@doc.filename}/files/images")).to be true 75 | end 76 | end 77 | describe '#add_styles' do 78 | before :each do 79 | @builder.domain = 'ffn' 80 | @builder.add_styles 81 | end 82 | it "adds the main css file to the folder" do 83 | expect(File.exist?("/tmp/#{@doc.filename}/files/css/main.css")).to be true 84 | end 85 | it "adds the domain specific css file to the folder" do 86 | expect(File.exist?("/tmp/#{@doc.filename}/files/css/#{@builder.domain}.css")).to be true 87 | end 88 | end 89 | describe "image handling", speed: 'slow' do 90 | before :all do 91 | @cover = create(:image, story_id: @story.id, source_url: 'url1', cover: true) 92 | @image = create(:png, story_id: @story.id, source_url: 'url2') 93 | FileUtils.cp(Rails.root.join("spec", "support", "images", "dice.png"), 94 | "#{@image.path}.temp") 95 | FileUtils.cp(Rails.root.join("spec", "support", "images", "lake.jpg"), 96 | "#{@cover.path}.temp") 97 | @cover.compress 98 | @image.compress 99 | @cover.upload 100 | @image.upload 101 | @builder.add_images 102 | end 103 | describe '#add_images' do 104 | it "downloads images to images subdir" do 105 | expect(File.exist?("/tmp/#{@doc.filename}/files/images/#{@image.name}")).to be true 106 | end 107 | it "only applies to images which are not a cover" do 108 | expect(File.exist?("/tmp/#{@doc.filename}/files/images/#{@cover.name}")).not_to be true 109 | end 110 | context "domain is sb or sv" do 111 | it "copies the smilies image to images subdir" do 112 | @builder.domain = 'sv' 113 | @builder.add_images 114 | expect(File.exist?("/tmp/#{@doc.filename}/files/images/xenforo-smilies-sprite.png")).to be true 115 | File.delete("/tmp/#{@doc.filename}/files/images/xenforo-smilies-sprite.png") 116 | end 117 | end 118 | context "domain is not sb or sv" do 119 | it "does not copy the smilies image to images subdir" do 120 | @builder.domain = 'ffn' 121 | @builder.add_images 122 | expect(File.exist?("/tmp/#{@doc.filename}/files/images/xenforo-smilies-sprite.png")).to be false 123 | end 124 | end 125 | end 126 | 127 | describe '#create_cover' do 128 | context "story has a cover image" do 129 | it "downloads cover image to images subdir" do 130 | @builder.create_cover 131 | expect(File.exist?("/tmp/#{@doc.filename}/files/images/#{@cover.name}")).to be true 132 | end 133 | end 134 | context "story does not have a cover image" do 135 | before :all do 136 | @story.images.delete_all 137 | end 138 | it "copies the domain specific placeholder image to images subdir" do 139 | @builder.create_cover 140 | expect(File.exist?("/tmp/#{@doc.filename}/files/images/#{@builder.domain}.png")).to be true 141 | end 142 | it "defaults to the omnibuser favicon if no domain is specified" do 143 | @builder.domain = nil 144 | @builder.create_cover 145 | expect(File.exist?("/tmp/#{@doc.filename}/files/images/favicon.png")).to be true 146 | end 147 | end 148 | end 149 | end 150 | 151 | describe '#create_story' do 152 | it "creates the story file" do 153 | @builder.create_story 154 | expect(File.exist?("/tmp/#{@doc.filename}/story.html")).to be true 155 | end 156 | end 157 | describe '#zip_directory' do 158 | it "creates a new zip file with the doc filename and zip extension" do 159 | File.delete("/tmp/#{@doc.filename}.zip") if File.exist?("/tmp/#{@doc.filename}.zip") 160 | @builder.input = [] 161 | @builder.zip_directory 162 | expect(File.exist?("/tmp/#{@doc.filename}.zip")).to be true 163 | end 164 | end 165 | end 166 | -------------------------------------------------------------------------------- /spec/models/builders/mobi_builder_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe MOBIBuilder do 4 | before :all do 5 | @story = create(:story) 6 | @doc = Document.new(story_id: @story.id, filename: @story.title, 7 | extension: 'mobi') 8 | @doc.sanitize_filename 9 | 3.times {create(:chapter, story_id: @story.id)} 10 | FileUtils.rm_r("/tmp/#{@doc.filename}") if Dir.exist?("/tmp/#{@doc.filename}") 11 | @builder = MOBIBuilder.new(doc: @doc) 12 | @builder.build 13 | end 14 | 15 | after :all do 16 | FileUtils.rm_r("/tmp/#{@doc.filename}") if Dir.exist?("/tmp/#{@doc.filename}") 17 | 18 | File.delete("/tmp/#{@doc.filename}.epub") if File.exist?("/tmp/#{@doc.filename}.epub") 19 | 20 | File.delete("/tmp/#{@doc.filename}.mobi") if File.exist?("/tmp/#{@doc.filename}.mobi") 21 | end 22 | 23 | describe '#build' do 24 | it "builds an epub of the doc" do 25 | expect(File.exist?("/tmp/#{@doc.filename}.epub")).to be true 26 | end 27 | it "converts to mobi" do 28 | expect(File.exist?("/tmp/#{@doc.filename}.mobi")).to be true 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /spec/models/builders/pdf_builder_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe PDFBuilder do 4 | before :all do 5 | @story = create(:story) 6 | @doc = Document.new(story_id: @story.id, filename: @story.title, 7 | extension: 'pdf') 8 | @doc.sanitize_filename 9 | 30.times {create(:chapter, story_id: @story.id)} 10 | FileUtils.rm_r("/tmp/#{@doc.filename}") if Dir.exist?("/tmp/#{@doc.filename}") 11 | @builder = PDFBuilder.new(doc: @doc) 12 | @builder.template_dir = Rails.root.join("app", "templates", 'html') 13 | @builder.domain = @story.domain 14 | @builder.create_directory_structure 15 | end 16 | 17 | after :all do 18 | FileUtils.rm_r("/tmp/#{@doc.filename}") if Dir.exist?("/tmp/#{@doc.filename}") 19 | 20 | File.delete("/tmp/#{@doc.filename}.pdf") if File.exist?("/tmp/#{@doc.filename}.pdf") 21 | end 22 | 23 | describe '#build' do 24 | after :each do 25 | allow_any_instance_of(PDFBuilder).to receive :convert_to_pdf 26 | allow_any_instance_of(PDFBuilder).to receive :combine_pdfs 27 | @builder.build 28 | end 29 | it "sets @domain" do 30 | allow_any_instance_of(PDFBuilder).to receive :convert_to_pdf 31 | allow_any_instance_of(PDFBuilder).to receive :combine_pdfs 32 | @builder.domain = nil 33 | @builder.build 34 | expect(@builder.domain).to eq @story.domain 35 | end 36 | it "creates directory structure" do 37 | expect_any_instance_of(PDFBuilder).to receive :create_directory_structure 38 | end 39 | it "adds styles" do 40 | expect_any_instance_of(PDFBuilder).to receive :add_styles 41 | end 42 | it "adds cover" do 43 | expect_any_instance_of(PDFBuilder).to receive :create_cover 44 | end 45 | it "adds images" do 46 | expect_any_instance_of(PDFBuilder).to receive :add_images 47 | end 48 | it "adds frontmatter" do 49 | expect_any_instance_of(PDFBuilder).to receive :create_frontmatter 50 | end 51 | it "adds content" do 52 | expect_any_instance_of(PDFBuilder).to receive :create_content 53 | end 54 | it "copies filler file" do 55 | expect(File.exist?("/tmp/#{@doc.filename}/filler.html")).to be true 56 | end 57 | it "converts to pdf" do 58 | expect_any_instance_of(PDFBuilder).to receive :convert_to_pdf 59 | end 60 | it "combines pdfs" do 61 | expect_any_instance_of(PDFBuilder).to receive :combine_pdfs 62 | end 63 | end 64 | describe '#create_frontmatter' do 65 | it "creates frontmatter file" do 66 | @builder.create_frontmatter 67 | expect(File.exist?("/tmp/#{@doc.filename}/frontmatter.html")).to be true 68 | end 69 | end 70 | describe '#create_content' do 71 | it "creates a html file for each 10 chapters" do 72 | @builder.create_content 73 | expect(File.exist?("/tmp/#{@doc.filename}/content_1.html")).to be true 74 | expect(File.exist?("/tmp/#{@doc.filename}/content_2.html")).to be true 75 | expect(File.exist?("/tmp/#{@doc.filename}/content_3.html")).to be true 76 | expect(File.exist?("/tmp/#{@doc.filename}/content_4.html")).to be false 77 | end 78 | end 79 | describe "pdf creation" do 80 | before :all do 81 | @builder.create_frontmatter 82 | @builder.create_content 83 | @builder.render_template('../pdf/filler.html.erb', 'filler.html') 84 | @builder.convert_to_pdf 85 | end 86 | describe '#convert_to_pdf' do 87 | it "creates a pdf for each file in @files" do 88 | expect(File.exist?("/tmp/#{@doc.filename}/content_1.pdf")).to be true 89 | expect(File.exist?("/tmp/#{@doc.filename}/content_2.pdf")).to be true 90 | expect(File.exist?("/tmp/#{@doc.filename}/content_3.pdf")).to be true 91 | expect(File.exist?("/tmp/#{@doc.filename}/frontmatter.pdf")).to be true 92 | end 93 | end 94 | describe '#combine_pdfs' do 95 | it "creates a new pdf at the doc path" do 96 | @builder.combine_pdfs 97 | expect(File.exist?("/tmp/#{@doc.filename}.pdf")).to be true 98 | end 99 | end 100 | end 101 | 102 | end 103 | -------------------------------------------------------------------------------- /spec/models/chapter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe Chapter do 4 | it { should belong_to(:story) } 5 | it { should validate_uniqueness_of(:number).scoped_to(:story_id) } 6 | 7 | describe '#ensure_title' do 8 | it "updates record with default title if title is blank" do 9 | chapter = create(:untitled_chapter) 10 | expect(chapter.title).to eq("Chapter #{chapter.number}") 11 | end 12 | it "does not change title if title is not blank" do 13 | chapter = create(:chapter, title: "Awesome Chapter") 14 | expect(chapter.title).to eq("Awesome Chapter") 15 | end 16 | end 17 | 18 | describe '#epub' do 19 | it "changes image src's to include correct path" do 20 | chapter = create(:chapter_with_images) 21 | expect(chapter.epub).to eq "
\n
" 22 | end 23 | end 24 | 25 | describe '#html' do 26 | it "changes image src's to include correct path" do 27 | chapter = create(:chapter_with_images) 28 | expect(chapter.html).to eq "
\n
" 29 | end 30 | it "does not alter image src if src is blank" do 31 | chapter = create(:chapter_with_srcless_images) 32 | expect(chapter.html).to eq "
\n
" 33 | end 34 | end 35 | 36 | describe '#xhtml' do 37 | it "returns a Nokogiri nodeset" do 38 | chapter = create(:chapter_with_images) 39 | expect(chapter.xhtml).to be_an_instance_of Nokogiri::XML::Document 40 | end 41 | end 42 | 43 | end 44 | -------------------------------------------------------------------------------- /spec/models/document_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe Document do 4 | it { should belong_to(:story) } 5 | 6 | describe '#sanitize_filename' do 7 | before :each do 8 | allow_any_instance_of(Document).to receive(:add_chapter_numbers).and_return("cnums added") 9 | @doc = build(:document, filename: 'ad-hoc') 10 | end 11 | it "replaces non-alphanumeric characters with underscores" do 12 | expect(build_doc_with_name('ad-hoc').filename).to eq('ad_hoc') 13 | end 14 | it "does not have consecutive underscores" do 15 | expect(build_doc_with_name('ad--hoc').filename).to eq('ad_hoc') 16 | end 17 | it "does not have leading underscores" do 18 | expect(build_doc_with_name('-ad-hoc').filename).to eq('ad_hoc') 19 | end 20 | it "does not have trailing underscores" do 21 | expect(build_doc_with_name('ad-hoc-').filename).to eq('ad_hoc') 22 | end 23 | it "is shorter than 230 characters" do 24 | expect(build_doc_with_name('a'*300).filename.length).to be <= 230 25 | end 26 | it "adds chapter range if recent strategy was used" do 27 | @doc.story.request.update strategy: 'recent' 28 | expect(@doc.sanitize_filename).to eq 'cnums added' 29 | end 30 | it "does not add chapter range if recent strategy was not used" do 31 | @doc.story.request.update strategy: 'all' 32 | expect(@doc.sanitize_filename).not_to eq 'cnums added' 33 | end 34 | end 35 | 36 | describe '#add_chapter_numbers' do 37 | before :all do 38 | FactoryGirl.reload 39 | @story = create(:ffn_story) 40 | create(:chapter, story_id: @story.id) 41 | end 42 | it "only adds a single number if there is only one chapter" do 43 | doc = build(:document, story_id: @story.id, filename: 'single') 44 | doc.add_chapter_numbers 45 | expect(doc.filename).to eq('single_1') 46 | end 47 | it "adds first and last chapter numbers if more than one chapter" do 48 | create(:chapter, story_id: @story.id) 49 | create(:chapter, story_id: @story.id) 50 | doc = build(:document, story_id: @story.id, filename: 'multiple') 51 | doc.add_chapter_numbers 52 | expect(doc.filename).to eq('multiple_1-3') 53 | end 54 | end 55 | 56 | describe '#path' do 57 | it "returns doc's path" do 58 | doc = build(:document) 59 | expect(doc.path).to eq("/tmp/#{doc.filename}.#{doc.extension}") 60 | end 61 | end 62 | 63 | describe '#delete_file' do 64 | it "deletes the doc file if it exists" do 65 | doc = build(:document) 66 | FileUtils.touch(doc.path) 67 | expect(File.exist?(doc.path)).to be true 68 | doc.delete_file 69 | expect(File.exist?(doc.path)).to be false 70 | end 71 | it "returns nil if doc file does not exist" do 72 | doc = build(:document) 73 | expect(doc.delete_file).to be nil 74 | end 75 | end 76 | 77 | describe '#build' do 78 | before :each do 79 | allow_any_instance_of(PDFBuilder).to receive(:build) 80 | allow_any_instance_of(HTMLBuilder).to receive(:build) 81 | allow_any_instance_of(MOBIBuilder).to receive(:build) 82 | allow_any_instance_of(EPUBBuilder).to receive(:build) 83 | allow_any_instance_of(Document).to receive(:upload) 84 | end 85 | it "instantiates PDFBuilder if extension is pdf" do 86 | expect(PDFBuilder).to receive_message_chain(:new, :build) 87 | build(:document, extension: 'pdf').build 88 | end 89 | it "instantiates HTMLBuilder if extension is html" do 90 | expect(HTMLBuilder).to receive_message_chain(:new, :build) 91 | build(:document, extension: 'html').build 92 | end 93 | it "instantiates MOBIBuilder if extension is mobi" do 94 | expect(MOBIBuilder).to receive_message_chain(:new, :build) 95 | build(:document, extension: 'mobi').build 96 | end 97 | it "instantiates EPUBBuilder if extension is epub" do 98 | expect(EPUBBuilder).to receive_message_chain(:new, :build) 99 | build(:document, extension: 'epub').build 100 | end 101 | it "changes extension to zip if it is html" do 102 | doc = build(:document, extension: 'html') 103 | doc.build 104 | expect(doc.extension).to eq 'zip' 105 | end 106 | it "uploads after building" do 107 | doc = build(:document, extension: 'html') 108 | expect(doc).to receive(:upload) 109 | doc.build 110 | end 111 | end 112 | 113 | describe '#upload' do 114 | before :each do 115 | allow_any_instance_of(Document).to receive(:build) 116 | @doc = create(:document) 117 | FileUtils.touch(@doc.path) 118 | @aws_object = S3_BUCKET.objects["documents/#{@doc.id}/#{@doc.filename}.#{@doc.extension}"] 119 | @doc.upload 120 | end 121 | it "uploads document to documents folder on AWS" do 122 | expect(@aws_object.exists?).to be true 123 | end 124 | it "updates document record with AWS url and key" do 125 | expect(@doc.aws_url).to eq(@aws_object.public_url.to_s) 126 | expect(@doc.aws_key).to eq(@aws_object.key) 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /spec/models/image_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | include Colorscore 3 | 4 | describe Image do 5 | it { should belong_to(:story) } 6 | 7 | before :all do 8 | @story = create(:story) 9 | @jpg = create(:image, story_id: @story.id, source_url: 'url1') 10 | @gif = create(:gif, story_id: @story.id, source_url: 'url2') 11 | @png = create(:png, story_id: @story.id, source_url: 'url3') 12 | FileUtils.cp(Rails.root.join("spec", "support", "images", "dancing_banana.gif"), 13 | "#{@gif.path}.temp") 14 | @gif.compress 15 | FileUtils.cp(Rails.root.join("spec", "support", "images", "lake.jpg"), 16 | "#{@jpg.path}.temp") 17 | @jpg.compress 18 | end 19 | 20 | describe '#generate_name' do 21 | it "generates a random filename" do 22 | expect(@jpg.filename).not_to eq(@png.filename) 23 | end 24 | end 25 | 26 | describe '#name' do 27 | it "gives its own filename with extension" do 28 | expect(@jpg.name).to eq("#{@jpg.filename}.#{@jpg.extension}") 29 | end 30 | end 31 | 32 | describe '#path' do 33 | it "gives its path on the filesystem" do 34 | expect(@jpg.path).to eq("/tmp/#{@jpg.name}") 35 | end 36 | end 37 | 38 | describe '#compress' do 39 | context "gifs" do 40 | it "saves output to correct path" do 41 | expect(File.exist?(@gif.path)).to be true 42 | end 43 | it "compresses output" do 44 | expect(File.size(@gif.path)).to be < File.size("#{@gif.path}.temp") 45 | end 46 | end 47 | 48 | context "non-gif" do 49 | context 'png' do 50 | before :all do 51 | @white_png = @png 52 | @black_png = create(:png, story_id: @story.id) 53 | FileUtils.cp(Rails.root.join("spec", "support", "images", "dice.png"), 54 | "#{@black_png.path}.temp") 55 | FileUtils.cp(Rails.root.join("spec", "support", "images", "dice.png"), 56 | "#{@white_png.path}.temp") 57 | @white_png.compress 58 | @black_png.compress('#000000') 59 | end 60 | 61 | it "replaces alpha in png with user provided background color" do 62 | expect(Histogram.new(@black_png.path).scores.first[1].hex).to match('000000') 63 | end 64 | 65 | it "defaults background color to white" do 66 | expect(Histogram.new(@white_png.path).scores.first[1].hex).to match('ffffff') 67 | end 68 | 69 | it "converts to jpg" do 70 | expect(MiniMagick::Image.open(@white_png.path).data['mimeType']).to eq('image/jpeg') 71 | end 72 | 73 | it "updates extension to jpg" do 74 | expect(@white_png.extension).to eq('jpg') 75 | end 76 | end 77 | it "limits width to 1000px while maintaining aspect ratio" do 78 | expect(MiniMagick::Image.open(@jpg.path).data['geometry']['width']).to eq(1000) 79 | expect(MiniMagick::Image.open(@jpg.path).data['geometry']['height']).to eq(750) 80 | end 81 | it "compresses file" do 82 | expect(File.size(@jpg.path)).to be < File.size("#{@jpg.path}.temp") 83 | end 84 | it "saves file to correct path" do 85 | expect(@jpg.path).to eq("/tmp/#{@jpg.filename}.jpg") 86 | end 87 | end 88 | end 89 | 90 | describe 'AWS interactions', type: 'aws', speed: 'slow' do 91 | before :all do 92 | @aws_object = S3_BUCKET.objects["images/#{@jpg.name}"] 93 | @jpg.upload 94 | end 95 | describe '#upload' do 96 | it "uploads image to images folder on AWS" do 97 | expect(@aws_object.exists?).to be true 98 | end 99 | it "updates image record with AWS url and key" do 100 | expect(@jpg.aws_url).to eq(@aws_object.public_url.to_s) 101 | expect(@jpg.aws_key).to eq(@aws_object.key) 102 | end 103 | end 104 | 105 | describe '#download' do 106 | before :all do 107 | File.delete(@jpg.path) 108 | @download = @jpg.download 109 | end 110 | it "downloads image from AWS" do 111 | expect(File.exist?(@jpg.path)).to be true 112 | end 113 | it "saves to specified dir if provided" do 114 | Dir.mkdir('/tmp/test') unless Dir.exist?('/tmp/test') 115 | @jpg.download('/tmp/test') 116 | expect(File.exist?("/tmp/test/#{@jpg.name}")).to be true 117 | end 118 | it "returns path to file" do 119 | expect(@download).to eq(@jpg.path) 120 | end 121 | end 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /spec/models/request_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe Request do 4 | it { should belong_to(:story) } 5 | end 6 | -------------------------------------------------------------------------------- /spec/models/story_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe Story do 4 | it { should have_many(:chapters).dependent(:destroy) } 5 | it { should have_many(:documents).dependent(:destroy) } 6 | it { should have_many(:images).dependent(:destroy) } 7 | it { should have_one(:request) } 8 | 9 | before :all do 10 | @story = create(:story) 11 | @image1 = create(:image, story_id: @story.id, source_url: 'url1') 12 | @image2 = create(:image, story_id: @story.id, source_url: 'url2') 13 | @cover_image = create(:cover_image, story_id: @story.id, source_url: 'url2') 14 | @smilies = create(:image, story_id: @story.id, source_url: 'styles/sv_smiles') 15 | end 16 | 17 | describe '#build' do 18 | before :each do 19 | allow_any_instance_of(Document).to receive(:build) 20 | allow_any_instance_of(Document).to receive(:sanitize_filename) 21 | end 22 | 23 | it "creates a Document" do 24 | expect{@story.build('html')}.to change(Document, :count).by(1) 25 | end 26 | 27 | it "sets Document attributes correctly" do 28 | document = Document.find(@story.build('html')) 29 | expect(document.story_id).to eq(@story.id) 30 | expect(document.filename).to eq(@story.title) 31 | expect(document.extension).to eq('html') 32 | end 33 | 34 | it "returns the doc id" do 35 | expect(@story.build('html')).to eq(Document.last.id) 36 | end 37 | end 38 | 39 | describe '#cover_image' do 40 | it "returns the cover image" do 41 | expect(@story.cover_image).to eq(@cover_image) 42 | end 43 | end 44 | describe '#has_image' do 45 | it "returns the image at specified url" do 46 | expect(@story.has_image('url1')).to eq(@image1) 47 | end 48 | it "does not return a cover image" do 49 | expect(@story.has_image('url2')).to eq(@image2) 50 | end 51 | it "returns nil if no match found" do 52 | expect(@story.has_image('url3')).to be_nil 53 | end 54 | end 55 | 56 | describe '#add_domain' do 57 | it "sets domain based on url" do 58 | expect(create(:ffn_story).domain).to eq('ffn') 59 | expect(create(:fp_story).domain).to eq('fp') 60 | expect(create(:sv_story).domain).to eq('sv') 61 | expect(create(:sb_story).domain).to eq('sb') 62 | expect(create(:qq_story).domain).to eq('qq') 63 | expect(create(:story, url: 'bad_url').domain).to be_nil 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/rails_helper.rb: -------------------------------------------------------------------------------- 1 | # This file is copied to spec/ when you run 'rails generate rspec:install' 2 | ENV['RAILS_ENV'] ||= 'test' 3 | require File.expand_path('../../config/environment', __FILE__) 4 | # Prevent database truncation if the environment is production 5 | abort("The Rails environment is running in production mode!") if Rails.env.production? 6 | require 'spec_helper' 7 | require 'rspec/rails' 8 | require 'capybara/rails' 9 | Dir[Rails.root.join("spec/support/**/*.rb")].each {|f| require f} 10 | 11 | # Add additional requires below this line. Rails is not loaded until this point! 12 | 13 | # Requires supporting ruby files with custom matchers and macros, etc, in 14 | # spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are 15 | # run as spec files by default. This means that files in spec/support that end 16 | # in _spec.rb will both be required and run as specs, causing the specs to be 17 | # run twice. It is recommended that you do not name files matching this glob to 18 | # end with _spec.rb. You can configure this pattern with the --pattern 19 | # option on the command line or in ~/.rspec, .rspec or `.rspec-local`. 20 | # 21 | # The following line is provided for convenience purposes. It has the downside 22 | # of increasing the boot-up time by auto-requiring all files in the support 23 | # directory. Alternatively, in the individual `*_spec.rb` files, manually 24 | # require only the support files necessary. 25 | # 26 | # Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f } 27 | 28 | # Checks for pending migration and applies them before tests are run. 29 | # If you are not using ActiveRecord, you can remove this line. 30 | ActiveRecord::Migration.maintain_test_schema! 31 | 32 | Shoulda::Matchers.configure do |config| 33 | config.integrate do |with| 34 | with.test_framework :rspec 35 | with.library :rails 36 | end 37 | end 38 | 39 | RSpec.configure do |config| 40 | # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures 41 | #config.fixture_path = "#{::Rails.root}/spec/fixtures" 42 | config.include FactoryGirl::Syntax::Methods 43 | config.include DocumentMacros 44 | config.include FormMacros 45 | 46 | # If you're not using ActiveRecord, or you'd prefer not to run each of your 47 | # examples within a transaction, remove the following line or assign false 48 | # instead of true. 49 | config.use_transactional_fixtures = true 50 | 51 | # RSpec Rails can automatically mix in different behaviours to your tests 52 | # based on their file location, for example enabling you to call `get` and 53 | # `post` in specs under `spec/controllers`. 54 | # 55 | # You can disable this behaviour by removing the line below, and instead 56 | # explicitly tag your specs with their type, e.g.: 57 | # 58 | # RSpec.describe UsersController, :type => :controller do 59 | # # ... 60 | # end 61 | # 62 | # The different available types are documented in the features, such as in 63 | # https://relishapp.com/rspec/rspec-rails/docs 64 | config.infer_spec_type_from_file_location! 65 | 66 | # Filter lines from Rails gems in backtraces. 67 | config.filter_rails_from_backtrace! 68 | # arbitrary gems may also be filtered via: 69 | # config.filter_gems_from_backtrace("gem name") 70 | end 71 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'capybara/rspec' 2 | 3 | ENV['RAILS_ENV'] ||= 'test' 4 | 5 | Capybara.register_driver :chrome do |app| 6 | Selenium::WebDriver::Chrome.driver_path = "/usr/bin/chromedriver" 7 | Capybara::Selenium::Driver.new(app, browser: :chrome) 8 | end 9 | 10 | Capybara.javascript_driver = :chrome 11 | Capybara.app_host = 'http://localhost:3000' 12 | Capybara.server_host = "localhost" 13 | Capybara.server_port = "3001" 14 | Capybara.default_max_wait_time = 300 15 | # This file was generated by the `rails generate rspec:install` command. Conventionally, all 16 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 17 | # The generated `.rspec` file contains `--require spec_helper` which will cause 18 | # this file to always be loaded, without a need to explicitly require it in any 19 | # files. 20 | # 21 | # Given that it is always loaded, you are encouraged to keep this file as 22 | # light-weight as possible. Requiring heavyweight dependencies from this file 23 | # will add to the boot time of your test suite on EVERY test run, even for an 24 | # individual file that may not need all of that loaded. Instead, consider making 25 | # a separate helper file that requires the additional dependencies and performs 26 | # the additional setup, and require it from the spec files that actually need 27 | # it. 28 | # 29 | # The `.rspec` file also contains a few flags that are not defaults but that 30 | # users commonly want. 31 | # 32 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 33 | RSpec.configure do |config| 34 | 35 | config.before(:suite) do 36 | Rails.application.load_seed 37 | end 38 | # rspec-expectations config goes here. You can use an alternate 39 | # assertion/expectation library such as wrong or the stdlib/minitest 40 | # assertions if you prefer. 41 | config.expect_with :rspec do |expectations| 42 | # This option will default to `true` in RSpec 4. It makes the `description` 43 | # and `failure_message` of custom matchers include text for helper methods 44 | # defined using `chain`, e.g.: 45 | # be_bigger_than(2).and_smaller_than(4).description 46 | # # => "be bigger than 2 and smaller than 4" 47 | # ...rather than: 48 | # # => "be bigger than 2" 49 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 50 | end 51 | 52 | # rspec-mocks config goes here. You can use an alternate test double 53 | # library (such as bogus or mocha) by changing the `mock_with` option here. 54 | config.mock_with :rspec do |mocks| 55 | # Prevents you from mocking or stubbing a method that does not exist on 56 | # a real object. This is generally recommended, and will default to 57 | # `true` in RSpec 4. 58 | mocks.verify_partial_doubles = true 59 | end 60 | 61 | # This option will default to `:apply_to_host_groups` in RSpec 4 (and will 62 | # have no way to turn it off -- the option exists only for backwards 63 | # compatibility in RSpec 3). It causes shared context metadata to be 64 | # inherited by the metadata hash of host groups and examples, rather than 65 | # triggering implicit auto-inclusion in groups with matching metadata. 66 | config.shared_context_metadata_behavior = :apply_to_host_groups 67 | 68 | # The settings below are suggested to provide a good initial experience 69 | # with RSpec, but feel free to customize to your heart's content. 70 | =begin 71 | # This allows you to limit a spec run to individual examples or groups 72 | # you care about by tagging them with `:focus` metadata. When nothing 73 | # is tagged with `:focus`, all examples get run. RSpec also provides 74 | # aliases for `it`, `describe`, and `context` that include `:focus` 75 | # metadata: `fit`, `fdescribe` and `fcontext`, respectively. 76 | config.filter_run_when_matching :focus 77 | 78 | # Allows RSpec to persist some state between runs in order to support 79 | # the `--only-failures` and `--next-failure` CLI options. We recommend 80 | # you configure your source control system to ignore this file. 81 | config.example_status_persistence_file_path = "spec/examples.txt" 82 | 83 | # Limits the available syntax to the non-monkey patched syntax that is 84 | # recommended. For more details, see: 85 | # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ 86 | # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 87 | # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode 88 | config.disable_monkey_patching! 89 | 90 | # Many RSpec users commonly either run the entire suite or an individual 91 | # file, and it's useful to allow more verbose output when running an 92 | # individual spec file. 93 | if config.files_to_run.one? 94 | # Use the documentation formatter for detailed output, 95 | # unless a formatter has already been configured 96 | # (e.g. via a command-line flag). 97 | config.default_formatter = 'doc' 98 | end 99 | 100 | # Print the 10 slowest examples and example groups at the 101 | # end of the spec run, to help surface which specs are running 102 | # particularly slow. 103 | config.profile_examples = 10 104 | 105 | # Run specs in random order to surface order dependencies. If you find an 106 | # order dependency and want to debug it, you can fix the order by providing 107 | # the seed, which is printed after each run. 108 | # --seed 1234 109 | config.order = :random 110 | 111 | # Seed global randomization in this process using the `--seed` CLI option. 112 | # Setting this allows you to use `--seed` to deterministically reproduce 113 | # test failures related to randomization by passing the same `--seed` value 114 | # as the one that triggered the failure. 115 | Kernel.srand config.seed 116 | =end 117 | end 118 | 119 | module FormatterOverrides 120 | def dump_pending(_) 121 | end 122 | end 123 | 124 | RSpec::Core::Formatters::DocumentationFormatter.prepend FormatterOverrides 125 | -------------------------------------------------------------------------------- /spec/support/document_macros.rb: -------------------------------------------------------------------------------- 1 | module DocumentMacros 2 | def build_doc_with_name(name) 3 | doc = build(:document, filename: name) 4 | doc.sanitize_filename 5 | doc 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/support/form_macros.rb: -------------------------------------------------------------------------------- 1 | module FormMacros 2 | def get_story(url, recent: false, extension: nil, count: nil) 3 | fill_in('q', with: url) 4 | recent ? choose('Get most recent') : choose('Get all chapters') 5 | select extension, from: 'ext' if extension 6 | select count, from: 'recent_number' if count 7 | click_button 'Get' 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /spec/support/images/dancing_banana.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/spec/support/images/dancing_banana.gif -------------------------------------------------------------------------------- /spec/support/images/dice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/spec/support/images/dice.png -------------------------------------------------------------------------------- /spec/support/images/lake.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/spec/support/images/lake.jpg -------------------------------------------------------------------------------- /tmp/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/tmp/.keep -------------------------------------------------------------------------------- /vendor/assets/javascripts/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/vendor/assets/javascripts/.keep -------------------------------------------------------------------------------- /vendor/assets/stylesheets/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valedan/omnibuser/c413b67562b07974f7296b5d82a8efb95ee4ac3b/vendor/assets/stylesheets/.keep --------------------------------------------------------------------------------