├── log └── .keep ├── app ├── mailers │ └── .keep ├── models │ ├── .keep │ ├── concerns │ │ ├── .keep │ │ └── ondemand_retry.rb │ ├── wikipedia_category_item.rb │ ├── agonp_program.rb │ ├── onsen_program.rb │ ├── anitama_program.rb │ ├── hibiki_program.rb │ ├── hibiki_program_v2.rb │ ├── niconico_video_program.rb │ ├── key_value.rb │ ├── settings.rb │ ├── niconico_live_program.rb │ └── job.rb ├── assets │ ├── images │ │ └── .keep │ ├── javascripts │ │ └── application.js │ └── stylesheets │ │ └── application.css ├── controllers │ ├── concerns │ │ └── .keep │ └── application_controller.rb ├── helpers │ └── application_helper.rb └── views │ └── layouts │ └── application.html.erb ├── lib ├── tasks │ ├── .keep │ └── main.rake ├── niconico_video │ ├── downloading.rb │ └── scraping.rb ├── onsen │ ├── downloading.rb │ └── scraping.rb ├── radiru │ ├── scraping.rb │ └── recording.rb ├── wikipedia │ └── scraping.rb ├── hibiki │ ├── scraping.rb │ └── downloading.rb ├── radiko │ ├── scraping.rb │ └── recording.rb ├── ag │ ├── recording.rb │ └── scraping.rb ├── agonp │ ├── scraping.rb │ └── downloading.rb ├── niconico_live │ ├── scraping.rb │ └── downloading.rb ├── main │ ├── workaround.rb │ └── main.rb └── main.rb ├── public ├── favicon.ico ├── robots.txt ├── 500.html ├── 422.html └── 404.html ├── test ├── helpers │ └── .keep ├── mailers │ └── .keep ├── models │ ├── .keep │ ├── hibiki_program_2nd_gen_test.rb │ ├── wikipedia_category_item_test.rb │ ├── job_test.rb │ ├── hibiki_program_test.rb │ └── onsen_program_test.rb ├── controllers │ └── .keep ├── fixtures │ ├── .keep │ ├── hibiki_program_2nd_gens.yml │ ├── wikipedia_category_items.yml │ ├── jobs.yml │ ├── hibiki_programs.yml │ └── onsen_programs.yml ├── integration │ └── .keep └── test_helper.rb ├── .dockerignore ├── vendor └── assets │ ├── javascripts │ └── .keep │ └── stylesheets │ └── .keep ├── niconico ├── Rakefile ├── .gitignore ├── lib │ ├── niconico │ │ ├── version.rb │ │ ├── live │ │ │ ├── client │ │ │ │ ├── search_result.rb │ │ │ │ └── search_filters.rb │ │ │ ├── util.rb │ │ │ ├── client.rb │ │ │ ├── mypage.rb │ │ │ └── api.rb │ │ ├── channel.rb │ │ ├── mylist.rb │ │ ├── ranking.rb │ │ ├── deferrable.rb │ │ ├── nico_api.rb │ │ ├── video.rb │ │ └── live.rb │ └── niconico.rb ├── Gemfile ├── CONTRIBUTING.md ├── README.mkd └── niconico.gemspec ├── .gitmodules ├── bin ├── bundle ├── rake ├── rails └── spring ├── config ├── initializers │ ├── cookies_serializer.rb │ ├── session_store.rb │ ├── mime_types.rb │ ├── filter_parameter_logging.rb │ ├── assets.rb │ ├── backtrace_silencers.rb │ ├── wrap_parameters.rb │ └── inflections.rb ├── environment.rb ├── boot.rb ├── database.example.yml ├── locales │ └── en.yml ├── secrets.yml ├── deploy.rb ├── application.rb ├── environments │ ├── development.rb │ ├── test.rb │ └── production.rb ├── routes.rb ├── schedule.rb └── settings.example.yml ├── config.ru ├── Rakefile ├── db ├── migrate │ ├── 20151030150712_add_column_onsen_retry.rb │ ├── 20150602100911_nico_add_column.rb │ ├── 20150607130411_create_key_value.rb │ ├── 20150527135411_create_niconico_live_programs.rb │ ├── 20151231134624_create_wikipedia_category_items.rb │ ├── 20150515174349_create_job.rb │ ├── 20171105142758_create_niconico_video_programs.rb │ ├── 20150515174949_create_hibiki_program.rb │ ├── 20150521174524_create_anitama_program.rb │ ├── 20150515174821_create_onsen_program.rb │ ├── 20170505142758_create_agonp_programs.rb │ ├── 20151002075432_create_agon_program.rb │ └── 20151110023306_create_hibiki_program_v2.rb ├── seeds.rb └── schema.rb ├── .gitignore ├── LICENSE ├── Gemfile ├── Dockerfile ├── README.md └── Gemfile.lock /log/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/mailers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/tasks/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/helpers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/mailers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/assets/images/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/controllers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/integration/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/models/concerns/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | vendor 3 | -------------------------------------------------------------------------------- /app/controllers/concerns/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vendor/assets/javascripts/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vendor/assets/stylesheets/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /niconico/Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | -------------------------------------------------------------------------------- /niconico/.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | Gemfile.lock 4 | pkg/* 5 | -------------------------------------------------------------------------------- /app/helpers/application_helper.rb: -------------------------------------------------------------------------------- 1 | module ApplicationHelper 2 | end 3 | -------------------------------------------------------------------------------- /niconico/lib/niconico/version.rb: -------------------------------------------------------------------------------- 1 | class Niconico 2 | VERSION = "1.8.0.beta1" 3 | end 4 | -------------------------------------------------------------------------------- /app/models/wikipedia_category_item.rb: -------------------------------------------------------------------------------- 1 | class WikipediaCategoryItem < ActiveRecord::Base 2 | end 3 | -------------------------------------------------------------------------------- /app/models/agonp_program.rb: -------------------------------------------------------------------------------- 1 | class AgonpProgram < ActiveRecord::Base 2 | include OndemandRetry 3 | end 4 | -------------------------------------------------------------------------------- /app/models/onsen_program.rb: -------------------------------------------------------------------------------- 1 | class OnsenProgram < ActiveRecord::Base 2 | include OndemandRetry 3 | end 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "niconico"] 2 | path = niconico 3 | url = https://github.com/yayugu/niconico.git 4 | -------------------------------------------------------------------------------- /app/models/anitama_program.rb: -------------------------------------------------------------------------------- 1 | class AnitamaProgram < ActiveRecord::Base 2 | include OndemandRetry 3 | end 4 | -------------------------------------------------------------------------------- /app/models/hibiki_program.rb: -------------------------------------------------------------------------------- 1 | class HibikiProgram < ActiveRecord::Base 2 | include OndemandRetry 3 | end 4 | -------------------------------------------------------------------------------- /app/models/hibiki_program_v2.rb: -------------------------------------------------------------------------------- 1 | class HibikiProgramV2 < ActiveRecord::Base 2 | include OndemandRetry 3 | end 4 | -------------------------------------------------------------------------------- /app/models/niconico_video_program.rb: -------------------------------------------------------------------------------- 1 | class NiconicoVideoProgram < ActiveRecord::Base 2 | include OndemandRetry 3 | end 4 | -------------------------------------------------------------------------------- /niconico/Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | # Specify your gem's dependencies in niconico.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /app/models/key_value.rb: -------------------------------------------------------------------------------- 1 | class KeyValue < ActiveRecord::Base 2 | self.table_name = 'key_value' 3 | self.primary_key = 'key' 4 | end 5 | -------------------------------------------------------------------------------- /bin/bundle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 3 | load Gem.bin_path('bundler', 'bundle') 4 | -------------------------------------------------------------------------------- /app/models/settings.rb: -------------------------------------------------------------------------------- 1 | class Settings < Settingslogic 2 | source "#{Rails.root}/config/settings.yml" 3 | namespace Rails.env 4 | suppress_errors true 5 | end 6 | -------------------------------------------------------------------------------- /config/initializers/cookies_serializer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.action_dispatch.cookies_serializer = :json -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | # This file is used by Rack-based servers to start the application. 2 | 3 | require ::File.expand_path('../config/environment', __FILE__) 4 | run Rails.application 5 | -------------------------------------------------------------------------------- /config/environment.rb: -------------------------------------------------------------------------------- 1 | # Load the Rails application. 2 | require File.expand_path('../application', __FILE__) 3 | 4 | # Initialize the Rails application. 5 | Rails.application.initialize! 6 | -------------------------------------------------------------------------------- /config/initializers/session_store.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.session_store :cookie_store, key: '_net-radio-archive_session' 4 | -------------------------------------------------------------------------------- /config/boot.rb: -------------------------------------------------------------------------------- 1 | # Set up gems listed in the Gemfile. 2 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 3 | 4 | require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE']) 5 | -------------------------------------------------------------------------------- /test/models/hibiki_program_2nd_gen_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class HibikiProgram2ndGenTest < ActiveSupport::TestCase 4 | # test "the truth" do 5 | # assert true 6 | # end 7 | end 8 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | begin 3 | load File.expand_path("../spring", __FILE__) 4 | rescue LoadError 5 | end 6 | require_relative '../config/boot' 7 | require 'rake' 8 | Rake.application.run 9 | -------------------------------------------------------------------------------- /config/initializers/mime_types.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new mime types for use in respond_to blocks: 4 | # Mime::Type.register "text/richtext", :rtf 5 | -------------------------------------------------------------------------------- /test/models/wikipedia_category_item_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class WikipediaCategoryItemTest < ActiveSupport::TestCase 4 | # test "the truth" do 5 | # assert true 6 | # end 7 | end 8 | -------------------------------------------------------------------------------- /niconico/lib/niconico/live/client/search_result.rb: -------------------------------------------------------------------------------- 1 | class Niconico 2 | class Live 3 | class Client 4 | class SearchResult < Struct.new(:id, :title, :description) 5 | end 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | # 3 | # To ban all spiders from the entire site uncomment the next two lines: 4 | # User-agent: * 5 | # Disallow: / 6 | -------------------------------------------------------------------------------- /config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Configure sensitive parameters which will be filtered from the log file. 4 | Rails.application.config.filter_parameters += [:password] 5 | -------------------------------------------------------------------------------- /app/models/niconico_live_program.rb: -------------------------------------------------------------------------------- 1 | class NiconicoLiveProgram < ActiveRecord::Base 2 | STATE = { 3 | waiting: 'waiting', 4 | downloading: 'downloading', 5 | done: 'done', 6 | failed: 'failed', 7 | } 8 | RETRY_LIMIT = 3 9 | end 10 | -------------------------------------------------------------------------------- /bin/rails: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | begin 3 | load File.expand_path("../spring", __FILE__) 4 | rescue LoadError 5 | end 6 | APP_PATH = File.expand_path('../../config/application', __FILE__) 7 | require_relative '../config/boot' 8 | require 'rails/commands' 9 | -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | class ApplicationController < ActionController::Base 2 | # Prevent CSRF attacks by raising an exception. 3 | # For APIs, you may want to use :null_session instead. 4 | protect_from_forgery with: :exception 5 | end 6 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # Add your own tasks in files placed in lib/tasks ending in .rake, 2 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 3 | 4 | require File.expand_path('../config/application', __FILE__) 5 | 6 | Rails.application.load_tasks 7 | -------------------------------------------------------------------------------- /app/models/concerns/ondemand_retry.rb: -------------------------------------------------------------------------------- 1 | module OndemandRetry 2 | STATE = { 3 | waiting: 'waiting', 4 | downloading: 'downloading', 5 | done: 'done', 6 | failed: 'failed', 7 | not_downloadable: 'not_downloadable', 8 | outdated: 'outdated', 9 | } 10 | RETRY_LIMIT = 3 11 | end 12 | -------------------------------------------------------------------------------- /niconico/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | - __You should use English__ for commits, pull request's title, description, and comments basically. 2 | 3 | - I'll ask you to do translation if you've posted in Japanese. 4 | - Don't worry for poor English :) 5 | - 基本英語でコミットコメントを残し、pull request のタイトル、コメントを書いてください。 6 | - 日本語で書いた場合英語にするようにお願いをしています。 7 | 8 | -------------------------------------------------------------------------------- /db/migrate/20151030150712_add_column_onsen_retry.rb: -------------------------------------------------------------------------------- 1 | class AddColumnOnsenRetry < ActiveRecord::Migration 2 | def up 3 | sql = < 2 | 3 | 4 | NetRadioArchive 5 | <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track' => true %> 6 | <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %> 7 | <%= csrf_meta_tags %> 8 | 9 | 10 | 11 | <%= yield %> 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /config/initializers/assets.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Version of your assets, change this if you want to expire all your assets. 4 | Rails.application.config.assets.version = '1.0' 5 | 6 | # Precompile additional assets. 7 | # application.js, application.css, and all non-JS/CSS in app/assets folder are already added. 8 | # Rails.application.config.assets.precompile += %w( search.js ) 9 | -------------------------------------------------------------------------------- /test/models/hibiki_program_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class HibikiProgramTest < ActiveSupport::TestCase 4 | test "save" do 5 | p = HibikiProgram.new 6 | p.title = 'hibiki test title' 7 | p.comment = '第123回 1月23日更新!' 8 | p.rtmp_url = 'rtmpe://test.hibiki_program' 9 | p.state = HibikiProgram::STATE[:waiting] 10 | p.retry_count = 0 11 | assert p.save, p.errors.messages 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /test/fixtures/hibiki_program_2nd_gens.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | # This model initially had no columns defined. If you add columns to the 4 | # model remove the '{}' from the fixture names and add the columns immediately 5 | # below each fixture, per the syntax in the comments below 6 | # 7 | one: {} 8 | # column: value 9 | # 10 | two: {} 11 | # column: value 12 | -------------------------------------------------------------------------------- /test/fixtures/wikipedia_category_items.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | # This model initially had no columns defined. If you add columns to the 4 | # model remove the '{}' from the fixture names and add the columns immediately 5 | # below each fixture, per the syntax in the comments below 6 | # 7 | one: {} 8 | # column: value 9 | # 10 | two: {} 11 | # column: value 12 | -------------------------------------------------------------------------------- /test/models/onsen_program_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class OnsenProgramTest < ActiveSupport::TestCase 4 | test "save" do 5 | p = OnsenProgram.new 6 | p.title = 'onsen test title' 7 | p.number = 1 8 | p.date = DateTime.now 9 | p.file_url = 'http://test.onsen_program.com' 10 | p.personality = '花澤香菜' 11 | p.state = OnsenProgram::STATE[:waiting] 12 | assert p.save, p.errors.messages 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /niconico/README.mkd: -------------------------------------------------------------------------------- 1 | # Niconico - yet another nicovideo.gem 2 | 3 | ## Description 4 | 5 | Wrapper of `Mechanize`, optimized for . 6 | 7 | ## Feature 8 | 9 | * Login to nicovideo 10 | * Retrive a ranking page 11 | * Download a video 12 | 13 | ## Requirements 14 | 15 | * Ruby 2.0.0+ (2.1+ is supported) 16 | 17 | ## Install 18 | 19 | $ gem install niconico 20 | 21 | ## Author 22 | 23 | * Shota Fukumori (sora\_h) 24 | -------------------------------------------------------------------------------- /config/initializers/backtrace_silencers.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. 4 | # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } 5 | 6 | # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. 7 | # Rails.backtrace_cleaner.remove_silencers! 8 | -------------------------------------------------------------------------------- /db/migrate/20150602100911_nico_add_column.rb: -------------------------------------------------------------------------------- 1 | class NicoAddColumn < ActiveRecord::Migration 2 | def up 3 | sql = < 9 | start: <%= DateTime.now %> 10 | end: <%= DateTime.now + Rational(1, 24) %> 11 | title: 'one title' 12 | state: <%= Job::STATE[:scheduled] %> 13 | two: 14 | ch: 'QRR' 15 | start: <%= DateTime.now %> 16 | end: <%= DateTime.now + Rational(1, 24) %> 17 | title: 'two title' 18 | state: <%= Job::STATE[:scheduled] %> 19 | -------------------------------------------------------------------------------- /db/migrate/20150527135411_create_niconico_live_programs.rb: -------------------------------------------------------------------------------- 1 | class CreateNiconicoLivePrograms < ActiveRecord::Migration 2 | def up 3 | sql = < 12 | retry_count: 0 13 | two: 14 | title: 'two title' 15 | comment: 'two comment' 16 | rtmp_url: 'rtmpe://test.two.hibiki_program' 17 | state: <%= HibikiProgram::STATE[:waiting] %> 18 | retry_count: 0 19 | -------------------------------------------------------------------------------- /config/initializers/inflections.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new inflection rules using the following format. Inflections 4 | # are locale specific, and you may define rules for as many different 5 | # locales as you wish. All of these examples are active by default: 6 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 7 | # inflect.plural /^(ox)$/i, '\1en' 8 | # inflect.singular /^(ox)en/i, '\1' 9 | # inflect.irregular 'person', 'people' 10 | # inflect.uncountable %w( fish sheep ) 11 | # end 12 | 13 | # These inflection rules are supported but not enabled by default: 14 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 15 | # inflect.acronym 'RESTful' 16 | # end 17 | -------------------------------------------------------------------------------- /config/database.example.yml: -------------------------------------------------------------------------------- 1 | # SQLite version 3.x 2 | # gem install sqlite3 3 | # 4 | # Ensure the SQLite 3 gem is defined in your Gemfile 5 | # gem 'sqlite3' 6 | # 7 | default: &default 8 | adapter: mysql2 9 | encoding: utf8mb4 10 | pool: 5 11 | username: root 12 | password: 13 | host: localhost 14 | 15 | development: 16 | <<: *default 17 | database: net_radio_archive_dev 18 | 19 | # Warning: The database defined as "test" will be erased and 20 | # re-generated from your development database when you run "rake". 21 | # Do not set this db to the same as development or production. 22 | test: 23 | <<: *default 24 | database: net_radio_archive_test 25 | 26 | production: 27 | <<: *default 28 | database: net_radio_archive 29 | -------------------------------------------------------------------------------- /config/locales/en.yml: -------------------------------------------------------------------------------- 1 | # Files in the config/locales directory are used for internationalization 2 | # and are automatically loaded by Rails. If you want to use locales other 3 | # than English, add the necessary files in this directory. 4 | # 5 | # To use the locales, use `I18n.t`: 6 | # 7 | # I18n.t 'hello' 8 | # 9 | # In views, this is aliased to just `t`: 10 | # 11 | # <%= t('hello') %> 12 | # 13 | # To use a different locale, set it with `I18n.locale`: 14 | # 15 | # I18n.locale = :es 16 | # 17 | # This would use the information in config/locales/es.yml. 18 | # 19 | # To learn more, please read the Rails Internationalization guide 20 | # available at http://guides.rubyonrails.org/i18n.html. 21 | 22 | en: 23 | hello: "Hello world" 24 | -------------------------------------------------------------------------------- /app/assets/javascripts/application.js: -------------------------------------------------------------------------------- 1 | // This is a manifest file that'll be compiled into application.js, which will include all the files 2 | // listed below. 3 | // 4 | // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts, 5 | // or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path. 6 | // 7 | // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the 8 | // compiled file. 9 | // 10 | // Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details 11 | // about supported directives. 12 | // 13 | //= require jquery 14 | //= require jquery_ujs 15 | //= require turbolinks 16 | //= require_tree . 17 | -------------------------------------------------------------------------------- /app/assets/stylesheets/application.css: -------------------------------------------------------------------------------- 1 | /* 2 | * This is a manifest file that'll be compiled into application.css, which will include all the files 3 | * listed below. 4 | * 5 | * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets, 6 | * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path. 7 | * 8 | * You're free to add application-wide styles to this file and they'll appear at the bottom of the 9 | * compiled file so the styles you add here take precedence over styles defined in any styles 10 | * defined in the other CSS/SCSS files in this directory. It is generally better to create a new 11 | * file per style scope. 12 | * 13 | *= require_tree . 14 | *= require_self 15 | */ 16 | -------------------------------------------------------------------------------- /test/fixtures/onsen_programs.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | # This model initially had no columns defined. If you add columns to the 4 | # model remove the '{}' from the fixture names and add the columns immediately 5 | # below each fixture, per the syntax in the comments below 6 | # 7 | one: 8 | title: 'one title' 9 | number: 1 10 | date: <%= DateTime.now %> 11 | file_url: 'http://test.one.onsen_program.com' 12 | personality: '花澤香菜' 13 | state: <%= OnsenProgram::STATE[:waiting] %> 14 | two: 15 | title: 'two title' 16 | number: 1 17 | date: <%= DateTime.now %> 18 | file_url: 'http://test.two.onsen_program.com' 19 | personality: '花澤香菜' 20 | state: <%= OnsenProgram::STATE[:waiting] %> 21 | -------------------------------------------------------------------------------- /lib/niconico_video/downloading.rb: -------------------------------------------------------------------------------- 1 | module NiconicoVideo 2 | class Downloading 3 | CH_NAME = 'nicodou' 4 | 5 | def download(program) 6 | Main::prepare_working_dir(CH_NAME) 7 | path = filepath(program) 8 | begin 9 | %Q(youtube-dl -v -f "[vbr<599]+[abr>127]" -u#{username} -p#{password} http://www.nicovideo.jp/watch/1509345736) 10 | rescue => e 11 | Rails.logger.error e 12 | return false 13 | end 14 | Main::move_to_archive_dir(CH_NAME, program.update_time, path) 15 | true 16 | end 17 | 18 | def filepath(program) 19 | date = program.update_time.strftime('%Y_%m_%d') 20 | title = "#{date}_#{program.title}" 21 | Main::file_path_working(CH_NAME, title, 'mp3') 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /niconico/niconico.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "niconico/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "niconico" 7 | s.version = Niconico::VERSION 8 | s.authors = ["Shota Fukumori (sora_h)"] 9 | s.email = ["her@sorah.jp"] 10 | s.homepage = "" 11 | s.summary = "wrapper of Mechanize, optimized for nicovideo." 12 | s.description = "wrapper of Mechanize, optimized for nicovideo. :)" 13 | 14 | s.add_dependency "mechanize", '>= 2.7.3' 15 | s.add_dependency "nokogiri", '>= 1.6.1' 16 | 17 | s.files = Dir['**/*'] 18 | s.test_files = Dir['{test,spec,features}/**/*'] 19 | s.executables = Dir['bin/*'].map{ |f| File.basename(f) } 20 | s.require_paths = ["lib"] 21 | end 22 | -------------------------------------------------------------------------------- /db/migrate/20151231134624_create_wikipedia_category_items.rb: -------------------------------------------------------------------------------- 1 | class CreateWikipediaCategoryItems < ActiveRecord::Migration 2 | def up 3 | # 検索用キーワードとして抽出するため 4 | # あまり長いワードは取得できても意味がないためvarcharの長さ制限は短めに 5 | sql = < 23 | -------------------------------------------------------------------------------- /config/deploy.rb: -------------------------------------------------------------------------------- 1 | lock '3.4.0' 2 | 3 | set :application, 'net-radio-archive' 4 | set :scm, :copy 5 | set :exclude_dir, %w|vendor/bundle .git/ .bundle/ log/* test/| 6 | 7 | set :log_level, :debug 8 | 9 | set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'tmp/working', 'vendor/bundle', 'public/system') 10 | 11 | set :keep_releases, 5 12 | 13 | namespace :deploy do 14 | desc 'Runs rake db:create if migrations are set' 15 | task :db_create => [:set_rails_env] do 16 | on primary fetch(:migration_role) do 17 | info '[deploy:db_create] Checking first deploy' 18 | if test("[ -d #{current_path} ]") 19 | info '[deploy:db_create] Skip `deploy:db_create` (not first deploy)' 20 | else 21 | info '[deploy:db_create] Run `rake db:create`' 22 | within release_path do 23 | with rails_env: fetch(:rails_env) do 24 | execute :rake, "db:create" 25 | end 26 | end 27 | end 28 | end 29 | end 30 | before 'deploy:migrate', 'deploy:db_create' 31 | end 32 | -------------------------------------------------------------------------------- /lib/niconico_video/scraping.rb: -------------------------------------------------------------------------------- 1 | require 'time' 2 | require 'mechanize' 3 | 4 | module NiconicoVideo 5 | ProgramBase = Struct.new(:video_id, :title) 6 | class Program < ProgramBase 7 | end 8 | 9 | class Scraping 10 | def initialize 11 | @a = Mechanize.new 12 | @a.user_agent_alias = 'Windows Chrome' 13 | end 14 | 15 | def main 16 | programs = [] 17 | if Settings.niconico.video.channels 18 | Settings.niconico.video.channels.each do |ch| 19 | programs += channel_videos(ch) 20 | end 21 | end 22 | programs 23 | end 24 | 25 | # https://github.com/sorah/niconico/blob/9379c60f08d0c811fcde3f0c0b41c0a579cc2633/lib/niconico/channel.rb#L8 26 | def channel_videos(ch) 27 | rss = Nokogiri::XML(open("http://ch.nicovideo.jp/#{ch}/video?rss=2.0", &:read)) 28 | 29 | rss.search('channel item').map do |item| 30 | title = item.at('title').inner_text 31 | link = item.at('link').inner_text 32 | Program.new(link.sub(/^.+\/watch\//, ''), title) 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/radiru/scraping.rb: -------------------------------------------------------------------------------- 1 | require 'date' 2 | require 'net/https' 3 | require 'json' 4 | 5 | module Radiru 6 | class Program < Struct.new(:start_time, :end_time, :title) 7 | end 8 | 9 | class Scraping 10 | def get(ch) 11 | id = get_id(ch) 12 | json = get_programs_json(id) 13 | parse(id, json) 14 | end 15 | 16 | def get_id(ch) 17 | case ch 18 | when 'r1' 19 | 'n1' 20 | when 'r2' 21 | 'n2' 22 | when 'fm' 23 | 'n3' 24 | end 25 | end 26 | 27 | def get_programs_json(id) 28 | today = Date.today.strftime("%Y-%m-%d") 29 | res = Net::HTTP.get(URI("https://api.nhk.or.jp/r2/pg/list/4/130/#{id}/#{today}.json")) 30 | JSON.parse(res); 31 | end 32 | 33 | def parse(id, json) 34 | json['list'][id].map do |program| 35 | parse_program(program) 36 | end 37 | end 38 | 39 | def parse_program(program) 40 | Program.new( 41 | program['start_time'], 42 | program['end_time'], 43 | program['title'], 44 | ) 45 | end 46 | end 47 | end 48 | 49 | -------------------------------------------------------------------------------- /niconico/lib/niconico/ranking.rb: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | class Niconico 4 | # options[:span] -> :hourly, :daily, :weekly, :monthly, :total 5 | # or -> :hour, :day, :week, :month, :all 6 | # default: daily 7 | # 8 | # options[:method] -> :fav, :view, :comment, :mylist 9 | # (or :all) (or :res) 10 | def ranking(category = 'all', options={}) 11 | login unless logged_in? 12 | 13 | span = options[:span] || :daily 14 | span = :hourly if span == :hour 15 | span = :daily if span == :day 16 | span = :weekly if span == :week 17 | span = :monthly if span == :month 18 | span = :total if span == :all 19 | 20 | method = options[:method] || :fav 21 | method = :res if method == :comment 22 | method = :fav if method == :all 23 | 24 | page = @agent.get(url = "http://www.nicovideo.jp/ranking/#{method}/#{span}/#{category}") 25 | page.search(".ranking.itemTitle a").map do |link| 26 | Video.new(self, link['href'].sub(/^.*?watch\//,""), title: link.inner_text) 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 yayugu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/wikipedia/scraping.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'json' 3 | require 'cgi/util' 4 | 5 | module Wikipedia 6 | class Scraping 7 | def main(category) 8 | members = get_all(category) 9 | ret = members.map do |m| 10 | m['title'].gsub(/^(.*) \(.*\)$/, '\1') 11 | end.delete_if do |t| 12 | t.match(/^Category:/) || 13 | t.match(/^Template:/) 14 | end 15 | end 16 | 17 | def get_all(category) 18 | category_members = [] 19 | continue = nil 20 | loop do 21 | r = get(category, continue) 22 | category_members += r['query']['categorymembers'] 23 | continue = r 24 | .try(:[], 'continue') 25 | .try(:[], 'cmcontinue') 26 | sleep 5 27 | unless continue 28 | break 29 | end 30 | end 31 | category_members 32 | end 33 | 34 | def get(category, continue) 35 | url_str = "https://ja.wikipedia.org/w/api.php?action=query&list=categorymembers&format=json&cmlimit=500&cmtitle=#{CGI.escape('Category:' + category)}" 36 | if continue 37 | url_str += "&cmcontinue=#{CGI.escape(continue)}" 38 | end 39 | 40 | res = HTTParty.get(url_str) 41 | JSON.parse(res.body) 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /config/application.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../boot', __FILE__) 2 | 3 | require 'rails/all' 4 | 5 | # Require the gems listed in Gemfile, including any gems 6 | # you've limited to :test, :development, or :production. 7 | Bundler.require(*Rails.groups) 8 | 9 | module NetRadioArchive 10 | class Application < Rails::Application 11 | # Settings in config/environments/* take precedence over those specified here. 12 | # Application configuration should go into files in config/initializers 13 | # -- all .rb files in that directory are automatically loaded. 14 | 15 | # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone. 16 | # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC. 17 | # config.time_zone = 'Central Time (US & Canada)' 18 | 19 | # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. 20 | # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] 21 | # config.i18n.default_locale = :de 22 | config.autoload_paths += %W(#{config.root}/lib) 23 | config.watchable_dirs['lib'] = [:rb] 24 | 25 | config.time_zone = 'Tokyo' 26 | config.active_record.default_timezone = :local 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/hibiki/scraping.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'time' 3 | require 'pp' 4 | 5 | module Hibiki 6 | class Program < Struct.new(:access_id, :episode_id, :title, :episode_name, :cast) 7 | end 8 | 9 | class Scraping 10 | def initialize 11 | @a = Mechanize.new 12 | @a.user_agent_alias = 'Windows Chrome' 13 | end 14 | 15 | def main 16 | get_list.reject do |program| 17 | program.episode_id == nil 18 | end 19 | end 20 | 21 | def get_list 22 | programs = [] 23 | page = 1 24 | begin 25 | res = @a.get( 26 | "https://vcms-api.hibiki-radio.jp/api/v1//programs?limit=8&page=#{page}", 27 | [], 28 | "https://hibiki-radio.jp/", 29 | 'X-Requested-With' => 'XMLHttpRequest', 30 | 'Origin' => 'https://hibiki-radio.jp' 31 | ) 32 | raws = JSON.parse(res.body) 33 | programs += raws.map{|raw| parse_program(raw) } 34 | sleep 1 35 | page += 1 36 | end while raws.size == 8 37 | programs 38 | end 39 | 40 | def parse_program(raw) 41 | Program.new( 42 | raw['access_id'], 43 | raw['latest_episode_id'], 44 | raw['name'], 45 | raw['latest_episode_name'], 46 | raw['cast'], 47 | ) 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/radiko/scraping.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'time' 3 | require 'chronic' 4 | require 'pp' 5 | require 'moji' 6 | 7 | module Radiko 8 | class Program < Struct.new(:start_time, :end_time, :title, :performers) 9 | end 10 | 11 | class Scraping 12 | def get(ch) 13 | dom = get_programs_dom(ch) 14 | programs = parse_dom(dom) 15 | validate_programs(programs) 16 | end 17 | 18 | def get_programs_dom(ch) 19 | xml = Net::HTTP.get(URI("http://radiko.jp/v2/api/program/station/weekly?station_id=#{ch}")) 20 | Nokogiri::XML(xml) 21 | end 22 | 23 | def parse_dom(dom) 24 | dom.css('prog').map do |program| 25 | parse_program(program) 26 | end 27 | end 28 | 29 | def validate_programs(programs) 30 | programs.delete_if do |program| 31 | program.title =~ /放送休止|番組休止/ 32 | end 33 | end 34 | 35 | def parse_program(dom) 36 | start_time = parse_time(dom.attribute('ft').value) 37 | end_time = parse_time(dom.attribute('to').value) 38 | title = dom.css('title').text 39 | performers = dom.css('pfm').text 40 | Program.new( 41 | start_time, 42 | end_time, 43 | title, 44 | performers 45 | ) 46 | end 47 | 48 | def parse_time(text) 49 | Time.strptime(text, '%Y%m%d%H%M%S') 50 | end 51 | end 52 | end 53 | 54 | -------------------------------------------------------------------------------- /lib/ag/recording.rb: -------------------------------------------------------------------------------- 1 | require 'shellwords' 2 | require 'fileutils' 3 | 4 | module Ag 5 | class Recording 6 | AGQR_STREAM_URL = 'https://fms2.uniqueradio.jp/agqr10/aandg1.m3u8' 7 | CH_NAME = 'ag' 8 | 9 | def record(job) 10 | exec_rec(job) 11 | end 12 | 13 | def exec_rec(job) 14 | Main::prepare_working_dir(CH_NAME) 15 | Main::sleep_until(job.start - 10.seconds) 16 | 17 | length = job.length_sec + 90 18 | file_path = Main::file_path_working(CH_NAME, title(job), 'mp4') 19 | arg = "\ 20 | -loglevel error \ 21 | -y \ 22 | -allowed_extensions ALL \ 23 | -protocol_whitelist file,crypto,http,https,tcp,tls \ 24 | -i #{AGQR_STREAM_URL} \ 25 | -t #{length} \ 26 | -vcodec copy -acodec copy -bsf:a aac_adtstoasc \ 27 | #{Shellwords.escape(file_path)}" 28 | exit_status, output = Main::ffmpeg(arg) 29 | unless exit_status.success? 30 | Rails.logger.error "rec failed. job:#{job}, exit_status:#{exit_status}, output:#{output}" 31 | return false 32 | end 33 | if output.present? 34 | Rails.logger.warn "ag ffmpeg command:#{arg} output:#{output}" 35 | end 36 | 37 | Main::move_to_archive_dir(CH_NAME, job.start, file_path) 38 | true 39 | end 40 | 41 | def title(job) 42 | date = job.start.strftime('%Y_%m_%d_%H%M') 43 | "#{date}_#{job.title}" 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /config/environments/development.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # In the development environment your application's code is reloaded on 5 | # every request. This slows down response time but is perfect for development 6 | # since you don't have to restart the web server when you make code changes. 7 | config.cache_classes = false 8 | 9 | # Do not eager load code on boot. 10 | config.eager_load = false 11 | 12 | # Show full error reports and disable caching. 13 | config.consider_all_requests_local = true 14 | config.action_controller.perform_caching = false 15 | 16 | # Don't care if the mailer can't send. 17 | config.action_mailer.raise_delivery_errors = false 18 | 19 | # Print deprecation notices to the Rails logger. 20 | config.active_support.deprecation = :log 21 | 22 | # Raise an error on page load if there are pending migrations. 23 | config.active_record.migration_error = :page_load 24 | 25 | # Debug mode disables concatenation and preprocessing of assets. 26 | # This option may cause significant delays in view rendering with a large 27 | # number of complex assets. 28 | config.assets.debug = true 29 | 30 | # Adds additional error checking when serving assets at runtime. 31 | # Checks for improperly declared sprockets dependencies. 32 | # Raises helpful error messages. 33 | config.assets.raise_runtime_errors = true 34 | 35 | # Raises error for missing translations 36 | # config.action_view.raise_on_missing_translations = true 37 | end 38 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | 4 | # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' 5 | gem 'rails', '4.2.10' 6 | # Use sqlite3 as the database for Active Record 7 | #gem 'sqlite3' 8 | # Use SCSS for stylesheets 9 | gem 'sass-rails', '~> 4.0.3' 10 | # Use Uglifier as compressor for JavaScript assets 11 | gem 'uglifier', '>= 1.3.0' 12 | # Use CoffeeScript for .js.coffee assets and views 13 | gem 'coffee-rails', '~> 4.0.0' 14 | # See https://github.com/sstephenson/execjs#readme for more supported runtimes 15 | # gem 'therubyracer', platforms: :ruby 16 | 17 | # Use jquery as the JavaScript library 18 | gem 'jquery-rails' 19 | # Turbolinks makes following links in your web application faster. Read more: https://github.com/rails/turbolinks 20 | gem 'turbolinks' 21 | # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder 22 | gem 'jbuilder', '~> 2.0' 23 | 24 | gem 'sdoc', '~> 0.4.0', group: :doc 25 | 26 | # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring 27 | gem 'spring', group: :development 28 | 29 | # Use ActiveModel has_secure_password 30 | # gem 'bcrypt', '~> 3.1.7' 31 | 32 | # Use unicorn as the app server 33 | # gem 'unicorn' 34 | 35 | # Use debugger 36 | # gem 'debugger', group: [:development, :test] 37 | 38 | gem 'nokogiri' 39 | gem 'moji' 40 | gem 'mysql2', '~> 0.3.18' 41 | gem 'chronic' 42 | gem 'whenever' 43 | gem 'settingslogic' 44 | gem 'mechanize' 45 | gem 'niconico', path: 'niconico' 46 | gem 'httparty' 47 | gem 'activerecord-import' 48 | gem 'pry' 49 | gem 'm3u8' 50 | -------------------------------------------------------------------------------- /niconico/lib/niconico/deferrable.rb: -------------------------------------------------------------------------------- 1 | class Niconico 2 | module Deferrable 3 | module ClassMethods 4 | def deferrable(*keys) 5 | keys.each do |key| 6 | binding.eval(<<-EOM, __FILE__, __LINE__.succ) 7 | define_method(:#{key}) do 8 | get() if @#{key}.nil? && !fetched? 9 | @#{key} 10 | end 11 | EOM 12 | end 13 | self.deferred_methods.push *keys 14 | end 15 | 16 | def deferred_methods 17 | @deferred_methods ||= [] 18 | end 19 | 20 | def lazy(key, &block) 21 | define_method(key) do 22 | case 23 | when fetched? 24 | self.instance_eval &block 25 | when @preload[key] 26 | @preload[key] 27 | else 28 | get() 29 | self.instance_eval &block 30 | end 31 | end 32 | self.lazy_methods.push key 33 | end 34 | 35 | def lazy_methods 36 | @lazy_methods ||= [] 37 | end 38 | end 39 | 40 | def self.included(klass) 41 | klass.extend ClassMethods 42 | end 43 | 44 | def fetched?; @fetched; end 45 | 46 | def get 47 | @fetched = true 48 | end 49 | 50 | private 51 | 52 | def preload_deffered_values(vars={}) 53 | @preload ||= {} 54 | vars.each do |k,v| 55 | case 56 | when self.class.deferred_methods.include?(k) 57 | instance_variable_set "@#{k}", v 58 | when self.class.lazy_methods.include?(k) 59 | @preload[k] = v 60 | end 61 | end 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /lib/agonp/scraping.rb: -------------------------------------------------------------------------------- 1 | module Agonp 2 | class Program < Struct.new(:title, :personality, :episode_id, :price) 3 | end 4 | 5 | class Scraping 6 | def initialize 7 | @a = Mechanize.new 8 | @a.user_agent_alias = 'Windows Chrome' 9 | end 10 | 11 | def main 12 | filter_list(get_list) 13 | end 14 | 15 | def get_list 16 | programs = [] 17 | page = 1 18 | 2.times do |i| # 一応2ページ分ほどみておく 19 | res = @a.get("https://agonp.jp/search?order=latest&tab=episodes&page=#{i + 1}") 20 | programs += parse_programs(res) 21 | sleep 1 22 | end 23 | programs 24 | end 25 | 26 | def filter_list(programs) 27 | programs.reject do |program| 28 | program.price != '無料' 29 | end 30 | end 31 | 32 | def parse_programs(page) 33 | page.search('.search-results__list-item.row').map do |program_row| 34 | parse_program(program_row) 35 | end 36 | end 37 | 38 | def parse_program(program_row) 39 | title = program_row.css('.search-results__title').first 40 | .children 41 | .inner_text.strip 42 | .sub(/無料\s+/,'') 43 | .sub(/^\s+/,'') 44 | .sub(/\s+$/,'') 45 | episode_id = program_row.css('a.search-results__button--play-now').attr('href').text.match(/play\/(\d+)/)[1] 46 | Program.new( 47 | title, 48 | program_row.css('.search-results__personality').first.text.strip.gsub('／', ' '), 49 | episode_id, 50 | program_row.css('.search-results__price').first.text.strip 51 | ) 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/niconico_live/scraping.rb: -------------------------------------------------------------------------------- 1 | module NiconicoLive 2 | class Scraping 3 | def main 4 | setup 5 | reject_ignore_keywords(search_keyword + search_keyword_category_bulk) 6 | end 7 | 8 | def setup 9 | @n = Niconico.new(Settings.niconico.username, Settings.niconico.password) 10 | @n.login 11 | @c = @n.live_client 12 | end 13 | 14 | def search_keyword 15 | keywords = Settings.niconico.live.keywords 16 | search(keywords) 17 | end 18 | 19 | def search_keyword_category_bulk 20 | ret = [] 21 | WikipediaCategoryItem.find_in_batches(batch_size: 10).each do |batches| 22 | search_word = batches.map do |item| 23 | item.title 24 | end.join(' OR ') 25 | ret_sub = search([search_word]) 26 | ret += ret_sub 27 | sleep 10 28 | end 29 | ret 30 | end 31 | 32 | def search(keywords) 33 | keywords.inject([]) do |ret, keyword| 34 | ret_sub = @c.search( 35 | keyword, 36 | [ 37 | Niconico::Live::Client::SearchFilters::CLOSED, 38 | Niconico::Live::Client::SearchFilters::OFFICIAL, 39 | Niconico::Live::Client::SearchFilters::CHANNEL, 40 | Niconico::Live::Client::SearchFilters::HIDE_TS_EXPIRED, 41 | ] 42 | ) 43 | ret + ret_sub 44 | end 45 | end 46 | 47 | def reject_ignore_keywords(search_results) 48 | ignore_keywords = Settings.niconico.live.ignore_keywords 49 | unless ignore_keywords 50 | return search_results 51 | end 52 | search_results.reject do |r| 53 | ignore_keywords.any? do |k| 54 | r.title.include? k 55 | end 56 | end 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /public/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | We're sorry, but something went wrong (500) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

We're sorry, but something went wrong.

62 |
63 |

If you are the application owner check the logs for more information.

64 |
65 | 66 | 67 | -------------------------------------------------------------------------------- /lib/onsen/scraping.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'time' 3 | require 'pp' 4 | require 'moji' 5 | 6 | module Onsen 7 | class Program < Struct.new(:title, :number, :update_date, :file_url, :personality) 8 | end 9 | 10 | class Scraping 11 | def initialize 12 | @a = Mechanize.new 13 | @a.user_agent_alias = 'Windows Chrome' 14 | end 15 | 16 | def main 17 | get_program_list 18 | end 19 | 20 | def get_program_list 21 | programs = get_programs() 22 | parse_programs(programs).reject do |program| 23 | program == nil 24 | end 25 | end 26 | 27 | def parse_programs(programs) 28 | programs.map do |program| 29 | parse_program(program) 30 | end 31 | end 32 | 33 | def parse_program(program) 34 | content = program['contents'].find do |content| 35 | content['latest'] && !content['premium'] 36 | end 37 | return nil if content.nil? 38 | 39 | title = Moji.normalize_zen_han(program['title']) 40 | number = Moji.normalize_zen_han(content['title']) 41 | update_date_str = content['delivery_date'] 42 | if update_date_str == "" 43 | return nil 44 | end 45 | update_date = Time.parse(update_date_str) 46 | 47 | file_url = content['streaming_url'] 48 | if file_url == "" 49 | return nil 50 | end 51 | 52 | personality = program['performers'].map do |performer| 53 | Moji.normalize_zen_han(performer['name']) 54 | end.join(',') 55 | Program.new(title, number, update_date, file_url, personality) 56 | end 57 | 58 | def get_programs() 59 | url = "https://www.onsen.ag/web_api/programs" 60 | res = @a.get(url) 61 | JSON.parse(res.body) 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /config/environments/test.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # The test environment is used exclusively to run your application's 5 | # test suite. You never need to work with it otherwise. Remember that 6 | # your test database is "scratch space" for the test suite and is wiped 7 | # and recreated between test runs. Don't rely on the data there! 8 | config.cache_classes = true 9 | 10 | # Do not eager load code on boot. This avoids loading your whole application 11 | # just for the purpose of running a single test. If you are using a tool that 12 | # preloads Rails for running tests, you may have to set it to true. 13 | config.eager_load = false 14 | 15 | # Configure static asset server for tests with Cache-Control for performance. 16 | config.serve_static_assets = true 17 | config.static_cache_control = 'public, max-age=3600' 18 | 19 | # Show full error reports and disable caching. 20 | config.consider_all_requests_local = true 21 | config.action_controller.perform_caching = false 22 | 23 | # Raise exceptions instead of rendering exception templates. 24 | config.action_dispatch.show_exceptions = false 25 | 26 | # Disable request forgery protection in test environment. 27 | config.action_controller.allow_forgery_protection = false 28 | 29 | # Tell Action Mailer not to deliver emails to the real world. 30 | # The :test delivery method accumulates sent emails in the 31 | # ActionMailer::Base.deliveries array. 32 | config.action_mailer.delivery_method = :test 33 | 34 | # Print deprecation notices to the stderr. 35 | config.active_support.deprecation = :stderr 36 | 37 | # Raises error for missing translations 38 | # config.action_view.raise_on_missing_translations = true 39 | end 40 | -------------------------------------------------------------------------------- /config/routes.rb: -------------------------------------------------------------------------------- 1 | Rails.application.routes.draw do 2 | # The priority is based upon order of creation: first created -> highest priority. 3 | # See how all your routes lay out with "rake routes". 4 | 5 | # You can have the root of your site routed with "root" 6 | # root 'welcome#index' 7 | 8 | # Example of regular route: 9 | # get 'products/:id' => 'catalog#view' 10 | 11 | # Example of named route that can be invoked with purchase_url(id: product.id) 12 | # get 'products/:id/purchase' => 'catalog#purchase', as: :purchase 13 | 14 | # Example resource route (maps HTTP verbs to controller actions automatically): 15 | # resources :products 16 | 17 | # Example resource route with options: 18 | # resources :products do 19 | # member do 20 | # get 'short' 21 | # post 'toggle' 22 | # end 23 | # 24 | # collection do 25 | # get 'sold' 26 | # end 27 | # end 28 | 29 | # Example resource route with sub-resources: 30 | # resources :products do 31 | # resources :comments, :sales 32 | # resource :seller 33 | # end 34 | 35 | # Example resource route with more complex sub-resources: 36 | # resources :products do 37 | # resources :comments 38 | # resources :sales do 39 | # get 'recent', on: :collection 40 | # end 41 | # end 42 | 43 | # Example resource route with concerns: 44 | # concern :toggleable do 45 | # post 'toggle' 46 | # end 47 | # resources :posts, concerns: :toggleable 48 | # resources :photos, concerns: :toggleable 49 | 50 | # Example resource route within a namespace: 51 | # namespace :admin do 52 | # # Directs /admin/products/* to Admin::ProductsController 53 | # # (app/controllers/admin/products_controller.rb) 54 | # resources :products 55 | # end 56 | end 57 | -------------------------------------------------------------------------------- /public/422.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The change you wanted was rejected (422) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The change you wanted was rejected.

62 |

Maybe you tried to change something you didn't have access to.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /config/schedule.rb: -------------------------------------------------------------------------------- 1 | # Use this file to easily define all of your cron jobs. 2 | # 3 | # It's helpful, but not entirely necessary to understand cron before proceeding. 4 | # http://en.wikipedia.org/wiki/Cron 5 | # 6 | # Learn more: http://github.com/javan/whenever 7 | 8 | set :output, {error: 'log/cron.log', standard: 'log/cron.log'} 9 | 10 | job_type :rake_not_silent, 'sleep $[ ( $RANDOM % 30 ) + 1 ]s; export PATH=/usr/local/bin:$PATH; export LANG=en_US.UTF-8; cd :path && :environment_variable=:environment bundle exec rake :task :output' 11 | 12 | every 1.minute do 13 | rake_not_silent 'main:rec_one' 14 | end 15 | 16 | every '3-50/21 * * * *' do 17 | rake_not_silent 'main:rec_ondemand' 18 | end 19 | 20 | #=== nico 21 | # maintenance on Thursday 22 | every '0 14 * * *' do 23 | rake_not_silent 'main:niconama_scrape' 24 | end 25 | 26 | # maintenance on Thursday 27 | # Use much traffics. avoid peek time. 28 | every '4-58 2-19 * * 0-3,5-6' do 29 | rake_not_silent 'main:rec_niconama' 30 | end 31 | every '4-58 12-19 * * 4' do 32 | rake_not_silent 'main:rec_niconama' 33 | end 34 | #=== 35 | 36 | every '0 15 * * *' do 37 | rake_not_silent 'main:ag_scrape' 38 | end 39 | 40 | every '4 10-22 * * *' do 41 | rake_not_silent 'main:onsen_scrape' 42 | end 43 | 44 | every '8 10-22 * * *' do 45 | rake_not_silent 'main:hibiki_scrape' 46 | end 47 | 48 | every '12 * * * *' do 49 | rake_not_silent 'main:radiko_scrape' 50 | end 51 | 52 | every '5 * * * *' do 53 | rake_not_silent 'main:radiru_scrape' 54 | end 55 | 56 | every '23 10-22 * * *' do 57 | rake_not_silent 'main:agonp_scrape' 58 | end 59 | 60 | every '38 15 1 * *' do 61 | rake_not_silent 'main:wikipedia_scrape' 62 | end 63 | 64 | every '37 15 * * *' do 65 | rake_not_silent 'main:kill_zombie_process' 66 | end 67 | 68 | every '7 16 * * *' do 69 | rake_not_silent 'main:rm_working_files' 70 | end 71 | -------------------------------------------------------------------------------- /lib/niconico_live/downloading.rb: -------------------------------------------------------------------------------- 1 | require 'shellwords' 2 | 3 | module NiconicoLive 4 | class Downloading 5 | CH_NAME = 'niconama' 6 | 7 | class NiconamaDownloadException < StandardError; end 8 | 9 | def download(program) 10 | @program = program 11 | 12 | begin 13 | _download 14 | rescue Exception => e 15 | Rails.logger.error e.class 16 | Rails.logger.warn e.inspect 17 | Rails.logger.warn e.backtrace.join("\n") 18 | program.state = NiconicoLiveProgram::STATE[:failed] 19 | return 20 | end 21 | program.state = NiconicoLiveProgram::STATE[:done] 22 | end 23 | 24 | def _download 25 | Main::prepare_working_dir(CH_NAME) 26 | 27 | exit_status, output = Main::shell_exec(livedl_command) 28 | unless exit_status.success? 29 | Rails.logger.warn "nico livedl failed: #{@program.id}, #{output}" 30 | end 31 | 32 | files = Dir.glob("#{Main::file_path_working_base(CH_NAME, '')}*.mp4") 33 | if files.empty? 34 | raise NiconamaDownloadException, "download failed: #{@program.title}" 35 | end 36 | 37 | files.each do |file| 38 | Main::move_to_archive_dir(CH_NAME, @program.created_at, file) 39 | end 40 | end 41 | 42 | def livedl_command 43 | "\ 44 | livedl \ 45 | -nico-login '#{Shellwords.escape(Settings.niconico.username)},#{Shellwords.escape(Settings.niconico.password)}'\ 46 | -nico-login-only=on \ 47 | -nico-force-reservation=on \ 48 | -nico-format '#{Main::file_path_working_base(CH_NAME, '')}?DAY8? ?HOUR??MINUTE? ?TITLE?' \ 49 | -nico-auto-convert=on \ 50 | -nico-auto-delete-mode 2 \ 51 | -nico-fast-ts \ 52 | lv#{Shellwords.escape(@program.id)} \ 53 | 2>&1 54 | " 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /public/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The page you were looking for doesn't exist (404) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The page you were looking for doesn't exist.

62 |

You may have mistyped the address or the page may have moved.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /lib/tasks/main.rake: -------------------------------------------------------------------------------- 1 | namespace :main do 2 | desc 'ag scrape' 3 | task :ag_scrape => :environment do 4 | Main::Main.new.ag_scrape 5 | end 6 | 7 | desc 'radiko scrape' 8 | task :radiko_scrape => :environment do 9 | Main::Main.new.radiko_scrape 10 | end 11 | 12 | desc 'radiru scrape' 13 | task :radiru_scrape => :environment do 14 | Main::Main.new.radiru_scrape 15 | end 16 | 17 | desc 'onsen scrape' 18 | task :onsen_scrape => :environment do 19 | Main::Main.new.onsen_scrape 20 | end 21 | 22 | desc 'hibiki scrape' 23 | task :hibiki_scrape => :environment do 24 | Main::Main.new.hibiki_scrape 25 | end 26 | 27 | desc 'niconama scrape' 28 | task :niconama_scrape => :environment do 29 | Main::Main.new.niconama_scrape 30 | end 31 | 32 | desc 'nicodou scrape' 33 | task :nicodou_scrape => :environment do 34 | Main::Main.new.nicodou_scrape 35 | end 36 | 37 | desc 'agonp scrape' 38 | task :agonp_scrape => :environment do 39 | Main::Main.new.agonp_scrape 40 | end 41 | 42 | desc 'wikipedia scape' 43 | task :wikipedia_scrape => :environment do 44 | Main::Main.new.wikipedia_scrape 45 | end 46 | 47 | desc 'rec one' 48 | task :rec_one => :environment do 49 | Main::Main.new.rec_one 50 | end 51 | 52 | desc 'rec ondemand' 53 | task :rec_ondemand => :environment do 54 | Main::Main.new.rec_ondemand 55 | end 56 | 57 | desc 'rec niconama timeshift' 58 | task :rec_niconama => :environment do 59 | Main::Main.new.niconama_download 60 | end 61 | 62 | desc 'kill zombie process (rtmpdump)' 63 | task :kill_zombie_process => :environment do 64 | Main::Workaround::kill_zombie_process 65 | end 66 | 67 | desc 'remove old working(temporary) files' 68 | task :rm_working_files => :environment do 69 | Main::Workaround::rm_working_files 70 | Main::Workaround::rm_latest_dir_symlinks 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/main/workaround.rb: -------------------------------------------------------------------------------- 1 | require 'time' 2 | 3 | module Main 4 | module Workaround 5 | def self.kill_zombie_process 6 | send_signal_to_zombie_processes('rtmpdump', :TERM) 7 | send_signal_to_zombie_processes('ffmpeg', :TERM) 8 | send_signal_to_zombie_processes('avconv', :TERM) 9 | sleep 5 10 | send_signal_to_zombie_processes('rtmpdump', :KILL) 11 | send_signal_to_zombie_processes('ffmpeg', :KILL) 12 | send_signal_to_zombie_processes('avconv', :KILL) 13 | end 14 | 15 | def self.rm_working_files 16 | if Settings.working_dir.strip.size < 2 17 | puts "working dir is maybe wrong: #{Settings.working_dir}" 18 | return 19 | end 20 | `find #{Settings.working_dir} -ctime +#{Settings.working_files_retention_period_days || 7} -name "*.flv" -exec rm {} \\;` 21 | `find #{Settings.working_dir} -ctime +#{Settings.working_files_retention_period_days || 7} -name "*.mp3" -exec rm {} \\;` 22 | `find #{Settings.working_dir} -ctime +#{Settings.working_files_retention_period_days || 7} -name "*.m4a" -exec rm {} \\;` 23 | end 24 | 25 | def self.rm_latest_dir_symlinks 26 | if Settings.archive_dir.strip.size < 2 27 | puts "archive dir is maybe wrong: #{Settings.archive_dir}" 28 | return 29 | end 30 | `find #{Settings.archive_dir}/*/#{Main::latest_dir_name} -ctime +30 -type l -exec rm {} \\;` 31 | end 32 | 33 | private 34 | 35 | def self.send_signal_to_zombie_processes(process_name, signal) 36 | pids = (`pgrep '#{process_name}'`).split("\n").map(&:to_i) 37 | pids.each do |pid| 38 | elapsed_sec = Time.now.to_i - Time.parse(`ps -o lstart --noheader -p #{pid}`).to_i 39 | if (60 * 60 * 25) < elapsed_sec # 24 + 1(margin) hours 40 | Process.kill(signal, pid) 41 | puts "kill pid:#{pid} elapsed_sec:#{elapsed_sec}" 42 | end 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /niconico/lib/niconico/nico_api.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | class Niconico 4 | class NicoAPI 5 | class AcquiringTokenError < Exception; end 6 | class ApiError < Exception 7 | def initialize(error) 8 | @description = error['description'] 9 | @code = error['code'] 10 | super "#{@code}: #{@description.inspect}" 11 | end 12 | 13 | attr_reader :code, :description 14 | end 15 | 16 | MYLIST_ITEM_TYPES = {video: 0, seiga: 5} 17 | 18 | def initialize(parent) 19 | @parent = parent 20 | end 21 | 22 | def agent; @parent.agent; end 23 | 24 | def token; @token ||= get_token; end 25 | 26 | def get_token 27 | page = agent.get(Niconico::URL[:my_mylist]) 28 | match = page.search("script").map(&:inner_text).grep(/\tNicoAPI\.token/) {|v| v.match(/\tNicoAPI\.token = "(.+)";\n/)}.first 29 | if match 30 | match[1] 31 | else 32 | raise AcquiringTokenError, "Couldn't find a token" 33 | end 34 | end 35 | 36 | def mylist_add(group_id, item_type, item_id, description='') 37 | !!post( 38 | '/api/mylist/add', 39 | { 40 | group_id: group_id, 41 | item_type: MYLIST_ITEM_TYPES[item_type], 42 | item_id: item_id, 43 | description: description, 44 | } 45 | ) 46 | end 47 | 48 | private 49 | 50 | def post(path, params) 51 | retried = false 52 | begin 53 | params = params.merge(token: token) 54 | uri = URI.join(Niconico::URL[:top], path) 55 | page = agent.post(uri, params) 56 | json = JSON.parse(page.body) 57 | 58 | raise ApiError.new(json['error']) unless json['status'] == 'ok' 59 | 60 | json 61 | rescue ApiError => e 62 | if (e.code == 'INVALIDTOKEN' || e.code == 'EXPIRETOKEN') && !retried 63 | retried = true 64 | @token = nil 65 | retry 66 | else 67 | raise e 68 | end 69 | end 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /niconico/lib/niconico/live/client.rb: -------------------------------------------------------------------------------- 1 | require 'niconico/live/client/search_result' 2 | require 'niconico/live/client/search_filters' 3 | 4 | class Niconico 5 | def live_client 6 | Live::Client.new(self.agent) 7 | end 8 | 9 | class Live 10 | class Client 11 | 12 | def initialize(agent) 13 | @agent = agent 14 | @api = API.new(agent) 15 | end 16 | 17 | def remove_timeshifts(ids) 18 | post_body = "delete=timeshift&confirm=#{Util::fetch_token(@agent)}" 19 | if ids.size == 0 20 | return 21 | end 22 | ids.each do |id| 23 | id = Util::normalize_id(id, with_lv: false) 24 | # mechanize doesn't support multiple values for the same key in query. 25 | post_body += "&vid%5B%5D=#{id}" 26 | end 27 | @agent.post( 28 | 'http://live.nicovideo.jp/my.php', 29 | post_body, 30 | 'Content-Type' => 'application/x-www-form-urlencoded' 31 | ) 32 | end 33 | 34 | def search(keyword, filters = []) 35 | filter = filters.join('+') 36 | page = @agent.get( 37 | 'http://live.nicovideo.jp/search', 38 | track: '', 39 | sort: 'recent', 40 | date: '', 41 | kind: '', 42 | keyword: keyword, 43 | filter: filter 44 | ) 45 | results_dom = page.at('.result-list') 46 | unless results_dom 47 | Rails.logger.error "nico scrape dom empty" 48 | Rails.logger.error page.inspect 49 | Rails.logger.error page.body 50 | end 51 | items = results_dom.css('.result-item') 52 | search_results = items.map do |item| 53 | title_dom = item.at('a.title') 54 | next nil unless title_dom 55 | id = Util::normalize_id(title_dom.attr(:href).scan(/lv[\d]+/).first, with_lv: false) 56 | title = title_dom.text.strip 57 | description = item.at('.description-text').text.strip 58 | SearchResult.new(id, title, description) 59 | end 60 | search_results.compact 61 | end 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /niconico/lib/niconico/live/mypage.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | require 'niconico/live' 3 | require 'niconico/live/api' 4 | 5 | class Niconico 6 | def live_mypage 7 | Niconico::Live::Mypage.new(self) 8 | end 9 | 10 | class Live 11 | class Mypage 12 | class UnknownStatus < Exception; end 13 | URL = 'http://live.nicovideo.jp/my'.freeze 14 | 15 | def initialize(client) 16 | @client = client 17 | end 18 | 19 | attr_reader :client 20 | 21 | def agent 22 | client.agent 23 | end 24 | 25 | def page 26 | @page ||= agent.get(URL) 27 | end 28 | 29 | def reservations 30 | return @reservations if @reservations 31 | lists = page.search("form[name=timeshift_list] .liveItems") 32 | @reservations = lists.flat_map do |list| 33 | list.search('.column').map do |column| 34 | link = column.at('.name a') 35 | id = link[:href].sub(/\A.*\//,'').sub(/\?.*\z/,'') 36 | status = column.at('.status').inner_text 37 | watch_button = column.at('.timeshift_watch a') 38 | 39 | preload = {} 40 | 41 | preload[:title] = link[:title] 42 | 43 | preload[:reservation] = Live::API.parse_reservation_message(status) 44 | raise UnknownStatus, "BUG, there's unknown message for reservation status: #{status.inspect}" unless preload[:reservation] 45 | 46 | # (試聴する) [試聴期限未定] 47 | if watch_button && watch_button[:onclick] && watch_button[:onclick].include?('confirm_watch_my') 48 | preload[:reservation][:status] = :reserved 49 | preload[:reservation][:available] = true 50 | end 51 | 52 | Niconico::Live.new(client, id, preload) 53 | end 54 | end 55 | end 56 | alias timeshift_list reservations 57 | 58 | def available_reservations 59 | reservations.select { |_| _.reservation_available? } 60 | end 61 | 62 | def unaccepted_reservations 63 | reservations.select { |_| _.reservation_unaccepted? } 64 | end 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/radiru/recording.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'shellwords' 3 | 4 | module Radiru 5 | class Recording 6 | 7 | def record(job) 8 | unless exec_rec(job) 9 | return false 10 | end 11 | exec_convert(job) 12 | 13 | true 14 | end 15 | 16 | def exec_rec(job) 17 | Main::prepare_working_dir(job.ch) 18 | download_hls(job) 19 | end 20 | 21 | def get_streams_dom 22 | xml = Net::HTTP.get(URI("http://www.nhk.or.jp/radio/config/config_web.xml")) 23 | Nokogiri::XML(xml) 24 | end 25 | 26 | def parse_dom(dom, ch) 27 | dom.css('data').map do |stream| 28 | parse_stream(stream, ch) 29 | end 30 | end 31 | 32 | def parse_stream(dom, ch) 33 | if dom.css('area').text == 'tokyo' 34 | @m3u8_url = dom.css(ch + 'hls').text 35 | end 36 | end 37 | 38 | def download_hls(job) 39 | dom = get_streams_dom 40 | parse_dom(dom, job.ch) 41 | 42 | Main::sleep_until(job.start - 10.seconds) 43 | 44 | length = job.length_sec + 60 45 | file_path = Main::file_path_working(job.ch, title(job), 'm4a') 46 | arg = "\ 47 | -loglevel warning \ 48 | -y \ 49 | -i #{Shellwords.escape(@m3u8_url)} \ 50 | -t #{length} \ 51 | -vcodec none -acodec copy -bsf:a aac_adtstoasc \ 52 | #{Shellwords.escape(file_path)}" 53 | 54 | exit_status, output = Main::ffmpeg(arg) 55 | unless exit_status.success? && output.blank? 56 | Rails.logger.error "rec failed. job:#{job.id}, exit_status:#{exit_status}, output:#{output}" 57 | return false 58 | end 59 | 60 | true 61 | end 62 | 63 | def exec_convert(job) 64 | m4a_path = Main::file_path_working(job.ch, title(job), 'm4a') 65 | if Settings.force_mp4 66 | mp4_path = Main::file_path_working(job.ch, title(job), 'mp4') 67 | Main::convert_ffmpeg_to_mp4_with_blank_video(m4a_path, mp4_path, job) 68 | dst_path = mp4_path 69 | else 70 | dst_path = m4a_path 71 | end 72 | Main::move_to_archive_dir(job.ch, job.start, dst_path) 73 | end 74 | 75 | def title(job) 76 | date = job.start.strftime('%Y_%m_%d_%H%M') 77 | "#{date}_#{job.title}" 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /config/settings.example.yml: -------------------------------------------------------------------------------- 1 | # cronからは常にproduction環境で起動します 2 | # 基本的にはこちらのみ設定してください 3 | production: 4 | 5 | # ファイルを保存(archive)するディレクトリを設定してください（フルパス） 6 | # このディレクトリ以下にサブディレクトリやファイルが作成されます 7 | # cronが実行されるユーザーにrwxのpermissionが与えられるようにしてください 8 | archive_dir: /archive 9 | 10 | # ダウンロード中などにファイルを一時的に置いておくディレクトリを設定してください（フルパス） 11 | # 一時的とはいえ、デフォルトで事故防止のためこのディレクトリのデータは30日ほど削除されません (lib/main/workaround.rbのrm_working_filesを参照) 12 | # そのためそれなりにおおきな容量となります 13 | # cronが実行されるユーザーにrwxのpermissionが与えられるようにしてください 14 | working_dir: /working 15 | 16 | # 作業時に残ったファイル(mp4に変換前のflv dumpなど)を残しておく日数です 17 | # 何か事故が起きた時に使えるかもしれない 18 | working_files_retention_period_days: 7 19 | 20 | # radikoで録音するチャンネルを設定してください 21 | # channel code (QRRなど) についてはREADME.mdのFAQを参照してください 22 | radiko_channels: 23 | - QRR # 文化放送 24 | - LFR # ニッポン放送 25 | 26 | # radiko premiumで録音するチャンネルを設定してください。 27 | # premiumでは同時視聴端末について3台の制限があるため 28 | # 3チャンネル以上の同時録画は失敗する可能性があります 29 | #radiko_premium: 30 | # mail: 'hoge@hoge.com' 31 | # password: 'hogehoge' 32 | # channels: 33 | # - QRR # ラジオ関西 34 | 35 | # らじるらじるで録音するチャンネルを設定してください 36 | radiru_channels: 37 | - r1 # ラジオ第1 38 | - r2 # ラジオ第2 39 | - fm # NHK-FM 40 | 41 | # ファイル名に含まれる単語からシムリンクを作成 42 | # archive_dirで設定したディレクトリ配下に 0_selections ディレクトリの作成、配下にシムリンク 43 | # 定期的に聴きたいラジオをより分けるのに便利です 44 | #selections: 45 | # - 花澤香菜 46 | # - 佐倉綾音 47 | # - モモノキ 48 | 49 | # trueの場合、音声のみのコンテンツ(音泉、アニたま、Radiko、らじるらじる)を保存時に強制的にmp4へ変換します(再エンコードはかけないので音声は劣化しません) 50 | # Googleフォトなどにバックアップする際に便利です 51 | force_mp4: false 52 | 53 | # AG-ON Premium 54 | # 録画しない場合はこの項目ごとコメントアウトすると何もしません 55 | #agonp: 56 | # # AG-ON Premiumにアカウント登録してそのアカウントを設定してください 57 | # # AG−ON 無印とは別のアカウント管理となっていますので移行する方は 58 | # # アカウントをPremiumであらためて取得してください 59 | # mail: 'foo@example.com' 60 | # password: 'XXXXXX' 61 | 62 | # ニコニコ(開発中。恐ろしく不安定) おそらくプレミアム会員必須 63 | # 録画しない場合はこの項目ごとコメントアウトすると何もしません 64 | #niconico: 65 | # username: 'USERNAME' 66 | # password: 'PASSWORD' 67 | # live: 68 | # # 録画(タイムシフトの取得)をしたい生放送をキーワードで指定 69 | # keywords: 70 | # - '村川梨衣' 71 | # - '麻倉もも' 72 | # # wikipediaのカテゴリで検索キーワードを一括指定できます 73 | # keyword_wikipedia_categories: 74 | # - '声優ユニット' 75 | # - '日本の女性声優' 76 | 77 | development: 78 | radiko_channels: 79 | - QRR # bunka housou 80 | - LFR # nippon housou 81 | archive_dir: /tmp/net-radio-archive 82 | working_dir: /tmp/net-radio-working 83 | -------------------------------------------------------------------------------- /lib/agonp/downloading.rb: -------------------------------------------------------------------------------- 1 | module Agonp 2 | class Downloading 3 | CH_NAME = 'agonp' 4 | 5 | def initialize 6 | @a = Mechanize.new 7 | @a.user_agent_alias = 'Windows Chrome' 8 | end 9 | 10 | def download(program) 11 | login 12 | program_id = get_program_id(program) 13 | url = get_video_url(program) 14 | nazo_auth(program, program_id) 15 | save_video(program, url) 16 | true 17 | end 18 | 19 | def login 20 | page = @a.get('https://agonp.jp/auth/login') 21 | form = page.forms.first 22 | form.email = Settings.agonp.mail 23 | form.password = Settings.agonp.password 24 | form.submit 25 | end 26 | 27 | def get_program_id(program) 28 | res = @a.get("https://agonp.jp/api/v2/episodes/info/#{program.episode_id}.json") 29 | resj = JSON.parse(res.body) 30 | resj['data']['episode']['program_id'] 31 | end 32 | 33 | def get_video_url(program) 34 | res = @a.get("https://agonp.jp/api/v1/episodes/media_url.json?episode_id=#{program.episode_id}&format=mp4&size=small") 35 | resj = JSON.parse(res.body) 36 | resj['data']['url'] 37 | end 38 | 39 | def nazo_auth(program, program_id) 40 | res = @a.post("https://agonp.jp/api/v1/programs/episodes/view.json", { 41 | format: 'mp4', 42 | program_id: program_id, 43 | episode_id: program.episode_id, 44 | time: '-1', 45 | fuel_csrf_token: '', 46 | }) 47 | fuel_csrf_token = @a.cookie_jar.jar['agonp.jp']['/']['fuel_csrf_token'].value 48 | res = @a.post("https://agonp.jp/api/v1/programs/episodes/view.json", { 49 | format: 'mp4', 50 | program_id: program_id, 51 | episode_id: program.episode_id, 52 | time: '-1', 53 | fuel_csrf_token: fuel_csrf_token, 54 | }) 55 | fuel_csrf_token = @a.cookie_jar.jar['agonp.jp']['/']['fuel_csrf_token'].value 56 | res = @a.get("https://agonp.jp/api/v1/slices/own/#{program.episode_id}.json?fuel_csrf_token=#{fuel_csrf_token}") 57 | fuel_csrf_token = @a.cookie_jar.jar['agonp.jp']['/']['fuel_csrf_token'].value 58 | res = @a.post("https://agonp.jp/api/v1/programs/episodes/view.json", { 59 | format: 'mp4', 60 | program_id: program_id, 61 | episode_id: program.episode_id, 62 | time: '0', 63 | see_rate: '0', 64 | fuel_csrf_token: fuel_csrf_token, 65 | }) 66 | end 67 | 68 | def save_video(program, url) 69 | file_path = Main::file_path_working(CH_NAME, title(program), 'mp4') 70 | Main::prepare_working_dir(CH_NAME) 71 | @a.get(url, [], "https://agonp.jp/episodes/view/#{program.episode_id}").save_as(file_path) 72 | Main::move_to_archive_dir(CH_NAME, program.created_at, file_path) 73 | end 74 | 75 | def title(program) 76 | date = program.created_at.strftime('%Y_%m_%d') 77 | title = "#{date}_#{program.title}" 78 | if program.personality 79 | title += "_#{program.personality}" 80 | end 81 | title 82 | end 83 | end 84 | end 85 | 86 | -------------------------------------------------------------------------------- /config/environments/production.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # Code is not reloaded between requests. 5 | config.cache_classes = true 6 | 7 | # Eager load code on boot. This eager loads most of Rails and 8 | # your application in memory, allowing both threaded web servers 9 | # and those relying on copy on write to perform better. 10 | # Rake tasks automatically ignore this option for performance. 11 | config.eager_load = true 12 | 13 | # Full error reports are disabled and caching is turned on. 14 | config.consider_all_requests_local = false 15 | config.action_controller.perform_caching = true 16 | 17 | # Enable Rack::Cache to put a simple HTTP cache in front of your application 18 | # Add `rack-cache` to your Gemfile before enabling this. 19 | # For large-scale production use, consider using a caching reverse proxy like nginx, varnish or squid. 20 | # config.action_dispatch.rack_cache = true 21 | 22 | # Disable Rails's static asset server (Apache or nginx will already do this). 23 | config.serve_static_files = false 24 | 25 | # Compress JavaScripts and CSS. 26 | config.assets.js_compressor = :uglifier 27 | # config.assets.css_compressor = :sass 28 | 29 | # Do not fallback to assets pipeline if a precompiled asset is missed. 30 | config.assets.compile = false 31 | 32 | # Generate digests for assets URLs. 33 | config.assets.digest = true 34 | 35 | # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb 36 | 37 | # Specifies the header that your server uses for sending files. 38 | # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache 39 | # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx 40 | 41 | # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. 42 | # config.force_ssl = true 43 | 44 | # Set to :debug to see everything in the log. 45 | config.log_level = :info 46 | 47 | # Prepend all log lines with the following tags. 48 | # config.log_tags = [ :subdomain, :uuid ] 49 | 50 | # Use a different logger for distributed setups. 51 | # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) 52 | 53 | # Use a different cache store in production. 54 | # config.cache_store = :mem_cache_store 55 | 56 | # Enable serving of images, stylesheets, and JavaScripts from an asset server. 57 | # config.action_controller.asset_host = "http://assets.example.com" 58 | 59 | # Ignore bad email addresses and do not raise email delivery errors. 60 | # Set this to true and configure the email server for immediate delivery to raise delivery errors. 61 | # config.action_mailer.raise_delivery_errors = false 62 | 63 | # Enable locale fallbacks for I18n (makes lookups for any locale fall back to 64 | # the I18n.default_locale when a translation cannot be found). 65 | config.i18n.fallbacks = true 66 | 67 | # Send deprecation notices to registered listeners. 68 | config.active_support.deprecation = :notify 69 | 70 | # Disable automatic flushing of the log to improve performance. 71 | # config.autoflush_log = false 72 | 73 | # Use default logging formatter so that PID and timestamp are not suppressed. 74 | config.log_formatter = ::Logger::Formatter.new 75 | 76 | # Do not dump schema after migrations. 77 | config.active_record.dump_schema_after_migration = false 78 | end 79 | -------------------------------------------------------------------------------- /niconico/lib/niconico.rb: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | require 'mechanize' 4 | require 'cgi' 5 | require 'niconico/version' 6 | 7 | class Niconico 8 | URL = { 9 | top: 'http://www.nicovideo.jp/', 10 | login: 'https://secure.nicovideo.jp/secure/login?site=niconico', 11 | watch: 'http://www.nicovideo.jp/watch/', 12 | getflv: 'http://flapi.nicovideo.jp/api/getflv', 13 | my_mylist: 'http://www.nicovideo.jp/my/mylist' 14 | } 15 | 16 | attr_reader :agent 17 | 18 | def logged_in?; @logged_in; end 19 | alias logined logged_in? 20 | 21 | def initialize(*args) 22 | case args.size 23 | when 2 24 | @mail, @pass = args 25 | when 1 26 | if args.first.kind_of?(Hash) 27 | @mail, @pass, @token = args.first.values_at(:mail, :password, :token) 28 | else 29 | @token = args.first 30 | end 31 | else 32 | raise ArgumentError, "wrong number of arguments (#{args.size} for 1..2)" 33 | end 34 | 35 | @logged_in = false 36 | 37 | @agent = Mechanize.new.tap do |agent| 38 | agent.user_agent = "Niconico.gem (#{Niconico::VERSION}, https://github.com/sorah/niconico)" 39 | agent.keep_alive = false 40 | 41 | agent.cookie_jar.add( 42 | HTTP::Cookie.new( 43 | domain: '.nicovideo.jp', path: '/', 44 | name: 'lang', value: 'ja-jp', 45 | ) 46 | ) 47 | end 48 | end 49 | 50 | def login(force=false) 51 | return false if !force && @logged_in 52 | 53 | if @token && @mail && @pass 54 | begin 55 | login_with_token 56 | rescue LoginError 57 | login_with_email 58 | end 59 | elsif @token 60 | login_with_token 61 | elsif @mail && @pass 62 | login_with_email 63 | else 64 | raise ArgumentError, 'Insufficient options for logging in (token or/and pair of mail and password required)' 65 | end 66 | end 67 | 68 | def inspect 69 | "#" 70 | end 71 | 72 | def token 73 | return @token if @token 74 | login unless logged_in? 75 | 76 | @token = agent.cookie_jar.each('https://www.nicovideo.jp').find{|_| _.name == 'user_session' }.value 77 | end 78 | 79 | def nico_api 80 | return @nico_api if @nico_api 81 | login unless logged_in? 82 | @nico_api = NicoAPI.new(self) 83 | end 84 | 85 | class LoginError < StandardError; end 86 | 87 | private 88 | 89 | def login_with_email 90 | page = @agent.post(URL[:login], 'mail_tel' => @mail, 'password' => @pass) 91 | 92 | raise LoginError, "Failed to log in (x-niconico-authflag is 0)" if page.header["x-niconico-authflag"] == '0' 93 | @token = nil 94 | @logged_in = true 95 | end 96 | 97 | def login_with_token 98 | @agent.cookie_jar.add( 99 | HTTP::Cookie.new( 100 | domain: '.nicovideo.jp', path: '/', 101 | name: 'user_session', value: @token 102 | ) 103 | ) 104 | 105 | page = @agent.get(URL[:top]) 106 | raise LoginError, "Failed to log in (x-niconico-authflag is 0)" if page.header["x-niconico-authflag"] == '0' 107 | 108 | @logged_in = true 109 | end 110 | 111 | end 112 | 113 | require 'niconico/video' 114 | require 'niconico/mylist' 115 | require 'niconico/ranking' 116 | require 'niconico/channel' 117 | require 'niconico/live' 118 | require 'niconico/live/client' 119 | require 'niconico/live/mypage' 120 | require 'niconico/nico_api' 121 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM buildpack-deps:xenial 2 | 3 | #============ 4 | # Packages 5 | #============ 6 | RUN echo "deb http://archive.ubuntu.com/ubuntu xenial main universe\n" > /etc/apt/sources.list \ 7 | && echo "deb http://archive.ubuntu.com/ubuntu xenial-updates main universe\n" >> /etc/apt/sources.list \ 8 | && echo "deb http://security.ubuntu.com/ubuntu xenial-security main universe\n" >> /etc/apt/sources.list 9 | RUN apt-get update -qqy \ 10 | && apt-get install -y --no-install-recommends nodejs swftools git xvfb wget bzip2 ca-certificates tzdata sudo unzip cron locales \ 11 | rsyslog \ 12 | coreutils 13 | # rsyslog: for get cron error logs 14 | # coreutils: for sleep command 15 | 16 | #========= 17 | # Ruby 18 | # see Dockerfiles on https://hub.docker.com/_/ruby/ 19 | #========= 20 | # skip installing gem documentation 21 | RUN mkdir -p /usr/local/etc \ 22 | && { \ 23 | echo 'install: --no-document'; \ 24 | echo 'update: --no-document'; \ 25 | } >> /usr/local/etc/gemrc 26 | 27 | ENV RUBY_MAJOR 2.4 28 | ENV RUBY_VERSION 2.4.2 29 | ENV RUBYGEMS_VERSION 2.7.2 30 | 31 | # some of ruby's build scripts are written in ruby 32 | # we purge system ruby later to make sure our final image uses what we just built 33 | RUN set -ex \ 34 | && buildDeps=' \ 35 | bison \ 36 | libgdbm-dev \ 37 | ruby \ 38 | ' \ 39 | && apt-get update \ 40 | && apt-get install -y --no-install-recommends $buildDeps \ 41 | && rm -rf /var/lib/apt/lists/* \ 42 | && wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz" \ 43 | && mkdir -p /usr/src/ruby \ 44 | && tar -xJf ruby.tar.xz -C /usr/src/ruby --strip-components=1 \ 45 | && rm ruby.tar.xz \ 46 | && cd /usr/src/ruby \ 47 | && { \ 48 | echo '#define ENABLE_PATH_CHECK 0'; \ 49 | echo; \ 50 | cat file.c; \ 51 | } > file.c.new \ 52 | && mv file.c.new file.c \ 53 | && autoconf \ 54 | && ./configure --disable-install-doc --enable-shared \ 55 | && make -j"$(nproc)" \ 56 | && make install \ 57 | && apt-get purge -y --auto-remove $buildDeps \ 58 | && cd / \ 59 | && rm -r /usr/src/ruby 60 | 61 | ENV BUNDLER_VERSION 1.16.0 62 | 63 | RUN gem install bundler --version "$BUNDLER_VERSION" 64 | 65 | # install things globally, for great justice 66 | # and don't create ".bundle" in all our apps 67 | ENV GEM_HOME /usr/local/bundle 68 | ENV BUNDLE_PATH="$GEM_HOME" \ 69 | BUNDLE_BIN="$GEM_HOME/bin" \ 70 | BUNDLE_SILENCE_ROOT_WARNING=1 \ 71 | BUNDLE_APP_CONFIG="$GEM_HOME" 72 | ENV PATH $BUNDLE_BIN:$PATH 73 | RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" \ 74 | && chmod 777 "$GEM_HOME" "$BUNDLE_BIN" 75 | 76 | #========= 77 | # ffmpeg 78 | #========= 79 | RUN wget --no-verbose -O /tmp/ffmpeg.tar.gz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz \ 80 | && tar -C /tmp -xf /tmp/ffmpeg.tar.gz \ 81 | && mv /tmp/ffmpeg-*-amd64-static/ffmpeg /usr/bin \ 82 | && rm -rf /tmp/ffmpeg* 83 | 84 | #========= 85 | # rtmpdump 86 | #========= 87 | RUN git clone git://git.ffmpeg.org/rtmpdump \ 88 | && cd /rtmpdump \ 89 | && make \ 90 | && make install 91 | 92 | #========= 93 | # youtube-dl 94 | #========= 95 | RUN wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl && chmod a+rx /usr/local/bin/youtube-dl 96 | 97 | #========= 98 | # livedl 99 | #========= 100 | RUN wget https://github.com/yayugu/livedl/releases/download/20181215.36/livedl -O /usr/local/bin/livedl \ 101 | && chmod a+rx /usr/local/bin/livedl 102 | 103 | #============ 104 | # Timezone 105 | # see: https://bugs.launchpad.net/ubuntu/+source/tzdata/+bug/1554806 106 | #============ 107 | ENV TZ "Asia/Tokyo" 108 | RUN echo 'Asia/Tokyo' > /etc/timezone \ 109 | && rm /etc/localtime \ 110 | && dpkg-reconfigure --frontend noninteractive tzdata 111 | 112 | #============ 113 | # Locale 114 | #============ 115 | ENV LC_ALL C.UTF-8 116 | 117 | #============ 118 | # Copy bundler env to /etc/environment to load on cron 119 | #============ 120 | RUN printenv | grep -E "^BUNDLE" >> /etc/environment 121 | 122 | #============ 123 | # Rails 124 | #============ 125 | RUN mkdir /myapp 126 | WORKDIR /myapp 127 | ADD Gemfile /myapp/Gemfile 128 | ADD Gemfile.lock /myapp/Gemfile.lock 129 | ADD niconico /myapp/niconico 130 | RUN bundle install -j4 --without development test 131 | ADD . /myapp 132 | RUN RAILS_ENV=production bundle exec rake db:create db:migrate \ 133 | && RAILS_ENV=production bundle exec whenever --update-crontab \ 134 | && chmod 0600 /var/spool/cron/crontabs/root 135 | 136 | CMD rsyslogd && /usr/sbin/cron -f 137 | -------------------------------------------------------------------------------- /niconico/lib/niconico/video.rb: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | require 'json' 3 | require 'niconico/deferrable' 4 | 5 | class Niconico 6 | def video(video_id) 7 | login unless logged_in? 8 | Video.new(self, video_id) 9 | end 10 | 11 | class Video 12 | include Niconico::Deferrable 13 | 14 | deferrable :id, :title, 15 | :description, :description_raw, 16 | :url, :video_url, :type, 17 | :tags, :mylist_comment, :api_data 18 | 19 | def initialize(parent, video_id, defer=nil) 20 | @parent = parent 21 | @agent = parent.agent 22 | @fetched = false 23 | @thread_id = @id = video_id 24 | @page = nil 25 | @url = "#{Niconico::URL[:watch]}#{@id}" 26 | 27 | if defer 28 | preload_deffered_values(defer) 29 | else 30 | get() 31 | end 32 | end 33 | 34 | def economy?; @eco; end 35 | 36 | def get(options = {}) 37 | begin 38 | @page = @agent.get(@url) 39 | rescue Mechanize::ResponseCodeError => e 40 | raise NotFound, "#{@id} not found" if e.message == "404 => Net::HTTPNotFound" 41 | raise e 42 | end 43 | 44 | if /^so/ =~ @id 45 | sleep 5 46 | @thread_id = @agent.get("#{Niconico::URL[:watch]}#{@id}").uri.path.sub(/^\/watch\//,"") 47 | end 48 | additional_params = nil 49 | if /^nm/ === @id && (!options.key?(:as3) || options[:as3]) 50 | additional_params = "&as3=1" 51 | end 52 | getflv = Hash[@agent.get_file("#{Niconico::URL[:getflv]}?v=#{@thread_id}#{additional_params}").scan(/([^&]+)=([^&]+)/).map{|(k,v)| [k.to_sym,CGI.unescape(v)] }] 53 | 54 | if api_data_node = @page.at("#watchAPIDataContainer") 55 | @api_data = JSON.parse(api_data_node.text()) 56 | video_detail = @api_data["videoDetail"] 57 | @title ||= video_detail["title"] if video_detail["title"] 58 | @description ||= video_detail["description"] if video_detail["description"] 59 | @tags ||= video_detail["tagList"].map{|e| e["tag"]} 60 | end 61 | 62 | t = @page.at("#videoTitle") 63 | @title ||= t.inner_text unless t.nil? 64 | d = @page.at("div#videoComment>div.videoDescription") 65 | @description ||= d.inner_html unless d.nil? 66 | 67 | @video_url = getflv[:url] 68 | if @video_url 69 | @eco = !(/low$/ =~ @video_url).nil? 70 | @type = case @video_url.match(/^http:\/\/(.+\.)?nicovideo\.jp\/smile\?(.+?)=.*$/).to_a[2] 71 | when 'm'; :mp4 72 | when 's'; :swf 73 | else; :flv 74 | end 75 | end 76 | @tags ||= @page.search("#video_tags a[rel=tag]").map(&:inner_text) 77 | @mylist_comment ||= nil 78 | 79 | @fetched = true 80 | @page 81 | end 82 | 83 | def available? 84 | !!video_url 85 | end 86 | 87 | def get_video 88 | raise VideoUnavailableError unless available? 89 | unless block_given? 90 | @agent.get_file(video_url) 91 | else 92 | cookies = video_cookies.map(&:to_s).join(';') 93 | uri = URI(video_url) 94 | http = Net::HTTP.new(uri.host, uri.port) 95 | http.request_get(uri.request_uri, 'Cookie' => cookies) do |res| 96 | res.read_body do |body| 97 | yield body 98 | end 99 | end 100 | end 101 | end 102 | 103 | def get_video_by_other 104 | raise VideoUnavailableError unless available? 105 | warn "WARN: Niconico::Video#get_video_by_other is deprecated. use Video#video_cookie_jar or video_cookie_jar_file, and video_cookies with video_url instead. (Called by #{caller[0]})" 106 | {cookie: @agent.cookie_jar.cookies(URI.parse(@video_url)), 107 | url: video_url} 108 | end 109 | 110 | def video_cookies 111 | return nil unless available? 112 | @agent.cookie_jar.cookies(URI.parse(video_url)) 113 | end 114 | 115 | def video_cookie_jar 116 | raise VideoUnavailableError unless available? 117 | video_cookies.map { |cookie| 118 | [cookie.domain, "TRUE", cookie.path, 119 | cookie.secure.inspect.upcase, cookie.expires.to_i, 120 | cookie.name, cookie.value].join("\t") 121 | }.join("\n") 122 | end 123 | 124 | def video_cookie_jar_file 125 | raise VideoUnavailableError unless available? 126 | Tempfile.new("niconico_cookie_jar_#{self.id}").tap do |io| 127 | io.puts(video_cookie_jar) 128 | io.flush 129 | end 130 | end 131 | 132 | def add_to_mylist(mylist_id, description='') 133 | @parent.nico_api.mylist_add(mylist_id, :video, @id, description) 134 | end 135 | 136 | def inspect 137 | "#" 138 | end 139 | 140 | class NotFound < StandardError; end 141 | class VideoUnavailableError < StandardError; end 142 | end 143 | end 144 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ねとらじあーかいぶ 2 | Net Radio Archive 3 | 4 | ## なにこれ 5 | ネットラジオを録画するやつ 6 | 7 | 今のところ対応しているラジオ 8 | 9 | - Radiko (エリアフリーも対応) 10 | - 超A&G+ 11 | - 響 12 | - 音泉 13 | - AG-ON Premium 14 | - らじる(NHK) 15 | - ニコ生（ニコニコ生放送） 16 | 17 | ## 特徴 18 | 「全部の番組」を取ります。 19 | 20 | 大抵の録画ソフトは時間していで録ったりしますがそうじゃない。 21 | 22 | 今ブレイクしてる新人声優の2年前のラジオ番組を掘り起こしたい! 23 | あと新番組を取り逃す心配がなかったり、ザッピングしていると意外とおもしろいラジオ発掘したりできて便利。 24 | 25 | ## 必要なもの 26 | - 常時起動しているマシン 27 | - LinuxなどUNIX的なOS 28 | - WindowsでもBash on Windows / Windows Subsystem for Linuxなら動きますがcronに依存しており、WSLではcronを動かすのが少し手間です 29 | - Ruby 2.4 or later 30 | - rtmpdump 31 | - swftools 32 | - あたらしめのffmpeg (HTTP Live Streaming の input に対応しているもの) 33 | - ※最新のffmpegの導入は面倒であることが多いです。自分はLinuxではstatic buildを使っています。 http://qiita.com/yayugu/items/d7f6a15a6f988064f51c 34 | - Macではhomebrewで導入できるバージョンで問題ありません 35 | - livedl 36 | - (ラジコエリアフリー利用者のみ) 37 | - ラジコプレミアム会員のアカウント 38 | - (AG-ON Premiumのみ) 39 | - AG-ON Premiumのアカウント 40 | - (ニコ生のみ) 41 | - プレミアム会員のアカウント 42 | 43 | ## セットアップ 44 | 45 | ### ふつうにセットアップ 46 | ``` 47 | # 必要なライブラリをインストール 48 | # Ubuntuの場合: 49 | $ # Mysqlは5.6以外でも可 50 | $ # Ubuntu 14.04だとrubyのversionが古いのでお好きな方法orこの辺(https://www.brightbox.com/blog/2016/01/06/ruby-2-3-ubuntu-packages/ ) を参考に新しめなバージョンをインストールしてください 51 | $ sudo apt-get install rtmpdump swftools ruby git mysql-server-5.6 mysql-client-5.6 libmysqld-dev 52 | $ sudo service mysql start # WSLだとっぽい表示がでるかもしれませんがプロセスが起動していればOK 53 | 54 | $ # libavがインストールされている場合には削除してから 55 | $ wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz 56 | $ tar xvf ffmpeg-release-amd64-static.tar.xz 57 | $ sudo cp ./ffmpeg-release-amd64-static/ffmpeg /usr/local/bin 58 | 59 | $ wget https://github.com/yayugu/livedl/releases/download/20181107.38/livedl 60 | $ sudo cp ./livedl /usr/local/bin/livedl 61 | $ sudo chmod +x /usr/local/bin/livedl 62 | # 取得したコンパイル済みバイナリが正常に動かない場合は、ここから(https://github.com/himananiito/livedl)ソースを取得して自前でコンパイルし、上記パスにインストールする 63 | 64 | $ git clone https://github.com/yayugu/net-radio-archive.git 65 | $ cd net-radio-archive 66 | $ (sudo) gem install bundler 67 | $ bundle install --without development test agon 68 | $ cp config/database.example.yml config/database.yml 69 | $ cp config/settings.example.yml config/settings.yml 70 | $ vi config/database.yml # 各自の環境に合わせて編集 71 | $ vi config/settings.yml # 各自の環境に合わせて編集 72 | $ RAILS_ENV=production bundle exec rake db:create db:migrate 73 | $ RAILS_ENV=production bundle exec whenever --update-crontab 74 | $ # (または) RAILS_ENV=production bundle exec whenever -u $YOUR-USERNAME --update-crontab 75 | 76 | # アップデート 77 | $ git pull origin master 78 | $ git submodule update --init --recursive 79 | $ bundle install --without development test agon 80 | $ RAILS_ENV=production bundle exec rake db:migrate 81 | $ RAILS_ENV=production bundle exec whenever --update-crontab 82 | ``` 83 | 84 | cronに 85 | `MAILTO='your-mail-address@foo.com'` 86 | のように記述してエラーが起きた時に検知しやすくしておくと便利です。 87 | 88 | 89 | ### Dockerでセットアップ 90 | Dockerの知識がある程度必要ですがわかっていれば楽です。 91 | 92 | まずMySQLサーバーを用意してください。 93 | ローカル用意してもdocker-composeとかで建ててもなんでもいいです 94 | そしてDockerコンテナからそのMySQLに疎通できるようにしておいてください 95 | 96 | ``` 97 | $ git clone https://github.com/yayugu/net-radio-archive.git 98 | $ cd net-radio-archive 99 | 100 | $ cp config/database.example.yml config/database.yml 101 | $ cp config/settings.example.yml config/settings.yml 102 | $ vi config/database.yml # 各自の環境に合わせて編集 103 | $ vi config/settings.yml # 各自の環境に合わせて編集 104 | 105 | $ docker build --network host -t yayugu/net-radio-archive . 106 | 107 | # 起動 108 | # いくつかのディレクトリはホストのものを使うことを推奨しています 109 | # /working : 作業用ディレクトリです。それなりに容量を消費します 110 | # /archive : 録画したファイルが置かれるディレクトリです。大事 111 | # /myapp/log : ログが置かれるディレクトリです 112 | $ docker run -d --rm --network host \ 113 | -v /host/path/to/working/dir:/working \ 114 | -v /host/path/to/archive/dir:/archive \ 115 | -v /host/path/to/log:/myapp/log \ 116 | yayugu/net-radio-archive 117 | 118 | # 長期運用する場合はlogrotateを入れておきましょう 119 | $ cat /etc/logrotate.d/net-radio-archive 120 | /host/path/to/log/*.log { 121 | daily 122 | missingok 123 | rotate 7 124 | notifempty 125 | copytruncate 126 | } 127 | ``` 128 | 129 | ## FAQ 130 | 131 | ### Q. 使い方でわからないところある 132 | A. Githubでissueつくってください。 133 | 134 | ### Q. ◯◯に対応してほしい 135 | A. Githubでissueつくってください。あとpull req募集中 136 | 137 | ### Q. radikoがうまく動かない 138 | A. Radikoはアクセスする側のIPによってどの局を聴けるかが変わります。 139 | ブラウザで開いてみたり、以下のページなどを参考にご自身が聞ける局をsettings.ymlに設定してください。 140 | 141 | http://d.hatena.ne.jp/zariganitosh/20130214/radiko_keyword_preset 142 | 143 | またプレミアム会員になることでエリアフリーですべての局を聴取することができます。 144 | ご自身のIPが希望する局のエリア外の場合にはラジコプレミアムに加入してradiko_premiumの設定を試してみてください 145 | 146 | ### Q. AG-ON Premiumで有料コンテンツを録画できない 147 | 自分が契約している月額コンテンツがないため、検証ができていません。 148 | そのため録画リストへの追加を行わないようにしています 149 | 150 | 対応してくれるpull reqを募集しております 151 | 152 | ### Q. rtmpdumpが不安定 / CPUを100%消費する 153 | gitで最新のソースを取得してきてビルドすると改善することが多いです。 154 | 155 | http://qiita.com/yayugu/items/12c0ffd92bc8539098b8 156 | 157 | ### Q. 録画がはじまらない / 特定のプラットフォーム or 局のみ録画がはじまらない 158 | 番組表の取得がまだ行われていない可能性があります。 config/schedule.rbを見ていただけるとわかるのですが番組表の取得は昼間中心となっています。お急ぎの場合は手動で 159 | 160 | ``` 161 | $ RAILS_ENV=production bundle exec rake main:XXXX_scrape 162 | ``` 163 | 164 | を実行してください 165 | 166 | ### Q. ニコ生の動作がいまいち 167 | ニコ生については色々制約が多いです 168 | - プレミアム会員必須 169 | - タイムシフトから取得するためタイムシフトに対応していない番組は対応不可 170 | - コメントはいまのところ取得できない 171 | - さまざまな理由でダウンロードに失敗することがある 172 | 173 | 改善のpull reqお待ちしております 174 | -------------------------------------------------------------------------------- /lib/main.rb: -------------------------------------------------------------------------------- 1 | require 'fileutils' 2 | require 'net/http' 3 | 4 | module Main 5 | def self.retry(limit = 3) 6 | exception = nil 7 | 8 | limit.times do 9 | begin 10 | return yield 11 | rescue => e 12 | exception = e 13 | end 14 | end 15 | raise exception 16 | end 17 | 18 | def self.sleep_until(time) 19 | now = Time.now 20 | if time - now <= 0 21 | Rails.logger.warn("rec start delayed? until:#{time} now:#{now}") 22 | return 23 | end 24 | sleep(time - Time.now) 25 | end 26 | 27 | def self.download(url, filename) 28 | uri = URI(url) 29 | use_ssl = uri.scheme == 'https' 30 | Net::HTTP.start(uri.host, uri.port, :use_ssl => use_ssl) do |http| 31 | request = Net::HTTP::Get.new(uri.request_uri) 32 | http.request(request) do |response| 33 | open(filename, 'wb') do |io| 34 | unless response.kind_of?(Net::HTTPSuccess) 35 | Rails.logger.error "download error: #{url}, #{response.code}" 36 | return false 37 | end 38 | 39 | response.read_body do |chunk| 40 | io.write chunk 41 | end 42 | end 43 | end 44 | end 45 | true 46 | end 47 | 48 | def self.shell_exec(command) 49 | output = `#{command}` 50 | exit_status = $? 51 | [exit_status, output] 52 | end 53 | 54 | def self.ffmpeg(arg) 55 | full = "ffmpeg #{arg} 2>&1" 56 | shell_exec(full) 57 | end 58 | 59 | def self.ffmpeg_with_timeout(duration, kill_duration, arg) 60 | full = "timeout -k #{kill_duration} #{duration} ffmpeg #{arg} 2>&1" 61 | shell_exec(full) 62 | end 63 | 64 | def self.convert_ffmpeg_to_mp4(src_path, dst_path, debug_obj) 65 | arg = "-loglevel error -y -i #{Shellwords.escape(src_path)} -acodec copy -vcodec copy #{Shellwords.escape(dst_path)}" 66 | convert_ffmpeg_to(arg, debug_obj) 67 | end 68 | 69 | def self.convert_ffmpeg_to_mp4_with_blank_video(src_path, dst_path, debug_obj) 70 | arg = "-loglevel error -y -s 320x240 -f rawvideo -pix_fmt rgb24 -r 1 -i /dev/zero -i #{Shellwords.escape(src_path)} -acodec copy -vcodec libx264 -shortest #{Shellwords.escape(dst_path)}" 71 | convert_ffmpeg_to(arg, debug_obj) 72 | end 73 | 74 | def self.convert_ffmpeg_to_m4a(src_path, dst_path, debug_obj) 75 | arg = "-loglevel error -y -i #{Shellwords.escape(src_path)} -acodec copy #{Shellwords.escape(dst_path)}" 76 | convert_ffmpeg_to(arg, debug_obj) 77 | end 78 | 79 | def self.convert_ffmpeg_to(arg, debug_obj) 80 | exit_status, output = ffmpeg(arg) 81 | unless output.empty? 82 | Rails.logger.info(output) 83 | end 84 | unless exit_status.success? 85 | Rails.logger.error "convert failed. debug_obj:#{debug_obj.inspect}, exit_status:#{exit_status}, output:#{output}" 86 | return false 87 | end 88 | if output.present? 89 | Rails.logger.warn "ffmpeg command:#{arg} output:#{output}" 90 | end 91 | true 92 | end 93 | 94 | def self.prepare_working_dir(ch_name) 95 | FileUtils.mkdir_p("#{Settings.working_dir}/#{ch_name}") 96 | end 97 | 98 | def self.latest_dir_name 99 | '0_latest' 100 | end 101 | 102 | def self.move_to_archive_dir(ch_name, date, src) 103 | filename = File.basename(src) 104 | dst_dir = "#{Settings.archive_dir}/#{ch_name}/#{month_str(date)}" 105 | dst = "#{dst_dir}/#{filename}" 106 | latest_dir = "#{Settings.archive_dir}/#{ch_name}/#{latest_dir_name}" 107 | latest_symlink = "#{latest_dir}/#{filename}" 108 | 109 | FileUtils.mkdir_p(dst_dir) 110 | FileUtils.mv(src, dst) 111 | 112 | # create symlink 113 | FileUtils.mkdir_p(latest_dir) 114 | unless File.exist?(latest_symlink) 115 | FileUtils.ln_s(dst, latest_symlink) 116 | end 117 | 118 | # create selections symlink 119 | if Settings.selections.present? 120 | if Settings.selections.any? {|s| !!filename.match(s) } 121 | selection_dir = "#{Settings.archive_dir}/0_selections" 122 | selection_symlink = "#{selection_dir}/#{filename}" 123 | FileUtils.mkdir_p(selection_dir) 124 | unless File.exist?(selection_symlink) 125 | FileUtils.ln_s(dst, selection_symlink) 126 | end 127 | end 128 | end 129 | end 130 | 131 | def self.file_path_archive(ch_name, title, ext) 132 | "#{Settings.archive_dir}/#{ch_name}/#{title_escape(title)}.#{ext}" 133 | end 134 | 135 | def self.file_path_working(ch_name, title, ext, date = nil) 136 | if date 137 | "#{Settings.working_dir}/#{ch_name}/#{month_str(date)}/#{title_escape(title)}.#{ext}" 138 | else 139 | "#{Settings.working_dir}/#{ch_name}/#{title_escape(title)}.#{ext}" 140 | end 141 | end 142 | 143 | def self.file_path_working_base(ch_name, title) 144 | "#{Settings.working_dir}/#{ch_name}/#{title_escape(title)}" 145 | end 146 | 147 | def self.title_escape(title) 148 | title 149 | .to_s 150 | .gsub(/\s/, '_') 151 | .gsub(/\//, '_') 152 | .gsub(/\?/, "？") # \/:*?"<>| 153 | .gsub(/\\/, "￥") 154 | .gsub(/:/, "：") 155 | .gsub(/\*/, "＊") 156 | .gsub(/\|/, "｜") 157 | .gsub(/"/, '”') 158 | .gsub(//, "＞") 160 | .byteslice(0, 200).scrub('') # safe length for filename 161 | end 162 | 163 | def self.month_str(date) 164 | date.strftime('%Y%m') 165 | end 166 | 167 | def self.check_file_size(path, expect_larger_than = (20 * 1024 * 1024)) # 20MB 168 | size = File.size?(path) 169 | size && size > expect_larger_than 170 | end 171 | end 172 | -------------------------------------------------------------------------------- /niconico/lib/niconico/live.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | require 'niconico/deferrable' 3 | require 'niconico/live/api' 4 | 5 | class Niconico 6 | def live(live_id) 7 | Live.new(self, live_id) 8 | end 9 | 10 | class Live 11 | include Niconico::Deferrable 12 | 13 | class ReservationOutdated < Exception; end 14 | class ReservationNotAccepted < Exception; end 15 | class TicketRetrievingFailed < Exception; end 16 | class AcceptingReservationFailed < Exception; end 17 | 18 | class << self 19 | def public_key 20 | @public_key ||= begin 21 | if ENV["NICONICO_LIVE_PUBLIC_KEY"] 22 | File.read(File.expand_path(ENV["NICONICO_LIVE_PUBLIC_KEY"])) 23 | else 24 | nil 25 | end 26 | end 27 | end 28 | 29 | def public_key=(other) 30 | @public_key = other 31 | end 32 | end 33 | 34 | def initialize(parent, live_id, preload = nil) 35 | @parent = parent 36 | @agent = parent.agent 37 | @id = @live_id = live_id 38 | @client = Niconico::Live::API.new(@agent) 39 | 40 | if preload 41 | preload_deffered_values(preload) 42 | else 43 | get 44 | end 45 | end 46 | 47 | attr_reader :id, :live, :ticket 48 | attr_writer :public_key 49 | 50 | def public_key 51 | @public_key || self.class.public_key 52 | end 53 | 54 | def fetched? 55 | !!@fetched 56 | end 57 | 58 | def get(force=false) 59 | return self if @fetched && !force 60 | @live = @client.get(@live_id) 61 | @fetched = true 62 | self 63 | end 64 | 65 | def seat(force=false) 66 | return @seat if @seat && !force 67 | raise ReservationNotAccepted if reserved? && !reservation_accepted? 68 | 69 | @seat = @client.get_player_status(self.id, self.public_key) 70 | 71 | raise TicketRetrievingFailed, @seat[:error] if @seat[:error] 72 | 73 | @seat 74 | end 75 | 76 | def accept_reservation 77 | return self if reservation_accepted? 78 | raise ReservationOutdated if reservation_outdated? 79 | 80 | result = @client.accept_watching_reservation(self.id) 81 | raise AcceptingReservationFailed unless result 82 | 83 | sleep 3 84 | 85 | # reload 86 | get(true) 87 | 88 | self 89 | end 90 | 91 | def inspect 92 | "#" 93 | end 94 | 95 | lazy :title do 96 | live[:title] 97 | end 98 | 99 | lazy :description do 100 | live[:description] 101 | end 102 | 103 | lazy :opens_at do 104 | live[:opens_at] 105 | end 106 | 107 | lazy :starts_at do 108 | live[:starts_at] 109 | end 110 | 111 | lazy :status do 112 | live[:status] 113 | end 114 | 115 | def scheduled? 116 | status == :scheduled 117 | end 118 | 119 | def on_air? 120 | status == :on_air 121 | end 122 | 123 | def closed? 124 | status == :closed 125 | end 126 | 127 | lazy :reservation do 128 | live[:reservation] 129 | end 130 | 131 | def reserved? 132 | !!reservation 133 | end 134 | 135 | def reservation_available? 136 | reserved? && reservation[:available] 137 | end 138 | 139 | def reservation_unaccepted? 140 | reservation_available? && reservation[:status] == :reserved 141 | end 142 | 143 | def reservation_accepted? 144 | reserved? && reservation[:status] == :accepted 145 | end 146 | 147 | def reservation_outdated? 148 | reserved? && reservation[:status] == :outdated 149 | end 150 | 151 | def reservation_expires_at 152 | reserved? ? reservation[:expires_at] : nil 153 | end 154 | 155 | lazy :channel do 156 | live[:channel] 157 | end 158 | 159 | def premium? 160 | !!seat[:premium?] 161 | end 162 | 163 | def rtmp_url 164 | seat[:rtmp][:url] 165 | end 166 | 167 | def ticket 168 | seat[:rtmp][:ticket] 169 | end 170 | 171 | def quesheet 172 | seat[:quesheet] 173 | end 174 | 175 | def execute_rtmpdump(file_base, ignore_failure = false) 176 | rtmpdump_commands(file_base).map do |cmd| 177 | system *cmd 178 | retval = $? 179 | raise RtmpdumpFailed, "#{cmd.inspect} failed" if !retval.success? && !ignore_failure 180 | [cmd, retval] 181 | end 182 | end 183 | 184 | def rtmpdump_infos(file_base) 185 | file_base = File.expand_path(file_base) 186 | 187 | publishes = quesheet.select{ |_| /^\/publish / =~ _[:body] }.map do |publish| 188 | publish[:body].split(/ /).tap(&:shift) 189 | end 190 | 191 | plays = quesheet.select{ |_| /^\/play / =~ _[:body] } 192 | 193 | infos = [] 194 | plays.flat_map.with_index do |play, i| 195 | cases = play[:body].sub(/^case:/,'').split(/ /)[1].split(/,/) 196 | publish_id = nil 197 | 198 | publish_id = cases.find { |_| _.start_with?('premium:') } if premium? 199 | publish_id ||= cases.find { |_| _.start_with?('default:') } 200 | publish_id ||= cases[0] 201 | 202 | publish_id = publish_id.split(/:/).last 203 | 204 | contents = publishes.select{ |_| _[0] == publish_id } 205 | contents.map.with_index do |content, j| 206 | content = content[1] 207 | rtmp = "#{self.rtmp_url}/mp4:#{content}" 208 | 209 | seq = 0 210 | begin 211 | file = "#{file_base}.#{i}.#{j}.#{seq}.flv" 212 | seq += 1 213 | end while File.exist?(file) 214 | app = URI.parse(self.rtmp_url).path.sub(/^\//,'') 215 | infos << { 216 | file_path: file, 217 | rtmp_url: rtmp, 218 | ticket: ticket, 219 | content: content, 220 | app: app 221 | } 222 | end 223 | infos 224 | end 225 | end 226 | end 227 | end 228 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: niconico 3 | specs: 4 | niconico (1.8.0.beta1) 5 | mechanize (>= 2.7.3) 6 | nokogiri (>= 1.6.1) 7 | 8 | GEM 9 | remote: https://rubygems.org/ 10 | specs: 11 | actionmailer (4.2.10) 12 | actionpack (= 4.2.10) 13 | actionview (= 4.2.10) 14 | activejob (= 4.2.10) 15 | mail (~> 2.5, >= 2.5.4) 16 | rails-dom-testing (~> 1.0, >= 1.0.5) 17 | actionpack (4.2.10) 18 | actionview (= 4.2.10) 19 | activesupport (= 4.2.10) 20 | rack (~> 1.6) 21 | rack-test (~> 0.6.2) 22 | rails-dom-testing (~> 1.0, >= 1.0.5) 23 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 24 | actionview (4.2.10) 25 | activesupport (= 4.2.10) 26 | builder (~> 3.1) 27 | erubis (~> 2.7.0) 28 | rails-dom-testing (~> 1.0, >= 1.0.5) 29 | rails-html-sanitizer (~> 1.0, >= 1.0.3) 30 | activejob (4.2.10) 31 | activesupport (= 4.2.10) 32 | globalid (>= 0.3.0) 33 | activemodel (4.2.10) 34 | activesupport (= 4.2.10) 35 | builder (~> 3.1) 36 | activerecord (4.2.10) 37 | activemodel (= 4.2.10) 38 | activesupport (= 4.2.10) 39 | arel (~> 6.0) 40 | activerecord-import (0.21.0) 41 | activerecord (>= 3.2) 42 | activesupport (4.2.10) 43 | i18n (~> 0.7) 44 | minitest (~> 5.1) 45 | thread_safe (~> 0.3, >= 0.3.4) 46 | tzinfo (~> 1.1) 47 | arel (6.0.4) 48 | builder (3.2.3) 49 | chronic (0.10.2) 50 | coderay (1.1.2) 51 | coffee-rails (4.0.1) 52 | coffee-script (>= 2.2.0) 53 | railties (>= 4.0.0, < 5.0) 54 | coffee-script (2.4.1) 55 | coffee-script-source 56 | execjs 57 | coffee-script-source (1.12.2) 58 | concurrent-ruby (1.0.5) 59 | crass (1.0.3) 60 | domain_name (0.5.20170404) 61 | unf (>= 0.0.5, < 1.0.0) 62 | erubis (2.7.0) 63 | execjs (2.7.0) 64 | globalid (0.4.1) 65 | activesupport (>= 4.2.0) 66 | hike (1.2.3) 67 | http-cookie (1.0.3) 68 | domain_name (~> 0.5) 69 | httparty (0.15.6) 70 | multi_xml (>= 0.5.2) 71 | i18n (0.9.1) 72 | concurrent-ruby (~> 1.0) 73 | jbuilder (2.7.0) 74 | activesupport (>= 4.2.0) 75 | multi_json (>= 1.2) 76 | jquery-rails (4.3.1) 77 | rails-dom-testing (>= 1, < 3) 78 | railties (>= 4.2.0) 79 | thor (>= 0.14, < 2.0) 80 | json (1.8.6) 81 | loofah (2.1.1) 82 | crass (~> 1.0.2) 83 | nokogiri (>= 1.5.9) 84 | m3u8 (0.8.2) 85 | mail (2.7.0) 86 | mini_mime (>= 0.1.1) 87 | mechanize (2.7.5) 88 | domain_name (~> 0.5, >= 0.5.1) 89 | http-cookie (~> 1.0) 90 | mime-types (>= 1.17.2) 91 | net-http-digest_auth (~> 1.1, >= 1.1.1) 92 | net-http-persistent (~> 2.5, >= 2.5.2) 93 | nokogiri (~> 1.6) 94 | ntlm-http (~> 0.1, >= 0.1.1) 95 | webrobots (>= 0.0.9, < 0.2) 96 | method_source (0.9.0) 97 | mime-types (3.1) 98 | mime-types-data (~> 3.2015) 99 | mime-types-data (3.2016.0521) 100 | mini_mime (1.0.0) 101 | mini_portile2 (2.3.0) 102 | minitest (5.10.3) 103 | moji (1.6) 104 | multi_json (1.12.2) 105 | multi_xml (0.6.0) 106 | mysql2 (0.3.21) 107 | net-http-digest_auth (1.4.1) 108 | net-http-persistent (2.9.4) 109 | nokogiri (1.8.1) 110 | mini_portile2 (~> 2.3.0) 111 | ntlm-http (0.1.1) 112 | pry (0.11.3) 113 | coderay (~> 1.1.0) 114 | method_source (~> 0.9.0) 115 | rack (1.6.8) 116 | rack-test (0.6.3) 117 | rack (>= 1.0) 118 | rails (4.2.10) 119 | actionmailer (= 4.2.10) 120 | actionpack (= 4.2.10) 121 | actionview (= 4.2.10) 122 | activejob (= 4.2.10) 123 | activemodel (= 4.2.10) 124 | activerecord (= 4.2.10) 125 | activesupport (= 4.2.10) 126 | bundler (>= 1.3.0, < 2.0) 127 | railties (= 4.2.10) 128 | sprockets-rails 129 | rails-deprecated_sanitizer (1.0.3) 130 | activesupport (>= 4.2.0.alpha) 131 | rails-dom-testing (1.0.8) 132 | activesupport (>= 4.2.0.beta, < 5.0) 133 | nokogiri (~> 1.6) 134 | rails-deprecated_sanitizer (>= 1.0.1) 135 | rails-html-sanitizer (1.0.3) 136 | loofah (~> 2.0) 137 | railties (4.2.10) 138 | actionpack (= 4.2.10) 139 | activesupport (= 4.2.10) 140 | rake (>= 0.8.7) 141 | thor (>= 0.18.1, < 2.0) 142 | rake (12.3.0) 143 | rdoc (4.3.0) 144 | sass (3.2.19) 145 | sass-rails (4.0.5) 146 | railties (>= 4.0.0, < 5.0) 147 | sass (~> 3.2.2) 148 | sprockets (~> 2.8, < 3.0) 149 | sprockets-rails (~> 2.0) 150 | sdoc (0.4.2) 151 | json (~> 1.7, >= 1.7.7) 152 | rdoc (~> 4.0) 153 | settingslogic (2.0.9) 154 | spring (2.0.2) 155 | activesupport (>= 4.2) 156 | sprockets (2.12.4) 157 | hike (~> 1.2) 158 | multi_json (~> 1.0) 159 | rack (~> 1.0) 160 | tilt (~> 1.1, != 1.3.0) 161 | sprockets-rails (2.3.3) 162 | actionpack (>= 3.0) 163 | activesupport (>= 3.0) 164 | sprockets (>= 2.8, < 4.0) 165 | thor (0.20.0) 166 | thread_safe (0.3.6) 167 | tilt (1.4.1) 168 | turbolinks (5.0.1) 169 | turbolinks-source (~> 5) 170 | turbolinks-source (5.0.3) 171 | tzinfo (1.2.4) 172 | thread_safe (~> 0.1) 173 | uglifier (3.2.0) 174 | execjs (>= 0.3.0, < 3) 175 | unf (0.1.4) 176 | unf_ext 177 | unf_ext (0.0.7.4) 178 | webrobots (0.1.2) 179 | whenever (0.10.0) 180 | chronic (>= 0.6.3) 181 | 182 | PLATFORMS 183 | ruby 184 | 185 | DEPENDENCIES 186 | activerecord-import 187 | chronic 188 | coffee-rails (~> 4.0.0) 189 | httparty 190 | jbuilder (~> 2.0) 191 | jquery-rails 192 | m3u8 193 | mechanize 194 | moji 195 | mysql2 (~> 0.3.18) 196 | niconico! 197 | nokogiri 198 | pry 199 | rails (= 4.2.10) 200 | sass-rails (~> 4.0.3) 201 | sdoc (~> 0.4.0) 202 | settingslogic 203 | spring 204 | turbolinks 205 | uglifier (>= 1.3.0) 206 | whenever 207 | 208 | BUNDLED WITH 209 | 1.16.0 210 | -------------------------------------------------------------------------------- /lib/hibiki/downloading.rb: -------------------------------------------------------------------------------- 1 | require 'shellwords' 2 | require 'fileutils' 3 | require 'm3u8' 4 | require 'uri' 5 | 6 | module Hibiki 7 | class Downloading 8 | CH_NAME = 'hibiki' 9 | 10 | class HibikiDownloadException < StandardError; end 11 | 12 | def initialize 13 | @a = Mechanize.new 14 | @a.user_agent_alias = 'Windows Chrome' 15 | end 16 | 17 | def download(program) 18 | infos = get_infos(program) 19 | actual_episode_id = infos.try(:[], 'episode').try(:[], 'id') 20 | if actual_episode_id == nil 21 | program.state = HibikiProgramV2::STATE[:not_downloadable] 22 | return 23 | end 24 | if actual_episode_id != program.episode_id 25 | Rails.logger.error("episode outdated. title=#{program.title} expected_episode_id=#{program.episode_id} actual_episode_id=#{actual_episode_id}") 26 | program.state = HibikiProgramV2::STATE[:outdated] 27 | return 28 | end 29 | live_flg = infos['episode'].try(:[], 'video').try(:[], 'live_flg') 30 | if live_flg == nil || live_flg == true 31 | program.state = HibikiProgramV2::STATE[:not_downloadable] 32 | return 33 | end 34 | url = get_m3u8_url(infos['episode']['video']['id']) 35 | 36 | prepare_working_dir(program) 37 | path = process_m3u8(program, url) 38 | unless download_hls(program, path) 39 | clean_working_dir(program) 40 | program.state = HibikiProgramV2::STATE[:failed] 41 | return 42 | end 43 | clean_working_dir(program) 44 | program.state = HibikiProgramV2::STATE[:done] 45 | end 46 | 47 | def get_infos(program) 48 | res = get_api("https://vcms-api.hibiki-radio.jp/api/v1/programs/#{program.access_id}") 49 | infos = JSON.parse(res.body) 50 | end 51 | 52 | def get_m3u8_url(video_id) 53 | res = get_api("https://vcms-api.hibiki-radio.jp/api/v1/videos/play_check?video_id=#{video_id}") 54 | play_infos = JSON.parse(res.body) 55 | url = play_infos['playlist_url'] 56 | if play_infos['token'] 57 | url += "&token=#{play_infos['token']}" 58 | end 59 | url 60 | end 61 | 62 | def process_m3u8(program, m3u8_url) 63 | 64 | playlist_m3u8 = get_api(m3u8_url).body 65 | playlist = M3u8::Playlist.read(playlist_m3u8) 66 | 67 | ts_audio_m3u8_url = playlist.items.first.uri 68 | if ts_audio_m3u8_url =~ %r(^https://ad.hibiki-radio.jp/m3u8/dynamic_media) 69 | uri = URI(ts_audio_m3u8_url) 70 | queries = URI::decode_www_form(uri.query).to_h 71 | ts_audio_m3u8_url = queries['url2'] 72 | end 73 | ts_audio_m3u8 = get_api(ts_audio_m3u8_url).body 74 | ts_audio = M3u8::Playlist.read(ts_audio_m3u8) 75 | 76 | uri = URI.parse(ts_audio_m3u8_url) 77 | path = "#{uri.scheme}://#{uri.host}#{File.dirname(uri.path)}/" 78 | 79 | ts_audio.items.each do |item| 80 | if item.kind_of?(M3u8::KeyItem) 81 | key_url = item.uri 82 | key_body = get_api(key_url).body.force_encoding('BINARY') 83 | key_path = working_dir(program) + 'key' 84 | IO.binwrite(key_path, key_body) 85 | item.uri = "file:/#{key_path}" 86 | elsif item.kind_of?(M3u8::SegmentItem) 87 | url = path + item.segment 88 | dst_path = working_dir(program) + item.segment 89 | download_ts(program, url, dst_path) 90 | item.segment = "file:/#{dst_path}" 91 | end 92 | end 93 | 94 | m3u8_path = working_dir(program) + 'ts_audio.m3u8' 95 | File.write(m3u8_path, ts_audio.to_s) 96 | m3u8_path 97 | end 98 | 99 | def download_ts(program, url, dst_path) 100 | command = "curl \ 101 | --silent \ 102 | --show-error \ 103 | --connect-timeout 10 \ 104 | --max-time 30 \ 105 | --retry 5 \ 106 | #{Shellwords.escape(url)} \ 107 | -o #{Shellwords.escape(dst_path)} \ 108 | 2>&1" 109 | exit_status, output = Main::shell_exec(command) 110 | unless exit_status.success? 111 | raise HibikiDownloadException, "ts download faild, hibiki program:#{program.id}, exit_status:#{exit_status}, output:#{output}" 112 | end 113 | end 114 | 115 | def download_hls(program, m3u8_path) 116 | file_path = Main::file_path_working(CH_NAME, title(program), 'mp4') 117 | arg = "\ 118 | -loglevel error \ 119 | -y \ 120 | -allowed_extensions ALL \ 121 | -protocol_whitelist file,crypto,http,https,tcp,tls \ 122 | -i #{Shellwords.escape(m3u8_path)} \ 123 | -timeout 10000000 \ 124 | -vcodec copy -acodec copy -bsf:a aac_adtstoasc \ 125 | #{Shellwords.escape(file_path)}" 126 | 127 | exit_status, output = Main::ffmpeg_with_timeout('5h', '1m', arg) 128 | unless exit_status.success? 129 | Rails.logger.error "rec failed. hibiki program:#{program.id}, exit_status:#{exit_status}, output:#{output}" 130 | return false 131 | end 132 | if output.present? 133 | Rails.logger.warn "hibiki ffmpeg command:#{arg} output:#{output}" 134 | end 135 | 136 | Main::move_to_archive_dir(CH_NAME, program.created_at, file_path) 137 | 138 | true 139 | end 140 | 141 | def get_api(url) 142 | @a.get( 143 | url, 144 | [], 145 | "http://hibiki-radio.jp/", 146 | 'X-Requested-With' => 'XMLHttpRequest', 147 | 'Origin' => 'http://hibiki-radio.jp' 148 | ) 149 | end 150 | 151 | def title(program) 152 | date = program.created_at.strftime('%Y_%m_%d') 153 | title = "#{date}_#{program.title}_#{program.episode_name}" 154 | if program.cast.present? 155 | cast = program.cast.gsub(',', ' ') 156 | title += "_#{cast}" 157 | end 158 | title 159 | end 160 | 161 | def working_dir(program) 162 | "#{Settings.working_dir}/#{CH_NAME}/#{program.id}/" 163 | end 164 | 165 | def prepare_working_dir(program) 166 | FileUtils.mkdir_p(working_dir(program)) 167 | end 168 | 169 | def clean_working_dir(program) 170 | FileUtils.rm_rf(working_dir(program)) 171 | end 172 | end 173 | end 174 | -------------------------------------------------------------------------------- /lib/ag/scraping.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'time' 3 | require 'chronic' 4 | require 'pp' 5 | require 'moji' 6 | 7 | module Ag 8 | class Program < Struct.new(:start_time, :minutes, :title) 9 | end 10 | 11 | class ProgramTime < Struct.new(:wday, :time) 12 | SAME_DAY_LINE_HOUR = 5 13 | 14 | # convert human friendly time to computer friendly time 15 | def self.parse(wday, time_str) 16 | m = time_str.match(/(\d+):(\d+)/) 17 | hour = m[1].to_i 18 | min = m[2].to_i 19 | over_24_oclock = false 20 | if hour >= 24 # 25:00 とかの表記用 21 | over_24_oclock = true 22 | hour -= 24 23 | wday = (wday + 1) % 7 24 | end 25 | time_str_fixed = sprintf("%02d:%02d", hour, min) 26 | time = Time.parse(time_str_fixed) 27 | if !over_24_oclock && time.hour < SAME_DAY_LINE_HOUR # 01:00 とかの表記用。現在は使われていないが一応 28 | wday = (wday + 1) % 7 29 | end 30 | self.new(wday, time) 31 | end 32 | 33 | def next_on_air 34 | time = chronic(wday_for_chronic_include_today(self[:wday])) 35 | if time > Time.now 36 | return time 37 | else 38 | chronic(wday_to_s(self[:wday])) 39 | end 40 | end 41 | 42 | def chronic(day_str) 43 | Chronic.parse( 44 | "#{day_str} #{self[:time].strftime("%H:%M")}", 45 | context: :future, 46 | ambiguous_time_range: :none, 47 | hours24: true, 48 | guess: :begin 49 | ) 50 | end 51 | 52 | def wday_for_chronic_include_today(wday) 53 | if Time.now.wday == wday 54 | return 'today' 55 | end 56 | wday_to_s(wday) 57 | end 58 | 59 | def wday_to_s(wday) 60 | %w(Sunday Monday Tuesday Wednesday Thursday Friday Saturday)[wday] 61 | end 62 | end 63 | 64 | class Scraping 65 | def main 66 | programs = scraping_page 67 | programs = validate_programs(programs) 68 | programs 69 | end 70 | 71 | def validate_programs(programs) 72 | if programs.size < 20 73 | puts "Error: Number of programs is too few!" 74 | exit 75 | end 76 | programs.delete_if do |program| 77 | program.title == '放送休止' 78 | end 79 | end 80 | 81 | 82 | def scraping_page 83 | res = HTTParty.get('http://www.agqr.jp/timetable/streaming.html') 84 | dom = Nokogiri::HTML.parse(res.body) 85 | tbody = dom.css('.timetb-ag tbody') # may be 30minutes belt 86 | td_list_list = parse_broken_table(tbody) 87 | two_dim_array = table_to_two_dim_array(td_list_list) 88 | day_time_array = join_box_program(transpose(two_dim_array)) 89 | day_time_array.each_with_index.inject([]) do |programs, (programs_day, index)| 90 | programs + parse_day(programs_day, index) 91 | end 92 | end 93 | 94 | def parse_broken_table(tbody) 95 | # time table HTML is broken!!!!!! some row aren't opened by . 96 | td_list_list = [] 97 | td_list_tmp = [] 98 | tbody.children.each do |tag| 99 | if tag.name == 'td' 100 | td_list_tmp.push tag 101 | elsif tag.name == 'tr' || tag.name == 'th' 102 | unless td_list_tmp.empty? 103 | td_list_list.push td_list_tmp 104 | td_list_tmp = [] 105 | end 106 | if tag.name == 'tr' 107 | td_list_list.push tag.css('td') 108 | end 109 | end 110 | end 111 | unless td_list_tmp.empty? 112 | td_list_list.push td_list_tmp 113 | end 114 | td_list_list 115 | end 116 | 117 | def parse_day(programs_day, index) 118 | wday = (index + 1) % 7 # monday start 119 | programs_day.map do |td| 120 | parse_td_dom(td, wday) 121 | end 122 | end 123 | 124 | def table_to_two_dim_array(td_list_list) 125 | aa = [] 126 | span = {} 127 | td_list_list.each_with_index do |td_list, row_n| 128 | a = [] 129 | col_n = 0 130 | td_list.each do |td| 131 | while span[[row_n, col_n]] 132 | a.push(nil) 133 | col_n += 1 134 | end 135 | a.push(td) 136 | cspan = 1 137 | if td['colspan'] =~ /(\d+)/ 138 | cspan = $1.to_i 139 | end 140 | rspan = 1 141 | if td['rowspan'] =~ /(\d+)/ 142 | rspan = $1.to_i 143 | end 144 | (row_n...(row_n + rspan)).each do |r| 145 | (col_n...(col_n + cspan)).each do |c| 146 | span[[r, c]] = true 147 | end 148 | end 149 | col_n += 1 150 | end 151 | aa.push(a) 152 | end 153 | aa 154 | end 155 | 156 | def transpose(two_dim_array) 157 | max_size = two_dim_array.max_by{|i| i.size }.size 158 | filled = two_dim_array.map{|i| i.fill(nil, i.size...max_size) } 159 | filled.transpose 160 | end 161 | 162 | def join_box_program(day_time_array) 163 | day_time_array.map do |day| 164 | day.inject([]) do |programs, td| 165 | unless td 166 | next programs 167 | end 168 | time = td.css('.time')[0].text 169 | if time.include?('頃') 170 | programs.last['rowspan'] = programs.last['rowspan'].to_i + td['rowspan'].to_i 171 | next programs 172 | end 173 | programs << td 174 | end 175 | end 176 | end 177 | 178 | def parse_td_dom(td, wday) 179 | start_time = parse_start_time(td, wday) 180 | minutes = parse_minutes(td) 181 | title = parse_title(td) 182 | Program.new(start_time, minutes, title) 183 | end 184 | 185 | def parse_minutes(td) 186 | rowspan = td.attribute('rowspan') 187 | if !rowspan || rowspan.value.blank? 188 | 30 189 | else 190 | td.attribute('rowspan').value.to_i 191 | end 192 | end 193 | 194 | def parse_start_time(td, wday) 195 | ProgramTime.parse(wday, td.css('.time')[0].text) 196 | end 197 | 198 | def parse_title(td) 199 | [td.css('.title-p')[0].text, td.css('.rp')[0].text].select do |text| 200 | !text.gsub(/\s/, '').empty? 201 | end.map do |text| 202 | Moji.normalize_zen_han(text).strip 203 | end.join(' ') 204 | end 205 | end 206 | end 207 | -------------------------------------------------------------------------------- /lib/radiko/recording.rb: -------------------------------------------------------------------------------- 1 | require 'net/https' 2 | require 'shellwords' 3 | require 'fileutils' 4 | require 'base64' 5 | 6 | module Radiko 7 | class Recording 8 | SWF_URL = 'http://radiko.jp/apps/js/flash/myplayer-release.swf' 9 | RTMP_URL = 'rtmpe://f-radiko.smartstream.ne.jp' 10 | WORK_DIR_NAME = 'radiko' 11 | 12 | def initialize 13 | @cookie = '' 14 | end 15 | 16 | def record(job) 17 | unless exec_rec(job) 18 | return false 19 | end 20 | exec_convert(job) 21 | 22 | true 23 | end 24 | 25 | def exec_rec(job) 26 | begin 27 | Main::prepare_working_dir(WORK_DIR_NAME) 28 | Main::prepare_working_dir(job.ch) 29 | Main::retry do 30 | auth(job) 31 | end 32 | rtmp(job) 33 | ensure 34 | logout 35 | end 36 | end 37 | 38 | def auth(job) 39 | dl_swf 40 | extract_swf 41 | login(job) 42 | auth1 43 | auth2 44 | end 45 | 46 | def dl_swf 47 | unless File.exists?(swf_path) 48 | Main::download(SWF_URL, swf_path) 49 | end 50 | end 51 | 52 | def extract_swf 53 | unless File.exists?(key_path) 54 | command = "swfextract -b 12 #{swf_path} -o #{key_path}" 55 | Main::shell_exec(command) 56 | end 57 | end 58 | 59 | def login(job) 60 | if !Settings.radiko_premium || 61 | !Settings.radiko_premium.mail || 62 | !Settings.radiko_premium.password || 63 | !Settings.radiko_premium.channels.include?(job.ch) 64 | return 65 | end 66 | uri = URI('https://radiko.jp/ap/member/login/login') 67 | https = Net::HTTP.new(uri.host, uri.port) 68 | https.use_ssl = true 69 | https.verify_mode = OpenSSL::SSL::VERIFY_NONE 70 | https.start do |h| 71 | req = Net::HTTP::Post.new(uri.path) 72 | req.set_form_data( 73 | 'mail' => Settings.radiko_premium.mail, 74 | 'pass' => Settings.radiko_premium.password, 75 | ) 76 | res = h.request(req) 77 | @cookie = res.response['set-cookie'] 78 | end 79 | end 80 | 81 | def auth1 82 | uri = URI('https://radiko.jp/v2/api/auth1_fms') 83 | https = Net::HTTP.new(uri.host, uri.port) 84 | https.use_ssl = true 85 | https.verify_mode = OpenSSL::SSL::VERIFY_NONE 86 | https.start do |h| 87 | res = h.post( 88 | uri.path, 89 | '', 90 | { 91 | 'pragma' => 'no-cache', 92 | 'X-Radiko-App' => 'pc_ts', 93 | 'X-Radiko-App-Version' => '4.0.0', 94 | 'X-Radiko-User' => 'test-stream', 95 | 'X-Radiko-Device' => 'pc', 96 | 'Cookie' => @cookie, 97 | } 98 | ) 99 | @auth_token = /x-radiko-authtoken=([\w-]+)/i.match(res.body)[1] 100 | @offset = /x-radiko-keyoffset=(\d+)/i.match(res.body)[1].to_i 101 | @length = /x-radiko-keylength=(\d+)/i.match(res.body)[1].to_i 102 | end 103 | end 104 | 105 | def partialkey 106 | open(key_path, 'rb:ASCII-8BIT') do |fp| 107 | fp.seek(@offset) 108 | return Base64.strict_encode64(fp.read(@length)) 109 | end 110 | end 111 | 112 | def auth2 113 | uri = URI('https://radiko.jp/v2/api/auth2_fms') 114 | https = Net::HTTP.new(uri.host, uri.port) 115 | https.use_ssl = true 116 | https.verify_mode = OpenSSL::SSL::VERIFY_NONE 117 | https.start do |h| 118 | res = h.post( 119 | uri.path, 120 | '', 121 | { 122 | 'pragma' => 'no-cache', 123 | 'X-Radiko-App' => 'pc_ts', 124 | 'X-Radiko-App-Version' => '4.0.0', 125 | 'X-Radiko-User' => 'test-stream', 126 | 'X-Radiko-Device' => 'pc', 127 | 'X-Radiko-Authtoken' => @auth_token, 128 | 'X-Radiko-Partialkey' => partialkey, 129 | 'Cookie' => @cookie, 130 | } 131 | ) 132 | @area_id = /^([^,]+),/.match(res.body)[1] 133 | end 134 | end 135 | 136 | def rtmp(job) 137 | Main::sleep_until(job.start - 10.seconds) 138 | 139 | length = job.length_sec + 60 140 | flv_path = Main::file_path_working(job.ch, title(job), 'flv') 141 | command = "\ 142 | rtmpdump \ 143 | -r #{Shellwords.escape(RTMP_URL)} \ 144 | --playpath 'simul-stream.stream' \ 145 | --app '#{job.ch}/_definst_' \ 146 | -W #{SWF_URL} \ 147 | -C S:'' -C S:'' -C S:'' -C S:#{@auth_token} \ 148 | --live \ 149 | --stop #{length} \ 150 | -o #{Shellwords.escape(flv_path)} \ 151 | 2>&1" 152 | 153 | exit_status, output = Main::shell_exec(command) 154 | unless exit_status.success? 155 | Rails.logger.error "rec failed. job:#{job.id}, exit_status:#{exit_status}, output:#{output}" 156 | return false 157 | end 158 | 159 | true 160 | end 161 | 162 | def logout 163 | if @cookie.empty? 164 | return 165 | end 166 | uri = URI('https://radiko.jp/ap/member/webapi/member/logout') 167 | https = Net::HTTP.new(uri.host, uri.port) 168 | https.use_ssl = true 169 | https.verify_mode = OpenSSL::SSL::VERIFY_NONE 170 | https.start do |h| 171 | res = h.get( 172 | uri.path, 173 | { 174 | 'pragma' => 'no-cache', 175 | 'Accept' => 'application/json, text/javascript, */*; q=0.01', 176 | 'X-Radiko-App-Version' => 'application/json, text/javascript, */*; q=0.01', 177 | 'X-Requested-With' => 'XMLHttpRequest', 178 | 'Cookie' => @cookie, 179 | } 180 | ) 181 | end 182 | end 183 | 184 | def exec_convert(job) 185 | flv_path = Main::file_path_working(job.ch, title(job), 'flv') 186 | if Settings.force_mp4 187 | mp4_path = Main::file_path_working(job.ch, title(job), 'mp4') 188 | Main::convert_ffmpeg_to_mp4_with_blank_video(flv_path, mp4_path, job) 189 | dst_path = mp4_path 190 | else 191 | m4a_path = Main::file_path_working(job.ch, title(job), 'm4a') 192 | Main::convert_ffmpeg_to_m4a(flv_path, m4a_path, job) 193 | dst_path = m4a_path 194 | end 195 | Main::move_to_archive_dir(job.ch, job.start, dst_path) 196 | end 197 | 198 | def title(job) 199 | date = job.start.strftime('%Y_%m_%d_%H%M') 200 | "#{date}_#{job.title}" 201 | end 202 | 203 | def swf_path 204 | Main::file_path_working_base(WORK_DIR_NAME, "player2.swf") 205 | end 206 | 207 | def key_path 208 | Main::file_path_working_base(WORK_DIR_NAME, "radiko_key2.png") 209 | end 210 | end 211 | end 212 | -------------------------------------------------------------------------------- /db/schema.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | # This file is auto-generated from the current state of the database. Instead 3 | # of editing this file, please use the migrations feature of Active Record to 4 | # incrementally modify your database, and then regenerate this schema definition. 5 | # 6 | # Note that this schema.rb definition is the authoritative source for your 7 | # database schema. If you need to create the application database on another 8 | # system, you should be using db:schema:load, not running all the migrations 9 | # from scratch. The latter is a flawed and unsustainable approach (the more migrations 10 | # you'll amass, the slower it'll run and the greater likelihood for issues). 11 | # 12 | # It's strongly recommended that you check this file into your version control system. 13 | 14 | ActiveRecord::Schema.define(version: 20170505142758) do 15 | 16 | create_table "agon_programs", force: true do |t| 17 | t.string "title", limit: 250, null: false 18 | t.string "personality", limit: 250, null: false 19 | t.string "episode_id", limit: 250, null: false 20 | t.string "page_url", limit: 767, null: false 21 | t.string "state", limit: 100, null: false 22 | t.integer "retry_count", null: false 23 | t.datetime "created_at", null: false 24 | t.datetime "updated_at", null: false 25 | end 26 | 27 | add_index "agon_programs", ["episode_id"], name: "episode_id", unique: true, using: :btree 28 | add_index "agon_programs", ["page_url"], name: "page_url", using: :btree 29 | 30 | create_table "agonp_programs", force: true do |t| 31 | t.string "title", limit: 250, null: false 32 | t.string "personality", limit: 250, null: false 33 | t.string "episode_id", limit: 250, null: false 34 | t.string "price", limit: 100, null: false 35 | t.string "state", limit: 100, null: false 36 | t.integer "retry_count", null: false 37 | t.datetime "created_at", null: false 38 | t.datetime "updated_at", null: false 39 | end 40 | 41 | add_index "agonp_programs", ["episode_id"], name: "episode_id", unique: true, using: :btree 42 | 43 | create_table "anitama_programs", force: true do |t| 44 | t.string "book_id", limit: 250, null: false 45 | t.string "title", limit: 250, null: false 46 | t.datetime "update_time", null: false 47 | t.string "state", limit: 100, null: false 48 | t.integer "retry_count", null: false 49 | t.datetime "created_at", null: false 50 | t.datetime "updated_at", null: false 51 | end 52 | 53 | add_index "anitama_programs", ["book_id", "update_time"], name: "book_id", unique: true, using: :btree 54 | 55 | create_table "ar_internal_metadata", primary_key: "key", force: true do |t| 56 | t.string "value" 57 | t.datetime "created_at", null: false 58 | t.datetime "updated_at", null: false 59 | end 60 | 61 | create_table "hibiki_program_v2s", force: true do |t| 62 | t.string "access_id", limit: 100, null: false 63 | t.integer "episode_id", null: false 64 | t.string "title", limit: 250, null: false 65 | t.string "episode_name", limit: 250, null: false 66 | t.string "cast", limit: 250, null: false 67 | t.string "state", limit: 100, null: false 68 | t.integer "retry_count", null: false 69 | t.datetime "created_at", null: false 70 | t.datetime "updated_at", null: false 71 | end 72 | 73 | add_index "hibiki_program_v2s", ["access_id", "episode_id"], name: "access_id", unique: true, using: :btree 74 | 75 | create_table "hibiki_programs", force: true do |t| 76 | t.string "title", limit: 250, null: false 77 | t.string "comment", limit: 150, null: false 78 | t.string "rtmp_url", limit: 767, null: false 79 | t.string "state", limit: 100, null: false 80 | t.integer "retry_count", null: false 81 | t.datetime "created_at", null: false 82 | t.datetime "updated_at", null: false 83 | end 84 | 85 | add_index "hibiki_programs", ["rtmp_url"], name: "rtmp_url", unique: true, using: :btree 86 | 87 | create_table "jobs", force: true do |t| 88 | t.string "ch", limit: 100, null: false 89 | t.datetime "start", null: false 90 | t.datetime "end", null: false 91 | t.string "title", limit: 250, null: false 92 | t.string "state", limit: 100, null: false 93 | t.datetime "created_at", null: false 94 | t.datetime "updated_at", null: false 95 | end 96 | 97 | add_index "jobs", ["ch", "end", "state"], name: "end_index", using: :btree 98 | add_index "jobs", ["ch", "start", "state"], name: "start_index", using: :btree 99 | 100 | create_table "key_value", primary_key: "key", force: true do |t| 101 | t.string "value", limit: 250, null: false 102 | t.datetime "created_at", null: false 103 | t.datetime "updated_at", null: false 104 | end 105 | 106 | create_table "niconico_live_programs", force: true do |t| 107 | t.string "title", limit: 250, null: false 108 | t.string "state", limit: 100, null: false 109 | t.boolean "cannot_recovery", null: false 110 | t.text "memo", null: false 111 | t.integer "retry_count", null: false 112 | t.datetime "created_at", null: false 113 | t.datetime "updated_at", null: false 114 | end 115 | 116 | create_table "onsen_programs", force: true do |t| 117 | t.string "title", limit: 250, null: false 118 | t.string "number", limit: 100, null: false 119 | t.datetime "date", null: false 120 | t.string "file_url", limit: 767, null: false 121 | t.string "personality", limit: 250, null: false 122 | t.string "state", limit: 100, null: false 123 | t.integer "retry_count", null: false 124 | t.datetime "created_at", null: false 125 | t.datetime "updated_at", null: false 126 | end 127 | 128 | add_index "onsen_programs", ["file_url"], name: "file_url", unique: true, using: :btree 129 | 130 | create_table "wikipedia_category_items", force: true do |t| 131 | t.string "category", limit: 100, null: false 132 | t.string "title", limit: 100, null: false 133 | t.datetime "created_at", null: false 134 | t.datetime "updated_at", null: false 135 | end 136 | 137 | add_index "wikipedia_category_items", ["category", "title"], name: "category", unique: true, using: :btree 138 | 139 | end 140 | -------------------------------------------------------------------------------- /niconico/lib/niconico/live/api.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | require 'time' 3 | require 'openssl' 4 | require 'niconico/live/util' 5 | 6 | class Niconico 7 | class Live 8 | class API 9 | class NoPublicKeyProvided < Exception; end 10 | 11 | URL_GETPLAYERSTATUS = 'http://ow.live.nicovideo.jp/api/getplayerstatus'.freeze 12 | URL_WATCHINGRESERVATION_LIST = 'http://live.nicovideo.jp/api/watchingreservation?mode=list' 13 | 14 | def initialize(agent) 15 | @agent = agent 16 | end 17 | 18 | attr_reader :agent 19 | 20 | def self.parse_reservation_message(message) 21 | valid_until_message = message.match(/(?:使用|利用)?期限: (.+?)まで|(?:期限中、何度でも視聴できます|視聴権(?:利用|使用)期限が切れています|タイムシフト利用期間は終了しました)\s*\[(.+?)まで\]/) 22 | valid_message = message.match(/\[視聴期限未定\]/) 23 | 24 | case 25 | when message.match(/予約中\s*\[/) 26 | {status: :reserved, available: false} 27 | when valid_until_message || valid_message 28 | {}.tap do |reservation| 29 | if valid_until_message 30 | reservation[:expires_at] = Time.parse("#{valid_until_message[1] || valid_until_message[2]} +0900") 31 | end 32 | 33 | if message.include?('視聴権を使用し、タイムシフト視聴を行いますか？') 34 | reservation[:status] = :reserved 35 | reservation[:available] = true 36 | elsif message.include?('本番組は、タイムシフト視聴を行う事が可能です。') \ 37 | || message.include?('期限中、何度でも視聴できます') 38 | reservation[:status] = :accepted 39 | reservation[:available] = true 40 | elsif message.include?('タイムシフト視聴をこれ以上行う事は出来ません。') \ 41 | || message.include?('視聴権の利用期限が過ぎています。') \ 42 | || message.include?('視聴権利用期限が切れています') \ 43 | || message.include?('視聴権使用期限が切れています') \ 44 | || message.include?('タイムシフト利用期間は終了しました') \ 45 | || message.include?('アーカイブ公開期限は終了しました。') 46 | reservation[:status] = :outdated 47 | reservation[:available] = false 48 | end 49 | end 50 | else 51 | nil 52 | end 53 | end 54 | 55 | def get(id) 56 | id = Util::normalize_id(id) 57 | 58 | page = agent.get("http://live.nicovideo.jp/gate/#{id}") 59 | 60 | comment_area = page.at("#comment_area#{id}").inner_text 61 | 62 | result = { 63 | title: page.at('h2 span[itemprop="name"]').inner_text, 64 | id: id, 65 | description: page.at('.stream_description .text_area').inner_html, 66 | } 67 | 68 | kaijo = page.search('.kaijo strong').map(&:inner_text) 69 | result[:opens_at] = Time.parse("#{kaijo[0]} #{kaijo[1]} +0900") 70 | result[:starts_at] = Time.parse("#{kaijo[0]} #{kaijo[2]} +0900") 71 | 72 | result[:status] = :scheduled if comment_area.include?('開場まで、あと') 73 | result[:status] = :on_air if comment_area.include?('現在放送中') 74 | close_message = comment_area.match(/この番組は(.+?)に終了いたしました。/) 75 | if close_message 76 | result[:status] = :closed 77 | result[:closed_at] = Time.parse("#{close_message[1]} +0900") 78 | end 79 | 80 | result[:reservation] = self.class.parse_reservation_message(comment_area) 81 | if !result[:reservation] && page.search(".watching_reservation_reserved").any? { |_| _['onclick'].include?(id) } 82 | result[:reservation] = {status: :reserved, available: false} 83 | end 84 | 85 | channel = page.at('div.chan') 86 | if channel 87 | result[:channel] = { 88 | name: channel.at('.shosai a').inner_text, 89 | id: channel.at('.shosai a')['href'].split('/').last, 90 | link: channel.at('.shosai a')['href'], 91 | } 92 | end 93 | 94 | result 95 | end 96 | 97 | def heartbeat 98 | raise NotImplementedError 99 | end 100 | 101 | def get_player_status(id, public_key = nil) 102 | id = Util::normalize_id(id) 103 | page = agent.get("http://ow.live.nicovideo.jp/api/getplayerstatus?locale=GLOBAL&lang=ja%2Djp&v=#{id}&seat%5Flocale=JP") 104 | if page.body[0] == 'c' # encrypted 105 | page = Nokogiri::XML(decrypt_encrypted_player_status(page.body, public_key)) 106 | end 107 | 108 | status = page.at('getplayerstatus') 109 | 110 | if status['status'] == 'fail' 111 | error = page.at('error code').inner_text 112 | return {error: error} 113 | end 114 | 115 | result = {} 116 | 117 | # Strings 118 | %w(id title description provider_type owner_name 119 | bourbon_url full_video kickout_video).each do |key| 120 | item = status.at(key) 121 | result[key.to_sym] = item.inner_text if item 122 | end 123 | 124 | # Integers 125 | %w(watch_count comment_count owner_id watch_count comment_count).each do |key| 126 | item = status.at(key) 127 | result[key.to_sym] = item.inner_text.to_i if item 128 | end 129 | 130 | # Flags 131 | %w(is_premium is_reserved is_owner international is_rerun_stream is_archiveplayserver 132 | archive allow_netduetto 133 | is_nonarchive_timeshift_enabled is_timeshift_reserved).each do |key| 134 | item = status.at(key) 135 | result[key.sub(/^is_/,'').concat('?').to_sym] = item.inner_text == '1' if item 136 | end 137 | 138 | # Datetimes 139 | %w(base_time open_time start_time end_time).each do |key| 140 | item = status.at(key) 141 | result[key.to_sym] = Time.at(item.inner_text.to_i) if item 142 | end 143 | 144 | rtmp = status.at('rtmp') 145 | result[:rtmp] = { 146 | url: rtmp.at('url').inner_text, 147 | ticket: rtmp.at('ticket').inner_text, 148 | } 149 | 150 | ms = status.at('ms') 151 | result[:ms] = { 152 | address: ms.at('addr').inner_text, 153 | port: ms.at('port').inner_text.to_i, 154 | thread: ms.at('thread').inner_text, 155 | } 156 | 157 | quesheet = status.search('quesheet que') 158 | result[:quesheet] = quesheet.map do |que| 159 | {vpos: que['vpos'].to_i, mail: que['mail'], name: que['name'], body: que.inner_text} 160 | end 161 | 162 | result 163 | end 164 | 165 | def watching_reservations 166 | page = agent.get(URL_WATCHINGRESERVATION_LIST) 167 | page.search('vid').map(&:inner_text).map{ |_| Util::normalize_id(_) } 168 | end 169 | 170 | def accept_watching_reservation(id_) 171 | id = Util::normalize_id(id_, with_lv: false) 172 | 173 | token = Util::fetch_token_for_watching_reservation(@agent, id) 174 | page = agent.post("http://live.nicovideo.jp/api/watchingreservation", 175 | mode: 'auto_register', vid: id, token: token, '_' => '') 176 | 177 | token = Util::fetch_token_for_watching_reservation(@agent, id) 178 | page = agent.post("http://live.nicovideo.jp/api/watchingreservation", 179 | accept: 'true', mode: 'use', vid: id, token: token) 180 | end 181 | 182 | def decrypt_encrypted_player_status(body, public_key) 183 | unless public_key 184 | raise NoPublicKeyProvided, 185 | 'You should provide proper public key to decrypt ' \ 186 | 'encrypted player status' 187 | end 188 | 189 | lines = body.lines 190 | pubkey = OpenSSL::PKey::RSA.new(public_key) 191 | 192 | encrypted_shared_key = lines[1].unpack('m*')[0] 193 | shared_key_raw = pubkey.public_decrypt(encrypted_shared_key) 194 | shared_key = shared_key_raw.unpack('L>*')[0].to_s 195 | 196 | cipher = OpenSSL::Cipher.new('bf-ecb').decrypt 197 | cipher.padding = 0 198 | cipher.key_len = shared_key.size 199 | cipher.key = shared_key 200 | 201 | encrypted_body = lines[2].unpack('m*')[0] 202 | 203 | body = cipher.update(encrypted_body) + cipher.final 204 | body.force_encoding('utf-8') 205 | end 206 | end 207 | end 208 | end 209 | -------------------------------------------------------------------------------- /lib/main/main.rb: -------------------------------------------------------------------------------- 1 | module Main 2 | class Main 3 | def ag_scrape 4 | programs = Ag::Scraping.new.main 5 | programs.each do |p| 6 | Job.new( 7 | ch: Job::CH[:ag], 8 | title: p.title, 9 | start: p.start_time.next_on_air, 10 | end: p.start_time.next_on_air + p.minutes.minutes 11 | ).schedule 12 | end 13 | end 14 | 15 | def radiko_scrape 16 | channels = [] 17 | channels += Settings.radiko_channels if Settings.radiko_channels 18 | channels += Settings.radiko_premium.channels if Settings.try(:radiko_premium).try(:channels) 19 | channels.each do |ch| 20 | programs = Radiko::Scraping.new.get(ch) 21 | programs.each do |p| 22 | title = p.title 23 | title += " #{p.performers}" if p.performers.present? 24 | Job.new( 25 | ch: ch, 26 | title: title.slice(0, 240), 27 | start: p.start_time, 28 | end: p.end_time 29 | ).schedule 30 | end 31 | end 32 | end 33 | 34 | def radiru_scrape 35 | unless Settings.radiru_channels 36 | exit 0 37 | end 38 | 39 | Settings.radiru_channels.each do |ch| 40 | programs = Radiru::Scraping.new.get(ch) 41 | programs.each do |p| 42 | Job.new( 43 | ch: ch, 44 | title: p.title, 45 | start: p.start_time, 46 | end: p.end_time 47 | ).schedule 48 | end 49 | end 50 | end 51 | 52 | def onsen_scrape 53 | program_list = Onsen::Scraping.new.main 54 | 55 | program_list.each do |program| 56 | if program.update_date.blank? || program.file_url.blank? 57 | next 58 | end 59 | ActiveRecord::Base.transaction do 60 | if OnsenProgram.where(file_url: program.file_url).first 61 | next 62 | end 63 | 64 | p = OnsenProgram.new 65 | p.title = program.title 66 | p.number = program.number 67 | p.date = program.update_date 68 | p.file_url = program.file_url 69 | p.personality = program.personality 70 | p.state = OnsenProgram::STATE[:waiting] 71 | p.retry_count = 0 72 | p.save 73 | end 74 | end 75 | end 76 | 77 | def hibiki_scrape 78 | program_list = Hibiki::Scraping.new.main 79 | 80 | program_list.each do |program| 81 | ActiveRecord::Base.transaction do 82 | if HibikiProgramV2 83 | .where(access_id: program.access_id) 84 | .where(episode_id: program.episode_id) 85 | .first 86 | next 87 | end 88 | 89 | p = HibikiProgramV2.new 90 | p.access_id = program.access_id 91 | p.episode_id = program.episode_id 92 | p.title = program.title 93 | p.episode_name = program.episode_name 94 | p.cast = program.cast 95 | p.state = HibikiProgramV2::STATE[:waiting] 96 | p.retry_count = 0 97 | p.save 98 | end 99 | end 100 | end 101 | 102 | def niconama_scrape 103 | if !Settings.niconico || !Settings.niconico.live 104 | exit 0 105 | end 106 | 107 | program_list = NiconicoLive::Scraping.new.main 108 | 109 | program_list.each do |program| 110 | ActiveRecord::Base.transaction do 111 | if NiconicoLiveProgram.where(id: program.id).first 112 | next 113 | end 114 | 115 | p = NiconicoLiveProgram.new 116 | p.id = program.id 117 | p.title = program.title 118 | p.state = NiconicoLiveProgram::STATE[:waiting] 119 | p.cannot_recovery = false 120 | p.memo = '' 121 | p.retry_count = 0 122 | p.save 123 | end 124 | end 125 | end 126 | 127 | def agonp_scrape 128 | unless Settings.agonp 129 | exit 0 130 | end 131 | 132 | program_list = Agonp::Scraping.new.main 133 | 134 | program_list.each do |program| 135 | ActiveRecord::Base.transaction do 136 | if AgonpProgram.where(episode_id: program.episode_id).first 137 | next 138 | end 139 | 140 | p = AgonpProgram.new 141 | p.title = program.title 142 | p.personality = program.personality 143 | p.episode_id = program.episode_id 144 | p.price = program.price 145 | p.state = OndemandRetry::STATE[:waiting] 146 | p.retry_count = 0 147 | p.save 148 | end 149 | end 150 | end 151 | 152 | def wikipedia_scrape 153 | unless Settings.niconico && Settings.niconico.live.keyword_wikipedia_categories 154 | exit 0 155 | end 156 | 157 | Settings.niconico.live.keyword_wikipedia_categories.each do |category| 158 | items = Wikipedia::Scraping.new.main(category) 159 | items = items.map do |item| 160 | [category, item] 161 | end 162 | WikipediaCategoryItem.import( 163 | [:category, :title], 164 | items, 165 | on_duplicate_key_update: [:title] 166 | ) 167 | end 168 | end 169 | 170 | def nicodou_scrape 171 | if !Settings.niconico || !Settings.niconico.video 172 | exit 0 173 | end 174 | 175 | program_list = NiconicoVideo::Scraping.new.main 176 | 177 | program_list.each do |program| 178 | ActiveRecord::Base.transaction do 179 | if NiconicoVideoProgram.where(video_id: program.video_id).first 180 | next 181 | end 182 | 183 | p = NiconicoVideoProgram.new 184 | p.video_id = program.video_id 185 | p.title = program.title 186 | p.state = OndemandRetry::STATE[:waiting] 187 | p.retry_count = 0 188 | p.save 189 | end 190 | end 191 | end 192 | 193 | def rec_one 194 | jobs = nil 195 | ActiveRecord::Base.connection_pool.with_connection do 196 | ActiveRecord::Base.transaction do 197 | jobs = Job 198 | .where( 199 | "? <= `start` and `start` <= ?", 200 | 2.minutes.ago, 201 | 5.minutes.from_now 202 | ) 203 | .where(state: Job::STATE[:scheduled]) 204 | .order(:start) 205 | .lock 206 | .all 207 | if jobs.empty? 208 | return 0 209 | end 210 | jobs.each do |j| 211 | j.state = Job::STATE[:recording] 212 | j.save! 213 | end 214 | end 215 | end 216 | 217 | threads_from_records(jobs) do |j| 218 | Rails.logger.debug "rec thread created. job:#{j.id}" 219 | 220 | succeed = false 221 | if j.ch == Job::CH[:ag] 222 | succeed = Ag::Recording.new.record(j) 223 | elsif Settings.radiru_channels && Settings.radiru_channels.include?(j.ch) 224 | succeed = Radiru::Recording.new.record(j) 225 | else 226 | succeed = Radiko::Recording.new.record(j) 227 | end 228 | 229 | ActiveRecord::Base.connection_pool.with_connection do 230 | j.state = 231 | if succeed 232 | Job::STATE[:done] 233 | else 234 | Job::STATE[:failed] 235 | end 236 | j.save! 237 | end 238 | 239 | Rails.logger.debug "rec thread end. job:#{j.id}" 240 | end 241 | 242 | return 0 243 | end 244 | 245 | def rec_ondemand 246 | onsen_download 247 | hibiki_download 248 | agonp_download 249 | end 250 | 251 | LOCK_NICONAMA_DOWNLOAD = 'lock_niconama_download' 252 | def niconama_download 253 | unless Settings.niconico 254 | return 0 255 | end 256 | ActiveRecord::Base.transaction do 257 | l = KeyValue.where(key: LOCK_NICONAMA_DOWNLOAD).lock.first 258 | if !l 259 | l = KeyValue.new 260 | l.key = LOCK_NICONAMA_DOWNLOAD 261 | l.value = 'true' 262 | l.save! 263 | elsif l.value == 'false' 264 | l.value = 'true' 265 | l.save! 266 | elsif l.updated_at < 1.hours.ago 267 | l.touch 268 | else 269 | return 0 270 | end 271 | end 272 | 273 | p = nil 274 | ActiveRecord::Base.transaction do 275 | # ニコ生は検索オプションで「タイムシフト視聴可」を付けても 276 | # 実際にはまだタイムシフトが用意されていない場合がある 277 | # これに対応するため検索で発見しても一定時間待つ 278 | p = NiconicoLiveProgram 279 | .where(state: NiconicoLiveProgram::STATE[:waiting]) 280 | .where('`created_at` <= ?', 2.hours.ago) 281 | .lock 282 | .first 283 | if p 284 | p.state = NiconicoLiveProgram::STATE[:downloading] 285 | p.save! 286 | end 287 | end 288 | 289 | if p 290 | NiconicoLive::Downloading.new.download(p) 291 | p.save! 292 | end 293 | 294 | ActiveRecord::Base.transaction do 295 | l = KeyValue.lock.find(LOCK_NICONAMA_DOWNLOAD) 296 | l.value = 'false' 297 | l.save! 298 | end 299 | 300 | return 0 301 | end 302 | 303 | private 304 | 305 | def threads_from_records(records) 306 | thread_array = [] 307 | records.each do |record| 308 | thread_array << Thread.start(record) do |r| 309 | begin 310 | yield r 311 | rescue => e 312 | Rails.logger.error %W|#{e.class}\n#{e.inspect}\n#{e.backtrace.join("\n")}| 313 | end 314 | end 315 | sleep 1 316 | end 317 | 318 | thread_array.each do |th| 319 | th.join 320 | end 321 | end 322 | 323 | def onsen_download 324 | download(OnsenProgram, Onsen::Downloading.new) 325 | end 326 | 327 | def hibiki_download 328 | download2(HibikiProgramV2, Hibiki::Downloading.new) 329 | end 330 | 331 | def agonp_download 332 | unless Settings.agonp 333 | exit 0 334 | end 335 | download(AgonpProgram, Agonp::Downloading.new) 336 | end 337 | 338 | def download(model_klass, downloader) 339 | p = nil 340 | ActiveRecord::Base.transaction do 341 | p = fetch_downloadable_program(model_klass) 342 | unless p 343 | return 0 344 | end 345 | 346 | p.state = model_klass::STATE[:downloading] 347 | p.save! 348 | end 349 | 350 | download_(model_klass, downloader, p) 351 | 352 | return 0 353 | end 354 | 355 | def download2(model_klass, downloader) 356 | p = nil 357 | ActiveRecord::Base.transaction do 358 | # Hibikiで古いデータのキャッシュが残っているのかepisode_idが一致せず 359 | # outdatedと誤判定してしまうケースがあった 360 | # 対策として時間を置くことでprograms APIと各個別program APIのepisode_idが一致すること狙う 361 | p = fetch_downloadable_program(model_klass, 30.minutes.ago) 362 | unless p 363 | return 0 364 | end 365 | 366 | p.state = model_klass::STATE[:downloading] 367 | p.save! 368 | end 369 | 370 | download_(model_klass, downloader, p) 371 | 372 | return 0 373 | end 374 | 375 | def download_(klass, downloader, p) 376 | succeed = false 377 | begin 378 | succeed = downloader.download(p) 379 | rescue => e 380 | Rails.logger.error %W|#{e.class}\n#{e.inspect}\n#{e.backtrace.join("\n")}| 381 | end 382 | if p.state == klass::STATE[:downloading] 383 | p.state = 384 | if succeed 385 | klass::STATE[:done] 386 | else 387 | klass::STATE[:failed] 388 | end 389 | end 390 | unless succeed 391 | p.retry_count += 1 392 | if p.retry_count > klass::RETRY_LIMIT 393 | Rails.logger.error "#{klass.name} rec failed. exceeded retry_limit. #{p.id}: #{p.title}" 394 | end 395 | end 396 | p.save! 397 | end 398 | 399 | def fetch_downloadable_program(klass, older_than = nil) 400 | p = klass 401 | .where(state: klass::STATE[:waiting]) 402 | if older_than 403 | p = p.where('`created_at` <= ?', older_than) 404 | end 405 | p = p 406 | .lock 407 | .first 408 | return p if p 409 | 410 | klass 411 | .where(state: [ 412 | klass::STATE[:failed], 413 | klass::STATE[:downloading], 414 | ]) 415 | .where('`retry_count` <= ?', klass::RETRY_LIMIT) 416 | .where('`updated_at` <= ?', 1.day.ago) 417 | .lock 418 | .first 419 | end 420 | end 421 | end 422 | --------------------------------------------------------------------------------