├── .gitignore ├── Aptfile ├── Gruntfile.js ├── LICENSE ├── Procfile ├── README.md ├── TECHNOTE.md ├── lib ├── getbooks.js ├── getids.js ├── git_utils.js ├── repo_bitbucket.js └── server.js ├── package.json ├── public ├── books │ └── whatsnew.html ├── css │ └── aozora.css ├── images │ ├── arrow-left.png │ ├── arrow-right.png │ ├── bg_hr.png │ ├── blacktocat.png │ ├── icon_download.png │ ├── search.png │ └── sprite_download.png └── js │ └── aozora.min.js ├── scraper ├── getbooks.coffee ├── getids.coffee └── package.json ├── spec └── pubserver.raml ├── src ├── git_utils.coffee ├── js │ ├── aozora.js │ ├── jquery.columns.js │ └── mustache.js ├── repo_bitbucket.coffee ├── scss │ └── aozora.scss └── server.coffee ├── test └── data │ └── pkg.zip ├── updatedb.sh └── upload_books.coffee /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | src/test_* 3 | lib/test_* 4 | *.zip 5 | *.csv 6 | repo/ 7 | .sass-cache/ 8 | -------------------------------------------------------------------------------- /Aptfile: -------------------------------------------------------------------------------- 1 | http://mirrors.kernel.org/ubuntu/pool/main/g/gcc-4.9/gcc-4.9_4.9.2-10ubuntu13_amd64.deb 2 | http://mirrors.kernel.org/ubuntu/pool/main/g/gcc-4.9/libstdc%2b%2b6_4.9.2-10ubuntu13_amd64.deb 3 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | module.exports = function(grunt) { 2 | grunt.initConfig({ 3 | coffee: { 4 | glob_to_multiple: { 5 | expand: true, 6 | flatten: true, 7 | cwd: './', 8 | src: ['src/*.coffee', 'scraper/*.coffee'], 9 | dest: 'lib/', 10 | ext: '.js' 11 | }}, 12 | uglify: { 13 | my_target: { 14 | files: { 15 | 'public/js/aozora.min.js': [ 16 | 'src/js/jquery.columns.js', 17 | 'src/js/mustache.js', 18 | 'src/js/aozora.js' 19 | ] 20 | } 21 | }}, 22 | sass: { 23 | dist: { 24 | options: { 25 | style: 'compressed', 26 | sourcemap: 'none' 27 | }, 28 | files: { 29 | 'public/css/aozora.css': 'src/scss/aozora.scss' 30 | } 31 | }}, 32 | watch: { 33 | files: ["src/js/*.js"], 34 | tasks: ['uglify'] 35 | } 36 | }); 37 | grunt.loadNpmTasks('grunt-contrib-coffee'); 38 | grunt.loadNpmTasks('grunt-contrib-uglify'); 39 | grunt.loadNpmTasks('grunt-contrib-sass'); 40 | grunt.loadNpmTasks('grunt-contrib-watch'); 41 | grunt.registerTask('default', ['coffee', 'uglify', 'sass']); 42 | }; 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Kenichi Sato 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: node lib/server.js 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pubserver 2 | Prototype of Aozora-bunko package management server prototype 3 | 4 | 青空文庫の書籍パッケージを受け取り、配布するためのサーバのプロトタイプです 5 | 6 | ## 動かし方 7 | 8 | ### 前提条件 9 | * MongoDB (2.6と3.0で確認しています) 10 | * foreman (`gem install foreman`) 11 | 12 | 13 | ### コマンドラインでの起動 14 | ``` 15 | npm install 16 | grunt coffee 17 | foreman start web 18 | ``` 19 | 20 | ### 環境変数 21 | 22 | * `AOZORA_MONGODB_CREDENTIAL` MongoDBにアクセスするユーザ名・パスワード "*username*:*password*@" (default: "") 23 | * `AOZORA_MONGODB_HOST` MongoDBのホスト名 (default: "localhost") 24 | * `AOZORA_MONGODB_PORT` MongoDBのポート番号 (default: 27017) 25 | * `PORT` pubserverの待ち受けポート番号 (default: 5000) 26 | 27 | 28 | 29 | ## ブラウザからのアクセス 30 | 31 | - 新規登録作品のリスト http://www.aozorahack.net/books/whatsnew.html 32 | 33 | 34 | ## APIアクセス方法 35 | 36 | 以下は heroku.com で仮稼働しているプロトタイプサーバのURLです。 37 | ローカルで動かす時にはホスト名を "localhost:5000"で適宜読み替えてください。 38 | 39 | #### 本のリストの取得 40 | ``` 41 | curl http://www.aozorahack.net/api/v0.1/books 42 | ``` 43 | 44 | 追加パラメータ 45 | - `title`: タイトル名でのフィルタ 46 | - ~~`author`: 著者名でのフィルタ~~ (正しく動作していません) 47 | - `fields`: 取得する属性を指定 48 | - `limit`: 取得するアイテム数を制限 49 | - `skip`: 指定した分のアイテムをスキップしてそれ以降を取得 50 | - `after`: release_dateがこの日付よりも新しいモノのみを返す(YYYY-MM-DD) 51 | 52 | #### 個別の本の情報の取得 53 | ``` 54 | curl http://www.aozorahack.net/api/v0.1/books/{book_id} 55 | ``` 56 | 57 | #### 本のカードを取得 58 | ``` 59 | curl http://www.aozorahack.net/api/v0.1/books/{book_id}/card 60 | ``` 61 | 62 | #### 本の中身をテキストで取得 63 | ``` 64 | curl http://www.aozorahack.net/api/v0.1/books/{book_id}/content?format=txt 65 | ``` 66 | 67 | #### 本の中身をhtmlで取得 68 | ``` 69 | curl http://www.aozorahack.net/api/v0.1/books/{book_id}/content?format=html 70 | ``` 71 | 72 | #### 本の情報をアップロード 73 | ``` 74 | curl -Fpackage=@{package_file} http://www.aozorahack.net/api/v0.1/books 75 | ``` 76 | 77 | `package_file`はaozora.txtとaozora.jsonが含まれるzipファイル。 78 | 79 | #### 人物情報のリストの取得 80 | ``` 81 | curl http://www.aozorahack.net/api/v0.1/persons 82 | ``` 83 | 84 | 追加パラメータ 85 | - `name`: 著者名でのフィルタ 86 | 87 | 88 | #### 個別の人物の情報の取得 89 | ``` 90 | curl http://www.aozorahack.net/api/v0.1/persons/{person_id} 91 | ``` 92 | 93 | #### 工作員情報のリストの取得 94 | ``` 95 | curl http://www.aozorahack.net/api/v0.1/workers 96 | ``` 97 | 98 | #### 個別の工作員の情報の取得 99 | ``` 100 | curl http://www.aozorahack.net/api/v0.1/workers/{worker_id} 101 | ``` 102 | 103 | ## 仕様 104 | * [RAML](http://raml.org/)で記述してみたAPI仕様が[ここ](./spec/pubserver.raml)にあります 105 | 106 | ## DBにデータ登録するためのスクリプト 107 | 108 | #### 書籍情報取得 109 | https://github.com/aozorabunko/aozorabunko/raw/master/index_pages/list_person_all_extended_utf8.zip をダウンロード、そこに含まれるCSVファイルから情報取得し、DBに投入。 110 | ``` 111 | npm install -g coffee 112 | coffee scraper/getbooks.coffee 113 | ``` 114 | 115 | #### 人物情報、工作員情報取得 116 | 117 | http//reception.aozora.gr.jp/{pidlist|widlist}.php からダウンロードしたHTMLファイルをscrapingしてDBに投入。結果は上記のAPIから取得できる。 118 | 119 | ``` 120 | npm install -g coffee 121 | coffee scraper/getids.coffee 122 | ``` 123 | -------------------------------------------------------------------------------- /TECHNOTE.md: -------------------------------------------------------------------------------- 1 | #要素技術に関するメモ 2 | 3 | プロトタイプづくりをしていく中で調べたことをまとめておきたい。 4 | 5 | 比較的長くなることが多いので、[Qiita](http://qiita.com/)に書いた上で、ここにリンクを張ることにします。 6 | 7 | ## バージョン管理システム 8 | 9 | - [APIでBitbucketにアクセスしてみる](http://qiita.com/ksato9700/items/bbf89abb7acbac717267) 10 | 11 | -------------------------------------------------------------------------------- /lib/getbooks.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var AdmZip, MongoClient, async, get_bookobj, list_url_base, list_url_pub, listfile_inp, mongo_url, mongodb, mongodb_credential, mongodb_host, mongodb_port, parse, person_extended_attrs, request, role_map; 3 | 4 | AdmZip = require('adm-zip'); 5 | 6 | parse = require('csv-parse'); 7 | 8 | async = require('async'); 9 | 10 | request = require('request'); 11 | 12 | mongodb = require('mongodb'); 13 | 14 | MongoClient = mongodb.MongoClient; 15 | 16 | mongodb_credential = process.env.AOZORA_MONGODB_CREDENTIAL || ''; 17 | 18 | mongodb_host = process.env.AOZORA_MONGODB_HOST || 'localhost'; 19 | 20 | mongodb_port = process.env.AOZORA_MONGODB_PORT || '27017'; 21 | 22 | mongo_url = "mongodb://" + mongodb_credential + mongodb_host + ":" + mongodb_port + "/aozora"; 23 | 24 | list_url_base = 'https://github.com/aozorabunko/aozorabunko/raw/master/index_pages/'; 25 | 26 | listfile_inp = 'list_inp_person_all_utf8.zip'; 27 | 28 | list_url_pub = 'list_person_all_extended_utf8.zip'; 29 | 30 | person_extended_attrs = ['book_id', 'title', 'title_yomi', 'title_sort', 'subtitle', 'subtitle_yomi', 'original_title', 'first_appearance', 'ndc_code', 'font_kana_type', 'copyright', 'release_date', 'last_modified', 'card_url', 'person_id', 'last_name', 'first_name', 'last_name_yomi', 'first_name_yomi', 'last_name_sort', 'first_name_sort', 'last_name_roman', 'first_name_roman', 'role', 'date_of_birth', 'date_of_death', 'author_copyright', 'base_book_1', 'base_book_1_publisher', 'base_book_1_1st_edition', 'base_book_1_edition_input', 'base_book_1_edition_proofing', 'base_book_1_parent', 'base_book_1_parent_publisher', 'base_book_1_parent_1st_edition', 'base_book_2', 'base_book_2_publisher', 'base_book_2_1st_edition', 'base_book_2_edition_input', 'base_book_2_edition_proofing', 'base_book_2_parent', 'base_book_2_parent_publisher', 'base_book_2_parent_1st_edition', 'input', 'proofing', 'text_url', 'text_last_modified', 'text_encoding', 'text_charset', 'text_updated', 'html_url', 'html_last_modified', 'html_encoding', 'html_charset', 'html_updated']; 31 | 32 | role_map = { 33 | '著者': 'authors', 34 | '翻訳者': 'translators', 35 | '編者': 'editors', 36 | '校訂者': 'revisers' 37 | }; 38 | 39 | get_bookobj = function(entry, cb) { 40 | var book, person, role; 41 | book = {}; 42 | role = null; 43 | person = {}; 44 | person_extended_attrs.forEach(function(e, i) { 45 | var value; 46 | value = entry[i]; 47 | if (value !== '') { 48 | if (e === 'book_id' || e === 'person_id' || e === 'text_updated' || e === 'html_updated') { 49 | value = parseInt(value); 50 | } else if (e === 'copyright' || e === 'author_copyright') { 51 | value = value !== 'なし'; 52 | } else if (e === 'release_date' || e === 'last_modified' || e === 'date_of_birth' || e === 'date_of_death' || e === 'text_last_modified' || e === 'html_last_modified') { 53 | value = new Date(value); 54 | } 55 | if (e === 'person_id' || e === 'first_name' || e === 'last_name' || e === 'last_name_yomi' || e === 'first_name_yomi' || e === 'last_name_sort' || e === 'first_name_sort' || e === 'last_name_roman' || e === 'first_name_roman' || e === 'date_of_birth' || e === 'date_of_death' || e === 'author_copyright') { 56 | person[e] = value; 57 | return; 58 | } else if (e === 'role') { 59 | role = role_map[value]; 60 | if (!role) { 61 | console.log(value); 62 | } 63 | return; 64 | } 65 | return book[e] = value; 66 | } 67 | }); 68 | return cb(book, role, person); 69 | }; 70 | 71 | MongoClient.connect(mongo_url, { 72 | connectTimeoutMS: 120000, 73 | socketTimeoutMS: 120000 74 | }, function(err, db) { 75 | var books, persons; 76 | if (err) { 77 | console.log(err); 78 | return -1; 79 | } 80 | db = db; 81 | books = db.collection('books'); 82 | persons = db.collection('persons'); 83 | return request.get(list_url_base + list_url_pub, { 84 | encoding: null 85 | }, function(err, resp, body) { 86 | var buf, entries, zip; 87 | if (err) { 88 | return -1; 89 | } 90 | zip = AdmZip(body); 91 | entries = zip.getEntries(); 92 | if (entries.length !== 1) { 93 | return -1; 94 | } 95 | buf = zip.readFile(entries[0]); 96 | return parse(buf, function(err, data) { 97 | return books.findOne({}, { 98 | fields: { 99 | release_date: 1 100 | }, 101 | sort: { 102 | release_date: -1 103 | } 104 | }, function(err, item) { 105 | var books_batch_list, last_release_date, persons_batch_list, updated; 106 | if (err || item === null) { 107 | last_release_date = new Date('1970-01-01'); 108 | } else { 109 | last_release_date = item.release_date; 110 | } 111 | updated = data.slice(1).filter(function(entry) { 112 | var release_date; 113 | release_date = new Date(entry[11]); 114 | return last_release_date < release_date; 115 | }); 116 | console.log(updated.length + " entries are updated"); 117 | if (updated.length > 0) { 118 | books_batch_list = {}; 119 | persons_batch_list = {}; 120 | return async.eachSeries(updated, function(entry, cb) { 121 | return async.setImmediate(function(entry, cb) { 122 | return get_bookobj(entry, function(book, role, person) { 123 | if (!books_batch_list[book.book_id]) { 124 | books_batch_list[book.book_id] = book; 125 | } 126 | if (!books_batch_list[book.book_id][role]) { 127 | books_batch_list[book.book_id][role] = []; 128 | } 129 | person.full_name = person.last_name + person.first_name; 130 | books_batch_list[book.book_id][role].push({ 131 | person_id: person.person_id, 132 | last_name: person.last_name, 133 | first_name: person.first_name, 134 | full_name: person.full_name 135 | }); 136 | if (!persons_batch_list[person.person_id]) { 137 | persons_batch_list[person.person_id] = person; 138 | } 139 | return cb(null); 140 | }); 141 | }, entry, cb); 142 | }, function(err) { 143 | if (err) { 144 | console.log(err); 145 | return -1; 146 | } 147 | return async.parallel([ 148 | function(cb) { 149 | var book, book_id, books_batch; 150 | books_batch = books.initializeUnorderedBulkOp(); 151 | for (book_id in books_batch_list) { 152 | book = books_batch_list[book_id]; 153 | books_batch.find({ 154 | book_id: book_id 155 | }).upsert().updateOne(book); 156 | } 157 | return books_batch.execute(cb); 158 | }, function(cb) { 159 | var person, person_id, persons_batch; 160 | persons_batch = persons.initializeUnorderedBulkOp(); 161 | for (person_id in persons_batch_list) { 162 | person = persons_batch_list[person_id]; 163 | persons_batch.find({ 164 | person_id: person_id 165 | }).upsert().updateOne(person); 166 | } 167 | return persons_batch.execute(cb); 168 | } 169 | ], function(err, result) { 170 | if (err) { 171 | console.log('err', err); 172 | } 173 | return db.close(); 174 | }); 175 | }); 176 | } else { 177 | return db.close(); 178 | } 179 | }); 180 | }); 181 | }); 182 | }); 183 | 184 | }).call(this); 185 | -------------------------------------------------------------------------------- /lib/getids.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var MongoClient, async, idurls, mongo_url, mongodb, mongodb_credential, mongodb_host, mongodb_port, scrape_url, scraperjs; 3 | 4 | scraperjs = require('scraperjs'); 5 | 6 | async = require('async'); 7 | 8 | mongodb = require('mongodb'); 9 | 10 | MongoClient = mongodb.MongoClient; 11 | 12 | mongodb = require('mongodb'); 13 | 14 | mongodb_credential = process.env.AOZORA_MONGODB_CREDENTIAL || ''; 15 | 16 | mongodb_host = process.env.AOZORA_MONGODB_HOST || 'localhost'; 17 | 18 | mongodb_port = process.env.AOZORA_MONGODB_PORT || '27017'; 19 | 20 | mongo_url = "mongodb://" + mongodb_credential + mongodb_host + ":" + mongodb_port + "/aozora"; 21 | 22 | scrape_url = function(idurl, cb) { 23 | return scraperjs.StaticScraper.create(idurl).scrape(function($) { 24 | return $("tr[valign]").map(function() { 25 | var $row, ret; 26 | $row = $(this); 27 | return ret = { 28 | id: $row.find(':nth-child(1)').text().trim(), 29 | name: $row.find(':nth-child(2)').text().trim().replace('　', ' ') 30 | }; 31 | }).get(); 32 | }, function(items) { 33 | return cb(null, items.slice(1)); 34 | }); 35 | }; 36 | 37 | idurls = { 38 | 'workers': 'http://reception.aozora.gr.jp/widlist.php?page=1&pagerow=-1' 39 | }; 40 | 41 | MongoClient.connect(mongo_url, function(err, db) { 42 | if (err) { 43 | console.log(err); 44 | return -1; 45 | } 46 | return async.map(Object.keys(idurls), function(idname, cb) { 47 | var collection, idurl; 48 | collection = db.collection(idname); 49 | idurl = idurls[idname]; 50 | console.log(idurl); 51 | return scrape_url(idurl, function(err, results) { 52 | if (err) { 53 | cb(err); 54 | } 55 | return async.map(results, function(result, cb2) { 56 | result.id = parseInt(result.id); 57 | return collection.update({ 58 | id: result.id 59 | }, result, { 60 | upsert: true 61 | }, cb2); 62 | }, function(err, results2) { 63 | if (err) { 64 | return cb(err); 65 | } else { 66 | return cb(null, results2.length); 67 | } 68 | }); 69 | }); 70 | }, function(err, result) { 71 | if (err) { 72 | console.log(err); 73 | return -1; 74 | } 75 | console.log(result); 76 | return db.close(); 77 | }); 78 | }); 79 | 80 | }).call(this); 81 | -------------------------------------------------------------------------------- /lib/git_utils.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var Git, Promise, fse, open_or_clone, path, promisify, remote_callbacks, the_sig; 3 | 4 | Git = require('nodegit'); 5 | 6 | path = require('path'); 7 | 8 | promisify = require('promisify-node'); 9 | 10 | fse = promisify(require('fs-extra')); 11 | 12 | Promise = require('nodegit-promise'); 13 | 14 | remote_callbacks = null; 15 | 16 | the_sig = null; 17 | 18 | open_or_clone = function(repo_local_path, origin_url) { 19 | return Git.Repository.open(repo_local_path)["catch"](function(error) { 20 | return Git.Clone.clone(origin_url, repo_local_path, { 21 | remoteCallbacks: remote_callbacks 22 | }); 23 | }); 24 | }; 25 | 26 | exports.set_credential = function(user, pass, email) { 27 | remote_callbacks = { 28 | certificateCheck: function() { 29 | return 1; 30 | }, 31 | credentials: function() { 32 | return Git.Cred.userpassPlaintextNew(user, pass); 33 | } 34 | }; 35 | return the_sig = Git.Signature.now(user, email); 36 | }; 37 | 38 | exports.setup_repo = function(origin_url, book_id, initial_files, cb) { 39 | return open_or_clone("repo/" + book_id, origin_url).then(function(repo) { 40 | if (repo.isEmpty()) { 41 | return repo.openIndex().then(function(index) { 42 | var filenames; 43 | filenames = Object.keys(initial_files); 44 | return Promise.all(filenames.map(function(filename) { 45 | return fse.writeFile(path.join(repo.workdir(), filename), initial_files[filename]); 46 | })).then(function() { 47 | return index.addAll(); 48 | }).then(function() { 49 | return index.write(); 50 | }).then(function() { 51 | return index.writeTree(); 52 | }); 53 | }).then(function(oid) { 54 | return Git.Tree.lookup(repo, oid); 55 | }).then(function(tree) { 56 | return the_sig.dup().then(function(sig) { 57 | return Git.Commit.create(repo, "HEAD", sig, sig, null, "initial commit", tree, 0, []); 58 | }).then(function(oid) { 59 | return repo; 60 | }); 61 | }); 62 | } else { 63 | return repo; 64 | } 65 | }).then(function(repo) { 66 | return Git.Remote.lookup(repo, 'origin'); 67 | }).then(function(remote) { 68 | remote.addPush("refs/heads/master:refs/heads/master"); 69 | return remote.getPushRefspecs().then(function(specs) { 70 | remote.setCallbacks(remote_callbacks); 71 | return the_sig.dup().then(function(sig) { 72 | return remote.push(specs, {}, sig, null); 73 | }).then(function(error) { 74 | if (error) { 75 | console.log('push error:', error); 76 | } 77 | return cb(error === void 0); 78 | }); 79 | }); 80 | })["catch"](function(error) { 81 | console.log('catched error', error); 82 | console.log(origin_url, book_id); 83 | return cb(false); 84 | }); 85 | }; 86 | 87 | }).call(this); 88 | -------------------------------------------------------------------------------- /lib/repo_bitbucket.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var BITBUCKET_APIBASE, BITBUCKET_EMAIL, BITBUCKET_PASS, BITBUCKET_USER, async, git_utils, iconv, request, sjis; 3 | 4 | request = require('request'); 5 | 6 | async = require('async'); 7 | 8 | git_utils = require('./git_utils'); 9 | 10 | iconv = require('iconv'); 11 | 12 | sjis = new iconv.Iconv('UTF-8', 'Shift_JIS'); 13 | 14 | BITBUCKET_APIBASE = "https://api.bitbucket.org/2.0"; 15 | 16 | BITBUCKET_USER = process.env.AOZORA_BITBUCKET_USER; 17 | 18 | BITBUCKET_PASS = process.env.AOZORA_BITBUCKET_PASS; 19 | 20 | BITBUCKET_EMAIL = process.env.AOZORA_BITBUCKET_EMAIL; 21 | 22 | exports.init_repo = function(title, author, book_id, is_private, cb) { 23 | var init_files, r, repo_url; 24 | if (!(title && author && book_id)) { 25 | cb(400); 26 | return; 27 | } 28 | repo_url = BITBUCKET_APIBASE + ("/repositories/" + BITBUCKET_USER + "/" + book_id); 29 | init_files = { 30 | 'aozora.json': JSON.stringify({ 31 | id: book_id, 32 | author: { 33 | name: author 34 | }, 35 | title: { 36 | name: title 37 | } 38 | }), 39 | 'head.txt': sjis.convert(title + "\r\n" + author) 40 | }; 41 | r = request.defaults({ 42 | auth: { 43 | user: BITBUCKET_USER, 44 | pass: BITBUCKET_PASS 45 | }, 46 | json: true 47 | }); 48 | return r.post(repo_url, { 49 | body: { 50 | scm: "git", 51 | is_private: is_private, 52 | fork_policy: is_private ? "no_public_forks" : "allow_forks" 53 | } 54 | }, function(err, resp, body) { 55 | if (err || (body.error && body.error.message === !'Repository already exists.')) { 56 | console.log(err || body.error); 57 | cb(400); 58 | return; 59 | } 60 | return r.get(repo_url, function(err, resp, body) { 61 | return async.some(body.links.clone, function(entry, cb2) { 62 | if (entry.name === 'https') { 63 | git_utils.set_credential(BITBUCKET_USER, BITBUCKET_PASS, BITBUCKET_EMAIL); 64 | return git_utils.setup_repo(entry.href, book_id, init_files, function(repo) { 65 | return cb2(true); 66 | }); 67 | } else { 68 | return cb2(false); 69 | } 70 | }, function(result) { 71 | return cb(result ? 201 : 500); 72 | }); 73 | }); 74 | }); 75 | }; 76 | 77 | }).call(this); 78 | -------------------------------------------------------------------------------- /lib/server.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var AdmZip, DATA_LIFETIME, DEFAULT_LIMIT, FB_MESSENGER_VERIFY_TOKEN, GridStore, MongoClient, add_ogp, api_root, app, bodyParser, check_archive, compression, content_type, encodings, express, fs, get_from_cache, get_ogpcard, get_zipped, iconv, methodOverride, mongo_url, mongodb, mongodb_credential, mongodb_host, mongodb_port, morgan, re_or_str, redis, rel_to_abs_path, request, upload_content, upload_content_data, version, yaml, zlib; 3 | 4 | fs = require('fs'); 5 | 6 | express = require('express'); 7 | 8 | morgan = require('morgan'); 9 | 10 | bodyParser = require('body-parser'); 11 | 12 | methodOverride = require('method-override'); 13 | 14 | compression = require('compression'); 15 | 16 | mongodb = require('mongodb'); 17 | 18 | AdmZip = require('adm-zip'); 19 | 20 | yaml = require('js-yaml'); 21 | 22 | request = require('request'); 23 | 24 | zlib = require('zlib'); 25 | 26 | redis = require('redis'); 27 | 28 | iconv = require('iconv-lite'); 29 | 30 | MongoClient = mongodb.MongoClient; 31 | 32 | GridStore = mongodb.GridStore; 33 | 34 | mongodb_credential = process.env.AOZORA_MONGODB_CREDENTIAL || ''; 35 | 36 | mongodb_host = process.env.AOZORA_MONGODB_HOST || 'localhost'; 37 | 38 | mongodb_port = process.env.AOZORA_MONGODB_PORT || '27017'; 39 | 40 | mongo_url = "mongodb://" + mongodb_credential + mongodb_host + ":" + mongodb_port + "/aozora"; 41 | 42 | DEFAULT_LIMIT = 100; 43 | 44 | DATA_LIFETIME = 3600; 45 | 46 | app = express(); 47 | 48 | version = 'v0.1'; 49 | 50 | api_root = '/api/' + version; 51 | 52 | app.use(express["static"](__dirname + '/../public')); 53 | 54 | app.use(morgan('dev')); 55 | 56 | app.use(bodyParser.urlencoded({ 57 | extended: false 58 | })); 59 | 60 | app.use(bodyParser.json()); 61 | 62 | app.use(methodOverride()); 63 | 64 | app.use(compression()); 65 | 66 | check_archive = function(path, cb) { 67 | var bookobj, data, err, error, textpath; 68 | try { 69 | data = fs.readFileSync(path + 'aozora.json'); 70 | } catch (error) { 71 | err = error; 72 | if (err.code === 'ENOENT') { 73 | cb("Cannot find aozora.json\n"); 74 | } else { 75 | cb(err); 76 | } 77 | return; 78 | } 79 | textpath = path + 'aozora.txt'; 80 | if (!fs.existsSync(textpath)) { 81 | cb("Cannot find aozora.txt\n"); 82 | return; 83 | } 84 | console.log(data); 85 | bookobj = yaml.safeLoad(data); 86 | console.log(bookobj); 87 | return cb(null, bookobj, textpath); 88 | }; 89 | 90 | upload_content = function(db, book_id, source_file, cb) { 91 | var gs; 92 | gs = new GridStore(db, book_id, book_id + ".txt", 'w'); 93 | return gs.writeFile(source_file, cb); 94 | }; 95 | 96 | upload_content_data = function(rc, key, data, cb) { 97 | return zlib.deflate(data, function(err, zdata) { 98 | if (err) { 99 | return cb(err); 100 | } else { 101 | return rc.setex(key, DATA_LIFETIME, zdata, cb); 102 | } 103 | }); 104 | }; 105 | 106 | re_or_str = function(src) { 107 | if (src[0] === '/' && src.slice(-1) === '/') { 108 | return { 109 | "$in": [new RegExp(src.slice(1, -1))] 110 | }; 111 | } else { 112 | return src; 113 | } 114 | }; 115 | 116 | app.route(api_root + '/books').get(function(req, res) { 117 | var options, query; 118 | query = {}; 119 | if (req.query.title) { 120 | query['title'] = re_or_str(req.query.title); 121 | } 122 | if (req.query.author) { 123 | query['authors.full_name'] = re_or_str(req.query.author); 124 | } 125 | if (req.query.after) { 126 | query['release_date'] = { 127 | "$gte": new Date(req.query.after) 128 | }; 129 | } 130 | options = { 131 | sort: { 132 | release_date: -1 133 | }, 134 | fields: { 135 | _id: 0 136 | } 137 | }; 138 | if (req.query.fields) { 139 | req.query.fields.split(',').forEach(function(a) { 140 | return options.fields[a] = 1; 141 | }); 142 | } 143 | if (req.query.limit) { 144 | options.limit = parseInt(req.query.limit); 145 | } else { 146 | options.limit = DEFAULT_LIMIT; 147 | } 148 | if (req.query.skip) { 149 | options.skip = parseInt(req.query.skip); 150 | } 151 | return app.my.books.find(query, options, function(err, items) { 152 | return items.toArray(function(err, docs) { 153 | if (err) { 154 | console.log(err); 155 | return res.status(500).end(); 156 | } else { 157 | return res.json(docs); 158 | } 159 | }); 160 | }); 161 | }).post(function(req, res) { 162 | var path, pkg, zip; 163 | pkg = req.files["package"]; 164 | if (!pkg) { 165 | return res.status(400).send("parameter package is not specified"); 166 | } 167 | zip = new AdmZip(pkg.path); 168 | path = process.env.TMPDIR + '/' + pkg.name.split('.')[0] + '-unzip/'; 169 | zip.extractAllTo(path); 170 | return check_archive(path, function(err, bookobj, source_file) { 171 | var book_id; 172 | if (err) { 173 | return res.status(400).send(err); 174 | } 175 | book_id = bookobj.id; 176 | return app.my.books.update({ 177 | id: book_id 178 | }, bookobj, { 179 | upsert: true 180 | }, function(err, doc) { 181 | if (err) { 182 | console.log(err); 183 | return res.sendStatus(500); 184 | } 185 | return upload_content(app.my.db, book_id, source_file, function(err) { 186 | console.log(err); 187 | if (err) { 188 | console.log(err); 189 | return res.sendStatus(500); 190 | } 191 | res.location("/books/" + book_id); 192 | return res.sendStatus(201); 193 | }); 194 | }); 195 | }); 196 | }); 197 | 198 | app.route(api_root + '/books/:book_id').get(function(req, res) { 199 | var book_id; 200 | book_id = parseInt(req.params.book_id); 201 | return app.my.books.findOne({ 202 | book_id: book_id 203 | }, { 204 | _id: 0 205 | }, function(err, doc) { 206 | if (err || doc === null) { 207 | console.log(err); 208 | return res.status(404).end(); 209 | } else { 210 | return res.json(doc); 211 | } 212 | }); 213 | }); 214 | 215 | content_type = { 216 | 'txt': 'text/plain; charset=shift_jis' 217 | }; 218 | 219 | get_from_cache = function(my, book_id, get_file, ext, cb) { 220 | var key; 221 | key = "" + ext + book_id; 222 | return my.rc.get(key, function(err, result) { 223 | if (err || !result) { 224 | if (get_file) { 225 | return get_file(my, book_id, ext, function(err, data) { 226 | if (err) { 227 | return cb(err); 228 | } else { 229 | return upload_content_data(my.rc, key, data, function(err) { 230 | if (err) { 231 | return cb(err); 232 | } else { 233 | return cb(null, data); 234 | } 235 | }); 236 | } 237 | }); 238 | } else { 239 | return cb(err); 240 | } 241 | } else { 242 | return zlib.inflate(result, function(err, data) { 243 | if (err) { 244 | return cb(err); 245 | } else { 246 | return cb(null, data); 247 | } 248 | }); 249 | } 250 | }); 251 | }; 252 | 253 | add_ogp = function(body, title, author) { 254 | var ogp_headers; 255 | ogp_headers = ['', '', '', '', '', '', '', '', "/, ogp_headers); 257 | }; 258 | 259 | rel_to_abs_path = function(body, ext) { 260 | if (ext === 'card') { 261 | return body.replace(/\.\.\/\.\.\//g, 'http://www.aozora.gr.jp/').replace(/\.\.\//g, 'http://www.aozora.gr.jp/cards/'); 262 | } else { 263 | return body.replace(/\.\.\/\.\.\//g, 'http://www.aozora.gr.jp/cards/'); 264 | } 265 | }; 266 | 267 | encodings = { 268 | 'card': 'utf-8', 269 | 'html': 'shift_jis' 270 | }; 271 | 272 | get_ogpcard = function(my, book_id, ext, cb) { 273 | return my.books.findOne({ 274 | book_id: book_id 275 | }, { 276 | card_url: 1, 277 | html_url: 1, 278 | title: 1, 279 | authors: 1 280 | }, function(err, doc) { 281 | if (err || doc === null) { 282 | cb(err); 283 | return; 284 | } 285 | console.log(doc[ext + "_url"]); 286 | return request.get(doc[ext + "_url"], { 287 | encoding: null, 288 | headers: { 289 | 'User-Agent': 'Mozilla/5.0', 290 | 'Accept': '*/*' 291 | } 292 | }, function(err, res, body) { 293 | var bodystr, encoding; 294 | if (err) { 295 | return cb(err); 296 | } else { 297 | encoding = encodings[ext]; 298 | bodystr = iconv.decode(body, encoding); 299 | bodystr = add_ogp(bodystr, doc.title, doc.authors[0].full_name); 300 | bodystr = rel_to_abs_path(bodystr, ext); 301 | return cb(null, iconv.encode(bodystr, encodings[ext])); 302 | } 303 | }); 304 | }); 305 | }; 306 | 307 | get_zipped = function(my, book_id, ext, cb) { 308 | return my.books.findOne({ 309 | book_id: book_id 310 | }, { 311 | text_url: 1 312 | }, function(err, doc) { 313 | if (err || doc === null) { 314 | cb(err); 315 | return; 316 | } 317 | return request.get(doc.text_url, { 318 | encoding: null, 319 | headers: { 320 | 'User-Agent': 'Mozilla/5.0', 321 | 'Accept': '*/*' 322 | } 323 | }, function(err, res, body) { 324 | var entry, zip; 325 | if (err) { 326 | return cb(err); 327 | } else { 328 | zip = new AdmZip(body); 329 | entry = zip.getEntries()[0]; 330 | return cb(null, zip.readFile(entry)); 331 | } 332 | }); 333 | }); 334 | }; 335 | 336 | app.route(api_root + '/books/:book_id/card').get(function(req, res) { 337 | var book_id; 338 | book_id = parseInt(req.params.book_id); 339 | return get_from_cache(app.my, book_id, get_ogpcard, 'card', function(err, result) { 340 | if (err) { 341 | console.log(err); 342 | return res.status(404).end(); 343 | } else { 344 | res.set('Content-Type', 'text/html'); 345 | return res.send(result); 346 | } 347 | }); 348 | }); 349 | 350 | app.route(api_root + '/books/:book_id/content').get(function(req, res) { 351 | var book_id, ext; 352 | book_id = parseInt(req.params.book_id); 353 | ext = req.query.format; 354 | if (ext === 'html') { 355 | return get_from_cache(app.my, book_id, get_ogpcard, 'html', function(err, result) { 356 | if (err) { 357 | console.log(err); 358 | return res.status(404).end(); 359 | } else { 360 | res.set('Content-Type', 'text/html; charset=shift_jis'); 361 | return res.send(result); 362 | } 363 | }); 364 | } else { 365 | ext = 'txt'; 366 | return get_from_cache(app.my, book_id, get_zipped, ext, function(err, result) { 367 | if (err) { 368 | console.log(err); 369 | return res.status(404).end(); 370 | } else { 371 | res.set('Content-Type', content_type[ext] || 'application/octet-stream'); 372 | return res.send(result); 373 | } 374 | }); 375 | } 376 | }); 377 | 378 | app.route(api_root + '/persons').get(function(req, res) { 379 | var query; 380 | query = {}; 381 | if (req.query.name) { 382 | query['full_name'] = re_or_str(req.query.name); 383 | } 384 | return app.my.persons.find(query, { 385 | _id: 0 386 | }, function(err, items) { 387 | return items.toArray(function(err, docs) { 388 | if (err) { 389 | console.log(err); 390 | return res.status(500).end(); 391 | } else { 392 | return res.json(docs); 393 | } 394 | }); 395 | }); 396 | }); 397 | 398 | app.route(api_root + '/persons/:person_id').get(function(req, res) { 399 | var person_id; 400 | person_id = parseInt(req.params.person_id); 401 | return app.my.persons.findOne({ 402 | person_id: person_id 403 | }, { 404 | _id: 0 405 | }, function(err, doc) { 406 | if (err || doc === null) { 407 | console.log(err); 408 | return res.status(404).end(); 409 | } else { 410 | return res.json(doc); 411 | } 412 | }); 413 | }); 414 | 415 | app.route(api_root + '/workers').get(function(req, res) { 416 | var query; 417 | query = {}; 418 | if (req.query.name) { 419 | query.name = re_or_str(req.query.name); 420 | } 421 | return app.my.workers.find(query, { 422 | _id: 0 423 | }, function(err, items) { 424 | return items.toArray(function(err, docs) { 425 | if (err) { 426 | console.log(err); 427 | return res.status(500).end(); 428 | } else { 429 | return res.json(docs); 430 | } 431 | }); 432 | }); 433 | }); 434 | 435 | app.route(api_root + '/workers/:worker_id').get(function(req, res) { 436 | var worker_id; 437 | worker_id = parseInt(req.params.worker_id); 438 | return app.my.workers.findOne({ 439 | id: worker_id 440 | }, { 441 | _id: 0 442 | }, function(err, doc) { 443 | if (err || doc === null) { 444 | console.log(err); 445 | return res.status(404).end(); 446 | } else { 447 | return res.json(doc); 448 | } 449 | }); 450 | }); 451 | 452 | FB_MESSENGER_VERIFY_TOKEN = process.env.FB_MESSENGER_VERIFY_TOKEN; 453 | 454 | app.route('/callback/:service').get(function(req, res) { 455 | if (req.params.service === 'fb') { 456 | console.dir(req.query); 457 | if (req.query['hub.mode'] === 'subscribe' && req.query['hub.verify_token'] === FB_MESSENGER_VERIFY_TOKEN) { 458 | return res.send(req.query['hub.challenge']); 459 | } else { 460 | return res.status(401).send(); 461 | } 462 | } else { 463 | return res.status(404).send(); 464 | } 465 | }).post(function(req, res) { 466 | console.dir(req.body); 467 | return res.sendStatus(200); 468 | }); 469 | 470 | MongoClient.connect(mongo_url, function(err, db) { 471 | var port, redis_url; 472 | if (err) { 473 | console.log(err); 474 | return -1; 475 | } 476 | port = process.env.PORT || 5000; 477 | app.my = {}; 478 | app.my.db = db; 479 | redis_url = process.env.REDIS_URL || "redis://127.0.0.1:6379"; 480 | app.my.rc = redis.createClient(redis_url, { 481 | return_buffers: true 482 | }); 483 | app.my.books = db.collection('books'); 484 | app.my.authors = db.collection('authors'); 485 | app.my.persons = db.collection('persons'); 486 | app.my.workers = db.collection('workers'); 487 | return app.listen(port, function() { 488 | return console.log("Magic happens on port " + port); 489 | }); 490 | }); 491 | 492 | }).call(this); 493 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pubserver", 3 | "version": "0.2.0", 4 | "description": "", 5 | "main": "lib/server.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "adm-zip": "^0.4.7", 11 | "async": "^2.4.1", 12 | "body-parser": "~1.17.2", 13 | "compression": "^1.6.2", 14 | "express": "^4.15.3", 15 | "fs-extra": "^3.0.1", 16 | "iconv-lite": "0.4.18", 17 | "js-yaml": "^3.8.4", 18 | "method-override": "~2.3.9", 19 | "mongodb": "^2.2.28", 20 | "morgan": "~1.8.2", 21 | "multer": "^1.3.0", 22 | "nodegit": "0.18.3", 23 | "nodegit-promise": "^4.0.0", 24 | "promisify-node": "^0.4.0", 25 | "redis": "2.7.1", 26 | "request": "^2.81.0", 27 | "csv-parse": "1.2.0", 28 | "scraperjs": "1.2.0" 29 | }, 30 | "devDependencies": { 31 | "grunt": "^1.0.1", 32 | "grunt-contrib-coffee": "^1.0.0", 33 | "grunt-contrib-sass": "^1.0.0", 34 | "grunt-contrib-uglify": "^3.0.1", 35 | "grunt-contrib-watch": "^1.0.0" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /public/books/whatsnew.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 新規公開作品 5 | 6 | 7 | 38 | 39 | 40 |

新規公開作品 2016年公開分

41 |
42 |


43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /public/css/aozora.css: -------------------------------------------------------------------------------- 1 | .columns .ui-table{border-collapse:collapse;width:100%;margin:1em 0;background-color:transparent;border:solid 1px #D5D5D5}.columns .ui-table thead{background-color:#ffffcc}.columns .ui-table thead th{background-color:transparent;color:#222222;padding:10px;border-right:solid 1px #D5D5D5;cursor:pointer}.columns .ui-table thead th.ui-table-sort-up,.columns .ui-table thead th.ui-table-sort-down{background-color:#CCCCCC}.columns .ui-table thead th:last-child{border-right:0px}.columns .ui-table thead th .ui-arrow{float:right;font-size:10px}.columns .ui-table tbody{background-color:transparent}.columns .ui-table tbody tr.ui-table-rows-odd{background-color:#F2F2F2}.columns .ui-table tbody tr.ui-table-rows-even{background:#FFFFFF}.columns .ui-table tbody tr td{background:transparent;border-right:solid 1px #D5D5D5;color:#656565;padding:10px}.columns .ui-table-controls span{font-size:12px;padding:5px;vertical-align:middle}.columns .ui-table-controls span.ui-table-control-next,.columns .ui-table-controls span.ui-table-control-prev{cursor:pointer;font-family:"Arial Narrow";font-size:16px}.columns .ui-table-controls span.ui-table-control-disabled{color:#999999;font-family:"Arial Narrow";font-size:16px}.columns .ui-table-footer{width:100%;padding:8px 0;font-size:11px;text-align:left;color:#333}.columns .ui-table-footer span{vertical-align:middle}.columns .ui-table-footer .ui-table-size,.columns .ui-table-footer .ui-table-results,.columns .ui-table-footer .ui-table-controls{display:inline-block;width:32%}.columns .ui-table-footer .ui-table-size{padding-left:20px}.columns .ui-table-footer .ui-table-results{text-align:center}.columns .ui-table-footer .ui-table-controls{text-align:right}.columns .ui-table-footer .ui-table-control-next,.columns .ui-table-footer .ui-table-control-prev,.columns .ui-table-footer .ui-table-control-disabled{display:inline-block;background-color:transparent;padding:5px;vertical-align:middle;cursor:pointer;text-align:center}.columns .ui-table-footer .ui-table-control-disabled img{opacity:0.5}.columns .ui-columns-search{text-align:right}.columns .ui-columns-search input{width:200px;border-radius:10px;padding:4px 10px 4px 25px;border:2px solid #ccc;background-image:url(../images/search.png);background-position:5px center;background-repeat:no-repeat}.columns .ui-columns-search input:focus{border:2px solid #6196CD;outline:none} 2 | -------------------------------------------------------------------------------- /public/images/arrow-left.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aozorahack/pubserver/4056df0ff338971d9f71f4418b4e6e66d6ca2d6d/public/images/arrow-left.png -------------------------------------------------------------------------------- /public/images/arrow-right.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aozorahack/pubserver/4056df0ff338971d9f71f4418b4e6e66d6ca2d6d/public/images/arrow-right.png -------------------------------------------------------------------------------- /public/images/bg_hr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aozorahack/pubserver/4056df0ff338971d9f71f4418b4e6e66d6ca2d6d/public/images/bg_hr.png -------------------------------------------------------------------------------- /public/images/blacktocat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aozorahack/pubserver/4056df0ff338971d9f71f4418b4e6e66d6ca2d6d/public/images/blacktocat.png -------------------------------------------------------------------------------- /public/images/icon_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aozorahack/pubserver/4056df0ff338971d9f71f4418b4e6e66d6ca2d6d/public/images/icon_download.png -------------------------------------------------------------------------------- /public/images/search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aozorahack/pubserver/4056df0ff338971d9f71f4418b4e6e66d6ca2d6d/public/images/search.png -------------------------------------------------------------------------------- /public/images/sprite_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aozorahack/pubserver/4056df0ff338971d9f71f4418b4e6e66d6ca2d6d/public/images/sprite_download.png -------------------------------------------------------------------------------- /public/js/aozora.min.js: -------------------------------------------------------------------------------- 1 | if(!window.console)var console={log:function(){}};!function(a){a.fn.columns=function(c){var d=[],e=Array.prototype.slice.call(arguments,1);return"string"==typeof c?this.each(function(){var b=a.data(this,"columns");if("undefined"==typeof b||!a.isFunction(b[c]))return a.error('No such method "'+c+'" for Columns');var f=b[c].apply(b,e);void 0!==f&&f!==b&&d.push(f)}):this.each(function(){a.data(this,"columns")||a.data(this,"columns",new b(this,c))}),0===d.length?this.data("columns"):1===d.length?d[0]:d};var b=function(b,c){this.$el=a(b),c&&a.extend(this,c),this.VERSION="2.2.2",this.sort=function(){function a(a,b,d){return b=b?-1:1,function(e,f){return e=e[a],f=f[a],c.test(e)&&c.test(f)?(e=new Date(e),e=Date.parse(e),f=new Date(f),f=Date.parse(f)):"undefined"!=typeof d&&(e=d(e),f=d(f)),f>e?-1*b:e>f?1*b:0}}var b=this,c=/^(Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December|(0?\d{1})|(10|11|12))(-|\s|\/|\.)(0?[1-9]|(1|2)[0-9]|3(0|1))(-|\s|\/|\.|,\s)(19|20)?\d\d$/i;b.total&&b.sortBy&&"undefined"!=typeof b.data[0][b.sortBy]&&b.data.sort(a(b.sortBy,b.reverse))},this.filter=function(){var b=this,c=b.searchableFields.length;if(b.query){var d=new RegExp(b.query,"gi");b.data=a.grep(b.data,function(a){for(var e=0;c>e;e++)if("string"==typeof a[b.searchableFields[e]]){if(a[b.searchableFields[e]].match(d))return!0}else if("number"==typeof a[b.searchableFields[e]]&&a[b.searchableFields[e]]==b.query)return!0;return!1})}b.total=b.data.length},this.paginate=function(){var a=this;a.pages=Math.ceil(a.data.length/a.size),a.page=a.page<=a.pages?a.page:1,a.setRange(),a.data=a.data.slice(a.range.start-1,a.range.end)},this.condition=function(){var a=this,b=[];if(a.schema){for(var c=a.data.length,d=a.schema.length,e=0;c>e;e++){for(var f=a.data[e],g={},h=0;d>h;h++){var i=a.schema[h];if(i.condition&&!i.condition(f[i.key])){g=null;break}g[i.key]=f[i.key]}g&&b.push(g)}a.data=b}},this.chevron=function(a,b){return Mustache.render(a,b)},this.create=function(){function b(){f.thead=[],a.each(f.schema,function(b,c){if(!c.hide){var d={};-1===a.inArray(c.key,f.sortableFields)?d.notSortable=!0:f.sortBy===c.key?f.reverse?d.sortedDown=!0:d.sortedUp=!0:d.sortable=!0,d.key=c.key,d.header=c.header,f.thead.push(d)}})}function c(b,c){var d=[];return b%2===0?d.push(''):d.push(''),a.each(f.schema,function(a,b){b.hide||(b.template?d.push(""+f.chevron(b.template,c)+""):d.push(""+c[b.key]+""))}),d.push(""),d}function d(){var b=[];b.push(""),f.showRowsMenu=b.join("")}function e(){f.rows=[],f.total?a.each(f.data,function(a,d){0===a&&b(),f.rows.push(c(a,d).join(""))}):f.rows.push('No Results')}var f=this;f.resetData(),f.searching&&f.filter(),f.sorting&&f.sort(),f.paginating&&f.paginate(),e(),d();var g={prevPage:f.page-1,nextPage:f.page+1,prevPageExists:f.pageExists(f.page-1),nextPageExists:f.pageExists(f.page+1),resultRange:f.range,tableTotal:f.total,showRowsMenu:f.showRowsMenu,rows:f.rows,headers:f.thead,query:f.query,search:f.search,table:f.table};return a.extend(f.view,g),f.plugins&&a.each(f.plugins,function(a,b){"undefined"!=typeof ColumnsPlugins&&"undefined"!=typeof ColumnsPlugins[b]&&ColumnsPlugins[b].create.call(f)}),f.search?(f.$el.html(f.chevron(f.template,f.view)),f.search=!1):(a("[data-columns-table]",f.$el).remove(),f.$el.append(f.chevron(f.template,f.view))),!0},this.init=function(){function b(){f.schema=[],a.each(f.data[0],function(a){f.schema.push({header:a,key:a})})}function c(){f.searchableFields=[],a.each(f.data[0],function(a){f.searchableFields.push(a)})}function d(){f.sortableFields=[],a.each(f.data[0],function(a){f.sortableFields.push(a)})}function e(){a.ajax({url:f.templateFile,async:!1,success:function(a){f.template=a},error:function(){a.error("Template could not be found.")}})}var f=this;a.isArray(f.data)?(f.master=[],f.view={},f.$el.addClass("columns"),f.$el.on("click",".ui-table-sortable",function(b){var c=a(this).data("columns-sortby");f.sortBy===c&&(f.reverse=f.reverse?!1:!0),f.sortBy=c,f.sortHandler(b)}),f.$el.on("click",".ui-table-control-next, .ui-table-control-prev",function(b){f.page=a(this).data("columns-page"),f.pageHandler(b)}),f.$el.on("keyup",".ui-table-search",function(b){f.query=a(this).val(),f.searchHandler(b)}),f.$el.on("change",".ui-table-size select",function(b){f.size=parseInt(a(this).val()),f.sizeHandler(b)}),f.plugins&&a.each(f.plugins,function(a,b){"undefined"!=typeof ColumnsPlugins&&"undefined"!=typeof ColumnsPlugins[b]&&ColumnsPlugins[b].init.call(f)}),f.conditioning&&f.condition(),f.schema||b(),f.searchableFields||c(),f.sortableFields||d(),f.templateFile&&e(),a.extend(f.master,f.data),f.create()):a.error('The "data" parameter must be an array.')},this.init()};b.prototype={evenRowClass:"ui-table-rows-even",oddRowClass:"ui-table-rows-odd",liveSearch:!0,page:1,pages:1,plugins:null,query:null,reverse:!1,pagination:!0,schema:null,search:!0,searchableFields:null,showRows:[5,10,25,50],size:5,sortableFields:null,sortBy:null,table:!0,templateFile:null,template:' {{#search}}
{{/search}} {{#table}}
{{#headers}} {{#sortable}} {{/sortable}} {{#notSortable}} {{/notSortable}} {{#sortedUp}} {{/sortedUp}} {{#sortedDown}} {{/sortedDown}} {{/headers}} {{#rows}} {{{.}}} {{/rows}}
{{header}}{{header}}{{header}} ▲{{header}} ▼
Show rows: {{{showRowsMenu}}} Results: {{resultRange.start}} – {{resultRange.end}} of {{tableTotal}} {{#prevPageExists}} {{/prevPageExists}} {{^prevPageExists}} {{/prevPageExists}} {{#nextPageExists}} {{/nextPageExists}} {{^nextPageExists}} {{/nextPageExists}}
{{/table}} ',conditioning:!0,paginating:!0,searching:!0,sorting:!0,pageHandler:function(){this.create()},searchHandler:function(a){this.liveSearch?this.create():"13"==a.keyCode&&this.create()},sizeHandler:function(){this.create()},sortHandler:function(){this.page=1,this.create()},destroy:function(){return this.$el.data("columns",null),this.$el.empty(),!0},getObject:function(){return this},getPage:function(){return this.page},getQuery:function(){return this.query},getRange:function(){return this.range},getRows:function(){return this.rows},getShowRowsMenu:function(){return this.showRowsMenu},getTemplate:function(){return this.template},getThead:function(){return this.thead},getTotal:function(){return this.total},getVersion:function(){return this.VERSION},getView:function(){return this.view},gotoPage:function(a){return this.pageExists(a)?(this.page=a,this.create(),!0):!1},pageExists:function(a){return a>0&&a<=this.pages?!0:!1},resetData:function(a){return this.data=this.master.slice(0),this.data},setMaster:function(b){return a.isArray(b)?(this.master=b,!0):!1},setPage:function(a){return this.page=this.pageExists(a)?a:this.page,this.page},setRange:function(){var a=(this.page-1)*this.size,b=a+this.size"'\/]/g,function(a){return q[a]})}function g(b,d){function f(){if(w&&!x)for(;q.length;)delete p[q.pop()];else q=[];w=!1,x=!1}function g(a){if("string"==typeof a&&(a=a.split(s,2)),!n(a)||2!==a.length)throw new Error("Invalid tags: "+a);k=new RegExp(c(a[0])+"\\s*"),l=new RegExp("\\s*"+c(a[1])),m=new RegExp("\\s*"+c("}"+a[1]))}if(!b)return[];var k,l,m,o=[],p=[],q=[],w=!1,x=!1;g(d||a.tags);for(var y,z,A,B,C,D,E=new j(b);!E.eos();){if(y=E.pos,A=E.scanUntil(k))for(var F=0,G=A.length;G>F;++F)B=A.charAt(F),e(B)?q.push(p.length):x=!0,p.push(["text",B,y,y+1]),y+=1,"\n"===B&&f();if(!E.scan(k))break;if(w=!0,z=E.scan(v)||"name",E.scan(r),"="===z?(A=E.scanUntil(t),E.scan(t),E.scanUntil(l)):"{"===z?(A=E.scanUntil(m),E.scan(u),E.scanUntil(l),z="&"):A=E.scanUntil(l),!E.scan(l))throw new Error("Unclosed tag at "+E.pos);if(C=[z,A,y,E.pos],p.push(C),"#"===z||"^"===z)o.push(C);else if("/"===z){if(D=o.pop(),!D)throw new Error('Unopened section "'+A+'" at '+y);if(D[1]!==A)throw new Error('Unclosed section "'+D[1]+'" at '+y)}else"name"===z||"{"===z||"&"===z?x=!0:"="===z&&g(A)}if(D=o.pop())throw new Error('Unclosed section "'+D[1]+'" at '+E.pos);return i(h(p))}function h(a){for(var b,c,d=[],e=0,f=a.length;f>e;++e)b=a[e],b&&("text"===b[0]&&c&&"text"===c[0]?(c[1]+=b[1],c[3]=b[3]):(d.push(b),c=b));return d}function i(a){for(var b,c,d=[],e=d,f=[],g=0,h=a.length;h>g;++g)switch(b=a[g],b[0]){case"#":case"^":e.push(b),f.push(b),e=b[4]=[];break;case"/":c=f.pop(),c[5]=b[2],e=f.length>0?f[f.length-1][4]:d;break;default:e.push(b)}return d}function j(a){this.string=a,this.tail=a,this.pos=0}function k(a,b){this.view=null==a?{}:a,this.cache={".":this.view},this.parent=b}function l(){this.cache={}}var m=Object.prototype.toString,n=Array.isArray||function(a){return"[object Array]"===m.call(a)},o=RegExp.prototype.test,p=/\S/,q={"&":"&","<":"<",">":">",'"':""","'":"'","/":"/"},r=/\s*/,s=/\s+/,t=/\s*=/,u=/\s*\}/,v=/#|\^|\/|>|\{|&|=|!/;j.prototype.eos=function(){return""===this.tail},j.prototype.scan=function(a){var b=this.tail.match(a);if(!b||0!==b.index)return"";var c=b[0];return this.tail=this.tail.substring(c.length),this.pos+=c.length,c},j.prototype.scanUntil=function(a){var b,c=this.tail.search(a);switch(c){case-1:b=this.tail,this.tail="";break;case 0:b="";break;default:b=this.tail.substring(0,c),this.tail=this.tail.substring(c)}return this.pos+=b.length,b},k.prototype.push=function(a){return new k(a,this)},k.prototype.lookup=function(a){var c,d=this.cache;if(a in d)c=d[a];else{for(var e,f,g=this;g;){if(a.indexOf(".")>0)for(c=g.view,e=a.split("."),f=0;null!=c&&fl;++l)switch(h=c[l],h[0]){case"#":if(i=d.lookup(h[1]),!i)continue;if(n(i))for(var o=0,p=i.length;p>o;++o)j+=this.renderTokens(h[4],d.push(i[o]),e,f);else if("object"==typeof i||"string"==typeof i)j+=this.renderTokens(h[4],d.push(i),e,f);else if(b(i)){if("string"!=typeof f)throw new Error("Cannot use higher-order sections without the original template");i=i.call(d.view,f.slice(h[3],h[5]),g),null!=i&&(j+=i)}else j+=this.renderTokens(h[4],d,e,f);break;case"^":i=d.lookup(h[1]),(!i||n(i)&&0===i.length)&&(j+=this.renderTokens(h[4],d,e,f));break;case">":if(!e)continue;i=b(e)?e(h[1]):e[h[1]],null!=i&&(j+=this.renderTokens(this.parse(i),d,e,i));break;case"&":i=d.lookup(h[1]),null!=i&&(j+=i);break;case"name":i=d.lookup(h[1]),null!=i&&(j+=a.escape(i));break;case"text":j+=h[1]}return j},a.name="mustache.js",a.version="0.8.1",a.tags=["{{","}}"];var w=new l;return a.clearCache=function(){return w.clearCache()},a.parse=function(a,b){return w.parse(a,b)},a.render=function(a,b,c){return w.render(a,b,c)},a.to_html=function(c,d,e,f){var g=a.render(c,d,e);return b(f)?void f(g):g},a.escape=f,a.Scanner=j,a.Context=k,a.Writer=l,a}),extract_meta=function(a){return now=new Date,a.map(function(a){return rdate=new Date(a.release_date),a.release_date=rdate.getFullYear()+"-"+(rdate.getMonth()+1)+"-"+rdate.getDate(),now-rdate<6048e5&&(a.release_date=''+a.release_date+""),a.title=''+a.title+"",a.subtitle&&(a.title=a.title+"
"+a.subtitle),authors=a.authors.map(function(a){return a.last_name+" "+a.first_name}),a.author=authors.join(", "),a.proofing=a.proofing||"",a})},whatsnew=function(a){$.ajax({url:"/api/v0.1/books?fields=release_date,title,subtitle,card_url,authors,input,proofing&after="+a,dataType:"json",success:function(a){tbl=$("#tbl").columns({data:extract_meta(a),schema:[{header:"公開日",key:"release_date"},{header:"作品名/副題",key:"title"},{header:"著者名",key:"author"},{header:"入力者名",key:"input"},{header:"校正者名",key:"proofing"}],showRows:[10,25,50,100],size:10})}})}; -------------------------------------------------------------------------------- /scraper/getbooks.coffee: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015 Kenichi Sato 3 | # 4 | AdmZip = require 'adm-zip' 5 | parse = require 'csv-parse' 6 | async = require 'async' 7 | request = require 'request' 8 | 9 | mongodb = require 'mongodb' 10 | MongoClient = mongodb.MongoClient 11 | 12 | mongodb_credential = process.env.AOZORA_MONGODB_CREDENTIAL || '' 13 | mongodb_host = process.env.AOZORA_MONGODB_HOST || 'localhost' 14 | mongodb_port = process.env.AOZORA_MONGODB_PORT || '27017' 15 | mongo_url = "mongodb://#{mongodb_credential}#{mongodb_host}:#{mongodb_port}/aozora" 16 | 17 | list_url_base = 'https://github.com/aozorabunko/aozorabunko/raw/master/index_pages/' 18 | listfile_inp = 'list_inp_person_all_utf8.zip' 19 | list_url_pub = 'list_person_all_extended_utf8.zip' 20 | 21 | person_extended_attrs = [ 22 | 'book_id', 23 | 'title', 24 | 'title_yomi', 25 | 'title_sort', 26 | 'subtitle', 27 | 'subtitle_yomi', 28 | 'original_title', 29 | 'first_appearance', 30 | 'ndc_code', 31 | 'font_kana_type', 32 | 'copyright', 33 | 'release_date', 34 | 'last_modified', 35 | 'card_url', 36 | 'person_id', 37 | 'last_name', 38 | 'first_name', 39 | 'last_name_yomi', 40 | 'first_name_yomi', 41 | 'last_name_sort', 42 | 'first_name_sort', 43 | 'last_name_roman', 44 | 'first_name_roman', 45 | 'role', 46 | 'date_of_birth', 47 | 'date_of_death', 48 | 'author_copyright', 49 | 'base_book_1', 50 | 'base_book_1_publisher', 51 | 'base_book_1_1st_edition', 52 | 'base_book_1_edition_input', 53 | 'base_book_1_edition_proofing', 54 | 'base_book_1_parent', 55 | 'base_book_1_parent_publisher', 56 | 'base_book_1_parent_1st_edition', 57 | 'base_book_2', 58 | 'base_book_2_publisher', 59 | 'base_book_2_1st_edition', 60 | 'base_book_2_edition_input', 61 | 'base_book_2_edition_proofing', 62 | 'base_book_2_parent', 63 | 'base_book_2_parent_publisher', 64 | 'base_book_2_parent_1st_edition', 65 | 'input', 66 | 'proofing', 67 | 'text_url', 68 | 'text_last_modified', 69 | 'text_encoding', 70 | 'text_charset', 71 | 'text_updated', 72 | 'html_url', 73 | 'html_last_modified', 74 | 'html_encoding', 75 | 'html_charset', 76 | 'html_updated' 77 | ] 78 | 79 | 80 | role_map = 81 | '著者': 'authors' 82 | '翻訳者': 'translators' 83 | '編者': 'editors' 84 | '校訂者': 'revisers' 85 | 86 | get_bookobj = (entry, cb)-> 87 | book = {} 88 | role = null 89 | person = {} 90 | 91 | person_extended_attrs.forEach (e,i)-> 92 | value = entry[i] 93 | if value != '' 94 | if e in ['book_id', 'person_id', 'text_updated', 'html_updated'] 95 | value = parseInt value 96 | else if e in ['copyright', 'author_copyright'] 97 | value = value != 'なし' 98 | else if e in ['release_date', 'last_modified', 'date_of_birth', 'date_of_death', 99 | 'text_last_modified', 'html_last_modified'] 100 | value = new Date value 101 | 102 | if e in ['person_id', 'first_name', 'last_name', 'last_name_yomi', 'first_name_yomi', 103 | 'last_name_sort', 'first_name_sort', 'last_name_roman', 'first_name_roman', 104 | 'date_of_birth', 'date_of_death', 'author_copyright'] 105 | person[e] = value 106 | return 107 | else if e is 'role' 108 | role = role_map[value] 109 | if not role 110 | console.log value 111 | return 112 | 113 | book[e] = value 114 | 115 | cb book, role, person 116 | 117 | MongoClient.connect mongo_url, 118 | connectTimeoutMS: 120000, 119 | socketTimeoutMS: 120000, 120 | , (err, db)-> 121 | if err 122 | console.log err 123 | return -1 124 | db = db 125 | books = db.collection('books') 126 | persons = db.collection('persons') 127 | 128 | # list_url_base = 'http://localhost:8000/' 129 | # list_url_pub = 'list_person_all_extended_utf8_short.zip' 130 | request.get list_url_base + list_url_pub, {encoding: null}, (err, resp, body)-> 131 | if err 132 | return -1 133 | zip = AdmZip body 134 | entries = zip.getEntries() 135 | if entries.length != 1 136 | return -1 137 | buf = zip.readFile entries[0] 138 | parse buf, (err, data)-> 139 | books.findOne {}, {fields: {release_date: 1}, sort: {release_date: -1}}, (err, item)-> 140 | if err or item is null 141 | last_release_date = new Date '1970-01-01' 142 | else 143 | last_release_date = item.release_date 144 | updated = data[1..].filter (entry)-> 145 | release_date = new Date entry[11] 146 | return last_release_date < release_date 147 | console.log "#{updated.length} entries are updated" 148 | if updated.length > 0 149 | books_batch_list = {} 150 | persons_batch_list = {} 151 | async.eachSeries updated, (entry, cb)-> 152 | async.setImmediate (entry, cb)-> 153 | get_bookobj entry, (book, role, person)-> 154 | if not books_batch_list[book.book_id] 155 | books_batch_list[book.book_id] = book 156 | if not books_batch_list[book.book_id][role] 157 | books_batch_list[book.book_id][role] = [] 158 | person.full_name = person.last_name + person.first_name 159 | books_batch_list[book.book_id][role].push 160 | person_id: person.person_id 161 | last_name: person.last_name 162 | first_name: person.first_name 163 | full_name: person.full_name 164 | if not persons_batch_list[person.person_id] 165 | persons_batch_list[person.person_id] = person 166 | cb null, 167 | , entry, cb 168 | , (err)-> 169 | if err 170 | console.log err 171 | return -1 172 | 173 | async.parallel [ 174 | (cb)-> 175 | books_batch = books.initializeUnorderedBulkOp() 176 | for book_id, book of books_batch_list 177 | books_batch.find({book_id: book_id}).upsert().updateOne book 178 | books_batch.execute cb 179 | ,(cb)-> 180 | persons_batch = persons.initializeUnorderedBulkOp() 181 | for person_id, person of persons_batch_list 182 | persons_batch.find({person_id: person_id}).upsert().updateOne person 183 | persons_batch.execute cb 184 | ], (err, result)-> 185 | if err 186 | console.log 'err', err 187 | db.close() 188 | else 189 | db.close() 190 | -------------------------------------------------------------------------------- /scraper/getids.coffee: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015 Kenichi Sato 3 | # 4 | scraperjs = require 'scraperjs' 5 | async = require 'async' 6 | 7 | mongodb = require 'mongodb' 8 | MongoClient = mongodb.MongoClient 9 | 10 | mongodb = require 'mongodb' 11 | mongodb_credential = process.env.AOZORA_MONGODB_CREDENTIAL || '' 12 | mongodb_host = process.env.AOZORA_MONGODB_HOST || 'localhost' 13 | mongodb_port = process.env.AOZORA_MONGODB_PORT || '27017' 14 | mongo_url = "mongodb://#{mongodb_credential}#{mongodb_host}:#{mongodb_port}/aozora" 15 | 16 | scrape_url = (idurl, cb)-> 17 | scraperjs.StaticScraper.create idurl 18 | .scrape ($)-> 19 | $("tr[valign]").map -> 20 | $row = $(this) 21 | ret = 22 | id: $row.find(':nth-child(1)').text().trim() 23 | name: $row.find(':nth-child(2)').text().trim().replace('　',' ') 24 | .get() 25 | , (items)-> 26 | cb null, items[1...] 27 | 28 | 29 | idurls = 30 | # 'persons': 'http://reception.aozora.gr.jp/pidlist.php?page=1&pagerow=-1', 31 | 'workers': 'http://reception.aozora.gr.jp/widlist.php?page=1&pagerow=-1' 32 | 33 | MongoClient.connect mongo_url, (err, db)-> 34 | if err 35 | console.log err 36 | return -1 37 | async.map Object.keys(idurls), (idname, cb)-> 38 | collection = db.collection idname 39 | idurl = idurls[idname] 40 | console.log idurl 41 | scrape_url idurl, (err, results)-> 42 | if err 43 | cb err 44 | async.map results, (result, cb2)-> 45 | result.id = parseInt(result.id) 46 | collection.update {id: result.id}, result, {upsert: true}, cb2 47 | , (err, results2)-> 48 | if err 49 | cb err 50 | else 51 | cb null, results2.length 52 | , (err, result)-> 53 | if err 54 | console.log err 55 | return -1 56 | console.log result 57 | db.close() 58 | -------------------------------------------------------------------------------- /scraper/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scraper", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "async": "^2.0.1", 13 | "csv-parse": "1.1.7", 14 | "scraperjs": "1.2.0" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /spec/pubserver.raml: -------------------------------------------------------------------------------- 1 | #%RAML 0.8 2 | title: publishing public server 3 | baseUri: http://pubserver-master.herokuapp.com/api/{version} 4 | #baseUri: http://localhost:8000/api/{version} 5 | version: v0.1 6 | mediaType: application/json 7 | /books: 8 | get: 9 | description: get a list of books 10 | queryParameters: 11 | name: 12 | description: filter by name 13 | post: 14 | description: register a book 15 | /{book_id}: 16 | get: 17 | description: get a book 18 | /content: 19 | get: 20 | description: get contents of a book 21 | queryParameters: 22 | format: 23 | enum: 24 | - txt 25 | - html 26 | - epub 27 | - archive 28 | /persons: 29 | get: 30 | description: get a list of persons 31 | queryParameters: 32 | name: 33 | description: filter by name 34 | post: 35 | description: register a person 36 | /{person_id}: 37 | get: 38 | description: get a person 39 | /workers: 40 | get: 41 | description: get a list of workers 42 | queryParameters: 43 | name: 44 | description: filter by name 45 | post: 46 | description: register a worker 47 | /{worker_id}: 48 | get: 49 | description: get a worker 50 | -------------------------------------------------------------------------------- /src/git_utils.coffee: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015 Kenichi Sato 3 | # 4 | Git = require 'nodegit' 5 | path = require 'path' 6 | promisify = require 'promisify-node' 7 | fse = promisify require 'fs-extra' 8 | Promise = require 'nodegit-promise' 9 | 10 | remote_callbacks = null 11 | the_sig = null 12 | 13 | open_or_clone = (repo_local_path, origin_url)-> 14 | Git.Repository.open repo_local_path 15 | .catch (error)-> 16 | Git.Clone.clone origin_url, repo_local_path, 17 | remoteCallbacks: remote_callbacks 18 | 19 | exports.set_credential = (user, pass, email)-> 20 | remote_callbacks = 21 | certificateCheck: -> 1 22 | credentials: -> 23 | Git.Cred.userpassPlaintextNew user, pass 24 | the_sig = Git.Signature.now user, email 25 | 26 | exports.setup_repo = (origin_url, book_id, initial_files, cb)-> 27 | # 28 | # make initial commit 29 | # 30 | open_or_clone "repo/#{book_id}", origin_url 31 | .then (repo)-> 32 | if repo.isEmpty() 33 | repo.openIndex() 34 | .then (index)-> 35 | filenames = Object.keys initial_files 36 | Promise.all filenames.map (filename)-> 37 | fse.writeFile path.join(repo.workdir(), filename), initial_files[filename] 38 | .then -> 39 | index.addAll() 40 | .then -> 41 | index.write() 42 | .then -> 43 | index.writeTree() 44 | .then (oid)-> 45 | Git.Tree.lookup repo, oid 46 | .then (tree)-> 47 | the_sig.dup() 48 | .then (sig)-> 49 | Git.Commit.create repo, "HEAD", sig, sig, null, "initial commit", tree, 0, [] 50 | .then (oid)-> 51 | return repo 52 | else 53 | return repo 54 | # 55 | # push the change 56 | # 57 | .then (repo)-> 58 | Git.Remote.lookup repo, 'origin' 59 | .then (remote)-> 60 | remote.addPush("refs/heads/master:refs/heads/master") 61 | remote.getPushRefspecs() 62 | .then (specs)-> 63 | remote.setCallbacks remote_callbacks 64 | the_sig.dup() 65 | .then (sig)-> 66 | remote.push specs, {}, sig, null 67 | .then (error)-> 68 | if error 69 | console.log 'push error:', error 70 | cb error == undefined 71 | 72 | # 73 | # error catcher 74 | # 75 | .catch (error)-> 76 | console.log 'catched error', error 77 | console.log origin_url, book_id 78 | cb false 79 | -------------------------------------------------------------------------------- /src/js/aozora.js: -------------------------------------------------------------------------------- 1 | extract_meta = function (json_data) { 2 | now = new Date() 3 | return json_data.map(function(item) { 4 | //release date 5 | rdate = new Date(item.release_date) 6 | item.release_date = rdate.getFullYear() + "-" + (rdate.getMonth()+1) + "-" + rdate.getDate() 7 | if ( now-rdate < 604800000 ) { 8 | item.release_date = "" + item.release_date + "" 9 | } 10 | 11 | // title 12 | item.title = "" + item.title + "" 13 | if (item.subtitle) { 14 | item.title = item.title + "
" + item.subtitle; 15 | } 16 | // author 17 | authors = item.authors.map(function(author) { 18 | return author.last_name + " " + author.first_name; 19 | }); 20 | item.author = authors.join(", "); 21 | 22 | item.proofing = item.proofing || "" 23 | return item; 24 | }); 25 | } 26 | 27 | whatsnew = function (start_date) { 28 | $.ajax({ 29 | url:'/api/v0.1/books?fields=release_date,title,subtitle,card_url,authors,input,proofing&after='+start_date, 30 | dataType: 'json', 31 | success: function(json_data) { 32 | tbl = $('#tbl').columns({ 33 | data: extract_meta(json_data), 34 | schema: [ 35 | {"header": "公開日", "key": "release_date"}, 36 | {"header": "作品名/副題", "key": "title"}, 37 | {"header": "著者名", "key": "author"}, 38 | {"header": "入力者名", "key": "input"}, 39 | {"header": "校正者名", "key": "proofing"}, 40 | ], 41 | showRows: [10, 25, 50, 100], 42 | size: 10 43 | }); 44 | } 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /src/js/jquery.columns.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * Copyright (c) 2014 3 | * Licensed under the MIT License. 4 | * 5 | * Author: Michael Eisenbraun 6 | * Version: 2.2.2 7 | * Requires: jQuery 1.7.2+ 8 | * Documentation: http://eisenbraun.github.io/columns/ 9 | */ 10 | 11 | if (!window.console) { 12 | var console = { 13 | log: function() { } 14 | }; 15 | } 16 | 17 | (function($) { 18 | $.fn.columns = function(options) { 19 | var val = []; 20 | var args = Array.prototype.slice.call(arguments, 1); 21 | 22 | if (typeof options === 'string') { 23 | this.each(function() { 24 | 25 | var instance = $.data(this, 'columns'); 26 | if (typeof instance !== 'undefined' && $.isFunction(instance[options])) { 27 | var methodVal = instance[options].apply(instance, args); 28 | if (methodVal !== undefined && methodVal !== instance) { 29 | val.push(methodVal); 30 | } 31 | } else { 32 | return $.error('No such method "' + options + '" for Columns'); 33 | } 34 | }); 35 | 36 | } else { 37 | this.each(function() { 38 | if (!$.data(this, 'columns')) { 39 | $.data(this, 'columns', new Columns(this, options)); 40 | } 41 | }); 42 | } 43 | 44 | if (val.length === 0) { 45 | return this.data('columns'); 46 | } else if (val.length === 1) { 47 | return val[0]; 48 | } else { 49 | return val; 50 | } 51 | }; 52 | 53 | var Columns = function(element, options) { 54 | this.$el = $(element); 55 | 56 | if (options) { 57 | $.extend( this, options ); 58 | } 59 | 60 | /** PLUGIN CONSTANTS */ 61 | this.VERSION = '2.2.2'; 62 | 63 | /** PLUGIN METHODS */ 64 | 65 | /** 66 | * SORT: 67 | * Arranges the data object in the order based on the object key 68 | * stored in the variable `sortBy` and the direction stored in the 69 | * variable `reverse`. 70 | * 71 | * A date primer has been created. If the object value matches the 72 | * date pattern, it be sorted as a date instead or a string or number. 73 | */ 74 | this.sort = function() { 75 | var $this = this; 76 | var date = /^(Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December|(0?\d{1})|(10|11|12))(-|\s|\/|\.)(0?[1-9]|(1|2)[0-9]|3(0|1))(-|\s|\/|\.|,\s)(19|20)?\d\d$/i; 77 | 78 | function objectSort(field, reverse, primer){ 79 | reverse = (reverse) ? -1 : 1; 80 | 81 | return function(a,b){ 82 | 83 | a = a[field]; 84 | b = b[field]; 85 | 86 | if (date.test(a) && date.test(b)) { 87 | a = new Date(a); 88 | a = Date.parse(a); 89 | 90 | b = new Date(b); 91 | b = Date.parse(b); 92 | } else if (typeof(primer) !== 'undefined'){ 93 | a = primer(a); 94 | b = primer(b); 95 | } 96 | 97 | if (ab) { 102 | return reverse * 1; 103 | } 104 | 105 | return 0; 106 | }; 107 | } 108 | 109 | if ($this.total && $this.sortBy && typeof $this.data[0][$this.sortBy] !== 'undefined') { 110 | $this.data.sort(objectSort($this.sortBy, $this.reverse)); 111 | } 112 | }; 113 | 114 | /** 115 | * FILTER: 116 | * Filters out all row from the data object that does not match the 117 | * the search query stored in the `query`. 118 | * 119 | * If the data object value is a string, the query can be anywhere in value 120 | * regardless of case. 121 | * 122 | * If the data object value is a number, the query must match value only, not 123 | * data type. 124 | */ 125 | this.filter = function() { 126 | var $this = this, 127 | length = $this.searchableFields.length; 128 | 129 | if ($this.query) { 130 | var re = new RegExp($this.query, "gi"); 131 | 132 | $this.data = $.grep($this.data, function(obj) { 133 | for (var key = 0; key < length; key++) { 134 | if (typeof obj[$this.searchableFields[key]] === 'string') { 135 | if (obj[$this.searchableFields[key]].match(re)) { 136 | return true; 137 | } 138 | } else if (typeof obj[$this.searchableFields[key]] === 'number') { 139 | if (obj[$this.searchableFields[key]] == $this.query) { 140 | return true; 141 | } 142 | } 143 | } 144 | return false; 145 | }); 146 | } 147 | 148 | /** setting data total */ 149 | $this.total = $this.data.length; 150 | }; 151 | 152 | /** 153 | * PAGINATE: 154 | * Calculates the number of pages, the current page number and the exact 155 | * rows to display. 156 | */ 157 | this.paginate = function() { 158 | var $this = this; 159 | 160 | /** calculate the number of pages */ 161 | $this.pages = Math.ceil($this.data.length/$this.size); 162 | 163 | /** retrieve page number */ 164 | $this.page = ($this.page <= $this.pages ? $this.page : 1); 165 | 166 | /** set range of rows */ 167 | $this.setRange(); 168 | 169 | $this.data = $this.data.slice($this.range.start-1,$this.range.end); 170 | 171 | }; 172 | 173 | /** 174 | * CONDITION: 175 | * Only displays the data object rows that meet the given criteria. 176 | * 177 | * Condition vs Filter: 178 | * Condition is true if the value meets a determined conditional statement, 179 | * which is found in the schema. Condition is column specific. Since conditions 180 | * are not subject to the end users actions, condition is only checked once during 181 | * initialization. 182 | * 183 | * Filter is true if the value matches the query. A query is compared across 184 | * all searchable columns. Filter is checked every time there is a query value. 185 | * 186 | * 187 | */ 188 | this.condition = function() { 189 | var $this = this, 190 | schema = []; 191 | 192 | if ($this.schema) { 193 | var dataLength = $this.data.length, 194 | schemaLength = $this.schema.length; 195 | 196 | for (var row = 0; row < dataLength; row++) { 197 | var data = $this.data[row], 198 | temp = {}; 199 | 200 | for (var key = 0; key < schemaLength; key++) { 201 | var val = $this.schema[key]; 202 | 203 | if(val.condition) { 204 | if(!val.condition(data[val.key])) { 205 | temp = null; 206 | break; 207 | } 208 | } 209 | 210 | temp[val.key] = data[val.key]; 211 | } 212 | 213 | if (temp) { 214 | schema.push(temp); 215 | } 216 | } 217 | 218 | $this.data = schema; 219 | } 220 | }; 221 | 222 | /** 223 | * CHEVRON 224 | * This a shortcut for compiling and render a Mustache template and data. 225 | */ 226 | this.chevron = function(template, data) { 227 | return Mustache.render(template, data); 228 | }; 229 | 230 | this.create = function() { 231 | var $this = this; 232 | 233 | //Building Data 234 | $this.resetData(); 235 | 236 | if($this.searching) { 237 | $this.filter(); 238 | } 239 | 240 | if($this.sorting) { 241 | $this.sort(); 242 | } 243 | 244 | if($this.paginating) { 245 | $this.paginate(); 246 | } 247 | 248 | 249 | /** Building Column Elements */ 250 | function buildThead() { 251 | $this.thead = []; 252 | 253 | $.each($this.schema, function(key, col) { 254 | if (!col.hide) { 255 | var th = {}; 256 | 257 | if ($.inArray(col.key,$this.sortableFields) === -1) { 258 | th.notSortable = true; 259 | } else if ($this.sortBy === col.key) { 260 | if ($this.reverse) { 261 | th.sortedDown = true; 262 | } else { 263 | th.sortedUp = true; 264 | } 265 | } else { 266 | th.sortable = true; 267 | } 268 | 269 | th.key = col.key; 270 | th.header = col.header; 271 | 272 | $this.thead.push(th); 273 | } 274 | }); 275 | } 276 | 277 | function buildRows(key, row) { 278 | var tr = []; 279 | 280 | if (key%2 === 0) { 281 | tr.push(''); 282 | } else { 283 | tr.push(''); 284 | } 285 | 286 | $.each($this.schema, function(key, col) { 287 | if (!col.hide) { 288 | if (col.template) { 289 | tr.push(''+$this.chevron(col.template, row)+''); 290 | } else { 291 | tr.push(''+row[col.key]+''); 292 | } 293 | } 294 | }); 295 | 296 | tr.push(''); 297 | 298 | return tr; 299 | } 300 | 301 | function buildShowRowsMenu() { 302 | var menu = []; 303 | 304 | menu.push(''); 319 | 320 | $this.showRowsMenu = menu.join(''); 321 | } 322 | 323 | function buildTable() { 324 | $this.rows = []; 325 | 326 | if($this.total) { 327 | $.each($this.data, function(key, row) { 328 | if (key === 0) { 329 | buildThead(); 330 | } 331 | $this.rows.push(buildRows(key, row).join('')); 332 | }); 333 | } else { 334 | $this.rows.push('No Results'); 335 | 336 | } 337 | } 338 | 339 | buildTable(); 340 | buildShowRowsMenu(); 341 | 342 | /** Creating Table from Mustache Template */ 343 | var view = { 344 | prevPage: $this.page-1, 345 | nextPage: $this.page+1, 346 | prevPageExists: $this.pageExists($this.page-1), 347 | nextPageExists: $this.pageExists($this.page+1), 348 | resultRange: $this.range, 349 | tableTotal: $this.total, 350 | showRowsMenu: $this.showRowsMenu, 351 | rows: $this.rows, 352 | headers: $this.thead, 353 | query: $this.query, 354 | search: $this.search, 355 | table: $this.table 356 | }; 357 | 358 | $.extend($this.view, view); 359 | 360 | /** Calling plugins, if any */ 361 | if ($this.plugins) { 362 | $.each($this.plugins, function(key, val) { 363 | if (typeof ColumnsPlugins !== 'undefined') { 364 | if (typeof ColumnsPlugins[val] !== 'undefined') { 365 | ColumnsPlugins[val].create.call($this); 366 | } 367 | } 368 | }); 369 | } 370 | 371 | if ($this.search) { 372 | $this.$el.html($this.chevron($this.template, $this.view)); 373 | $this.search = false; 374 | } else { 375 | $('[data-columns-table]', $this.$el).remove(); 376 | $this.$el.append($this.chevron($this.template, $this.view)); 377 | } 378 | 379 | return true; 380 | }; 381 | 382 | this.init = function() { 383 | var $this = this; 384 | 385 | function buildSchema() { 386 | $this.schema = []; 387 | $.each($this.data[0], function(key) { 388 | $this.schema.push({"header":key, "key":key}); 389 | }); 390 | } 391 | 392 | function buildSearchableFields() { 393 | $this.searchableFields = []; 394 | $.each($this.data[0], function(key) { 395 | $this.searchableFields.push(key); 396 | }); 397 | } 398 | 399 | function buildSortableFields() { 400 | $this.sortableFields = []; 401 | $.each($this.data[0], function(key) { 402 | $this.sortableFields.push(key); 403 | }); 404 | } 405 | 406 | function getTemplateFile() { 407 | $.ajax({ 408 | url: $this.templateFile, 409 | async: false, 410 | success: function(template) { 411 | $this.template = template; 412 | }, 413 | error: function() { 414 | $.error('Template could not be found.'); 415 | } 416 | }); 417 | } 418 | 419 | if ($.isArray($this.data)) { 420 | $this.master = []; 421 | $this.view = {}; 422 | 423 | /** setting up DOM */ 424 | $this.$el.addClass('columns'); 425 | 426 | /** creating listeners */ 427 | 428 | /** sort listener */ 429 | $this.$el.on('click', '.ui-table-sortable', function(event) { 430 | var sortBy = $(this).data('columns-sortby'); 431 | 432 | if ($this.sortBy === sortBy) { 433 | $this.reverse = ($this.reverse) ? false : true; 434 | } 435 | 436 | $this.sortBy = sortBy 437 | 438 | $this.sortHandler(event); 439 | }); 440 | 441 | /** page listener */ 442 | $this.$el.on('click', '.ui-table-control-next, .ui-table-control-prev', function(event) { 443 | $this.page = $(this).data('columns-page'); 444 | 445 | $this.pageHandler(event); 446 | }); 447 | 448 | /** search listener */ 449 | $this.$el.on('keyup', '.ui-table-search', function(event) { 450 | $this.query = $(this).val(); 451 | 452 | $this.searchHandler(event); 453 | }); 454 | 455 | /** size listener */ 456 | $this.$el.on('change', '.ui-table-size select', function(event) { 457 | $this.size = parseInt($(this).val()); 458 | 459 | $this.sizeHandler(event); 460 | }); 461 | 462 | /** Calling plugins, if any */ 463 | if ($this.plugins) { 464 | $.each($this.plugins, function(key, val) { 465 | if (typeof ColumnsPlugins !== 'undefined') { 466 | if (typeof ColumnsPlugins[val] !== 'undefined') { 467 | ColumnsPlugins[val].init.call($this); 468 | } 469 | } 470 | }); 471 | } 472 | 473 | /** condition never change, so only checked once. */ 474 | if($this.conditioning) { 475 | $this.condition(); 476 | } 477 | 478 | /** updating defaults */ 479 | if (!$this.schema) { 480 | buildSchema(); 481 | } 482 | 483 | if (!$this.searchableFields) { 484 | buildSearchableFields(); 485 | } 486 | 487 | if (!$this.sortableFields) { 488 | buildSortableFields(); 489 | } 490 | 491 | if ($this.templateFile) { 492 | getTemplateFile(); 493 | } 494 | 495 | /** making a master copy of data */ 496 | $.extend($this.master, $this.data); 497 | 498 | /** creating columns table */ 499 | $this.create(); 500 | 501 | } else { 502 | $.error('The "data" parameter must be an array.'); 503 | } 504 | 505 | }; 506 | 507 | this.init(); 508 | }; 509 | 510 | Columns.prototype = { 511 | 512 | //defaults 513 | evenRowClass: "ui-table-rows-even", 514 | oddRowClass: "ui-table-rows-odd", 515 | liveSearch: true, 516 | page: 1, 517 | pages: 1, 518 | plugins: null, 519 | query: null, 520 | reverse: false, 521 | pagination: true, 522 | schema: null, 523 | search: true, 524 | searchableFields: null, 525 | showRows: [5, 10, 25, 50], 526 | size: 5, 527 | sortableFields: null, 528 | sortBy: null, 529 | table: true, 530 | templateFile: null, 531 | template: ' {{#search}}
{{/search}} {{#table}}
{{#headers}} {{#sortable}} {{/sortable}} {{#notSortable}} {{/notSortable}} {{#sortedUp}} {{/sortedUp}} {{#sortedDown}} {{/sortedDown}} {{/headers}} {{#rows}} {{{.}}} {{/rows}}
{{header}}{{header}}{{header}} ▲{{header}} ▼
Show rows: {{{showRowsMenu}}} Results: {{resultRange.start}} – {{resultRange.end}} of {{tableTotal}} {{#prevPageExists}} {{/prevPageExists}} {{^prevPageExists}} {{/prevPageExists}} {{#nextPageExists}} {{/nextPageExists}} {{^nextPageExists}} {{/nextPageExists}}
{{/table}} ', 532 | 533 | //functionality 534 | conditioning: true, 535 | paginating: true, 536 | searching: true, 537 | sorting: true, 538 | 539 | 540 | //Handlers 541 | pageHandler: function() { 542 | this.create(); 543 | }, 544 | searchHandler: function(event) { 545 | if(this.liveSearch) { 546 | this.create(); 547 | } else { 548 | if(event.keyCode == '13') { 549 | this.create(); 550 | } 551 | } 552 | }, 553 | sizeHandler: function() { 554 | this.create(); 555 | }, 556 | sortHandler: function() { 557 | this.page = 1; 558 | this.create(); 559 | }, 560 | 561 | //API 562 | destroy: function() { 563 | this.$el.data('columns', null); 564 | this.$el.empty(); 565 | return true; 566 | }, 567 | getObject: function() { 568 | return this; 569 | }, 570 | getPage: function() { 571 | return this.page; 572 | }, 573 | getQuery: function() { 574 | return this.query; 575 | }, 576 | getRange: function() { 577 | return this.range; 578 | }, 579 | getRows: function() { 580 | return this.rows; 581 | }, 582 | getShowRowsMenu: function() { 583 | return this.showRowsMenu; 584 | }, 585 | getTemplate: function() { 586 | return this.template; 587 | }, 588 | getThead: function() { 589 | return this.thead; 590 | }, 591 | getTotal: function() { 592 | return this.total; 593 | }, 594 | getVersion: function() { 595 | return this.VERSION; 596 | }, 597 | getView: function() { 598 | return this.view; 599 | }, 600 | gotoPage: function(p) { 601 | if(this.pageExists(p)) { 602 | this.page = p; 603 | this.create(); 604 | return true; 605 | } 606 | 607 | return false; 608 | }, 609 | pageExists: function(p) { 610 | return (p > 0 && p <= this.pages) ? true : false; 611 | }, 612 | resetData: function(d) { 613 | this.data = this.master.slice(0); 614 | return this.data; 615 | }, 616 | setMaster: function(d) { 617 | if ($.isArray(d)) { 618 | this.master = d; 619 | return true; 620 | } 621 | 622 | return false; 623 | }, 624 | setPage: function(p) { 625 | this.page = (this.pageExists(p) ? p : this.page); 626 | return this.page; 627 | }, 628 | setRange: function() { 629 | var start = ((this.page -1) * (this.size)); 630 | var end = (start + this.size < this.total) ? start + this.size : this.total; 631 | 632 | this.range = {"start":start+1, "end":end}; 633 | }, 634 | setTotal: function(t) { 635 | this.total = t; 636 | 637 | return true; 638 | }, 639 | 640 | //performance tracking 641 | startTime: null, 642 | endTime: null, 643 | startTimer: function() { 644 | var now = new Date(); 645 | this.startTime = now.getTime(); 646 | }, 647 | endTimer: function() { 648 | var now = new Date(); 649 | this.endTime = now.getTime(); 650 | }, 651 | getTimer: function() { 652 | console.log((this.endTime - this.startTime)/1000); 653 | } 654 | }; 655 | 656 | })(jQuery); 657 | -------------------------------------------------------------------------------- /src/js/mustache.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * mustache.js - Logic-less {{mustache}} templates with JavaScript 3 | * http://github.com/janl/mustache.js 4 | */ 5 | 6 | /*global define: false*/ 7 | 8 | (function (global, factory) { 9 | global.Mustache = factory({}); //