' 55 | text += '已同步 ' + d.book_synced_n + '/' + d.book_n + ' 本图书收藏，'; 56 | if (d.queue_length > 2) { 57 | text += '共有' + d.queue_length + '人同时在排队'; 58 | } 59 | } 60 | 61 | if (t > 60000) { 62 | text += '
预计还需要' + datetime.mili2chinese(t, true) + '左右'; 63 | } else if (t > 5000) { 64 | text += '
只剩下大概' + datetime.mili2chinese(t); 65 | } else if (t) { 66 | text = '马上就好'; 67 | } 68 | text = text || '
同步仍在进行，请耐心等待..
'; 69 | if (ad) { 70 | text += '太慢了? 捐点钱让服务器更快!' 71 | } 72 | remains.html(text); 73 | } 74 | 75 | // refresh page after five minutes, no matter what 76 | setTimeout(function() { 77 | location.reload(); 78 | }, 300000); 79 | }); 80 | -------------------------------------------------------------------------------- /templates/people/click/mods/done.jade: -------------------------------------------------------------------------------- 1 | include ./mixins.jade 2 | 3 | h2 都读过的书 4 | if clicks.ratios.done[0] 5 | small (共#{clicks.done.length}本, 占#{people._name}全部读过的 #{(clicks.ratios.done[0] * 100).toFixed(2)}%) 6 | mixin book_list(clicks.done.slice(0, 6), null, mixed_rating_stars_mixin) 7 | hr 8 | h2 都想读的书 9 | if clicks.ratios.wish[0] 10 | small (共#{clicks.wish.length}本, 占#{people._name}全部读过的 #{(clicks.ratios.wish[0] * 100).toFixed(2)}%) 11 | mixin book_list(clicks.wish.slice(0, 6)) 12 | hr 13 | h2 都喜欢的书 14 | small （四星以上 15 | if clicks.ratios.love[0] 16 | | ，共#{clicks.love.length}本, 占#{people._name}全部喜欢的书之 #{(clicks.ratios.love[0] * 100).toFixed(2)}% 17 | | ） 18 | mixin book_list(clicks.love.slice(0, 6), null, mixed_rating_stars_mixin) 19 | hr 20 | h2 都不喜欢的书 21 | small （两星以下 22 | if clicks.ratios.hate[0] 23 | ，共#{clicks.hate.length}本, 占#{people._name}全部不喜欢的书之 #{(clicks.ratios.hate[0] * 100).toFixed(2)}% 24 | | ） 25 | mixin book_list(clicks.hate.slice(0, 6), null, mixed_rating_stars_mixin) 26 | hr 27 | h2 #{people._name}读过的#{other._name}想读的书 28 | if clicks.done_wish.length && other.book_stats.wish 29 | - n = clicks.done_wish.length 30 | small (共#{n}本, 占#{other._name}全部想读的 #{(n / other.book_stats.wish.total).toFixed(3)}%) 31 | mixin book_list(clicks.done_wish.slice(0, 6)) 32 | hr 33 | h2 #{other._name}读过的#{people._name}想读的书 34 | if clicks.wish_done.length && people.book_stats.wish 35 | - n = clicks.wish_done.length 36 | small (共#{n}本, 占#{people._name}全部想读的 #{(n / people.book_stats.wish.total).toFixed(3)}%) 37 | mixin book_list(clicks.wish_done.slice(0, 6)) 38 | hr 39 | 40 | if clicks.ratios.love_hate[0] 41 | h2 #{people._name}喜欢#{other._name}却不喜欢的书 42 | - n = clicks.love_hate.length 43 | small (共#{n}本, 占#{people._name}全部喜欢的#{(clicks.ratios.love_hate[0] * 100).toFixed(2)}%) 44 | mixin book_list(clicks.love_hate.slice(0, 6), null, mixed_rating_stars_mixin) 45 | hr 46 | 47 | if clicks.ratios.hate_love[0] 48 | h2 #{people._name}不喜欢#{other._name}却喜欢的书 49 | - n = clicks.hate_love.length 50 | small (共#{n}本, 占#{people._name}全部不喜欢的#{(clicks.ratios.hate_love[0] * 100).toFixed(2)}%) 51 | mixin book_list(clicks.hate_love.slice(0, 6), null, mixed_rating_stars_mixin) 52 | hr 53 | -------------------------------------------------------------------------------- /static/js/mine/booter.js: -------------------------------------------------------------------------------- 1 | Do('lodash', function() { 2 | var lodash = require('lodash')._; 3 | var tmpl_friends = lodash.template($('#tmpl-friends').html()); 4 | var tmpl_friends_items = lodash.template($('#tmpl-friends-items').html()); 5 | 6 | var uid = window._uid_; 7 | 8 | var API_FOLLOWINGS = '/api/mine/followings'; 9 | 10 | function preprocess(items) { 11 | lodash.each(items, function(item) { 12 | item.url = '/people/' + item.uid + '/'; 13 | }); 14 | } 15 | 16 | var followings = $('#followings'); 17 | var loading_html = followings.html(); 18 | 19 | var start = 48, limit = 24; 20 | 21 | function first_pull(fresh) { 22 | $.getJSON(API_FOLLOWINGS, { 23 | fresh: fresh, 24 | limit: 48 25 | }, function(res, err) { 26 | if (res.r) { 27 | if (res.msg === 'pulling') { 28 | res.msg = '同步正在进行..'; 29 | setTimeout(first_pull, 10000); 30 | } 31 | followings.find('.alert').addClass('alert-error').html(res.msg || '出错啦!'); 32 | return; 33 | } 34 | if (res.items) { 35 | preprocess(res.items); 36 | } 37 | followings.html(tmpl_friends(res)); 38 | followings.find('ul').html(tmpl_friends_items(res)); 39 | }); 40 | } 41 | first_pull(); 42 | 43 | followings.delegate('.btn-more', 'click', function(e) { 44 | e.preventDefault(); 45 | 46 | var node = $(this); 47 | if (node.hasClass('disabled')) return; 48 | 49 | node.addClass('disabled').html('加载中...'); 50 | 51 | $.getJSON(API_FOLLOWINGS, { start: start, limit: limit }, function(res, a) { 52 | start += limit; 53 | if (res.r) { 54 | node.remove(); 55 | alert('出错啦！' + res.msg); 56 | return; 57 | } 58 | if (res.items) { 59 | preprocess(res.items); 60 | } 61 | if (res.items.length < limit) { 62 | node.html('没有更多了'); 63 | } else { 64 | node.removeClass('disabled').html('加载更多友邻'); 65 | } 66 | followings.find('ul').append(tmpl_friends_items(res)); 67 | }); 68 | }); 69 | 70 | $('body').delegate('.clear-followings', 'click', function(e) { 71 | e.preventDefault(); 72 | followings.html(loading_html); 73 | first_pull(true); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /conf/default.conf.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var env = process.env; 3 | 4 | var MONGO_HOST = env.MONGODB_HOST ? (env.MONGODB_HOST + ':' + (env.MONGODB_PORT || 27017)) : null; 5 | var ARR_SPLITTER = /\s*,\s*/g; 6 | var is_prod = env.NODE_ENV == 'production'; 7 | 8 | function getDoubanMore() { 9 | var ret = []; 10 | if (env.DOUBAN_APP_MORE) { 11 | ret = env.DOUBAN_APP_MORE.trim().split(ARR_SPLITTER).map(function(item, i) { 12 | var tmp = item.split(':'); 13 | return { 14 | key: tmp[0], 15 | secret: tmp[1], 16 | limit: env.DOUBAN_APP_MORE_LIMIT || 10 17 | } 18 | }); 19 | } 20 | return ret; 21 | } 22 | 23 | /** 24 | * some default settings 25 | */ 26 | module.exports = { 27 | debug: false, 28 | 29 | site_name: '豆瓣酱', 30 | 31 | // the port of the root server 32 | port: env.PORT || 5000, 33 | ssl_root: env.SSL_ROOT, 34 | site_root: env.SITE_ROOT || (is_prod ? 'http://doubanj.yjc.me' : 'http://localhost:5000'), 35 | assets_root: env.ASSETS_ROOT || (is_prod ? 'http://doubanj.yjc.me' : 'http://localhost:5000'), 36 | 37 | salt: env.COOKIE_SALT || 'keyboardcatndog', 38 | 39 | // the Sentry client auth url 40 | // neccessray for tracking events in Sentry 41 | raven: null, 42 | 43 | mongo: { 44 | url: env.MONGO_URL, 45 | dbname: env.MONGODB_DATABASE || 'doubanj', 46 | username: env.MONGODB_USERNAME || null, 47 | password: env.MONGODB_PASSWORD || null, 48 | servers: [ MONGO_HOST || '127.0.0.1:27017'] 49 | }, 50 | 51 | redis: { 52 | url: env.REDIS_URL || 'redis://127.0.0.1:6379', 53 | port: env.REDIS_PORT || '6379', 54 | host: env.REDIS_IP || '127.0.0.1', 55 | prefix: 'doubanj_', 56 | // only set default ttl when there is a memory limit 57 | ttl: 7 * 24 * 60 * 60, // in seconds 58 | }, 59 | 60 | // 管理员的豆瓣 uid 61 | admin_users: env.ADMIN_USERS ? env.ADMIN_USERS.split(ARR_SPLITTER) : ['yajc'], 62 | 63 | douban: { 64 | limit: env.DOUBAN_APP_LIMIT || 10, // request limit per minute 65 | key: env.DOUBAN_APP_KEY || 'my', 66 | secret: env.DOUBAN_APP_SECRET || 'god' 67 | }, 68 | // more random api keys for public informations 69 | douban_more: getDoubanMore(), 70 | 71 | // google analytics id 72 | ga_id: '' 73 | }; 74 | -------------------------------------------------------------------------------- /templates/monitor/index.jade: -------------------------------------------------------------------------------- 1 | extend ../layout/basic 2 | 3 | block main 4 | style. 5 | .monitor-overview { font-size: 20px; } 6 | .monitor-overview .mod { margin-bottom: 50px; } 7 | .users a { margin: 0 4px; display: inline-block; font-size: 14px; } 8 | h1 监控中心 9 | if err 10 | pre.alert.alert-danger 11 | !{err} 12 | hr 13 | .monitor-overview.row 14 | .col.col-lg-6 15 | h2 队列 16 | mixin queue_item('interest', '采集') 17 | mixin queue_item('compute', '统计') 18 | - queue = get_queue('click').queue 19 | .mod 20 | p 契合：#{String(queue.length)} 21 | if queue.length 22 | mixin click_queue_users(queue) 23 | .col.col-lg-6 24 | h2 统计 25 | .mod 26 | p 全部用户 27 | strong #{total} 28 | 人，已统计 29 | strong #{n_succeed} 30 | ，队列中 31 | strong #{n_ing} 32 | if timeouted.length 33 | .mod 34 | h3 超时 35 | strong #{timeouted.length} 36 | 人 37 | p.users 38 | each u in timeouted 39 | a(href="/people/#{u.uid}/", target="dbj-monitor-user") #{u.name || u.uid} 40 | .mod 41 | | 图书 42 | span.num #{n_book} 43 | | 本，收藏记录 44 | span.num #{n_book_interest} 45 | | 条 46 | p csrf to use: 47 | pre 48 | | #{_csrf} 49 | 50 | 51 | mixin queue_item(queue, title) 52 | - queue = get_queue(queue).queue 53 | .mod 54 | p #{title}：#{String(queue.length)} 55 | if queue.length 56 | mixin queue_users(queue) 57 | 58 | mixin queue_users(queue, param_name) 59 | - param_name = param_name || 'user' 60 | p.users 61 | each uid in queue.map(function(arg) { return arg[1][param_name]; }).slice(0, 40) 62 | if (typeof uid === 'object') 63 | uid = uid.uid 64 | a(href="/people/#{uid}/", target="dbj-monitor-user") #{uid} 65 | 66 | 67 | mixin click_queue_users(queue) 68 | p.users 69 | each uid in queue.map(function(arg) { return arg[1].users; }).slice(0, 40) 70 | - uid = uid.map(function(item) { if (typeof item === 'object') { item = item.uid } return item; }) 71 | a(href="/people/#{uid[0]}/click/#{uid[1]}", target="dbj-monitor-user") #{uid[0]} + #{uid[1]} 72 | 73 | -------------------------------------------------------------------------------- /database/index.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var debug = require('debug'); 3 | var log = debug('dbj:database'); 4 | 5 | 6 | // unique key index option 7 | var index_options = { 8 | unique: true, 9 | dropDups: true, 10 | w: 1 11 | }; 12 | var sparse_option = { 13 | background: true, 14 | sparse: 1, 15 | }; 16 | 17 | module.exports = function(db, next) { 18 | db.collection('book_interest', function(err, r) { 19 | log('ensuring database "book_interest"...'); 20 | var n = 4; 21 | function _tick(err, r) { 22 | n--; 23 | if (err) console.error(err); 24 | if (n <= 0) tick(); 25 | } 26 | r.ensureIndex({ 'user_id': 1, 'updated': 1, }, sparse_option, _tick); 27 | r.ensureIndex({ 'user_id': 1, 'rating.value': -1, }, sparse_option, _tick); 28 | r.ensureIndex({ 'user_id': 1, 'status': 1, }, sparse_option, _tick); 29 | r.ensureIndex({ 'status': 1, 'commented': 1 }, sparse_option, _tick); 30 | }); 31 | 32 | db.collection('book', function(err, r) { 33 | log('ensuring database "book"...'); 34 | var n = 5; 35 | function _tick(err, r) { 36 | n--; 37 | if (err) console.error(err); 38 | if (n <= 0) tick(); 39 | } 40 | 41 | r.ensureIndex({ 'raters': -1 }, sparse_option, _tick); 42 | r.ensureIndex({ 'rated': -1 }, sparse_option, _tick); 43 | r.ensureIndex({ 'pages': -1 }, sparse_option, _tick); 44 | r.ensureIndex({ 'price': -1 }, sparse_option, _tick); 45 | r.ensureIndex({ 'pubdate': -1 }, sparse_option, _tick); 46 | }); 47 | 48 | db.collection('user', function(err, r) { 49 | log('ensuring database "users"...'); 50 | var n = 5; 51 | function _tick(err, r) { 52 | n--; 53 | if (err) console.error(err); 54 | if (n <= 0) tick(); 55 | } 56 | r.ensureIndex({ 'uid': 1 }, index_options, _tick); 57 | r.ensureIndex({ 'book_synced_n': 1 }, sparse_option, _tick); 58 | r.ensureIndex({ 'last_statsed': 1 }, sparse_option, _tick); 59 | r.ensureIndex({ 'last_synced': 1 }, sparse_option, _tick); 60 | r.ensureIndex({ 'last_synced_status': 1, 'book_stats.n_done': -1 }, sparse_option, _tick); 61 | //r.ensureIndex({ 'mtime': 1 }, _tick); 62 | }); 63 | 64 | var n = 3; 65 | function tick() { 66 | n--; 67 | if (n === 0) next(null, db); 68 | } 69 | }; 70 | -------------------------------------------------------------------------------- /tools/update.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /** 4 | * Update all users in background. 5 | */ 6 | var log = require('debug')('dbj:tool:update') 7 | 8 | var User = require('../models/user') 9 | var cached = require('../lib/cached') 10 | var tasks = require('../tasks') 11 | 12 | tasks.setKeyPrefix('dbj-cron-update-') 13 | 14 | var oneday = 60 * 60 * 24 * 1000 15 | var oneweek = oneday * 7 16 | 17 | function updateAll(query) { 18 | var now = new Date() 19 | var canExit = true 20 | var blacklist = [] 21 | var counter = 0 22 | 23 | if (tasks.getQueueLength()) { 24 | log('There are unfinished task. Exit.') 25 | return 26 | } 27 | 28 | cached.get('user_blacklist', function(err, res) { 29 | if (res && res.length) { 30 | blacklist = res 31 | } 32 | }) 33 | 34 | User.stream(query, { limit: null }, function(stream) { 35 | function done(e) { 36 | if (e) { 37 | log('Sync and compute error: %s', e) 38 | } 39 | if (canExit) { 40 | log('==== Updated %s users, exit. ======', counter) 41 | process.exit() 42 | } 43 | } 44 | function resume() { 45 | if (tasks.getQueueLength() < 4) { 46 | stream.resume() 47 | } else { 48 | log('[warning] Too many tasks running, wait next..') 49 | } 50 | } 51 | stream.on('data', function(u) { 52 | if (~blacklist.indexOf(u._id)) { 53 | return resume() 54 | } 55 | canExit = false 56 | stream.pause() 57 | counter += 1 58 | u.pull(function() { 59 | resume() 60 | tasks.interest.collect_book({ 61 | user: u, 62 | force: true, 63 | fresh: false 64 | }) 65 | u.once('computed', function(e) { 66 | resume() 67 | done(e) 68 | }) 69 | }) 70 | log('Queue user %s [%s]', u.uid, u.name) 71 | }) 72 | stream.on('end', function() { 73 | log('===== Stream ended. %s users in queue. =====', counter) 74 | canExit = true 75 | if (counter === 0) { 76 | done() 77 | } 78 | }) 79 | }) 80 | } 81 | 82 | setTimeout(updateAll, 1000, { 83 | last_synced_status: { 84 | $ne: 'ing' 85 | }, 86 | book_n: { 87 | $gt: 500 88 | }, 89 | // 一周之内更新过的用户就不再更新 90 | last_synced: { 91 | $lt: new Date(new Date() - oneweek) 92 | }, 93 | }) 94 | -------------------------------------------------------------------------------- /static/css/widgets/rbox.styl: -------------------------------------------------------------------------------- 1 | .rbox-list 2 | margin 0 3 | a 4 | display: block; 5 | padding: 8px 6 | border 1px solid #e0e0e0 7 | color: #333; 8 | font-size: 12px; 9 | border-radius(3px) 10 | 11 | &:hover 12 | text-decoration: none; 13 | background: #fffdfa; 14 | border-color: #f98; 15 | .title 16 | color: #45b; 17 | font-size: 14px; 18 | margin: 0.8em 0 0; 19 | .rbox-supple 20 | border-top: 1px solid #f98; 21 | 22 | p 23 | margin: 0.5em 0 0.3em; 24 | line-height: 1.3em 25 | 26 | .pic 27 | margin: 0.6em auto; 28 | text-align: center; 29 | img 30 | border: 1px solid #eee; 31 | padding: 3px; 32 | border-radius(2px) 33 | 34 | li 35 | position: relative; 36 | vertical-align: top; 37 | display inline-block 38 | margin 10px 8px 30px 39 | 40 | .rbox-supple 41 | border: 1px solid #ddd; 42 | border-top: 0; 43 | background: #fafafa; 44 | padding: 10px; 45 | font-size: 12px; 46 | position: absolute; 47 | right: 0; 48 | display: none; 49 | width: 106px; 50 | z-index: 1; 51 | border-radius(0 0 3px 3px); 52 | box-sizing(content-box); 53 | 54 | li:hover .rbox-supple 55 | display: block; 56 | 57 | .mixed-stars 58 | text-align: center 59 | .splitter 60 | color: #999; 61 | 62 | .interest-rboxes 63 | img 64 | width: 100px; 65 | .pic, p 66 | width: 110px; 67 | 68 | .people-rboxes 69 | li 70 | text-align: center; 71 | img 72 | height: 48px; 73 | width: 48px; 74 | 75 | p.title 76 | font-size: 13px; 77 | p 78 | width: 80px; 79 | height: 1.3em; 80 | overflow: hidden; 81 | 82 | p.tags 83 | font-size: 12px; 84 | font-weight: 100; 85 | color: #b3b3b3; 86 | height: 2.4em; 87 | text-align: left; 88 | 89 | .rbox-supple 90 | width: 70px; 91 | 92 | a 93 | padding: 3px 5px 7px; 94 | .rbox-supple 95 | margin-left: -6px; 96 | margin-top: 7px; 97 | 98 | 99 | a:hover 100 | background: #fafdff; 101 | border-color: #90d0ec; 102 | .rbox-supple 103 | border-color: #90d0ec; 104 | border-top-color: #d9e3ef; 105 | background: #fff; 106 | 107 | 108 | @media (max-width: 768px) 109 | .rbox-list 110 | margin: 0 -8px; 111 | -------------------------------------------------------------------------------- /static/js/chart/treemap.js: -------------------------------------------------------------------------------- 1 | function Treemap(container, options) { 2 | if (!container) throw new Error('Must give a container for Treemap.'); 3 | if (!(this instanceof Treemap)) return new Treemap(container, options); 4 | options = lodash.defaults(options, Treemap.defaultOptions); 5 | 6 | var self = this; 7 | self.options = options; 8 | self.container = d3.select(container); 9 | self.data = options.data; 10 | 11 | d3.rebind(self, self.container, 'on'); 12 | } 13 | 14 | Treemap.defaultOptions = { 15 | colors: DEFAULT_COLORS, 16 | width: 600, 17 | height: 400, 18 | name_prop: '_id', 19 | href: null, 20 | value: function(d) { return 1; }, 21 | text: null, 22 | position: function() { 23 | this.style("left", function(d) { return d.x + "px"; }) 24 | .style("top", function(d) { return d.y + "px"; }) 25 | .style("width", function(d) { return Math.max(0, d.dx - 1) + "px"; }) 26 | .style("height", function(d) { return Math.max(0, d.dy - 1) + "px"; }); 27 | } 28 | }; 29 | 30 | Treemap.prototype.draw = function(data) { 31 | var self = this; 32 | 33 | data = data || this.data; 34 | 35 | var options = self.options 36 | , colors = options.colors 37 | , name_prop = options.name_prop 38 | , container = self.container 39 | , width = options.width 40 | , height = options.height; 41 | 42 | var color = colors ? d3_colors(colors) : d3.scale.category20c(); 43 | var text = options.text || function(d) { return d.children ? null : d[name_prop]; }; 44 | 45 | var treemap = self.treemap = d3.layout.treemap() 46 | .size([width, height]) 47 | .sticky(true) 48 | .value(options.value); 49 | 50 | var nodes = self.nodes = container.datum(data).selectAll('.node') 51 | .data(treemap.nodes) 52 | .enter().append('div') 53 | .attr('class', function(d) { return d.children ? 'node parent' : 'node'; }) 54 | .call(options.position) 55 | .style('background', function(d) { 56 | if (d.children) return color(d[name_prop]); 57 | var kid_len = d.parent.children.length; 58 | var factor = Math.pow(d.value / d.parent.value * kid_len, 0.29); 59 | return d3.hsl(color(d.parent[name_prop])).brighter(1.57).darker(factor); 60 | }) 61 | .append('a') 62 | .text(text); 63 | 64 | if (options.href) { 65 | nodes.attr('href', options.href); 66 | } 67 | 68 | return self; 69 | }; 70 | -------------------------------------------------------------------------------- /templates/people/click/mods/mixins.jade: -------------------------------------------------------------------------------- 1 | include ../../mixins/interest 2 | 3 | mixin mixed_ratings(subject_id) 4 | each u in [people, other] 5 | item = u._interest_by_subject_id[subject_id] 6 | - if (item && item.rating) 7 | .rating A: 8 | mixin stars(item.rating) 9 | - else 10 | .rating B: 未评价 11 | 12 | mixin rating_text(user, subject_id) 13 | - item = user._interest_by_subject_id[subject_id] 14 | if !item 15 | - return 16 | if item.rating 17 | | #{user._name}的评价: 18 | mixin stars(item.rating) 19 | else 20 | | #{user._name}未评价 21 | 22 | mixin mixed_rating_text(subject_id) 23 | - item1 = people._interest_by_subject_id[subject_id] 24 | - item2 = other._interest_by_subject_id[subject_id] 25 | - item = item1 && item1.rating && item1 || item2 26 | if !item 27 | - return 28 | div.mixed-ratings 29 | - if (item1 && item2 && item1.rated && item2.rated) 30 | - if (item1.rated === item2.rated) 31 | p #{people._name}和#{other._name}都给了#{item.rated}星 32 | - else 33 | - if (item1 && item1.rated) 34 | p #{people._name}给了#{item1.rated}星 35 | - if (item2 && item2.rated) 36 | p #{other._name}给了#{item2.rated}星 37 | 38 | mixin rate_stars(i) 39 | if i 40 | mixin stars(i) 41 | else 42 | small.text-muted (暂未评价) 43 | 44 | mixin mixed_rating_stars(subject_id) 45 | - item1 = people._interest_by_subject_id[subject_id] 46 | - item2 = other._interest_by_subject_id[subject_id] 47 | div.mixed-ratings.mixed-stars 48 | mixin rate_stars(item1.rated) 49 | .splitter v.s. 50 | mixin rate_stars(item2.rated) 51 | 52 | mixin book_list(ids, head, tail) 53 | if (!ids || !ids.length) 54 | p.no-result 暂时没有诶 ... 55 | - return 56 | ol.rbox-list.interest-rboxes 57 | each i in ids 58 | if !(i in all_books) 59 | - continue 60 | - subject = all_books[i] 61 | li 62 | if head 63 | .rbox-supple 64 | !{head(i)} 65 | a(href="http://book.douban.com/subject/#{subject.id}/", target="db-#{subject.type}") 66 | if subject.images 67 | .pic 68 | img(src=subject.images.medium) 69 | p.title #{subject.title} 70 | if subject.author 71 | p.meta.authors !{subject.author.join(', ')} 72 | if subject.publisher 73 | p.meta.publisher #{subject.publisher} 74 | if tail 75 | .rbox-supple 76 | !{tail(i)} 77 | -------------------------------------------------------------------------------- /models/user/click.js: -------------------------------------------------------------------------------- 1 | var redis = central.redis; 2 | var User = require('./index') 3 | 4 | // 过期时间，单位 秒 5 | var CLICK_EXPIRES = 60 * 60 * 12; 6 | 7 | if (central.conf.debug) { 8 | CLICK_EXPIRES = 30; 9 | } 10 | 11 | var grades = { 12 | 0: '话不投机', 13 | 100: '形同陌路', 14 | 300: '貌合神离', 15 | 600: '志同道合', 16 | 800: '情投意合', 17 | 1200: '情同手足', 18 | 1600: '心有灵犀', 19 | 2000: '同甘共苦', 20 | 3000: '生死与共', 21 | }; 22 | 23 | 24 | /** 25 | * progress of calculating click 26 | */ 27 | User.prototype.clickProgress = function(other, cb) { 28 | if (!(other instanceof this.constructor)) return null; 29 | this.getClick(other, function(err, r) { 30 | if (err) return cb(err); 31 | if (!r) return cb(null, 0); 32 | return cb(null, r.p); 33 | }); 34 | }; 35 | 36 | /** 37 | * return a click grading text 38 | */ 39 | User.prototype.clickGrade = function(num) { 40 | num = parseInt(num); 41 | for (var k in grades) { 42 | if (parseInt(k) < num) continue; 43 | return grades[k]; 44 | } 45 | return '爆表'; 46 | }; 47 | 48 | function swap(obj, a, b) { 49 | var tmp1 = obj[a], tmp2 = obj[b]; 50 | obj[b] = tmp1; 51 | obj[a] = tmp2; 52 | } 53 | 54 | /** 55 | * Get click for the other one 56 | */ 57 | User.prototype.getClick = function(other, cb) { 58 | var self = this; 59 | redis.client.get(self._clickKey(other), function(err, ret) { 60 | try { 61 | ret = JSON.parse(ret); 62 | } catch (e) {} 63 | //ret = null; 64 | if (!err && ret && typeof ret.ratios === 'object') { 65 | // 这种比较与计算结果时的 sort 一致 66 | if (self.id > other.id) { 67 | swap(ret, 'love_hate', 'hate_love'); 68 | swap(ret, 'done_wish', 'wish_done'); 69 | for (var k in ret.ratios) { 70 | swap(ret.ratios[k], 0, 1); 71 | } 72 | } 73 | ret.score = ret.score || ret.index || '0'; 74 | } 75 | return cb(err, ret); 76 | }); 77 | }; 78 | User.prototype.setClick = function(other, val, cb) { 79 | var self = this; 80 | var key = self._clickKey(other); 81 | redis.client.set(key, JSON.stringify(val), cb); 82 | redis.client.expire(key, val && val.expires_in || CLICK_EXPIRES); 83 | }; 84 | User.prototype.click_url = function(other, tail) { 85 | var other_uid = typeof other !== 'object' ? other : (other.uid || other.id); 86 | return this.url() + 'click/' + other_uid + (tail || ''); 87 | }; 88 | 89 | User.prototype._clickKey = function(other) { 90 | // key 只需保证唯一，先后顺序其实并不重要 91 | return 'click_' + [this.id, other.id].sort().join('_'); 92 | }; 93 | -------------------------------------------------------------------------------- /serve/api/index.js: -------------------------------------------------------------------------------- 1 | var error = require('debug')('dbj:api') 2 | var tasks = require('../../tasks') 3 | var cached = require('../../lib/cached') 4 | 5 | module.exports = function(app, central) { 6 | var utils = central.utils 7 | var cwd = central.cwd 8 | var User = require(cwd + '/models/user').User 9 | 10 | app.use('/api/', function(err, req, res, next) { 11 | if (err == 404) { 12 | res.statusCode = 404 13 | return res.json({ r: 404 }) 14 | } 15 | 16 | if (err) { 17 | error(err) 18 | if (err.stack) { 19 | console.error(err.stack) 20 | } 21 | 22 | central.raven.express(err, req, res) 23 | 24 | res.statusCode = err.statusCode || 500 25 | return res.json({ 26 | err: err.code || 500, 27 | msg: err.msg 28 | }) 29 | } 30 | next() 31 | }) 32 | app.get('/api/people/:uid/progress', function(req, res, next) { 33 | var people = res.data.people 34 | if (people) { 35 | var remaining = people.remaining() 36 | var delay = central.task.API_REQ_DELAY 37 | var ret = { 38 | r: 0, 39 | interval: people.progressInterval(remaining, delay), 40 | is_ing: people.isIng(), 41 | queue_length: tasks.getQueueLength('interest'), 42 | book_n: people.book_n, 43 | book_synced_n: people.book_synced_n, 44 | //last_synced: people.last_synced, 45 | //last_synced_status: people.last_synced_status, 46 | stats_status: people.stats_status, 47 | percents: people.progresses(), 48 | remaining: people.remaining() 49 | } 50 | //console.log(ret) 51 | res.json(ret) 52 | } else { 53 | res.statusCode = 404 54 | res.json({ 55 | r: 404 56 | }) 57 | } 58 | }) 59 | app.get('/api/people/:uid/', function(req, res, next) { 60 | var people = res.data.people 61 | if (people) { 62 | res.json(people) 63 | } else { 64 | res.statusCode = 404 65 | res.json({ 66 | r: 404 67 | }) 68 | } 69 | }) 70 | app.get('/api/latest_synced', function(req, res, next) { 71 | User.latestSynced(function(err, users) { 72 | var people = users && users.map(function(item) { 73 | return { 74 | url: item.url(), 75 | name: item.name 76 | } 77 | }) || [] 78 | 79 | people = utils.shuffle(people).slice(0,12) 80 | 81 | res.json({ 82 | r: err ? 500 : 0, 83 | people: people, 84 | }) 85 | }) 86 | }) 87 | } 88 | -------------------------------------------------------------------------------- /lib/assets.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Static Assets files management 3 | */ 4 | var fs = require('fs'); 5 | var path = require('path'); 6 | var crypto = require('crypto'); 7 | var istatic = require('istatic'); 8 | var cwd = process.cwd(); 9 | var conf = require(cwd + '/conf'); 10 | 11 | var root = cwd + '/static/dist'; 12 | 13 | var reg_log = /_log\(.*?\);/g; 14 | istatic.default({ 15 | root: root, 16 | debug: conf.debug, 17 | compress: !conf.debug, 18 | ttl: conf.debug ? 0 : 60 * 60 * 24 * 7, 19 | fresh: conf.debug, 20 | js: { 21 | filter: function(str) { 22 | return str.replace(reg_log, ''); 23 | } 24 | } 25 | }); 26 | 27 | var hash_cache = {}; 28 | try { 29 | hash_cache = require(cwd + '/static/hash.json'); 30 | } catch (e) { 31 | if (e.code == 'MODULE_NOT_FOUND' || e.code == 'ENOENT') { 32 | console.error('No existing static hashmap.'); 33 | } else { 34 | throw e; 35 | } 36 | } 37 | 38 | var reg_css_js = /\.(css|js)$/; 39 | 40 | function fileUrl(p) { 41 | if (p[0] == '/') p = p.slice(1); 42 | if (conf.debug || !reg_css_js.test(p)) return conf.assets_root + '/' + p; 43 | 44 | var hash = hash_cache[p]; 45 | if (hash) { 46 | var ext = path.extname(p); 47 | p = path.join(path.dirname(p), path.basename(p, ext) + '_' + hash + ext); 48 | } 49 | return conf.assets_root + '/' + p; 50 | } 51 | 52 | function urlMap() { 53 | var files = [].slice.call(arguments); 54 | var ret = {}; 55 | files.forEach(function(f) { 56 | if (!path.extname(f)) f = path.join('js', f + '.js'); 57 | ret[conf.assets_root + '/' + f] = fileUrl(f); 58 | }); 59 | return ret; 60 | } 61 | 62 | function readFile(filepath, encoding) { 63 | var contents = ''; 64 | try { 65 | contents = fs.readFileSync(String(filepath), encoding); 66 | } catch (e) {} 67 | return contents; 68 | } 69 | function md5(contents) { 70 | var shasum = crypto.createHash('md5'); 71 | shasum.update(contents); 72 | return shasum.digest('hex').slice(0, 10); 73 | } 74 | 75 | // pass in some static files, get their hash value 76 | function hashMap() { 77 | var files = [].slice.call(arguments); 78 | var ret = {}; 79 | files.forEach(function(f) { 80 | ret[f] = getHash(f.indexOf('.') == -1 ? 'js/' + f + '.js' : f); 81 | }); 82 | return ret; 83 | } 84 | function getHash(f, fresh, r, encoding){ 85 | r = typeof r !== 'string' ? root : r; 86 | return fresh ? (hash_cache[f] = md5(readFile(path.join(r, f), encoding))) : hash_cache[f]; 87 | } 88 | 89 | module.exports = { 90 | root: root, 91 | istatic: istatic, 92 | hashMap: hashMap, 93 | urlMap: urlMap, 94 | getHash: getHash, 95 | fileUrl: fileUrl 96 | }; 97 | -------------------------------------------------------------------------------- /static/css/base/layout.styl: -------------------------------------------------------------------------------- 1 | .vmiddle 2 | margin: 10% auto 3 | padding: 20px 4 | max-width: 600px 5 | text-align: center 6 | 7 | pre 8 | text-align: left 9 | 10 | h1, h2, h3, h4 11 | font-family: sans 12 | margin-bottom: 0.5em 13 | line-height: 1.5 14 | 15 | h1 16 | font-size: h1_size 17 | text-align: center 18 | 19 | h2 20 | font-size: h2_size 21 | .btn 22 | margin: 0 0.5em 23 | vertical-align: text-top 24 | h3 25 | font-size: h3_size 26 | 27 | h4 28 | font-size: h4_size 29 | 30 | #footer 31 | border-top: 1px solid #e0e0e0 32 | padding: 20px 33 | text-align: right 34 | font-size: 0.8em 35 | a 36 | color: #666 37 | margin-left: 10px 38 | &:hover 39 | color: #333 40 | 41 | #main 42 | position: relative 43 | padding: 30px 0 44 | padding-bottom: 20px 45 | 46 | h1 47 | margin-top: 0 48 | 49 | .desc 50 | line-height: 1.4em 51 | font-size: desc_size 52 | word-wrap: break-word 53 | 54 | .people-intro 55 | .pull-right 56 | display: inline-block 57 | margin: 0 0 0.5em 0.5em 58 | 59 | .alert 60 | p 61 | margin: 0.5em 0 62 | .progress 63 | margin-bottom: 0 64 | 65 | .label-compact 66 | font-weight: 200 67 | padding: 0 0.2em 68 | line-height: 1.4em 69 | 70 | .mod 71 | margin-bottom: 2em 72 | 73 | .tagcloud 74 | margin-bottom: 1em 75 | 76 | .tag-item 77 | display: inline-block 78 | margin-right: 0.7em 79 | a 80 | margin-right: 0 0.1em 81 | .muted 82 | font-size: 0.9em 83 | 84 | .list-small 85 | font-size: 12px 86 | margin-left: 1.5em 87 | 88 | .breadcrumb 89 | margin: 20px 0 90 | 91 | a.anchor 92 | display: inline-block 93 | zoom: 1 94 | vertical-align: middle 95 | color: #aaa 96 | margin-left: 10px 97 | font-size: 16px 98 | line-height: 1em 99 | padding: 2px 5px 100 | font-family: arial 101 | 102 | &:hover 103 | text-decoration: none 104 | background: #aa 105 | color: #fff 106 | 107 | .people-info 108 | img 109 | width: 48px 110 | height: 48px 111 | 112 | .notice-more 113 | margin-top: 8em 114 | 115 | .pager-center 116 | text-align: center 117 | margin: 20px 0 118 | .pagination 119 | margin: 0 10px 120 | vertical-align: middle 121 | 122 | @media (max-width: 768px) 123 | .navbar:before, .navbar:after 124 | content: "" 125 | display: none 126 | .row 127 | margin: 0 128 | .mod, .chart 129 | padding: 0 130 | @media (min-width: 440px) and (max-width: 768px) 131 | #tops .span3 132 | float: left 133 | width: 50% 134 | -------------------------------------------------------------------------------- /serve/mine/index.js: -------------------------------------------------------------------------------- 1 | module.exports = function(app, central) { 2 | 3 | var utils = require('../utils'); 4 | var auth_utils = require('../auth/utils'); 5 | 6 | var KEY_CLICK_BOOK_SCORE = require('../../models/consts').KEY_CLICK_BOOK_SCORE; 7 | 8 | var lodash = require('lodash'); 9 | var cwd = central.cwd; 10 | var User = require(cwd + '/models/user').User; 11 | var Interest = require(cwd + '/models/interest'); 12 | var tasks = require(cwd + '/tasks'); 13 | 14 | function get_click_book_scores(req, res, next) { 15 | var c = res.data = res.data || {}; 16 | var user = c.user = req.user; 17 | 18 | user.data(KEY_CLICK_BOOK_SCORE, function(err, ret) { 19 | ret = ret || {}; 20 | c.click_book_scores = ret; 21 | next(); 22 | }); 23 | } 24 | 25 | app.get('/mine', auth_utils.require_login, 26 | function(req, res, next) { 27 | var user = req.user; 28 | if (!user.stats) { 29 | res.redirect(user.url()); 30 | } 31 | next(); 32 | }, 33 | get_click_book_scores, 34 | function(req, res, next) { 35 | var c = res.data; 36 | var book_scores = c.click_book_scores; 37 | 38 | var user_ids = Object.keys(book_scores).sort(function(a, b) { return book_scores[b] - book_scores[a] }); 39 | user_ids = user_ids.slice(0, 4); 40 | 41 | User.gets(user_ids, function(err, users) { 42 | 43 | c.top_click_users = users.filter(function(u) { 44 | if (!u) return false; 45 | 46 | u.book_score = book_scores[u.id]; 47 | 48 | return true; 49 | }); 50 | 51 | res.render('mine', c); 52 | }); 53 | }); 54 | 55 | var rj = auth_utils.require_login_json; 56 | 57 | app.get('/api/mine/followings', rj, function(req, res, next) { 58 | if (!req.user) { 59 | res.statusCode = 401; 60 | res.json({ r: 401 }); 61 | return; 62 | } 63 | req.query = req.query || {}; 64 | 65 | if (req.query.fresh) { 66 | req.user.clearFollowings(next); 67 | return; 68 | } 69 | return next(); 70 | }, 71 | get_click_book_scores, 72 | function(req, res, next) { 73 | req.user.listFollowings({ 74 | limit: req.query.limit || 24, 75 | start: req.query.start 76 | }, function(err, result) { 77 | if (err) { 78 | res.statusCode = err.code || 200; 79 | return res.json({ r: err.code || 500, msg: err.message }); 80 | } 81 | 82 | var all_scores = res.data.click_book_scores; 83 | 84 | result.forEach(function(item) { 85 | item.ready = (item.stats_p == 100); 86 | item.ing = item.last_synced_status === 'ing'; 87 | item.score = all_scores[item.id]; 88 | }); 89 | 90 | res.json({ 91 | r: 0, 92 | items: result 93 | }); 94 | }); 95 | }); 96 | 97 | }; 98 | -------------------------------------------------------------------------------- /templates/people/mixins/interest.jade: -------------------------------------------------------------------------------- 1 | mixin tagcloud(tags, ns) 2 | ns = ns || locals.ns || 'book' 3 | .tagcloud 4 | each tag in tags 5 | if tag && tag._id 6 | a.tag-item(href='/tag/' + encodeURIComponent(tag._id), target="_blank") 7 | strong #{tag._id} 8 | | 9 | span.text-muted (#{tag.count}#{tag.count_b ? ':' + tag.count_b : ''}) 10 | 11 | - ns_cats = { 'book': '1001' } 12 | mixin searchcloud(tags, ns) 13 | ns = ns || locals.ns 14 | cat = ns_cats[ns] 15 | .tagcloud 16 | each tag in tags 17 | if tag && tag._id 18 | span.tag-item 19 | a(href="http://" + ns + '.douban.com/subject_search?cat=' + cat + '&search_text=' + encodeURIComponent(tag._id), 20 | target="_blank") #{tag._id} 21 | span.text-muted (#{tag.count}) 22 | 23 | - make_stars = function(star) { if (!star || !star.value) return ''; var n = star.max || 5, s = ''; star = star.value; while (n > 0) { star--; n--; s += star >= 0 ? '✭' : '✩' } return s; } 24 | 25 | mixin stars(star, only_text) 26 | if star && star.value 27 | - star = star.value 28 | if !star 29 | - return 30 | - var n = star.max || 5, s = ''; 31 | while n > 0 32 | - star-- 33 | - n -- 34 | - s += star >= 0 ? '✭' : '✩' 35 | if only_text 36 | !{s} 37 | - return 38 | span.stars !{s} 39 | 40 | mixin interest_item(i, trimto) 41 | if !i 42 | - return 43 | trimto = trimto || 14 44 | span.label.label-default.label-status #{i.status_cn()} 45 | 46 | if i.subject 47 | a(href="http://#{i.subject_ns()}.douban.com/subject/#{i.subject_id}/", 48 | target="_blank", 49 | title="#{i.subject.title}#{i.subject.author ? ' - ' + i.subject.author.join(', ') : ''}") 50 | | #{trunc(i.subject.title, trimto)} 51 | else 52 | a(href="http://#{i.subject_ns()}.douban.com/subject/#{i.subject_id}/", target="_blank") 53 | | #{i.subject_id} 54 | 55 | 56 | mixin interests_list(items) 57 | ol.rbox-list.interest-rboxes 58 | each i in items 59 | li 60 | a(href="http://#{i.subject_ns()}.douban.com/subject/#{i.subject_id}/", target="db-#{i.subject_type}") 61 | div.status #{i.status_cn()} 62 | mixin stars(i.rating) 63 | if i.subject 64 | if i.subject.images 65 | .pic 66 | img(src=i.subject.images.medium) 67 | p.title #{i.subject.title} 68 | if i.subject.author 69 | p.meta.authors !{i.subject.author.join(', ')} 70 | if i.subject.publisher 71 | p.meta.publisher #{i.subject.publisher} 72 | else 73 | | #{i.subject_type}:#{i.subject_id} 74 | -------------------------------------------------------------------------------- /lib/raven.js: -------------------------------------------------------------------------------- 1 | /** 2 | * To send messages to Sentry. 3 | */ 4 | var conf = require('../conf'); 5 | 6 | var debug = require('debug'); 7 | var verbose = debug('dbj:raven:verbose'); 8 | var message = _message = debug('dbj:raven:message'); 9 | var error = _error = debug('dbj:raven:error'); 10 | 11 | var client; 12 | 13 | if (conf.raven) { 14 | var raven = require('raven'); 15 | client = new raven.Client(conf.raven, { 16 | logger: process.env.USER || 'general', 17 | site: conf.site_name, 18 | }); 19 | 20 | message = function(msg) { 21 | var args = [].slice.apply(arguments); 22 | 23 | _message.apply(debug, args); 24 | 25 | if (!client._enabled) return; 26 | 27 | setImmediate(function() { 28 | args.shift(); 29 | 30 | if (msg.indexOf('%s') !== -1) { 31 | msg = msg.replace(/\%s/g, function() { 32 | return args.shift(); 33 | }); 34 | } 35 | 36 | var extra = tags = null; 37 | if (args.length) { 38 | extra = args[args.length - 1]; 39 | args.pop(); 40 | } 41 | if (extra && typeof extra !== 'object') { 42 | extra = { extra: extra }; 43 | } 44 | if (extra) { 45 | tags = extra.tags; 46 | delete extra.tags; 47 | } 48 | var level = 'info'; 49 | if (extra && 'level' in extra) { 50 | tags = tags || {}; 51 | level = extra.level || level; 52 | tags.level = level; 53 | delete extra.level; 54 | } 55 | 56 | //console.log(level, tags, extra); 57 | client.captureMessage(msg, { 58 | level: level, 59 | tags: tags, 60 | extra: extra, 61 | }); 62 | }); 63 | }; 64 | error = function(err) { 65 | var args = [].slice.apply(arguments); 66 | 67 | if (!client._enabled) return; 68 | 69 | if (typeof err === 'number') err = String(err); 70 | if (typeof err === 'string') { 71 | if (err.indexOf('%s') !== -1) { 72 | err = err.replace(/\%s/g, function() { 73 | return args.shift(); 74 | }); 75 | } 76 | err = new Error(err); 77 | } 78 | args[0] = err; 79 | 80 | setImmediate(function() { 81 | client.captureError.apply(client, args); 82 | }); 83 | }; 84 | 85 | client.on('logged', function(){ 86 | _message('Sent raven message.'); 87 | }); 88 | client.on('error', function(e){ 89 | _error('sentry is broken: %s', e); 90 | }) 91 | } else { 92 | // just keep silent 93 | message = error = function() {} 94 | } 95 | 96 | module.exports = { 97 | client: client, 98 | message: message, 99 | error: error, 100 | express: function () {} 101 | }; 102 | -------------------------------------------------------------------------------- /static/js/people/crossfilter.js: -------------------------------------------------------------------------------- 1 | // ========== for crossfilter detailed view ======= 2 | function get_detailed(bar, list) { 3 | var crossfilter = require('crossfilter').crossfilter; 4 | 5 | function dataList(div) { 6 | var data = bar.parsed_data; 7 | } 8 | 9 | list = d3.selectAll(list).data([dataList]); 10 | 11 | var brush = d3.svg.brush(), 12 | brushDirty, 13 | dimension, 14 | group, 15 | round; 16 | 17 | var g = bar.svg.append('g.brush-layer'); 18 | 19 | // Initialize the brush component with pretty resize handles. 20 | var gBrush = g.append("g").attr("class", "brush").call(brush); 21 | 22 | gBrush.selectAll("rect").attr("height", height); 23 | gBrush.selectAll(".resize").append("path").attr("d", resizePath); 24 | 25 | // Only redraw the brush if set externally. 26 | if (brushDirty) { 27 | brushDirty = false; 28 | g.selectAll(".brush").call(brush); 29 | div.select(".title a").style("display", brush.empty() ? "none" : null); 30 | if (brush.empty()) { 31 | g.selectAll("#clip-" + id + " rect") 32 | .attr("x", 0) 33 | .attr("width", width); 34 | } else { 35 | var extent = brush.extent(); 36 | g.selectAll("#clip-" + id + " rect") 37 | .attr("x", x(extent[0])) 38 | .attr("width", x(extent[1]) - x(extent[0])); 39 | } 40 | } 41 | 42 | function barPath(groups) { 43 | var path = [], 44 | i = -1, 45 | n = groups.length, 46 | d; 47 | while (++i < n) { 48 | d = groups[i]; 49 | path.push("M", x(d.key), ",", height, "V", y(d.value), "h9V", height); 50 | } 51 | return path.join(""); 52 | } 53 | 54 | function resizePath(d) { 55 | var e = +(d == "e"), 56 | x = e ? 1 : -1, 57 | y = height / 3; 58 | return "M" + (.5 * x) + "," + y 59 | + "A6,6 0 0 " + e + " " + (6.5 * x) + "," + (y + 6) 60 | + "V" + (2 * y - 6) 61 | + "A6,6 0 0 " + e + " " + (.5 * x) + "," + (2 * y) 62 | + "Z" 63 | + "M" + (2.5 * x) + "," + (y + 8) 64 | + "V" + (2 * y - 8) 65 | + "M" + (4.5 * x) + "," + (y + 8) 66 | + "V" + (2 * y - 8); 67 | } 68 | 69 | 70 | brush.on("brushstart.chart", function() { 71 | var div = d3.select(this.parentNode.parentNode.parentNode); 72 | div.select(".title a").style("display", null); 73 | }); 74 | 75 | } 76 | 77 | function detail_expand() { 78 | var trigger = $(this); 79 | var elem = this; 80 | trigger.siblings('.details-content').show(300); 81 | 82 | if (elem.detailed) return elem.detailed.show(); 83 | 84 | Do('crossfilter', function() { 85 | var list = trigger.siblings('.details-content')[0]; 86 | var bar = $(elem.href.split(':')[1])[0]._bar; 87 | elem.detailed = get_detailed(bar, list); 88 | }); 89 | } 90 | 91 | function detail_collapse(e) { 92 | $(this).siblings('.details-content').hide(400); 93 | } 94 | 95 | $('.details-toggler').toggle(detail_expand, detail_collapse); 96 | 97 | 98 | -------------------------------------------------------------------------------- /models/interest/base.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Interest Base Class 3 | */ 4 | var debug = require('debug'); 5 | var util = require('util'); 6 | var log = debug('dbj:interest:info'); 7 | var verbose = debug('dbj:interest:verbose'); 8 | var error = debug('dbj:interest:error'); 9 | 10 | var conf = central.conf; 11 | var mongo = central.mongo; 12 | //var redis = central.redis; 13 | var utils = central.utils; 14 | 15 | var Subject = require('../subject'); 16 | 17 | var consts = require('../consts'); 18 | var DOUBAN_APPS = consts.DOUBAN_APPS; 19 | var INTEREST_STATUS_LABELS = consts.INTEREST_STATUS_LABELS; 20 | var APP_DATA_DEFAULT = { 21 | n_interest: 0 22 | }; 23 | 24 | function Interest(info) { 25 | if (!(this instanceof Interest)) return new Interest(info); 26 | 27 | var self = this; 28 | utils.extend(self, info); 29 | //for (var i in info) { 30 | //this[i] = info[i]; 31 | //} 32 | 33 | return this; 34 | } 35 | 36 | /** 37 | * Inherited functions: 38 | */ 39 | util.inherits(Interest, mongo.Model); 40 | utils.extend(Interest, mongo.Model); 41 | 42 | Interest.prototype.kind = Interest._collection = 'interest'; 43 | Interest.prototype.subject_type = 'general'; 44 | 45 | Interest._cache_ttl = 36000; 46 | Interest._default_sort = { 'updated': -1 }; 47 | //Interest.get = redis.cached.wrap(mongo.Model.get); 48 | 49 | Interest.findByUser = function(uid, opts, cb) { 50 | verbose('getting interests for [%s]: ', uid, opts); 51 | 52 | var query = { user_id: uid }; 53 | 54 | if (opts.query) { 55 | utils.extend(query, opts.query); 56 | delete opts.query; 57 | } 58 | 59 | return this.find(query, opts, cb); 60 | }; 61 | 62 | /** 63 | * Attach book subject 64 | */ 65 | Interest.book_attached = Interest.attached('subject_id', 'subject', Subject.book); 66 | //Interest.movie_attached = Interest.attached('subject_id', Subject.movie); 67 | 68 | Interest.prototype.toObject = function() { 69 | var self = this; 70 | var now = new Date(); 71 | var ret = { 72 | '_id': self['_id'], 73 | 'id': self['id'], 74 | 'uid': self['uid'] || self['user_id'], 75 | 'user_id': self['user_id'], 76 | 'updated': self['updated'], 77 | 'privacy': self['privacy'], 78 | 'subject_type': self['subject_type'], 79 | 'atime': now 80 | }; 81 | var s_key = ret['subject_type'] + '_id'; 82 | ret[s_key] = self[s_key]; 83 | return ret; 84 | }; 85 | Interest.prototype.subject_ns = function() { 86 | return this.subject_type; 87 | }; 88 | Interest.prototype.status_cn = function(unknown) { 89 | unknown = '' || unknown; 90 | return INTEREST_STATUS_LABELS[this.subject_type][this.status] || unknown; 91 | }; 92 | 93 | Object.defineProperty(Interest.prototype, 'rated', { 94 | get: function() { 95 | return this.rating && this.rating.value; 96 | }, 97 | enumerable: false 98 | }); 99 | 100 | module.exports = Interest; 101 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | //if (!module.parent) process.on('uncaughtException', function(err, next) { 2 | //var msg; 3 | //if (err instanceof Error) { 4 | //msg = '[err]: ' + err + '\n' + err.stack; 5 | //} else { 6 | //msg = (err.name || err.reason || err.message); 7 | //console.error(err); 8 | //} 9 | //console.error(msg); 10 | //next && next(); 11 | //}); 12 | 13 | var URL = require('url'); 14 | var express = require('express'); 15 | 16 | var central = require('./lib/central'); 17 | var passport = require('./lib/passport'); 18 | var ip2geo = require('./lib/ip2geo'); 19 | var serve = require('./serve'); 20 | var jade = require('jade'); 21 | var Redis = require('redis'); 22 | var session = require('express-session'); 23 | var RedisStore = require('connect-redis')(session); 24 | var methodOverride = require('method-override'); 25 | var serveStatic = require('serve-static'); 26 | var cookieParser = require('cookie-parser'); 27 | var bodyParser = require('body-parser'); 28 | var csrf = require('csurf'); 29 | 30 | var TWO_WEEKS = 60 * 60 * 24 * 14; 31 | 32 | // initial bootstraping, only serve the API 33 | module.exports.boot = function() { 34 | var app = express(); 35 | var conf = central.conf; 36 | app.enable('trust proxy') 37 | 38 | app.engine('jade', jade.renderFile); 39 | 40 | app.set('view engine', 'jade'); 41 | app.set('view cache', !central.conf.debug); 42 | app.set('views', __dirname + '/templates'); 43 | 44 | app.use(serveStatic(central.assets.root, { maxAge: TWO_WEEKS })); 45 | 46 | app.use(methodOverride()); 47 | app.use(cookieParser()); 48 | 49 | // parse application/x-www-form-urlencoded 50 | app.use(bodyParser.urlencoded({ extended: false })) 51 | // parse application/json 52 | app.use(bodyParser.json()) 53 | 54 | app.use(session({ 55 | secret: conf.salt, 56 | store: new RedisStore({ 57 | client: Redis.createClient(conf.redis) 58 | }) 59 | })); 60 | app.use(ip2geo.express()); 61 | app.use(passport.initialize()); 62 | app.use(passport.session()); 63 | 64 | app.locals = { 65 | ...app.locals, 66 | ...central.template_helpers 67 | } 68 | 69 | app.use(csrf()); 70 | 71 | app.use(function(req, res, next) { 72 | req.is_ssl = req.headers['x-forwarded-proto'] === 'https'; 73 | res.locals.req = req; 74 | res.locals._csrf = req.csrfToken(); 75 | res.locals.static = function(url) { 76 | if (req.is_ssl) return URL.resolve(central.conf.https_root, url); 77 | return central.assets.fileUrl(url); 78 | }; 79 | res.locals.strftime = central.template_helpers.strftime.bind(res.locals); 80 | next(); 81 | }); 82 | 83 | serve(app, central); 84 | 85 | app.listen(central.conf.port); 86 | central.log('Now listening on ' + central.conf.port); 87 | central.log('Site root: ' + central.conf.site_root); 88 | }; 89 | 90 | if (!module.parent) { 91 | //setTimeout(function() { 92 | module.exports.boot(); 93 | //}, 20000); 94 | } 95 | -------------------------------------------------------------------------------- /models/toplist/book.js: -------------------------------------------------------------------------------- 1 | var central = require('../../lib/central') 2 | var _ = require('../../lib/utils') 3 | var cached = require('../../lib/cached') 4 | var getToplistKey = require('../../models/toplist').getToplistKey 5 | var mongo = central.mongo 6 | var verbose = require('debug')('dbj:toplist:verbose') 7 | 8 | var User = require('../user') 9 | 10 | var people_fields = { _id: 1, uid: 1, name: 1, avatar: 1, 'book_stats.all.top_tags': 1, signature: 1 } 11 | 12 | var banned_tags = [ 13 | '耽美', 'BL', '青春文学', '耽美小说', 14 | 'BL漫画', '腐', 15 | '日本文学', '日本', 16 | '推理', '推理小说', '日系推理', '日本推理', 17 | '写真', '写真集', '摄影', 'PHOTOBOOK', '寫真', 18 | '绘本', '童话', '童书', '儿童文学', '图画书', 19 | '轻小说', '輕小說', '漫画与轻小说', 20 | '少年向', '乙女向', 21 | '热血', '恐怖', '穿越', '武侠', '奇幻', '九州', 22 | '经典', '名著', '师太', 23 | '少年漫画', '日漫', '港漫', '国漫', '青年漫画', 24 | '漫画', '日本漫画', '漫畫', 25 | '晋江', '现代都市', '网络小说', 26 | '青春', '言情', '小说', '爱情', '悬疑' 27 | ] 28 | function is_serious_reading(tags) { 29 | var counter = 0 30 | for (var i in tags) { 31 | var item = tags[i] 32 | if (banned_tags.indexOf(item._id) !== -1) { 33 | counter++ 34 | if (counter > 5) return false 35 | } 36 | } 37 | return true 38 | } 39 | 40 | var ONE_HOUR = 3600 41 | 42 | function get_hardest_reader(period, cb) { 43 | var key = getToplistKey('book', period) 44 | var cached_items_key = key + '_cached' 45 | 46 | cached.get(cached_items_key, function(err, items) { 47 | if (items) { 48 | return cb(err, items) 49 | } 50 | // get blacklist (currently, can only set it via redis-client) 51 | cached.get('user_blacklist', function(err, blacklist) { 52 | verbose('Got blacklist: %j', blacklist) 53 | // get ids 54 | cached.get(key, function(err, ids) { 55 | if (ids && ids.length && blacklist && blacklist.length) { 56 | ids = ids.filter(function(item) { 57 | if (~blacklist.indexOf(item._id)) { 58 | return false 59 | } 60 | return true 61 | }) 62 | } 63 | if (err || !ids || !ids.length) { 64 | return cb(err, []) 65 | } 66 | getReal(ids) 67 | }) 68 | }) 69 | }) 70 | 71 | function getReal(ids) { 72 | User.gets(ids, { 73 | fields: people_fields 74 | }, function(err, users) { 75 | if (err || !users) { 76 | return cb(err, []) 77 | } 78 | users = users.filter(function(item, i) { 79 | if (item) { 80 | try { 81 | // there are useless type of books in he/she's collection 82 | if (is_serious_reading(item.book_stats.all.top_tags.slice(0,12))) { 83 | item.book_quote_n = ids[i].value 84 | return true 85 | } 86 | } catch (e) {} 87 | } 88 | return false 89 | }) 90 | users = users.slice(0, 99) 91 | cb(err, users) 92 | cached.set(cached_items_key, users, function(){}) 93 | }) 94 | } 95 | 96 | } 97 | 98 | exports.hardest_reader = get_hardest_reader 99 | -------------------------------------------------------------------------------- /lib/mongo/pool.js: -------------------------------------------------------------------------------- 1 | // a mongodb model, like mongoose 2 | var debug = require('debug'); 3 | var log = debug('dbj:mongo:info'); 4 | var verbose = debug('dbj:mongo:verbose'); 5 | var error = debug('dbj:mongo:error'); 6 | 7 | var gpool = require('generic-pool'); 8 | var mongo = require('mongodb'); 9 | 10 | var cwd = process.cwd(); 11 | var conf = require(cwd + '/conf'); 12 | var utils = require(cwd + '/lib/utils'); 13 | 14 | var noop = utils.noop; 15 | 16 | var mongo_pool = gpool.createPool({ 17 | name: 'mongo', 18 | create: connect_db, 19 | destroy: function(db) { db.close(); }, 20 | max: 4, 21 | //min: 1, // always keep at least one idle connection 22 | priorityRange: 6, 23 | log: false, 24 | //log: verbose 25 | }); 26 | 27 | var db; 28 | function add_task(fn) { 29 | if (db && !db.closed) return fn(db, noop); 30 | connect_db(function(err, client) { 31 | db = add_task.db = client; 32 | fn(db, noop); 33 | }); 34 | } 35 | 36 | process.on('exit', function() { 37 | db && db.close(); 38 | }); 39 | 40 | /** 41 | * To connect the mongodb databse 42 | */ 43 | function connect_db(callback, throw_err) { 44 | var server, db; 45 | 46 | function parse_server(uri) { 47 | var tmp = uri.split(':'); 48 | return new mongo.Server(tmp[0], tmp[1] || mongo.Connection.DEFAULT_PORT); 49 | } 50 | 51 | if (conf.mongo.url) { 52 | mongo.MongoClient.connect(conf.mongo.url, callback); 53 | } else { 54 | if (conf.mongo.servers.length === 1) { 55 | server = parse_server(conf.mongo.servers[0]); 56 | } else { 57 | var ss = []; 58 | conf.mongo.servers.forEach(function(item) { 59 | ss.push(parse_server(item)); 60 | }); 61 | server = new mongo.ReplSetServers(ss); 62 | } 63 | db = new mongo.Db(conf.mongo.dbname, server, { w: 1 }); 64 | db.open(function(err, mongoClient) { 65 | if (err !== null) { 66 | error('db open failed: %s', err); 67 | console.trace(err); 68 | if (throw_err) throw err; 69 | //return callback(err, db); 70 | } 71 | 72 | if (conf.mongo.username) { 73 | db.authenticate(conf.mongo.username, conf.mongo.password, function(err, result) { 74 | verbose('mongodb pool connected database'); 75 | callback(err, mongoClient); 76 | }); 77 | } else { 78 | verbose('mongodb pool connected database'); 79 | callback(err, mongoClient); 80 | } 81 | }); 82 | db.on('close', function() { 83 | db.closed = true; 84 | }); 85 | } 86 | } 87 | // set up a database connection at start, 88 | // throw err if connection failed 89 | connect_db(function(err, _db) { 90 | db = _db; 91 | // prepare database, ensure indexes 92 | require(cwd + '/database')(_db, function(err) { 93 | if (err) error('Database preparation failed...', err); 94 | log('database is ready.'); 95 | }); 96 | }, true); 97 | 98 | module.exports = mongo_pool; 99 | module.exports.instant = add_task; 100 | -------------------------------------------------------------------------------- /lib/task.js: -------------------------------------------------------------------------------- 1 | /* 2 | * task pools 3 | */ 4 | var debug = require('debug'); 5 | var log = debug('dbj:pool:info'); 6 | var error = debug('dbj:pool:error'); 7 | 8 | var gpool = require('generic-pool'); 9 | const decorator = require('generic-pool-decorator') 10 | var douban = require('./douban'); 11 | 12 | var conf = require(process.cwd() + '/conf'); 13 | 14 | var mores = conf.douban_more || [ conf.douban ]; 15 | 16 | var i_tick = 0; 17 | var n_mores = mores.length; 18 | var n_main = 2; 19 | 20 | function oauth2_item(item) { 21 | var is_specified = true; 22 | if (!item) { 23 | item = mores[i_tick]; 24 | i_tick++; 25 | if (i_tick >= n_mores) i_tick = 0; 26 | is_specified = false; 27 | } 28 | var ret = new douban.OAuth2(item.key, item.secret); 29 | // default to maximium 10 request per limit 30 | var req_delay = 60000 / (item.limit || 10); 31 | if (is_specified) { 32 | req_delay = req_delay * n_main; 33 | } else { 34 | req_delay = req_delay / n_mores; 35 | } 36 | log('OAuth2 client request delay: %ss', req_delay / 1000) 37 | ret.req_delay = req_delay; 38 | return ret; 39 | } 40 | 41 | // http(s) request pool, mainly for douban api 42 | var api_pool = gpool.createPool({ 43 | name: 'api', 44 | create: function(callback) { 45 | var oauth2 = oauth2_item(conf.douban); 46 | callback(null, oauth2); 47 | }, 48 | destroy: function() { }, 49 | // 主请求池允许开多个 client ，防止单个请求挂起时影响后续所有请求 50 | max: n_main, 51 | //min: 1, 52 | priorityRange: 6, 53 | //log: conf.debug ? log : false 54 | }); 55 | 56 | var api_pool2 = gpool.createPool({ 57 | name: 'api', 58 | create: function(callback) { 59 | var oauth2 = oauth2_item(); 60 | callback(null, oauth2); 61 | }, 62 | destroy: function() { }, 63 | max: n_mores, 64 | //min: 1, 65 | priorityRange: 6, 66 | //log: conf.debug ? log : false 67 | }); 68 | 69 | var computings = { n: 0 }; 70 | var compute_pool = gpool.createPool({ 71 | name: 'compute', 72 | create: function(callback) { 73 | computings.n++; 74 | callback(null, computings); 75 | }, 76 | destroy: function() { computings.n--; }, 77 | max: 3, 78 | priorityRange: 6, 79 | }); 80 | compute_pool.pooled = decorator(compute_pool); 81 | 82 | function queue(pool, default_priority) { 83 | return function(fn, priority) { 84 | pool.acquire(function(err, client) { 85 | // `fn` defination is like `fn(db, next)`; 86 | if (fn.length === 2) { 87 | //log('async calling job'); 88 | fn(client, function(err) { 89 | if (err) error('async job:\n%s\nfailed because:\n%s', job.toString(), err); 90 | pool.release(client); 91 | }); 92 | } else { 93 | fn(client); 94 | pool.release(client); 95 | } 96 | }, typeof priority === 'undefined' ? default_priority : priority); 97 | } 98 | } 99 | 100 | module.exports = { 101 | api_pool: api_pool, 102 | api_pool2: api_pool2, 103 | compute_pool: compute_pool, 104 | compute: queue(compute_pool), 105 | api: queue(api_pool, 3), // default priority is 3 106 | api2: queue(api_pool2, 3), 107 | API_REQ_DELAY: 60000 / (conf.douban.limit || 10), 108 | API_REQ_PERPAGE: 100, 109 | queue: queue 110 | }; 111 | -------------------------------------------------------------------------------- /static/js/chart/pie.js: -------------------------------------------------------------------------------- 1 | // @import ./consts.js 2 | 3 | function Pie(container, options) { 4 | if (!container) throw new Error('Must give a container for Pie chart.'); 5 | if (!(this instanceof Pie)) return new Pie(container, options); 6 | options = options || {}; 7 | lodash.defaults(options, Pie.defaultOptions); 8 | this.options = options; 9 | var container = this.container = d3.select(container); 10 | this.data = options.data || container.attr('data-pie'); 11 | } 12 | 13 | Pie.defaultOptions = { 14 | colors: DEFAULT_COLORS, 15 | width: 150, 16 | height: 150, 17 | data: null, 18 | sort: null, 19 | val: function(d) { 20 | return d.value; 21 | }, 22 | textValue: function(d, i) { 23 | return d.data.value + '\n' + d.data.label; 24 | }, 25 | // offset from center point 26 | offsetUnit: 'em', 27 | textOffset: 'auto', 28 | // total offset 29 | textX: 0, 30 | textY: 0, 31 | textSize: "0.85em" 32 | }; 33 | 34 | Pie.prototype.draw = function() { 35 | var self = this; 36 | var options = self.options; 37 | var container = self.container; 38 | var width = options.width, height = options.height; 39 | var radius = options.radius || Math.min(width, height) / 2 - 10; 40 | var radiusInner = options.radiusInner || 0; 41 | 42 | var arc = self.arc = d3.svg.arc().outerRadius(radius).innerRadius(radiusInner); 43 | var pie = self.pie = d3.layout.pie().sort(options.sort).value(options.val); 44 | var color = self.color = d3_colors(options.colors); 45 | 46 | var svg = self.svg = self.container.append('svg') 47 | .attr('height', height).attr('width', width) 48 | .append('g') 49 | .attr("transform", "translate(" + width / 2 + "," + height / 2 + ")"); 50 | 51 | 52 | var data = d3.csv.parse(self.data); 53 | var g = svg.selectAll(".arc") 54 | .data(pie(data)) 55 | .enter().append("g") 56 | .attr("class", "arc"); 57 | 58 | g.append("path").attr("d", arc) 59 | .style("fill", function(d) { return color(d.data.label); }); 60 | 61 | // default text x y 62 | var textY = options.textY, textX = options.textX; 63 | var textZ = options.textOffset; 64 | var autoZ = textZ === 'auto'; 65 | var unit = autoZ ? '' : options.offsetUnit; 66 | 67 | var textValue = options.textValue; 68 | 69 | g.each(function(d, i) { 70 | var t = textValue(d); 71 | var r = autoZ ? radius / 6 : textZ; 72 | var c = (d.startAngle + d.endAngle) / 2; 73 | //* 90 / Math.PI; 74 | var x = Math.sin(c) * r; 75 | var y = - Math.cos(c) * r; 76 | x += textX; 77 | y += textY; 78 | 79 | var lines = t.split('\n'); 80 | // 0.17... = 10 * 3.14 / 180 81 | if (d.endAngle - d.startAngle < 0.17 * lines.length && c > 3.2) { 82 | y += lines.length * 7; 83 | } 84 | 85 | var text = svg.append("text") 86 | .attr("transform", "translate(" + arc.centroid(d) + ")") 87 | .attr("font-size", options.textSize) 88 | .style("text-anchor", "middle") 89 | 90 | if (options.textColor) { 91 | text.style("fill", options.textColor(d)); 92 | } 93 | 94 | lodash.each(lines, function(t, i) { 95 | text.append('tspan') 96 | .attr('x', 0) 97 | .attr('dx', x + unit) 98 | .attr('dy', i ? (i + 0.15 + 'em') : y + unit) 99 | .text(t); 100 | }); 101 | }); 102 | }; 103 | -------------------------------------------------------------------------------- /tasks/toplist/index.js: -------------------------------------------------------------------------------- 1 | var async = require('async') 2 | var tasks = require('../../tasks') 3 | var central = require('../../lib/central') 4 | var cached = require('../../lib/cached') 5 | var getToplistKey = require('../../models/toplist').getToplistKey 6 | var mongo = central.mongo 7 | 8 | var debug = require('debug') 9 | var log = debug('dbj:toplist:log') 10 | var verbose = debug('dbj:toplist:verbose') 11 | var error = debug('dbj:toplist:error') 12 | 13 | var ONE_MONTH = 60 * 60 * 24 * 1000 * 30.5 14 | 15 | function aggregate_hardest_reader(period, cb) { 16 | cb = cb || function(){} 17 | period = period || 'all_time' 18 | 19 | var key = getToplistKey('book', period) 20 | var now = new Date() 21 | var query = { 22 | status: 'done', 23 | commented: { $gt: 3 } 24 | } 25 | var maxTime = 600000 // defaults to 10 min timeout 26 | var limit = 500 27 | switch(period) { 28 | case 'last_30_days': 29 | query.updated = { 30 | $gte: new Date(now - ONE_MONTH) 31 | } 32 | maxTime = 60000 // max 1 min for last 20 days 33 | break 34 | case 'this_year': 35 | query.updated = { 36 | $gte: new Date('' + now.getFullYear()) 37 | } 38 | break 39 | case 'last_year': 40 | query.updated = { 41 | $gte: new Date('' + (now.getFullYear() - 1)), 42 | $lt: new Date('' + now.getFullYear()) 43 | } 44 | break 45 | case 'last_12_month': 46 | query.updated = { 47 | $gt: new Date(now - ONE_MONTH * 12), 48 | } 49 | break 50 | case 'all_time': 51 | limit = 2000 52 | maxTime = 1800000 // max 30min for calculating all time top reader 53 | break 54 | default: 55 | break 56 | } 57 | 58 | var pipe = [ 59 | { $match: query }, 60 | { $project: { user_id: 1 } }, 61 | { $group: { 62 | _id: "$user_id", 63 | value: { $sum: 1 } 64 | }}, 65 | { $sort: { value: -1 } }, 66 | { $limit: limit } 67 | ] 68 | 69 | mongo.queue(function(db, next) { 70 | log('Generating Toplist for %s...', key) 71 | db.collection('book_interest').aggregate(pipe, { 72 | allowDiskUse: true, 73 | maxTimeMS: maxTime 74 | }, function(err, results) { 75 | next() 76 | if (err) { 77 | error('Toplist failed: ', err) 78 | return cb(err) 79 | } 80 | log('Toplist for %s generated.', key) 81 | cached.set(key, results, cb) 82 | cached.del(key + '_cached', function(){}) 83 | }) 84 | }, 5) // 5 means low priority 85 | } 86 | 87 | exports.hardest_reader = aggregate_hardest_reader 88 | exports.by_tag = require('./by_tag') 89 | 90 | 91 | function breakable(period) { 92 | return function(callback) { 93 | // if some task is running, break 94 | if (tasks.getQueueLength()) { 95 | log('Exit compute toplist %s due to running computing.', period) 96 | return callback() 97 | } 98 | aggregate_hardest_reader(period, callback) 99 | } 100 | } 101 | 102 | exports.run = function(total, callback) { 103 | var jobs = [] 104 | // 收藏数量太少的用户对最终结果应该也没什么影响 105 | if (total > 200) { 106 | jobs.push(breakable('last_30_days')) 107 | } 108 | if (total > 500) { 109 | jobs.push(breakable('last_12_month')) 110 | } 111 | if (total > 2000) { 112 | jobs.push(breakable('all_time')) 113 | } 114 | if (jobs.length) { 115 | async.series(jobs, callback) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /models/user/progress.js: -------------------------------------------------------------------------------- 1 | var cwd = process.cwd() 2 | var task = require('../../lib/task') 3 | var tasks = require('../../tasks/') 4 | 5 | var User = require('./index') 6 | 7 | var DB_RESPOND_DELAY = 5000 8 | 9 | /** 10 | * reset queue status 11 | */ 12 | User.prototype.reset = function(cb) { 13 | this.update({ 14 | stats_p: 0, 15 | stats_status: null, 16 | book_n: null, 17 | book_synced_n: 0, 18 | last_synced: new Date(), 19 | last_synced_status: 'ing' 20 | }, cb) 21 | } 22 | 23 | /** 24 | * Mark sync start 25 | */ 26 | User.prototype.markSync = function(cb) { 27 | this.update({ 28 | stats_p: 0, 29 | stats_status: null, 30 | last_synced_status: 'ing' 31 | }, cb) 32 | } 33 | 34 | /** 35 | * is syncing or statsing, not finished yet 36 | */ 37 | User.prototype.isIng = function() { 38 | return this.last_synced_status === 'ing' || this.stats_status === 'ing' 39 | } 40 | 41 | /** 42 | * total progress in percent 43 | */ 44 | User.prototype.progress = function() { 45 | var ps = this.progresses() 46 | var n = 0 47 | ps.forEach(function(item) { 48 | n += item 49 | }) 50 | return n 51 | } 52 | /** 53 | * @return {array} 54 | */ 55 | User.prototype.progresses = function() { 56 | var ps = [0, 0] 57 | var user = this 58 | // got douban account info 59 | if (user.created) ps[0] = 5 60 | // starting to sync 61 | if (user.last_synced) ps[0] = 10 62 | 63 | if (user.book_n === 0) { 64 | ps[0] = 80 65 | } else if (user.book_n && user.book_synced_n) { 66 | // only book for now 67 | ps[0] = 10 + Math.ceil((user.book_synced_n / user.book_n) * 70) 68 | } 69 | // 20% percent is for computing 70 | ps[1] = ps[0] >= 80 ? Math.ceil((user.stats_p || 0) * 0.2) : 0 71 | return ps 72 | } 73 | /** 74 | * interval for checking progress 75 | */ 76 | User.prototype.progressInterval = function(remaining, delay) { 77 | var n = this.queue_length() 78 | delay = delay * n 79 | return remaining < 15000 ? DB_RESPOND_DELAY : delay + 4000 80 | } 81 | User.prototype.queue_length = function() { 82 | return tasks.getQueueLength('interest') 83 | } 84 | 85 | User.prototype.isEmpty = function(ns) { 86 | if (!ns) return null 87 | return this[ns+'_n'] === 0 88 | } 89 | 90 | /** 91 | * total remaining 92 | */ 93 | User.prototype.remaining = function() { 94 | var ret = this.syncRemaining() 95 | if (ret === null) return null 96 | 97 | var sr = this.statsRemaining() 98 | 99 | // 15 seconds for stats by default 100 | if (ret && sr === null) sr = 15000 101 | 102 | if (ret === 0 && sr === null) return null 103 | 104 | return ret + sr 105 | } 106 | /** 107 | * Approximate remaining time for computing statistics 108 | */ 109 | User.prototype.statsRemaining = function() { 110 | var user = this 111 | 112 | var total = user.book_n 113 | 114 | if (total === null) return null 115 | 116 | if (user.isIng()) { 117 | // At least five seconds 118 | return Math.round(Math.sqrt(total) * (100 - (user.stats_p || 0)) * 5) + 5000 119 | } 120 | 121 | return null 122 | } 123 | /** 124 | * expected remaing time for finish collectng job 125 | */ 126 | User.prototype.syncRemaining = function() { 127 | var user = this 128 | var total = user.book_n 129 | var synced = user.book_synced_n 130 | 131 | if (!total) return null 132 | 133 | var perpage = task.API_REQ_PERPAGE 134 | 135 | var n = this.queue_length() 136 | 137 | return (task.API_REQ_DELAY + DB_RESPOND_DELAY) * n * Math.ceil((total - synced) / perpage) 138 | } 139 | 140 | User.prototype.isSyncing = function() { 141 | return this.last_synced_status === 'ing' 142 | } 143 | 144 | /** 145 | * is syncing timeout 146 | */ 147 | User.prototype.syncTimeout = function() { 148 | var remaining = this.remaining() 149 | // 30 minutes by default 150 | if (remaining === null || this.last_synced === null) return false 151 | 152 | return new Date() - this.last_synced > remaining + 300000 153 | } 154 | User.prototype.syncFailed = function() { 155 | return this.last_synced_status !== 'ing' || this.last_synced_status !== 'succeed' 156 | } 157 | 158 | /** 159 | * syncing timeout or failed 160 | */ 161 | User.prototype.needResync = function() { 162 | var user = this 163 | return !user.isIng() || (!user.stats && user.syncTimeout()) 164 | } 165 | -------------------------------------------------------------------------------- /models/user/friends.js: -------------------------------------------------------------------------------- 1 | var debug = require('debug') 2 | var async = require('async') 3 | var util = require('util') 4 | var lodash = require('lodash') 5 | 6 | var verbose = debug('dbj:user:friends:verbose') 7 | var log = debug('dbj:user:friends:info') 8 | var error = debug('dbj:user:friends:error') 9 | 10 | var central = require('../../lib/central') 11 | 12 | var cwd = central.cwd 13 | var conf = central.conf 14 | var redis = central.redis 15 | var mongo = central.mongo 16 | var task = central.task 17 | 18 | var consts = require('../consts') 19 | var User = require('./index') 20 | 21 | var ONE_DAY = 60 * 60 * 24 * 100 22 | var FRIENDSHIP_EXPIRES = 10 * ONE_DAY 23 | 24 | 25 | /** 26 | * Get following friends from redis cache 27 | */ 28 | User.prototype.listFollowings = function(query, cb) { 29 | var self = this 30 | self.data('followings', function(err, ids) { 31 | if (err) { 32 | error('get followings from redis failed: ', err) 33 | return cb(err) 34 | } 35 | 36 | var start = parseInt(query.start, 10) || 0 37 | var limit = parseInt(query.limit, 10) || 20 38 | 39 | if (!Array.isArray(ids)) { 40 | ids = [] 41 | } 42 | 43 | var ending = start + limit 44 | 45 | function end(_ids) { 46 | _ids = _ids.slice(start, ending) 47 | cb(null, _ids) 48 | } 49 | 50 | // there's enough ids in local cache. 51 | if (ids.length >= ending) return end(ids) 52 | 53 | self.data('following_sites', function(err, sites) { 54 | var _start = ids.length 55 | if (_start && sites && typeof sites === 'object') { 56 | _start += Object.keys(sites).length 57 | } 58 | log('Pulling followings for [%s] from: %s (current: %s)', self.uid, _start, start) 59 | self.pullFollowings(_start, function(err, _ids) { 60 | if (err) { 61 | error('Pulling followings failed: ', err) 62 | return cb(err) 63 | } 64 | 65 | ids = lodash.union(ids, _ids) 66 | 67 | // save it 68 | self.data('followings', ids, function(err, r) { 69 | verbose('Save pulled followings: (%s, %s)', err, r) 70 | end(ids) 71 | }) 72 | self.expire('followings', FRIENDSHIP_EXPIRES) 73 | }) 74 | }) 75 | }) 76 | } 77 | 78 | /** 79 | * Get following friends list from douban 80 | */ 81 | User.prototype.pullFollowings = function(start, cb) { 82 | var self = this 83 | var cls = self.constructor 84 | 85 | verbose('Pulling followings for %s..', self.name) 86 | 87 | task.api2(function(oauth2, next) { 88 | var uid = self.id || self.uid 89 | oauth2.clientFromToken(self.douban_token).request( 90 | 'GET', '/shuo/v2/users/' + uid + '/following', 91 | { 92 | start: start || 0, 93 | count: 200 // 豆瓣API允许的最大值即为200 94 | }, function(err, ret) { 95 | 96 | // release task client 97 | setTimeout(next, oauth2.req_delay || 0) 98 | 99 | if (err) return cb(err) 100 | 101 | var ids = [], sites = {} 102 | 103 | // save all the users 104 | async.each(ret, function(item, callback) { 105 | if (item.type !== 'user') { 106 | verbose('Skipping none-user %s...', item.screen_name) 107 | sites[item.id] = item.original_site_id 108 | return callback() 109 | } 110 | 111 | ids.push(item.id) 112 | 113 | cls({ _id: item.id }).update({ 114 | name: item.screen_name, 115 | uid: item.uid, 116 | // +800 is the timezone 117 | created: new Date(item.created_at + '+800'), 118 | loc_name: item.city, 119 | signature: item.description, 120 | avatar: item.small_avatar 121 | }, callback) 122 | }, function(err) { 123 | self.data('following_sites', sites, function(err, r) { 124 | verbose('Save pulled followings sites: (%s, %s)', err, r) 125 | cb(err, ids) 126 | }) 127 | self.expire('following_sites', FRIENDSHIP_EXPIRES) 128 | }) 129 | 130 | }) 131 | }) 132 | } 133 | 134 | /** 135 | * clear local cache of followings 136 | */ 137 | User.prototype.clearFollowings = function(cb) { 138 | var self = this 139 | self._del_data(['followings', 'following_sites'], cb) 140 | } 141 | -------------------------------------------------------------------------------- /tasks/interest/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * aggregate user subject collections (called "interest") 3 | */ 4 | var central = require(process.cwd() + '/lib/central'); 5 | var utils = central.utils; 6 | var async = require('async'); 7 | 8 | var raven = central.raven; 9 | 10 | var User = require(central.cwd + '/models/user'); 11 | var FetchStream = require('./stream'); 12 | 13 | function error() { 14 | var args = [].slice.apply(arguments); 15 | var extra = args[args.length - 1] || {}; 16 | args[args.length - 1] = { tags: { task: 'collect' }, extra: extra }; 17 | raven.error.apply(raven, args); 18 | } 19 | function message() { 20 | var args = [].slice.apply(arguments); 21 | var extra = args[args.length - 1] || {}; 22 | args[args.length - 1] = { tags: { task: 'collect' }, extra: extra }; 23 | raven.message.apply(raven, args); 24 | } 25 | 26 | var collect, _collect; 27 | collect = User.ensured(function(user, arg) { 28 | if (!user) return arg.error('NO_USER'); 29 | 30 | arg.user = user; 31 | 32 | // try update user info 33 | if (arg.fresh && !arg._from_halt) { 34 | setImmediate(function() { 35 | user.pull(); 36 | }); 37 | } 38 | 39 | var uid = user.uid || user.id; 40 | 41 | var raven_extra = { ns: arg.ns, uid: uid }; 42 | message('START collect interests for %s', uid, raven_extra); 43 | 44 | var collector = new FetchStream(arg); 45 | 46 | // halt if syncing is already running 47 | if (user.last_synced_status === 'ing' && !arg.force && !arg._from_halt) { 48 | message('EXIT collect for %s due to runing..', uid, raven_extra); 49 | arg.error(new Error('Already running collecting process.')); 50 | return; 51 | } 52 | 53 | collector.on('error', function(err) { 54 | console.error(err); 55 | error(err, raven_extra); 56 | 57 | collector.status = 'failed'; 58 | collector.updateUser(function() { 59 | collector.end(); 60 | }); 61 | }); 62 | 63 | collector.on('saved', function(data) { 64 | collector.updateUser(); 65 | }); 66 | 67 | collector.once('end', function() { 68 | // wait for the really ends 69 | setTimeout(function() { 70 | if (collector.status == 'succeed') { 71 | message('SUCCEED collect interests for %s', uid, raven_extra); 72 | 73 | arg.success.call(collector, user); 74 | 75 | collector.emit('succeed'); 76 | } else { 77 | raven_extra.status = collector.status; 78 | error('Collect failed.', raven_extra); 79 | arg.error.call(collector, user); 80 | } 81 | }, 2000); 82 | }); 83 | 84 | collector.once('succeed', function() { 85 | // run compute task right after success 86 | require('../compute')[arg.ns]({ 87 | user: user, 88 | force: true, 89 | success: function() { 90 | user.emit('computed') 91 | }, 92 | error: function(err) { 93 | user.emit('computed', err) 94 | } 95 | }) 96 | }); 97 | 98 | collector.run(); 99 | }); 100 | 101 | function is_night() { 102 | var h = (new Date()).getHours(); 103 | return h > 1 && h < 9; 104 | } 105 | 106 | function collect_in_namespace(ns) { 107 | return function(arg) { 108 | // queue arguments safely 109 | exports.queue.safely('collect_' + ns, arg); 110 | 111 | arg.ns = ns; 112 | 113 | collect(arg.user, arg); 114 | }; 115 | } 116 | 117 | var exports = {}; 118 | 119 | central.DOUBAN_APPS.forEach(function(item) { 120 | exports['collect_' + item] = collect_in_namespace(item); 121 | }); 122 | 123 | // collect all the interest 124 | exports.collect_all = function(user, succeed_cb, error_cb) { 125 | var called = false; 126 | var error_next = function(err) { 127 | if (called) return; 128 | called = true; 129 | error_cb && error_cb(err); 130 | } 131 | if (!user) return error_next('NO_USER'); 132 | 133 | var apps = central.DOUBAN_APPS; 134 | var collectors = []; 135 | (function run(i) { 136 | var ns = apps[i]; 137 | // all apps proceeded 138 | if (!ns) succeed_cb && succeed_cb(collectors); 139 | exports['collect_' + item]({ 140 | user: user, 141 | success: function(collectors) { 142 | collectors.push(collector); 143 | run(i+1); 144 | }, 145 | error: error_next 146 | }); 147 | })(0); 148 | }; 149 | exports.collect = collect; 150 | 151 | module.exports = exports; 152 | -------------------------------------------------------------------------------- /lib/douban.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Douban API v2 SDK 3 | */ 4 | var util = require('util'); 5 | var mo_url = require('url'); 6 | var error = require('debug')('dbj:douban:error') 7 | 8 | function OAuth2(clientId, clientSecret, customHeaders) { 9 | this._clientId= clientId; 10 | this._clientSecret= clientSecret; 11 | this._baseSite= 'https://www.douban.com'; 12 | this._authorizeUrl= "/service/auth2/auth"; 13 | this._accessTokenUrl= "/service/auth2/token"; 14 | this._accessTokenName= "access_token"; 15 | this._authMethod= "Bearer"; 16 | this._customHeaders = {}; 17 | } 18 | util.inherits(OAuth2, require('oauth').OAuth2); 19 | 20 | OAuth2.prototype.clientFromToken = function(token) { 21 | var client = new Client(token, this); 22 | return client; 23 | }; 24 | 25 | var SEVEN_DAYS = 60 * 60 * 24 * 7; 26 | OAuth2.prototype.getToken = function(code, params, callback) { 27 | this.getOAuthAccessToken(code, params, function(err, access_token, refresh_token, results) { 28 | if (err) return callback(err); 29 | results = results || {}; 30 | results.access_token = access_token; 31 | results.refresh_token = refresh_token; 32 | // 30 seconds is time gap of douban server and local server 33 | results.expire_date = new Date((+new Date() + (results.expires_in || SEVEN_DAYS) - 30) * 1000); 34 | return callback(null, results); 35 | }); 36 | }; 37 | 38 | /* 39 | * get request client from token 40 | */ 41 | function Client(token, oauth2) { 42 | this.init(token); 43 | this.oauth2 = oauth2; 44 | this._accessTokenName = 'access_token'; 45 | this._customHeaders = {}; 46 | } 47 | 48 | Client.prototype.init = function(token) { 49 | token = token || {}; 50 | this.access_token = token.access_token; 51 | this.refresh_token = token.refresh_token; 52 | this.user_id = token.douban_user_id; 53 | this.expire_date = token.expire_date; 54 | 55 | this._base_url = 'https://api.douban.com'; 56 | }; 57 | 58 | Client.prototype._request = OAuth2.prototype._request; 59 | Client.prototype._executeRequest = OAuth2.prototype._executeRequest; 60 | 61 | Client.prototype._auth_headers = function() { 62 | if (!this.access_token) return {}; 63 | return { 64 | 'Authorization': 'Bearer ' + this.access_token 65 | }; 66 | }; 67 | 68 | Client.prototype.isExpired = function() { 69 | return this.dead || this.expire_date < new Date(); 70 | }; 71 | 72 | Client.prototype.request = function(method, url, data, callback) { 73 | var self = this; 74 | 75 | if (typeof data === 'function') { 76 | callback = data; 77 | data = ''; 78 | } 79 | 80 | method = method.toUpperCase(); 81 | 82 | var parsed_url = mo_url.parse(self._base_url + url); 83 | 84 | var qs = parsed_url.query || {}; 85 | if (self.oauth2) { 86 | qs.apikey = qs.client_id = self.oauth2._clientId; 87 | } 88 | if (method === 'GET' && data) { 89 | for (var k in data) { 90 | qs[k] = data[k]; 91 | } 92 | data = ''; 93 | } 94 | parsed_url.query = qs; 95 | 96 | function run() { 97 | self.oauth2._request(method, mo_url.format(parsed_url), 98 | self._auth_headers(), data, self.access_token || '', function(err, body, res) { 99 | if (err) return callback(err); 100 | var ret = {} 101 | try { 102 | ret = JSON.parse(body); 103 | } catch (e) { 104 | //error('Error parsing json: %s', body) 105 | return callback(e, ret, res) 106 | } 107 | callback(err, ret, res); 108 | }); 109 | } 110 | 111 | // if has expired 112 | if (self.isExpired()) { 113 | self.refresh(function(err, new_token) { 114 | if (err || !new_token) { 115 | self.dead = true; 116 | return; 117 | } 118 | self.init(new_token) = new_token; 119 | run(); 120 | }); 121 | } else { 122 | run(); 123 | } 124 | }; 125 | 126 | Client.prototype.refresh = function(cb) { 127 | var self = this; 128 | this.oauth2.getToken(self.refresh_token, { 129 | grand_type: 'refresh_token', 130 | }, function(err, new_token) { 131 | if (err) { 132 | self.emit('refresh_error', error); 133 | } else { 134 | self.emit('refresh', new_token); 135 | } 136 | return cb(err, new_token); 137 | }); 138 | }; 139 | 140 | ['get', 'delete', 'head'].forEach(function(item) { 141 | Client.prototype[item] = function(url, callback) { 142 | this.request(item, url, "", callback); 143 | }; 144 | }); 145 | ['post', 'put'].forEach(function(item) { 146 | Client.prototype[item] = function(url, data, callback) { 147 | this.request(item, url, data, callback); 148 | }; 149 | }); 150 | 151 | module.exports = { 152 | Client: Client, 153 | OAuth2: OAuth2 154 | }; 155 | -------------------------------------------------------------------------------- /tasks/compute/book.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Book Interests Analytics Configuration 3 | */ 4 | var async = require('async') 5 | var utils = require('../utils') 6 | var common = require('./common') 7 | var debug = require('debug') 8 | var log = debug('dbj:task:compute:book:info') 9 | var error = debug('dbj:task:compute:book:error') 10 | var verbose = debug('dbj:task:compute:book:verbose') 11 | 12 | var AggStream = common.AggStream 13 | 14 | var cwd = process.cwd() 15 | var raven = require(cwd + '/lib/raven') 16 | var consts = require(cwd + '/models/consts') 17 | 18 | var STATUSES = consts.INTEREST_STATUS_ORDERED.book 19 | 20 | var conf_book = { 21 | // count by appearence 22 | top: ['tags', 'author', 'translator', 'publisher'], 23 | // doesn't need unwind 24 | $top_u: ['publisher'], 25 | $top_k: { 26 | 'tags': 'name' 27 | }, 28 | $top_limit: 30, 29 | // sorting by prop value, return a 'most_xxx' list 30 | // by default, will count 'least_xxx' too 31 | // can set `[ { pages: -1 } ]` (means descending by pages) to avoid. 32 | // use 33 | // { 34 | // '$name': 'most_abced', 35 | // '$limit': 10, 36 | // 'prop1': 1, 37 | // 'prop2': -1 38 | // } 39 | // for more advanced settings 40 | most: ['pages', 'rated', 'raters', { 41 | price: -1, 42 | $name: 'most_price', 43 | $fields: { price: 1, ori_price: 1, title: 1 } 44 | }, { 45 | price: 1, 46 | $name: 'least_price', 47 | $fields: { price: 1, ori_price: 1, title: 1 } 48 | }], 49 | $most_fields: { title: 1 }, 50 | $most_limit: 20, 51 | range: { 52 | 'pages': [100, 300, 500, 800], 53 | 'price': [10, 20, 40, 60, 80, 100, 200, 500], 54 | }, 55 | date: { 56 | 'pubdate': { 57 | periods: ['year'] 58 | } 59 | } 60 | } 61 | var AGG_PARAM_BOOK = common.aggParam(conf_book) 62 | var AGG_PARAM_INTEREST = common.agg_param_interest 63 | 64 | var DB_INTEREST = 'book_interest' 65 | var DB_BOOK = 'book' 66 | 67 | module.exports = function(db, user, callback, progress) { 68 | callback = callback || function() {} 69 | progress = progress || function() {} 70 | 71 | var uid = user.uid || user.id 72 | 73 | // find out all collected books by user 74 | db.collection(DB_INTEREST).find( 75 | { user_id: user._id }, 76 | { fields: { subject_id: 1, status: 1 } } 77 | ).toArray(function(err, docs) { 78 | if (err) return callback(err); 79 | run(sidsByStatus(docs)) 80 | }) 81 | 82 | function sidsByStatus(docs) { 83 | // filter out subject ids by status 84 | var all_sids = [] 85 | var by_status = {} 86 | docs.forEach(function(i) { 87 | all_sids.push(i.subject_id) 88 | var arr = by_status[i.status] || [] 89 | arr.push(i.subject_id) 90 | by_status[i.status] = arr 91 | }) 92 | by_status['all'] = all_sids 93 | return by_status; 94 | } 95 | 96 | var total_step; 97 | 98 | function agg(i, name, options) { 99 | return function(callback) { 100 | var agger = new AggStream(options) 101 | agger.once('error', callback) 102 | agger.once('end', function() { 103 | this.fillup() 104 | progress(i / total_step * 100) 105 | callback(null, this.results) 106 | }) 107 | log('Agg %s for %s ...', name, uid) 108 | agger.run() 109 | } 110 | } 111 | 112 | function run(sids_by_status) { 113 | var statuses = ['all'].concat(STATUSES) 114 | var actions = statuses.map(function(st, i) { 115 | var sids = sids_by_status[st] 116 | if (!sids) { 117 | return function(next) { next(null) } 118 | } 119 | var options = { 120 | uid: uid, 121 | params: AGG_PARAM_BOOK, 122 | collection: DB_BOOK, 123 | prefilter: { 124 | $match: { _id: { $in: sids } }, 125 | } 126 | } 127 | return agg(i + 1, st + ' book', options) 128 | }) 129 | 130 | actions.push(agg(actions.length + 1, 'book interest', { 131 | uid: uid, 132 | params: AGG_PARAM_INTEREST, 133 | collection: DB_INTEREST, 134 | prefilter: { 135 | $match: { user_id: user._id } 136 | } 137 | })) 138 | 139 | total_step = actions.length 140 | 141 | async.series(actions, function(err, computed) { 142 | if (err) return callback(err) 143 | 144 | var interest = computed.pop() 145 | var n_all = sids_by_status['all'].length 146 | 147 | var results = { 148 | total: n_all, 149 | interest: interest 150 | } 151 | 152 | statuses.forEach(function(item, i) { 153 | results[item] = computed[i] 154 | var n = sids_by_status[item] 155 | n = n && n.length || 0 156 | results['n_' + item] = n 157 | results['ratio_' + item] = (n / n_all * 100).toFixed(2) 158 | }) 159 | callback(null, results) 160 | }) 161 | } 162 | } 163 | --------------------------------------------------------------------------------