├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── aid_finder.go ├── ansi ├── ansi_convert.go └── escape.go ├── article ├── const.go ├── index_mapper.go ├── render.go ├── render_test.go ├── segment.go ├── terminal.go └── util.go ├── atomfeed └── atomfeed.go ├── cache └── cache.go ├── cached_inproc.go ├── cached_ops.go ├── captcha ├── config.go └── handler.go ├── config.go ├── context.go ├── cookie.go ├── experiment └── struct.go ├── extcache └── extcache.go ├── gate ├── gate.go └── gate_test.go ├── go.mod ├── go.sum ├── page ├── ajax.go ├── context.go ├── names.go ├── pages.go ├── template.go └── wrapper.go ├── paging.go ├── proto ├── Makefile ├── api │ ├── Makefile │ └── board.proto └── man │ ├── Makefile │ └── man.proto ├── pttbbs ├── aid.go ├── aid_test.go ├── board_ref.go ├── grpc.go ├── search_predicate.go ├── string.go ├── string_test.go └── struct.go ├── pttweb.go ├── pushstream └── stream.go ├── richcontent ├── rich_content.go ├── url_based_finder.go ├── url_detect.go └── url_detect_test.go ├── struct.go ├── tmpl_funcs.go ├── useragent.go └── util.go /.gitignore: -------------------------------------------------------------------------------- 1 | templates/ 2 | proto/api/board.pb.go 3 | proto/api/board_grpc.pb.go 4 | proto/man/man.pb.go 5 | proto/man/man_grpc.pb.go 6 | pttweb 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.x 5 | - tip 6 | 7 | go_import_path: github.com/ptt/pttweb 8 | 9 | sudo: required 10 | dist: bionic 11 | addons: 12 | apt: 13 | update: true 14 | packages: 15 | - protobuf-compiler 16 | 17 | install: 18 | - go get google.golang.org/grpc 19 | - go get -u github.com/golang/protobuf/{proto,protoc-gen-go} 20 | - make -C $GOPATH/src/github.com/ptt/pttweb/proto 21 | - go get -u -t -f github.com/ptt/pttweb/... 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Robert Wang 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | pttweb 2 | ====== 3 | 4 | PTT BBS Web Frontend. 5 | 6 | In production on http://www.ptt.cc/ 7 | 8 | [![Build Status](https://travis-ci.org/ptt/pttweb.svg?branch=master)](https://travis-ci.org/ptt/pttweb) 9 | 10 | Features 11 | -------- 12 | 13 | - List board index 14 | - Show articles 15 | - Render ANSI colors 16 | - Templating support 17 | - Ask user if he/she is over age 18 when entering some areas. 18 | 19 | Configuration 20 | ------------- 21 | 22 | See `config.go`. 23 | Put them into a JSON-encoded file. 24 | 25 | $ ./pttweb -conf config.json 26 | 27 | Template 28 | -------- 29 | 30 | To be documented. 31 | 32 | License 33 | ------- 34 | 35 | MIT 36 | -------------------------------------------------------------------------------- /aid_finder.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "regexp" 6 | 7 | "github.com/ptt/pttweb/pttbbs" 8 | "github.com/ptt/pttweb/richcontent" 9 | 10 | "golang.org/x/net/context" 11 | ) 12 | 13 | var aidPatterns = []struct { 14 | Pattern *regexp.Regexp 15 | Handler func(ctx context.Context, input []byte, m richcontent.MatchIndices) (link string, err error) 16 | }{ 17 | { 18 | Pattern: regexp.MustCompile(`([0-9A-Za-z\-_]{1,12}) 看板 #([0-9A-Za-z\-_\@]{8,10})`), 19 | Handler: handleBoardAidText, 20 | }, 21 | { 22 | Pattern: regexp.MustCompile(`#([0-9A-Za-z\-_\@]{8,10}) \(([0-9A-Za-z\-_]{1,12})\)`), 23 | Handler: handleAidBoardText, 24 | }, 25 | { 26 | Pattern: regexp.MustCompile(`#([0-9A-Za-z\-_\@]{8,10})`), 27 | Handler: handleAidText, 28 | }, 29 | } 30 | 31 | func init() { 32 | richcontent.RegisterFinder(findAidText) 33 | } 34 | 35 | func findAidText(ctx context.Context, input []byte) (rcs []richcontent.RichContent, err error) { 36 | // Fast path. 37 | if bytes.IndexByte(input, '#') < 0 { 38 | return nil, nil 39 | } 40 | 41 | for _, p := range aidPatterns { 42 | all := p.Pattern.FindAllSubmatchIndex(input, -1) 43 | for _, m := range all { 44 | link, err := p.Handler(ctx, input, richcontent.MatchIndices(m)) 45 | if err != nil { 46 | return nil, err 47 | } 48 | if link != "" { 49 | rcs = append(rcs, richcontent.MakeRichContent(m[0], m[1], link, nil)) 50 | } 51 | } 52 | } 53 | return rcs, nil 54 | } 55 | 56 | type boardname interface { 57 | Boardname() string 58 | } 59 | 60 | func handleAidText(ctx context.Context, input []byte, m richcontent.MatchIndices) (string, error) { 61 | bn, ok := ctx.Value(CtxKeyBoardname).(boardname) 62 | if !ok { 63 | return "", nil // Silently fail 64 | } 65 | 66 | aidString := string(m.ByteSliceOf(input, 1)) 67 | return aidAndBrdnameToArticle(bn.Boardname(), aidString) 68 | } 69 | 70 | func handleAidBoardText(ctx context.Context, input []byte, m richcontent.MatchIndices) (string, error) { 71 | aidString := string(m.ByteSliceOf(input, 1)) 72 | brdname := string(m.ByteSliceOf(input, 2)) 73 | return aidAndBrdnameToArticle(brdname, aidString) 74 | } 75 | 76 | func handleBoardAidText(ctx context.Context, input []byte, m richcontent.MatchIndices) (string, error) { 77 | brdname := string(m.ByteSliceOf(input, 1)) 78 | aidString := string(m.ByteSliceOf(input, 2)) 79 | return aidAndBrdnameToArticle(brdname, aidString) 80 | } 81 | 82 | func aidAndBrdnameToArticle(brdname, aidString string) (string, error) { 83 | aid, err := pttbbs.ParseAid(aidString) 84 | if err != nil { 85 | return "", nil // Silently fail 86 | } 87 | 88 | u, err := router.Get("bbsarticle").URLPath("brdname", brdname, "filename", aid.Filename()) 89 | if err != nil { 90 | return "", err 91 | } 92 | return u.String(), nil 93 | } 94 | -------------------------------------------------------------------------------- /ansi/ansi_convert.go: -------------------------------------------------------------------------------- 1 | package ansi 2 | 3 | import ( 4 | "unicode/utf8" 5 | ) 6 | 7 | // States 8 | const ( 9 | Default = iota 10 | Escaping 11 | ParsingControl 12 | InControl 13 | SkipOne 14 | ) 15 | 16 | type AnsiParser struct { 17 | Rune func(r rune) 18 | Escape func(e EscapeSequence) 19 | } 20 | 21 | func (a *AnsiParser) ConvertFromUTF8(input []byte) error { 22 | s := Default 23 | buf := make([]rune, 0, 16) 24 | var esc EscapeSequence 25 | 26 | for i, n := 0, len(input); i < n; { 27 | r, sz := utf8.DecodeRune(input[i:]) 28 | if r == utf8.RuneError { 29 | i++ 30 | continue 31 | } 32 | switch s { 33 | case Default: 34 | switch r { 35 | case 033: 36 | s = Escaping 37 | buf = buf[0:0] 38 | esc.Reset() 39 | default: 40 | a.Rune(r) 41 | } 42 | case Escaping: 43 | switch r { 44 | case '*': 45 | // special case in ptt, not implemented here 46 | s = SkipOne 47 | case '[': 48 | // multi-byte control sequence 49 | s = ParsingControl 50 | case 'm': 51 | // XXX: some asciarts tend to use this as reset, but is not in the spec. 52 | a.Escape(esc) 53 | s = Default 54 | default: 55 | if r >= '@' && r <= '_' { 56 | // 2-char control code, not supported 57 | s = SkipOne 58 | } else { 59 | // error! but be nice 60 | a.Rune(r) 61 | s = Default 62 | } 63 | } 64 | case ParsingControl: 65 | switch { 66 | case r >= ' ' && r <= '/': 67 | esc.Trailings = append(esc.Trailings, r) 68 | case r >= '@' && r <= '~': 69 | esc.Mode = r 70 | esc.ParseNumbers(buf) 71 | a.Escape(esc) 72 | s = Default 73 | default: 74 | buf = append(buf, r) 75 | } 76 | case SkipOne: 77 | // just skip 78 | s = Default 79 | } 80 | i += sz 81 | } 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /ansi/escape.go: -------------------------------------------------------------------------------- 1 | package ansi 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type EscapeSequence struct { 8 | IsCSI bool 9 | PrivateModes []rune // not supported, will not be parsed 10 | Nums []int 11 | Trailings []rune 12 | Mode rune 13 | } 14 | 15 | func (e *EscapeSequence) Reset() { 16 | e.IsCSI = false 17 | 18 | if e.PrivateModes == nil { 19 | e.PrivateModes = make([]rune, 0, 4) 20 | } else { 21 | e.PrivateModes = e.PrivateModes[0:0] 22 | } 23 | 24 | if e.Nums == nil { 25 | e.Nums = make([]int, 0, 4) 26 | } else { 27 | e.Nums = e.Nums[0:0] 28 | } 29 | 30 | if e.Trailings == nil { 31 | e.Trailings = make([]rune, 0, 4) 32 | } else { 33 | e.Trailings = e.Trailings[0:0] 34 | } 35 | 36 | e.Mode = 0 37 | } 38 | 39 | func (e *EscapeSequence) ParseNumbers(buf []rune) { 40 | part := make([]rune, 0, 4) 41 | for i, r := range buf { 42 | if r != ';' { 43 | part = append(part, r) 44 | } 45 | if r == ';' || i == len(buf)-1 { 46 | switch len(part) { 47 | case 0: 48 | // Treat empty parameter as 0 (eg. "\e[;34m" as "\e[0;34m"). 49 | // It is not stated in the spec whether it's valid. But most 50 | // terminal does this and are being relied by ascii art 51 | // creators. Let's allow this. 52 | e.Nums = append(e.Nums, 0) 53 | default: 54 | num, err := strconv.Atoi(string(part)) 55 | if err != nil { 56 | continue // be nice 57 | } 58 | e.Nums = append(e.Nums, num) 59 | part = part[0:0] 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /article/const.go: -------------------------------------------------------------------------------- 1 | package article 2 | 3 | const ( 4 | ClassFgPrefix = `f` 5 | ClassBgPrefix = `b` 6 | ClassHighlight = `hl` 7 | ClassPushDiv = `push` 8 | ClassPushTag = `push-tag` 9 | ClassPushUserId = `push-userid` 10 | ClassPushContent = `push-content` 11 | ClassPushIpDatetime = `push-ipdatetime` 12 | 13 | ClassArticleMetaLine = `article-metaline` 14 | ClassArticleMetaLineRight = `article-metaline-right` 15 | ClassArticleMetaTag = `article-meta-tag` 16 | ClassArticleMetaValue = `article-meta-value` 17 | ) 18 | -------------------------------------------------------------------------------- /article/index_mapper.go: -------------------------------------------------------------------------------- 1 | package article 2 | 3 | const ( 4 | kIndexMapperPrealloc = 16 5 | ) 6 | 7 | type IndexMapper struct { 8 | indexArr []int 9 | elemLen int 10 | } 11 | 12 | func NewIndexMapper(elemLen int) *IndexMapper { 13 | L := elemLen * kIndexMapperPrealloc 14 | return &IndexMapper{ 15 | indexArr: make([]int, L, L), 16 | elemLen: elemLen, 17 | } 18 | } 19 | 20 | func (im *IndexMapper) Reset() { 21 | } 22 | 23 | func (im *IndexMapper) Record(from int, to ...int) { 24 | for im.elemLen*from+im.elemLen >= len(im.indexArr) { 25 | L := len(im.indexArr) 26 | newArr := make([]int, 2*L, 2*L) 27 | copy(newArr, im.indexArr) 28 | im.indexArr = newArr 29 | } 30 | for i := 0; i < im.elemLen; i++ { 31 | im.indexArr[from*im.elemLen+i] = to[i] 32 | } 33 | } 34 | 35 | func (im *IndexMapper) Get(from int) []int { 36 | return im.indexArr[from*im.elemLen : (from+1)*im.elemLen] 37 | } 38 | -------------------------------------------------------------------------------- /article/render.go: -------------------------------------------------------------------------------- 1 | package article 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "strings" 7 | 8 | "github.com/ptt/pttweb/ansi" 9 | "github.com/ptt/pttweb/pttbbs" 10 | "github.com/ptt/pttweb/richcontent" 11 | 12 | "golang.org/x/net/context" 13 | ) 14 | 15 | const ( 16 | kPreviewContentLines = 5 17 | ) 18 | 19 | type RenderOption func(*renderer) 20 | 21 | func WithContent(content []byte) RenderOption { 22 | return func(r *renderer) { 23 | r.content = content 24 | } 25 | } 26 | 27 | func WithContext(ctx context.Context) RenderOption { 28 | return func(r *renderer) { 29 | r.ctx = ctx 30 | } 31 | } 32 | 33 | func WithDisableArticleHeader() RenderOption { 34 | return func(r *renderer) { 35 | r.disableArticleHeader = true 36 | } 37 | } 38 | 39 | type RenderedArticle interface { 40 | ParsedTitle() string 41 | PreviewContent() string 42 | HTML() []byte 43 | } 44 | 45 | func Render(opts ...RenderOption) (RenderedArticle, error) { 46 | r := newRenderer() 47 | for _, opt := range opts { 48 | opt(r) 49 | } 50 | if err := r.Render(); err != nil { 51 | return nil, err 52 | } 53 | return r, nil 54 | } 55 | 56 | type renderer struct { 57 | // Options. 58 | content []byte 59 | disableArticleHeader bool 60 | ctx context.Context 61 | 62 | // Internal states. 63 | buf bytes.Buffer 64 | lineNo int 65 | 66 | mapper *IndexMapper 67 | lineBuf bytes.Buffer 68 | lineSegs []Segment 69 | segIndex int 70 | segOffset int 71 | segClosed bool 72 | 73 | terminalState TerminalState 74 | 75 | acceptMetaLines bool 76 | 77 | title string 78 | 79 | previewContent string 80 | previewLineCount int 81 | } 82 | 83 | func newRenderer() *renderer { 84 | ar := &renderer{ 85 | ctx: context.TODO(), 86 | mapper: NewIndexMapper(2), 87 | lineSegs: make([]Segment, 0, 8), 88 | } 89 | ar.init() 90 | return ar 91 | } 92 | 93 | func (r *renderer) init() { 94 | r.buf.Reset() 95 | r.lineNo = 1 96 | 97 | r.mapper.Reset() 98 | r.lineBuf.Reset() 99 | r.lineSegs = r.lineSegs[0:0] 100 | r.segIndex = 0 101 | r.segOffset = 0 102 | r.segClosed = true 103 | 104 | r.terminalState.Reset() 105 | 106 | r.acceptMetaLines = true 107 | 108 | r.title = "" 109 | 110 | r.previewContent = "" 111 | r.previewLineCount = 0 112 | } 113 | 114 | func (r *renderer) ParsedTitle() string { 115 | return r.title 116 | } 117 | 118 | func (r *renderer) PreviewContent() string { 119 | return r.previewContent 120 | } 121 | 122 | func (r *renderer) HTML() []byte { 123 | return r.buf.Bytes() 124 | } 125 | 126 | func (r *renderer) Render() error { 127 | converter := &ansi.AnsiParser{ 128 | Rune: r.oneRune, 129 | Escape: r.escape, 130 | } 131 | if err := converter.ConvertFromUTF8(r.content); err != nil { 132 | return err 133 | } 134 | // Simulate end of line if there isn't one at the end. 135 | if r.lineBuf.Len() > 0 { 136 | r.endOfLine() 137 | } 138 | return nil 139 | } 140 | 141 | func (r *renderer) currSeg() *Segment { 142 | if len(r.lineSegs) == 0 || r.segClosed { 143 | r.startSegment() 144 | } 145 | return &r.lineSegs[len(r.lineSegs)-1] 146 | } 147 | 148 | func (r *renderer) escape(esc ansi.EscapeSequence) { 149 | r.terminalState.ApplyEscapeSequence(esc) 150 | if r.segClosed || !r.terminalState.Equal(&r.currSeg().TermState) { 151 | r.startSegment() 152 | } 153 | } 154 | 155 | func (r *renderer) startSegment() { 156 | if !r.segClosed { 157 | r.endSegment() 158 | } 159 | r.lineSegs = append(r.lineSegs, Segment{ 160 | Tag: "span", 161 | TermState: r.terminalState, 162 | Buffer: &bytes.Buffer{}, 163 | }) 164 | r.segClosed = false 165 | } 166 | 167 | func (r *renderer) endSegment() { 168 | // Remove empty segment 169 | if r.lineSegs[len(r.lineSegs)-1].Len() == 0 { 170 | r.lineSegs = r.lineSegs[:len(r.lineSegs)-1] 171 | } 172 | 173 | r.segClosed = true 174 | } 175 | 176 | func (r *renderer) oneRune(ru rune) { 177 | seg := r.currSeg() 178 | r.mapper.Record(r.lineBuf.Len(), len(r.lineSegs)-1, seg.Len()) 179 | fastWriteHtmlEscapedRune(seg.Buffer, ru) 180 | r.lineBuf.WriteRune(ru) 181 | 182 | if ru == '\n' { 183 | r.endOfLine() 184 | } 185 | } 186 | 187 | func (r *renderer) outputToSegment(i, off int) { 188 | for ; r.segIndex < i; r.segIndex++ { 189 | s := &r.lineSegs[r.segIndex] 190 | r.maybeOpenCurrentSegment() 191 | r.buf.Write(s.InnerBytes()[r.segOffset:]) 192 | r.maybeCloseCurrentSegment() 193 | // advance to next segment at offset 0. 194 | r.segOffset = 0 195 | } 196 | if off > 0 { 197 | s := &r.lineSegs[r.segIndex] 198 | r.maybeOpenCurrentSegment() 199 | r.buf.Write(s.InnerBytes()[r.segOffset:off]) 200 | r.segOffset = off 201 | } 202 | } 203 | 204 | func (r *renderer) skipToSegment(i, off int) { 205 | r.maybeCloseCurrentSegment() 206 | r.segIndex = i 207 | r.segOffset = off 208 | } 209 | 210 | func (r *renderer) maybeOpenCurrentSegment() { 211 | if r.segClosed { 212 | r.lineSegs[r.segIndex].WriteOpen(&r.buf) 213 | r.segClosed = false 214 | } 215 | } 216 | 217 | func (r *renderer) maybeCloseCurrentSegment() { 218 | if !r.segClosed { 219 | r.lineSegs[r.segIndex].WriteClose(&r.buf) 220 | r.segClosed = true 221 | } 222 | } 223 | 224 | func (r *renderer) matchFirstLineAndOutput(line []byte) bool { 225 | tag1, val1, tag2, val2, ok := pttbbs.ParseArticleFirstLine(r.lineBuf.Bytes()) 226 | if !ok { 227 | return false 228 | } 229 | 230 | r.writeMetaLine(tag1, val1, ClassArticleMetaLine) 231 | r.writeMetaLine(tag2, val2, ClassArticleMetaLineRight) 232 | return true 233 | } 234 | 235 | func (r *renderer) writeMetaLine(tag, val []byte, divClass string) { 236 | r.buf.WriteString(`
`) 237 | fastWriteHtmlEscaped(&r.buf, string(tag)) 238 | r.buf.WriteString(``) 239 | r.buf.WriteString(``) 240 | fastWriteHtmlEscaped(&r.buf, string(val)) 241 | r.buf.WriteString(`
`) 242 | } 243 | 244 | func (r *renderer) endOfLine() { 245 | r.segClosed = true 246 | 247 | // Map pass the end of line to end of seg. 248 | r.mapper.Record(r.lineBuf.Len(), len(r.lineSegs), 0) 249 | line := r.lineBuf.Bytes() 250 | parsed := false 251 | 252 | if !r.disableArticleHeader && r.acceptMetaLines && r.lineNo < 5 { 253 | if r.lineNo == 1 && r.matchFirstLineAndOutput(line) { 254 | parsed = true 255 | } else if tag, val, ok := pttbbs.ParseArticleMetaLine(line); ok { 256 | if bytes.Equal(tag, []byte(pttbbs.ArticleTitle)) { 257 | r.title = string(val) 258 | } 259 | r.writeMetaLine(tag, val, ClassArticleMetaLine) 260 | parsed = true 261 | } else { 262 | r.acceptMetaLines = false 263 | } 264 | } 265 | 266 | if !parsed { 267 | isMainContent := false 268 | if len(r.lineSegs) > 0 { 269 | if pttbbs.MatchPrefixBytesToStrings(line, pttbbs.QuotePrefixStrings) { 270 | r.lineSegs[0].TermState.SetColor(6, DefaultBg, NoFlags) 271 | } else if pttbbs.MatchPrefixBytesToStrings(line, pttbbs.SignaturePrefixStrings) { 272 | r.lineSegs[0].TermState.SetColor(2, DefaultBg, NoFlags) 273 | } else { 274 | // Non-empty, not quote, and not signature line. 275 | isMainContent = true 276 | } 277 | } 278 | 279 | // Collect non-empty lines as preview starting at first main 280 | // content line. 281 | isEmpty := len(strings.TrimSpace(string(line))) == 0 282 | canCollect := !isEmpty && (r.previewLineCount == 0 && isMainContent || r.previewLineCount > 0) 283 | if canCollect && r.previewLineCount < kPreviewContentLines { 284 | r.previewContent += string(line) 285 | r.previewLineCount++ 286 | } 287 | r.processNormalContentLine(line) 288 | } 289 | 290 | // Reset and update variables 291 | r.mapper.Reset() 292 | r.lineBuf.Reset() 293 | r.lineSegs = r.lineSegs[0:0] 294 | r.segIndex = 0 295 | r.segOffset = 0 296 | r.segClosed = true 297 | r.lineNo++ 298 | } 299 | 300 | func (r *renderer) processNormalContentLine(line []byte) { 301 | // Detect push line 302 | isPush := false 303 | if matchPushLine(r.lineSegs) { 304 | r.lineSegs[0].ExtraFlags |= PushTag 305 | r.lineSegs[1].ExtraFlags |= PushUserId 306 | r.lineSegs[2].ExtraFlags |= PushContent 307 | r.lineSegs[3].ExtraFlags |= PushIpDateTime 308 | // Remove trailing spaces 309 | r.lineSegs[2].TrimRight(" ") 310 | r.buf.WriteString(`
`) 311 | isPush = true 312 | } 313 | 314 | rcs, err := richcontent.Find(r.ctx, line) 315 | if err != nil { 316 | rcs = nil 317 | log.Println("warning: rendering article: richcontent.Find:", err) 318 | } 319 | 320 | for _, rc := range rcs { 321 | linkBegin, linkEnd := makeExternalUrlLink(rc.URLString()) 322 | 323 | lbegin, lend := rc.Pos() 324 | begin := r.mapper.Get(lbegin) 325 | end := r.mapper.Get(lend) 326 | r.outputToSegment(begin[0], begin[1]) 327 | if begin[0] == end[0] { 328 | // same segment: embed 329 | r.maybeOpenCurrentSegment() 330 | r.buf.WriteString(linkBegin) 331 | r.outputToSegment(end[0], end[1]) 332 | r.buf.WriteString(linkEnd) 333 | } else { 334 | // different segments: split, wrap-around 335 | r.maybeCloseCurrentSegment() 336 | r.buf.WriteString(linkBegin) 337 | r.outputToSegment(end[0], end[1]) 338 | r.maybeCloseCurrentSegment() 339 | r.buf.WriteString(linkEnd) 340 | } 341 | } 342 | r.outputToSegment(len(r.lineSegs), 0) 343 | 344 | if isPush { 345 | r.buf.WriteString(`
`) 346 | } 347 | 348 | // Append rich contents to next line. 349 | for _, rc := range rcs { 350 | for _, comp := range rc.Components() { 351 | r.buf.WriteString(`
` + comp.HTML() + `
`) 352 | } 353 | } 354 | } 355 | -------------------------------------------------------------------------------- /article/render_test.go: -------------------------------------------------------------------------------- 1 | package article 2 | 3 | import "testing" 4 | 5 | func TestRender(t *testing.T) { 6 | tests := []struct { 7 | desc string 8 | input string 9 | wantErr error 10 | wantHTML string 11 | }{ 12 | { 13 | desc: "link crossing segments", 14 | input: "\033[31mhttp://exam\033[32mple.com/ bar\033[m", 15 | wantHTML: `http://example.com/ bar`, 16 | }, 17 | { 18 | desc: "link spans 2 segments", 19 | input: "\033[31mhttp://exam\033[32mple.com/", 20 | wantHTML: `http://example.com/`, 21 | }, 22 | { 23 | desc: "link at beginning of a segment", 24 | input: "\033[31mhttp://example.com/ bar\033[m", 25 | wantHTML: `http://example.com/ bar`, 26 | }, 27 | } 28 | for _, test := range tests { 29 | ra, err := Render(WithContent([]byte(test.input)), WithDisableArticleHeader()) 30 | if err != test.wantErr { 31 | t.Errorf("%v: Render(test.input) = _, %v; want _, %v", test.desc, err, test.wantErr) 32 | continue 33 | } else if err != nil { 34 | continue 35 | } 36 | if got, want := string(ra.HTML()), test.wantHTML; got != want { 37 | t.Errorf("%v: ra.HTML():\ngot = %v\nwant = %v", test.desc, got, want) 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /article/segment.go: -------------------------------------------------------------------------------- 1 | package article 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | type ExtraFlag int 11 | 12 | const ( 13 | PushTag ExtraFlag = 1 << iota 14 | PushUserId 15 | PushContent 16 | PushIpDateTime 17 | PushMaxVal // Dummy value 18 | ) 19 | 20 | type Segment struct { 21 | *bytes.Buffer 22 | Tag string 23 | ExtraFlags ExtraFlag 24 | TermState TerminalState 25 | } 26 | 27 | var extraFlagClasses = []string{ 28 | ClassPushTag, 29 | ClassPushUserId, 30 | ClassPushContent, 31 | ClassPushIpDatetime, 32 | } 33 | 34 | func (s *Segment) WriteOpen(w io.Writer) (int, error) { 35 | classes := make([]string, 0, 3) 36 | if s.TermState.Fg() != 7 { 37 | classes = append(classes, ClassFgPrefix+strconv.Itoa(s.TermState.Fg())) 38 | } 39 | if s.TermState.Bg() != 0 { 40 | classes = append(classes, ClassBgPrefix+strconv.Itoa(s.TermState.Bg())) 41 | } 42 | if s.TermState.HasFlags(Highlighted) { 43 | classes = append(classes, ClassHighlight) 44 | } 45 | for i, fl := 0, ExtraFlag(1); fl < PushMaxVal; i, fl = i+1, fl<<1 { 46 | if s.HasExtraFlags(fl) { 47 | classes = append(classes, extraFlagClasses[i]) 48 | } 49 | } 50 | if len(classes) > 0 { 51 | return w.Write([]byte(`<` + s.Tag + ` class="` + strings.Join(classes, ` `) + `">`)) 52 | } else { 53 | s.Tag = "" 54 | } 55 | return 0, nil 56 | } 57 | 58 | func (s *Segment) HasExtraFlags(fl ExtraFlag) bool { 59 | return s.ExtraFlags&fl == fl 60 | } 61 | 62 | func (s *Segment) WriteInner(w io.Writer) (int, error) { 63 | return w.Write(s.Bytes()) 64 | } 65 | 66 | func (s *Segment) WriteClose(w io.Writer) (int, error) { 67 | if s.Tag != "" { 68 | return w.Write([]byte(``)) 69 | } 70 | return 0, nil 71 | } 72 | 73 | func (s *Segment) WriteTo(w io.Writer) (total int64, err error) { 74 | wrote, err := s.WriteOpen(w) 75 | total += int64(wrote) 76 | if err != nil { 77 | return 78 | } 79 | 80 | wrote, err = s.WriteInner(w) 81 | total += int64(wrote) 82 | if err != nil { 83 | return 84 | } 85 | 86 | wrote, err = s.WriteClose(w) 87 | total += int64(wrote) 88 | return 89 | } 90 | 91 | func (s *Segment) InnerBytes() []byte { 92 | return s.Bytes() 93 | } 94 | 95 | func (s *Segment) TrimRight(cutset string) { 96 | s.Truncate(len(bytes.TrimRight(s.Bytes(), cutset))) 97 | } 98 | -------------------------------------------------------------------------------- /article/terminal.go: -------------------------------------------------------------------------------- 1 | package article 2 | 3 | import ( 4 | "github.com/ptt/pttweb/ansi" 5 | ) 6 | 7 | // Flags 8 | const ( 9 | NoFlags = 1 << iota 10 | Highlighted 11 | ) 12 | 13 | const ( 14 | DefaultFg = 7 15 | DefaultBg = 0 16 | ) 17 | 18 | type TerminalState struct { 19 | fg, bg, flags int 20 | } 21 | 22 | func (t *TerminalState) Reset() { 23 | t.fg = DefaultFg 24 | t.bg = DefaultBg 25 | t.flags = NoFlags 26 | } 27 | 28 | func (t *TerminalState) IsDefaultState() bool { 29 | return t.fg == DefaultFg && t.bg == DefaultBg && t.flags == NoFlags 30 | } 31 | 32 | func (t *TerminalState) SetColor(fg, bg, flags int) { 33 | t.fg = fg 34 | t.bg = bg 35 | t.flags = flags 36 | } 37 | 38 | func (t *TerminalState) ApplyEscapeSequence(esc ansi.EscapeSequence) { 39 | switch esc.Mode { 40 | case 'm': 41 | if len(esc.Nums) == 0 { 42 | t.Reset() 43 | return 44 | } 45 | fg, bg, flags := t.fg, t.bg, t.flags 46 | for _, ctl := range esc.Nums { 47 | switch { 48 | case ctl == 0: 49 | fg = DefaultFg 50 | bg = DefaultBg 51 | flags = NoFlags 52 | case ctl == 1: 53 | flags |= Highlighted 54 | case ctl == 22: 55 | flags &= ^Highlighted 56 | case ctl >= 30 && ctl <= 37: 57 | fg = ctl % 10 58 | case ctl >= 40 && ctl <= 47: 59 | bg = ctl % 10 60 | default: 61 | // be nice 62 | } 63 | } 64 | t.SetColor(fg, bg, flags) 65 | } 66 | } 67 | 68 | func (t *TerminalState) Equal(u *TerminalState) bool { 69 | return t.fg == u.fg && t.bg == u.bg && t.flags == u.flags 70 | } 71 | 72 | func (t *TerminalState) Fg() int { 73 | return t.fg 74 | } 75 | 76 | func (t *TerminalState) Bg() int { 77 | return t.bg 78 | } 79 | 80 | func (t *TerminalState) Flags() int { 81 | return t.flags 82 | } 83 | 84 | func (t *TerminalState) HasFlags(f int) bool { 85 | return t.flags&f == f 86 | } 87 | -------------------------------------------------------------------------------- /article/util.go: -------------------------------------------------------------------------------- 1 | package article 2 | 3 | import ( 4 | "bytes" 5 | "html" 6 | 7 | "github.com/ptt/pttweb/pttbbs" 8 | ) 9 | 10 | func fastWriteHtmlEscapedRune(buf *bytes.Buffer, ru rune) { 11 | if ru >= 256 { 12 | buf.WriteRune(ru) 13 | return 14 | } 15 | switch ru { 16 | case '&': 17 | buf.WriteString(`&`) 18 | case '\'': 19 | buf.WriteString(`'`) 20 | case '<': 21 | buf.WriteString(`<`) 22 | case '>': 23 | buf.WriteString(`>`) 24 | case '"': 25 | buf.WriteString(`"`) 26 | default: 27 | buf.WriteRune(ru) 28 | } 29 | } 30 | 31 | func fastWriteHtmlEscaped(buf *bytes.Buffer, str string) { 32 | for _, ru := range str { 33 | if ru == 0xFFFD { 34 | // Invalid UTF-8 sequence 35 | continue 36 | } 37 | fastWriteHtmlEscapedRune(buf, ru) 38 | } 39 | } 40 | 41 | func makeExternalUrlLink(urlString string) (begin, end string) { 42 | begin = `` 43 | end = `` 44 | return 45 | } 46 | 47 | func matchColor(t *TerminalState, fg, bg, flags int) bool { 48 | return t.Fg() == fg && t.Bg() == bg && t.HasFlags(flags) 49 | } 50 | 51 | func matchAny(b []byte, patt []string) bool { 52 | for _, p := range patt { 53 | if bytes.Equal(b, []byte(p)) { 54 | return true 55 | } 56 | } 57 | return false 58 | } 59 | 60 | func matchPushLine(segs []Segment) bool { 61 | return len(segs) == 4 && 62 | matchAny(segs[0].Bytes(), pttbbs.ArticlePushPrefixStrings) && 63 | (matchColor(&segs[0].TermState, 1, 0, Highlighted) || 64 | matchColor(&segs[0].TermState, 7, 0, Highlighted)) && 65 | matchColor(&segs[1].TermState, 3, 0, Highlighted) && 66 | matchColor(&segs[2].TermState, 3, 0, NoFlags) && 67 | matchColor(&segs[3].TermState, 7, 0, NoFlags) 68 | } 69 | -------------------------------------------------------------------------------- /atomfeed/atomfeed.go: -------------------------------------------------------------------------------- 1 | package atomfeed 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "html/template" 7 | "time" 8 | 9 | "github.com/ptt/pttweb/pttbbs" 10 | "golang.org/x/tools/blog/atom" 11 | ) 12 | 13 | type PostEntry struct { 14 | Article pttbbs.Article 15 | Snippet string 16 | } 17 | 18 | type Converter struct { 19 | FeedTitleTemplate *template.Template 20 | LinkFeed func(brdname string) (string, error) 21 | LinkArticle func(brdname, filename string) (string, error) 22 | } 23 | 24 | func (c *Converter) Convert(board pttbbs.Board, posts []*PostEntry) (*atom.Feed, error) { 25 | var title bytes.Buffer 26 | if err := c.FeedTitleTemplate.Execute(&title, board); err != nil { 27 | return nil, err 28 | } 29 | 30 | feedURL, err := c.LinkFeed(board.BrdName) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | var entries []*atom.Entry 36 | // Reverse (time) order. 37 | for i := len(posts) - 1; i >= 0; i-- { 38 | entry, err := c.convertArticle(posts[i], board.BrdName) 39 | if err != nil { 40 | // Ignore errors. 41 | continue 42 | } 43 | entries = append(entries, entry) 44 | } 45 | 46 | return &atom.Feed{ 47 | Title: title.String(), 48 | ID: feedURL, 49 | Link: []atom.Link{{ 50 | Rel: "self", 51 | Href: feedURL, 52 | }}, 53 | Updated: atom.Time(firstArticleTimeOrNow(posts)), 54 | Entry: entries, 55 | }, nil 56 | } 57 | 58 | func (c *Converter) convertArticle(p *PostEntry, brdname string) (*atom.Entry, error) { 59 | a := p.Article 60 | articleURL, err := c.LinkArticle(brdname, a.FileName) 61 | if err != nil { 62 | return nil, err 63 | } 64 | // Will use a zero time if unable to parse. 65 | published, _ := pttbbs.ParseFileNameTime(a.FileName) 66 | return &atom.Entry{ 67 | Author: &atom.Person{ 68 | Name: a.Owner, 69 | }, 70 | Title: a.Title, 71 | ID: articleURL, 72 | Link: []atom.Link{{ 73 | Rel: "alternate", 74 | Type: "text/html", 75 | Href: articleURL, 76 | }}, 77 | Published: atom.Time(published), 78 | Updated: atom.Time(a.Modified), 79 | Content: &atom.Text{ 80 | Type: "html", 81 | Body: fmt.Sprintf("
%v
", p.Snippet), 82 | }, 83 | }, nil 84 | } 85 | 86 | func firstArticleTimeOrNow(posts []*PostEntry) time.Time { 87 | for _, p := range posts { 88 | if t, err := pttbbs.ParseFileNameTime(p.Article.FileName); err == nil { 89 | return t 90 | } 91 | } 92 | return time.Now() 93 | } 94 | -------------------------------------------------------------------------------- /cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "sync" 7 | "time" 8 | 9 | "github.com/bradfitz/gomemcache/memcache" 10 | "github.com/ptt/pttweb/gate" 11 | ) 12 | 13 | const ( 14 | // Request and connect timeout 15 | DefaultTimeout = time.Second * 30 16 | ) 17 | 18 | var ( 19 | ErrTooBusy = errors.New("conn pool too busy") 20 | ) 21 | 22 | type Key interface { 23 | String() string 24 | } 25 | 26 | type NewableFromBytes interface { 27 | NewFromBytes(data []byte) (Cacheable, error) 28 | } 29 | 30 | type Cacheable interface { 31 | NewableFromBytes 32 | EncodeToBytes() ([]byte, error) 33 | } 34 | 35 | type GenerateFunc func(key Key) (Cacheable, error) 36 | 37 | type result struct { 38 | Obj Cacheable 39 | Err error 40 | } 41 | 42 | type resultChan chan result 43 | 44 | type CacheManager struct { 45 | server string 46 | mc *memcache.Client 47 | gate *gate.Gate 48 | 49 | mu sync.Mutex 50 | pending map[string][]resultChan 51 | } 52 | 53 | func NewCacheManager(server string, maxOpen int) *CacheManager { 54 | mc := memcache.New(server) 55 | mc.Timeout = DefaultTimeout 56 | mc.MaxIdleConns = maxOpen 57 | 58 | return &CacheManager{ 59 | server: server, 60 | mc: mc, 61 | gate: gate.New(maxOpen, maxOpen), 62 | pending: make(map[string][]resultChan), 63 | } 64 | } 65 | 66 | func (m *CacheManager) Get(key Key, tp NewableFromBytes, expire time.Duration, generate GenerateFunc) (Cacheable, error) { 67 | keyString := key.String() 68 | 69 | // Check if can be served from cache 70 | if data, err := m.getFromCache(keyString); err != nil { 71 | if err != memcache.ErrCacheMiss { 72 | log.Printf("getFromCache: key: %q, err: %v", keyString, err) 73 | } 74 | } else if data != nil { 75 | return tp.NewFromBytes(data) 76 | } 77 | 78 | ch := make(chan result) 79 | 80 | // No luck. Check if anyone is generating 81 | if first := m.putPendings(keyString, ch); first { 82 | // We are the one responsible for generating the result 83 | go m.doGenerate(key, keyString, expire, generate) 84 | } 85 | 86 | result := <-ch 87 | return result.Obj, result.Err 88 | } 89 | 90 | func (m *CacheManager) doGenerate(key Key, keyString string, expire time.Duration, generate GenerateFunc) { 91 | obj, err := generate(key) 92 | if err == nil { 93 | // There is no errors during generating, store result in cache 94 | if data, err := obj.EncodeToBytes(); err != nil { 95 | log.Printf("obj.EncodeToBytes: key: %q, err: %v", keyString, err) 96 | } else if err = m.storeResultCache(keyString, data, expire); err != nil { 97 | log.Printf("storeResultCache: key: %q, err: %v", keyString, err) 98 | } 99 | } 100 | 101 | // Respond to all audience 102 | result := result{ 103 | Obj: obj, 104 | Err: err, 105 | } 106 | for _, c := range m.removePendings(keyString) { 107 | c <- result 108 | } 109 | } 110 | 111 | func (m *CacheManager) putPendings(key string, ch resultChan) (first bool) { 112 | m.mu.Lock() 113 | defer m.mu.Unlock() 114 | 115 | if _, ok := m.pending[key]; !ok { 116 | first = true 117 | m.pending[key] = make([]resultChan, 0, 1) 118 | } 119 | m.pending[key] = append(m.pending[key], ch) 120 | return 121 | } 122 | 123 | func (m *CacheManager) removePendings(key string) []resultChan { 124 | m.mu.Lock() 125 | defer m.mu.Unlock() 126 | 127 | pendings := m.pending[key] 128 | delete(m.pending, key) 129 | return pendings 130 | } 131 | 132 | func (m *CacheManager) getFromCache(key string) ([]byte, error) { 133 | rsv, ok := m.gate.Reserve() 134 | if !ok { 135 | return nil, ErrTooBusy 136 | } 137 | rsv.Wait() 138 | defer rsv.Release() 139 | 140 | res, err := m.mc.Get(key) 141 | if err != nil { 142 | return nil, err 143 | } 144 | return res.Value, nil 145 | } 146 | 147 | func (m *CacheManager) storeResultCache(key string, data []byte, expire time.Duration) error { 148 | rsv, ok := m.gate.Reserve() 149 | if !ok { 150 | return ErrTooBusy 151 | } 152 | rsv.Wait() 153 | defer rsv.Release() 154 | 155 | return m.mc.Set(&memcache.Item{ 156 | Key: key, 157 | Value: data, 158 | Flags: uint32(0), 159 | Expiration: int32(expire.Seconds()), 160 | }) 161 | } 162 | -------------------------------------------------------------------------------- /cached_inproc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | "time" 7 | 8 | "github.com/ptt/pttweb/pttbbs" 9 | ) 10 | 11 | type brdCacheEntry struct { 12 | Board *pttbbs.Board 13 | Expire time.Time 14 | } 15 | 16 | var ( 17 | brdCache = make(map[string]*brdCacheEntry) 18 | brdCacheLk sync.Mutex 19 | ) 20 | 21 | const ( 22 | brdCacheExpire = time.Minute * 5 23 | ) 24 | 25 | func getBrdCache(brdname string) *pttbbs.Board { 26 | brdCacheLk.Lock() 27 | defer brdCacheLk.Unlock() 28 | 29 | brdname = strings.ToLower(brdname) 30 | entry := brdCache[brdname] 31 | if entry != nil { 32 | if time.Now().Before(entry.Expire) { 33 | return entry.Board 34 | } else { 35 | delete(brdCache, brdname) 36 | return nil 37 | } 38 | } 39 | return nil 40 | } 41 | 42 | func setBrdCache(brdname string, board *pttbbs.Board) { 43 | brdCacheLk.Lock() 44 | defer brdCacheLk.Unlock() 45 | 46 | brdCache[strings.ToLower(brdname)] = &brdCacheEntry{ 47 | Board: board, 48 | Expire: time.Now().Add(brdCacheExpire), 49 | } 50 | } 51 | 52 | func getBoardByNameCached(brdname string) (*pttbbs.Board, error) { 53 | if brd := getBrdCache(brdname); brd != nil { 54 | return brd, nil 55 | } 56 | 57 | board, err := pttbbs.OneBoard(ptt.GetBoards(pttbbs.BoardRefByName(brdname))) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | setBrdCache(brdname, &board) 63 | return &board, nil 64 | } 65 | -------------------------------------------------------------------------------- /cached_ops.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "errors" 7 | "fmt" 8 | "log" 9 | "net/url" 10 | "strconv" 11 | 12 | "github.com/ptt/pttweb/article" 13 | "github.com/ptt/pttweb/atomfeed" 14 | "github.com/ptt/pttweb/cache" 15 | "github.com/ptt/pttweb/extcache" 16 | "github.com/ptt/pttweb/pttbbs" 17 | 18 | "golang.org/x/net/context" 19 | ) 20 | 21 | const ( 22 | EntryPerPage = 20 23 | 24 | CtxKeyBoardname = `ContextBoardname` 25 | ) 26 | 27 | type BbsIndexRequest struct { 28 | Brd pttbbs.Board 29 | Page int 30 | } 31 | 32 | func (r *BbsIndexRequest) String() string { 33 | return fmt.Sprintf("pttweb:bbsindex/%v/%v", r.Brd.BrdName, r.Page) 34 | } 35 | 36 | func generateBbsIndex(key cache.Key) (cache.Cacheable, error) { 37 | r := key.(*BbsIndexRequest) 38 | page := r.Page 39 | 40 | bbsindex := &BbsIndex{ 41 | Board: r.Brd, 42 | IsValid: true, 43 | } 44 | 45 | // Handle paging 46 | paging := NewPaging(EntryPerPage, r.Brd.NumPosts) 47 | if page == 0 { 48 | page = paging.LastPageNo() 49 | paging.SetPageNo(page) 50 | } else if err := paging.SetPageNo(page); err != nil { 51 | return nil, NewNotFoundError(err) 52 | } 53 | 54 | // Fetch article list 55 | var err error 56 | bbsindex.Articles, err = ptt.GetArticleList(r.Brd.Ref(), paging.Cursor(), EntryPerPage) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | // Fetch bottoms when at last page 62 | if page == paging.LastPageNo() { 63 | bbsindex.Bottoms, err = ptt.GetBottomList(r.Brd.Ref()) 64 | if err != nil { 65 | return nil, err 66 | } 67 | } 68 | 69 | // Page links 70 | if u, err := router.Get("bbsindex").URLPath("brdname", r.Brd.BrdName); err == nil { 71 | bbsindex.LastPage = u.String() 72 | } 73 | pageLink := func(n int) string { 74 | u, err := router.Get("bbsindex_page").URLPath("brdname", r.Brd.BrdName, "page", strconv.Itoa(n)) 75 | if err != nil { 76 | return "" 77 | } 78 | return u.String() 79 | } 80 | bbsindex.FirstPage = pageLink(1) 81 | if page > 1 { 82 | bbsindex.PrevPage = pageLink(page - 1) 83 | } 84 | if page < paging.LastPageNo() { 85 | bbsindex.NextPage = pageLink(page + 1) 86 | } 87 | 88 | return bbsindex, nil 89 | } 90 | 91 | type BbsSearchRequest struct { 92 | Brd pttbbs.Board 93 | Page int 94 | Query string 95 | Preds []pttbbs.SearchPredicate 96 | } 97 | 98 | func (r *BbsSearchRequest) String() string { 99 | queryHash := sha256.Sum256([]byte(r.Query)) 100 | query := base64.URLEncoding.EncodeToString(queryHash[:]) 101 | return fmt.Sprintf("pttweb:bbssearch/%v/%v/%v", r.Brd.BrdName, r.Page, query) 102 | } 103 | 104 | func generateBbsSearch(key cache.Key) (cache.Cacheable, error) { 105 | r := key.(*BbsSearchRequest) 106 | page := r.Page 107 | if page == 0 { 108 | page = 1 109 | } 110 | offset := -EntryPerPage * page 111 | 112 | bbsindex := &BbsIndex{ 113 | Board: r.Brd, 114 | Query: r.Query, 115 | IsValid: true, 116 | } 117 | 118 | // Search articles 119 | articles, totalPosts, err := pttSearch.Search(r.Brd.Ref(), r.Preds, offset, EntryPerPage) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | // Handle paging 125 | paging := NewPaging(EntryPerPage, totalPosts) 126 | if lastPage := paging.LastPageNo(); page > lastPage { 127 | articles = nil 128 | bbsindex.IsValid = false 129 | } else if page == lastPage { 130 | // We may get extra entries for last page. 131 | n := totalPosts % EntryPerPage 132 | if n < len(articles) { 133 | articles = articles[:n] 134 | } 135 | } 136 | 137 | // Show the page in reverse order. 138 | for i, j := 0, len(articles)-1; i < j; i, j = i+1, j-1 { 139 | articles[i], articles[j] = articles[j], articles[i] 140 | } 141 | bbsindex.Articles = articles 142 | 143 | // Page links, in newest first order. 144 | pageLink := func(n int) string { 145 | u, err := router.Get("bbssearch").URLPath("brdname", r.Brd.BrdName) 146 | if err != nil { 147 | return "" 148 | } 149 | q := url.Values{} 150 | q.Set("q", r.Query) 151 | q.Set("page", strconv.Itoa(n)) 152 | u.RawQuery = q.Encode() 153 | return u.String() 154 | } 155 | bbsindex.FirstPage = pageLink(paging.LastPageNo()) 156 | bbsindex.LastPage = pageLink(1) 157 | if page > 1 { 158 | bbsindex.NextPage = pageLink(page - 1) 159 | } 160 | if page < paging.LastPageNo() { 161 | bbsindex.PrevPage = pageLink(page + 1) 162 | } 163 | 164 | return bbsindex, nil 165 | } 166 | 167 | type BoardAtomFeedRequest struct { 168 | Brd pttbbs.Board 169 | } 170 | 171 | func (r *BoardAtomFeedRequest) String() string { 172 | return fmt.Sprintf("pttweb:atomfeed/%v", r.Brd.BrdName) 173 | } 174 | 175 | func generateBoardAtomFeed(key cache.Key) (cache.Cacheable, error) { 176 | r := key.(*BoardAtomFeedRequest) 177 | 178 | if atomConverter == nil { 179 | return nil, errors.New("atom feed not configured") 180 | } 181 | 182 | // Fetch article list 183 | articles, err := ptt.GetArticleList(r.Brd.Ref(), -EntryPerPage, EntryPerPage) 184 | if err != nil { 185 | return nil, err 186 | } 187 | // Fetch snippets and contruct posts. 188 | var posts []*atomfeed.PostEntry 189 | for _, article := range articles { 190 | // Use an empty string when error. 191 | snippet, _ := getArticleSnippet(r.Brd, article.FileName) 192 | posts = append(posts, &atomfeed.PostEntry{ 193 | Article: article, 194 | Snippet: snippet, 195 | }) 196 | } 197 | 198 | feed, err := atomConverter.Convert(r.Brd, posts) 199 | if err != nil { 200 | log.Println("atomfeed: Convert:", err) 201 | // Don't return error but cache that it's invalid. 202 | } 203 | return &BoardAtomFeed{ 204 | Feed: feed, 205 | IsValid: err == nil, 206 | }, nil 207 | } 208 | 209 | const SnippetHeadSize = 16 * 1024 // Enough for 8 pages of 80x24. 210 | 211 | func getArticleSnippet(brd pttbbs.Board, filename string) (string, error) { 212 | p, err := ptt.GetArticleSelect(brd.Ref(), pttbbs.SelectHead, filename, "", 0, SnippetHeadSize) 213 | if err != nil { 214 | return "", err 215 | } 216 | if len(p.Content) == 0 { 217 | return "", pttbbs.ErrNotFound 218 | } 219 | 220 | ra, err := article.Render(article.WithContent(p.Content)) 221 | if err != nil { 222 | return "", err 223 | } 224 | return ra.PreviewContent(), nil 225 | } 226 | 227 | const ( 228 | TruncateSize = 1048576 229 | TruncateMaxScan = 1024 230 | 231 | HeadSize = 100 * 1024 232 | TailSize = 50 * 1024 233 | ) 234 | 235 | type ArticleRequest struct { 236 | Namespace string 237 | Brd pttbbs.Board 238 | Filename string 239 | Select func(m pttbbs.SelectMethod, offset, maxlen int) (*pttbbs.ArticlePart, error) 240 | } 241 | 242 | func (r *ArticleRequest) String() string { 243 | return fmt.Sprintf("pttweb:%v/%v/%v", r.Namespace, r.Brd.BrdName, r.Filename) 244 | } 245 | 246 | func (r *ArticleRequest) Boardname() string { 247 | return r.Brd.BrdName 248 | } 249 | 250 | func generateArticle(key cache.Key) (cache.Cacheable, error) { 251 | r := key.(*ArticleRequest) 252 | ctx := context.TODO() 253 | ctx = context.WithValue(ctx, CtxKeyBoardname, r) 254 | if config.Experiments.ExtCache.Enabled(fastStrHash64(r.Filename)) { 255 | ctx = extcache.WithExtCache(ctx, extCache) 256 | } 257 | 258 | p, err := r.Select(pttbbs.SelectHead, 0, HeadSize) 259 | if err != nil { 260 | return nil, err 261 | } 262 | 263 | // We don't want head and tail have duplicate content 264 | if p.FileSize > HeadSize && p.FileSize <= HeadSize+TailSize { 265 | p, err = r.Select(pttbbs.SelectPart, 0, p.FileSize) 266 | if err != nil { 267 | return nil, err 268 | } 269 | } 270 | 271 | if len(p.Content) == 0 { 272 | return nil, pttbbs.ErrNotFound 273 | } 274 | 275 | a := new(Article) 276 | 277 | a.IsPartial = p.Length < p.FileSize 278 | a.IsTruncated = a.IsPartial 279 | 280 | if a.IsPartial { 281 | // Get and render tail 282 | ptail, err := r.Select(pttbbs.SelectTail, -TailSize, TailSize) 283 | if err != nil { 284 | return nil, err 285 | } 286 | if len(ptail.Content) > 0 { 287 | ra, err := article.Render( 288 | article.WithContent(ptail.Content), 289 | article.WithContext(ctx), 290 | article.WithDisableArticleHeader(), 291 | ) 292 | if err != nil { 293 | return nil, err 294 | } 295 | a.ContentTailHtml = ra.HTML() 296 | } 297 | a.CacheKey = ptail.CacheKey 298 | a.NextOffset = ptail.FileSize - TailSize + ptail.Offset + ptail.Length 299 | } else { 300 | a.CacheKey = p.CacheKey 301 | a.NextOffset = p.Length 302 | } 303 | 304 | ra, err := article.Render( 305 | article.WithContent(p.Content), 306 | article.WithContext(ctx), 307 | ) 308 | if err != nil { 309 | return nil, err 310 | } 311 | a.ParsedTitle = ra.ParsedTitle() 312 | a.PreviewContent = ra.PreviewContent() 313 | a.ContentHtml = ra.HTML() 314 | a.IsValid = true 315 | return a, nil 316 | } 317 | 318 | type ArticlePartRequest struct { 319 | Brd pttbbs.Board 320 | Filename string 321 | CacheKey string 322 | Offset int 323 | } 324 | 325 | func (r *ArticlePartRequest) String() string { 326 | return fmt.Sprintf("pttweb:bbs/%v/%v#%v,%v", r.Brd.BrdName, r.Filename, r.CacheKey, r.Offset) 327 | } 328 | 329 | func (r *ArticlePartRequest) Boardname() string { 330 | return r.Brd.BrdName 331 | } 332 | 333 | func generateArticlePart(key cache.Key) (cache.Cacheable, error) { 334 | r := key.(*ArticlePartRequest) 335 | ctx := context.TODO() 336 | ctx = context.WithValue(ctx, CtxKeyBoardname, r) 337 | if config.Experiments.ExtCache.Enabled(fastStrHash64(r.Filename)) { 338 | ctx = extcache.WithExtCache(ctx, extCache) 339 | } 340 | 341 | p, err := ptt.GetArticleSelect(r.Brd.Ref(), pttbbs.SelectHead, r.Filename, r.CacheKey, r.Offset, -1) 342 | if err == pttbbs.ErrNotFound { 343 | // Returns an invalid result 344 | return new(ArticlePart), nil 345 | } 346 | if err != nil { 347 | return nil, err 348 | } 349 | 350 | ap := new(ArticlePart) 351 | ap.IsValid = true 352 | ap.CacheKey = p.CacheKey 353 | ap.NextOffset = r.Offset + p.Offset + p.Length 354 | 355 | if len(p.Content) > 0 { 356 | ra, err := article.Render( 357 | article.WithContent(p.Content), 358 | article.WithContext(ctx), 359 | article.WithDisableArticleHeader(), 360 | ) 361 | if err != nil { 362 | return nil, err 363 | } 364 | ap.ContentHtml = string(ra.HTML()) 365 | } 366 | 367 | return ap, nil 368 | } 369 | 370 | func truncateLargeContent(content []byte, size, maxScan int) []byte { 371 | if len(content) <= size { 372 | return content 373 | } 374 | for i := size - 1; i >= size-maxScan && i >= 0; i-- { 375 | if content[i] == '\n' { 376 | return content[:i+1] 377 | } 378 | } 379 | return content[:size] 380 | } 381 | -------------------------------------------------------------------------------- /captcha/config.go: -------------------------------------------------------------------------------- 1 | package captcha 2 | 3 | type Config struct { 4 | Enabled bool 5 | InsertSecret string 6 | ExpireSecs int 7 | Recaptcha RecaptchaConfig 8 | Redis RedisConfig 9 | } 10 | 11 | type RecaptchaConfig struct { 12 | SiteKey string 13 | Secret string 14 | } 15 | 16 | // See https://godoc.org/github.com/go-redis/redis#Options 17 | type RedisConfig struct { 18 | Network string 19 | Addr string 20 | Password string 21 | DB int 22 | } 23 | -------------------------------------------------------------------------------- /captcha/handler.go: -------------------------------------------------------------------------------- 1 | package captcha 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/go-redis/redis" 12 | "github.com/gorilla/mux" 13 | "github.com/ptt/pttweb/page" 14 | "github.com/rvelhote/go-recaptcha" 15 | ) 16 | 17 | const ( 18 | RecaptchaURL = `https://www.google.com/recaptcha/api/siteverify` 19 | GRecaptchaResponse = `g-recaptcha-response` 20 | CaptchaHandle = `handle` 21 | 22 | MaxCaptchaHandleLength = 80 23 | ) 24 | 25 | var ( 26 | ErrCaptchaHandleNotFound = &CaptchaErr{ 27 | error: errors.New("captcha handle not found"), 28 | SetCaptchaPage: func(p *page.Captcha) { p.CaptchaErr.IsNotFound = true }, 29 | } 30 | ErrCaptchaVerifyFailed = &CaptchaErr{ 31 | error: errors.New("captcha verfication failed"), 32 | SetCaptchaPage: func(p *page.Captcha) { p.CaptchaErr.IsVerifyFailed = true }, 33 | } 34 | ) 35 | 36 | type CaptchaErr struct { 37 | error 38 | SetCaptchaPage func(p *page.Captcha) 39 | } 40 | 41 | type Handler struct { 42 | config *Config 43 | router *mux.Router 44 | redisClient *redis.Client 45 | } 46 | 47 | func Install(cfg *Config, r *mux.Router) error { 48 | redisClient := redis.NewClient(&redis.Options{ 49 | Network: cfg.Redis.Network, 50 | Addr: cfg.Redis.Addr, 51 | Password: cfg.Redis.Password, 52 | DB: cfg.Redis.DB, 53 | }) 54 | h := &Handler{ 55 | config: cfg, 56 | router: r, 57 | redisClient: redisClient, 58 | } 59 | h.installRoutes(r) 60 | return nil 61 | } 62 | 63 | func (h *Handler) installRoutes(r *mux.Router) { 64 | r.Path(`/captcha`). 65 | Handler(page.ErrorWrapper(h.handleCaptcha)). 66 | Name("captcha") 67 | 68 | r.Path(`/captcha/insert`). 69 | Handler(page.ErrorWrapper(h.handleCaptchaInsert)). 70 | Name("captcha_insert") 71 | } 72 | 73 | func (h *Handler) handleCaptcha(ctx page.Context, w http.ResponseWriter) error { 74 | p, err := h.handleCaptchaInternal(ctx, w) 75 | if err != nil { 76 | return err 77 | } 78 | return page.ExecutePage(w, p) 79 | } 80 | 81 | func (h *Handler) handleCaptchaInternal(ctx page.Context, w http.ResponseWriter) (*page.Captcha, error) { 82 | p := &page.Captcha{ 83 | Handle: ctx.Request().FormValue(CaptchaHandle), 84 | RecaptchaSiteKey: h.config.Recaptcha.SiteKey, 85 | } 86 | if u, err := h.router.Get("captcha").URLPath(); err != nil { 87 | return nil, err 88 | } else { 89 | q := make(url.Values) 90 | q.Set(CaptchaHandle, ctx.Request().FormValue(CaptchaHandle)) 91 | p.PostAction = u.String() + "?" + q.Encode() 92 | } 93 | // Check if the handle is valid. 94 | if _, err := h.fetchVerificationKey(p.Handle); err != nil { 95 | return translateCaptchaErr(p, err) 96 | } 97 | if response := ctx.Request().PostFormValue(GRecaptchaResponse); response != "" { 98 | r := recaptcha.Recaptcha{ 99 | PrivateKey: h.config.Recaptcha.Secret, 100 | URL: RecaptchaURL, 101 | } 102 | verifyResp, errs := r.Verify(response, "") 103 | if len(errs) != 0 || !verifyResp.Success { 104 | p.InternalErrMessage = fmt.Sprintf("%v", errs) 105 | return translateCaptchaErr(p, ErrCaptchaVerifyFailed) 106 | } else if verifyResp.Success { 107 | var err error 108 | p.VerificationKey, err = h.fetchVerificationKey(p.Handle) 109 | if err != nil { 110 | return translateCaptchaErr(p, err) 111 | } 112 | } 113 | } 114 | return p, nil 115 | } 116 | 117 | func translateCaptchaErr(p *page.Captcha, err error) (*page.Captcha, error) { 118 | if ce, ok := err.(*CaptchaErr); ok && ce.SetCaptchaPage != nil { 119 | ce.SetCaptchaPage(p) 120 | if p.InternalErrMessage == "" { 121 | p.InternalErrMessage = fmt.Sprintf("%v", err) 122 | } 123 | return p, nil 124 | } 125 | return p, err 126 | } 127 | 128 | func (h *Handler) fetchVerificationKey(handle string) (string, error) { 129 | if len(handle) > MaxCaptchaHandleLength { 130 | return "", ErrCaptchaHandleNotFound 131 | } 132 | data, err := h.redisClient.Get(handle).Result() 133 | if err == redis.Nil { 134 | return "", ErrCaptchaHandleNotFound 135 | } else if err != nil { 136 | return "", err 137 | } 138 | e, err := decodeCaptchaEntry(data) 139 | if err != nil { 140 | return "", err 141 | } 142 | return e.Verify, nil 143 | } 144 | 145 | func (h *Handler) handleCaptchaInsert(ctx page.Context, w http.ResponseWriter) error { 146 | req := ctx.Request() 147 | secret := req.FormValue("secret") 148 | handle := req.FormValue("handle") 149 | verify := req.FormValue("verify") 150 | if secret == "" || handle == "" || verify == "" || len(handle) > MaxCaptchaHandleLength { 151 | w.WriteHeader(http.StatusBadRequest) 152 | return nil 153 | } 154 | if secret != h.config.InsertSecret { 155 | w.WriteHeader(http.StatusForbidden) 156 | return nil 157 | } 158 | data, err := encodeCaptchaEntry(&CaptchaEntry{ 159 | Handle: handle, 160 | Verify: verify, 161 | }) 162 | if err != nil { 163 | return err 164 | } 165 | expire := time.Duration(h.config.ExpireSecs) * time.Second 166 | r, err := h.redisClient.SetNX(handle, data, expire).Result() 167 | if err != nil { 168 | return err 169 | } 170 | if !r { 171 | w.WriteHeader(http.StatusConflict) 172 | return nil 173 | } 174 | return nil 175 | } 176 | 177 | type CaptchaEntry struct { 178 | Handle string `json:"h"` 179 | Verify string `json:"v"` 180 | } 181 | 182 | func encodeCaptchaEntry(e *CaptchaEntry) (string, error) { 183 | buf, err := json.Marshal(e) 184 | if err != nil { 185 | return "", err 186 | } 187 | return string(buf), nil 188 | } 189 | 190 | func decodeCaptchaEntry(data string) (*CaptchaEntry, error) { 191 | var e CaptchaEntry 192 | if err := json.Unmarshal([]byte(data), &e); err != nil { 193 | return nil, err 194 | } 195 | return &e, nil 196 | } 197 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/ptt/pttweb/captcha" 7 | "github.com/ptt/pttweb/experiment" 8 | "github.com/ptt/pttweb/extcache" 9 | ) 10 | 11 | type PttwebConfig struct { 12 | Bind []string 13 | BoarddAddress string 14 | SearchAddress string 15 | MandAddress string 16 | MemcachedAddress string 17 | TemplateDirectory string 18 | StaticPrefix string 19 | SitePrefix string 20 | 21 | MemcachedMaxConn int 22 | 23 | GAAccount string 24 | GADomain string 25 | 26 | EnableOver18Cookie bool 27 | 28 | // EnableLinkOriginalInAllPost indicates whether to parse the board name in 29 | // ALLPOST board and link to original posts. 30 | EnableLinkOriginalInAllPost bool 31 | 32 | FeedPrefix string 33 | AtomFeedTitleTemplate string 34 | 35 | EnablePushStream bool 36 | PushStreamSharedSecret string 37 | PushStreamSubscribeLocation string 38 | 39 | RecaptchaSiteKey string 40 | RecaptchaSecret string 41 | CaptchaInsertSecret string 42 | CaptchaExpireSecs int 43 | CaptchaRedisConfig *captcha.RedisConfig 44 | 45 | ExtCacheConfig extcache.Config 46 | 47 | Experiments Experiments 48 | } 49 | 50 | type Experiments struct { 51 | ExtCache experiment.OptIn 52 | } 53 | 54 | const ( 55 | DefaultBoarddMaxConn = 16 56 | DefaultMemcachedMaxConn = 16 57 | ) 58 | 59 | func (c *PttwebConfig) CheckAndFillDefaults() error { 60 | if c.BoarddAddress == "" { 61 | return errors.New("boardd address not specified") 62 | } 63 | 64 | if c.MemcachedAddress == "" { 65 | return errors.New("memcached address not specified") 66 | } 67 | 68 | if c.MemcachedMaxConn <= 0 { 69 | c.MemcachedMaxConn = DefaultMemcachedMaxConn 70 | } 71 | 72 | return nil 73 | } 74 | 75 | func (c *PttwebConfig) captchaConfig() *captcha.Config { 76 | enabled := c.RecaptchaSiteKey != "" && c.RecaptchaSecret != "" && c.CaptchaRedisConfig != nil 77 | return &captcha.Config{ 78 | Enabled: enabled, 79 | InsertSecret: c.CaptchaInsertSecret, 80 | ExpireSecs: c.CaptchaExpireSecs, 81 | Recaptcha: captcha.RecaptchaConfig{ 82 | SiteKey: c.RecaptchaSiteKey, 83 | Secret: c.RecaptchaSecret, 84 | }, 85 | Redis: *c.CaptchaRedisConfig, 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /context.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | type Context struct { 8 | R *http.Request 9 | 10 | skipOver18 bool 11 | hasOver18Cookie bool 12 | isCrawler bool 13 | } 14 | 15 | func (c *Context) Request() *http.Request { 16 | return c.R 17 | } 18 | 19 | func (c *Context) MergeFromRequest(r *http.Request) error { 20 | c.R = r 21 | c.hasOver18Cookie = checkOver18Cookie(r) 22 | c.isCrawler = isCrawlerUserAgent(r) 23 | return nil 24 | } 25 | 26 | func (c *Context) SetSkipOver18() { 27 | c.skipOver18 = true 28 | } 29 | 30 | func (c *Context) IsOver18() bool { 31 | return c.hasOver18Cookie || c.skipOver18 32 | } 33 | 34 | func (c *Context) IsCrawler() bool { 35 | return c.isCrawler 36 | } 37 | -------------------------------------------------------------------------------- /cookie.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | const ( 8 | Over18CookieName = "over18" 9 | ) 10 | 11 | func checkOver18Cookie(r *http.Request) bool { 12 | if cookie, err := r.Cookie(Over18CookieName); err == nil { 13 | return cookie.Value != "" 14 | } 15 | return false 16 | } 17 | 18 | func setOver18Cookie(w http.ResponseWriter) { 19 | http.SetCookie(w, &http.Cookie{ 20 | Name: Over18CookieName, 21 | Value: "1", 22 | Path: "/", 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /experiment/struct.go: -------------------------------------------------------------------------------- 1 | package experiment 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | ) 7 | 8 | const percentBase = uint64(10000) 9 | 10 | type Percent struct { 11 | threshold uint64 12 | } 13 | 14 | func (p *Percent) UnmarshalJSON(b []byte) error { 15 | var pct float64 16 | if err := json.Unmarshal(b, &pct); err != nil { 17 | return err 18 | } 19 | if pct < 0 { 20 | return fmt.Errorf("percent is negative: %v", pct) 21 | } 22 | p.threshold = uint64(pct * float64(percentBase) / 100) 23 | return nil 24 | } 25 | 26 | type OptIn struct { 27 | OptIn Percent 28 | } 29 | 30 | func (o *OptIn) Enabled(val uint64) bool { 31 | return val%percentBase < o.OptIn.threshold 32 | } 33 | -------------------------------------------------------------------------------- /extcache/extcache.go: -------------------------------------------------------------------------------- 1 | package extcache 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" 6 | "encoding/base64" 7 | "hash/fnv" 8 | "net/url" 9 | "strconv" 10 | "time" 11 | ) 12 | 13 | type ctxExtCacheKey struct{} 14 | 15 | var ctxKey = (*ctxExtCacheKey)(nil) 16 | 17 | func WithExtCache(ctx context.Context, cache ExtCache) context.Context { 18 | if cache == nil { 19 | return ctx 20 | } 21 | return context.WithValue(ctx, ctxKey, cache) 22 | } 23 | 24 | func FromContext(ctx context.Context) (ExtCache, bool) { 25 | c, ok := ctx.Value(ctxKey).(ExtCache) 26 | return c, ok 27 | } 28 | 29 | type ExtCache interface { 30 | Generate(urlStr string) (string, error) 31 | } 32 | 33 | type Config struct { 34 | Enabled bool 35 | Prefix string 36 | HashPrefix string 37 | Secret string 38 | Expires int 39 | } 40 | 41 | type extCache struct { 42 | cfg Config 43 | } 44 | 45 | func New(cfg Config) ExtCache { 46 | if !cfg.Enabled { 47 | return nil 48 | } 49 | return &extCache{cfg: cfg} 50 | } 51 | 52 | func (c *extCache) Generate(urlStr string) (string, error) { 53 | u, err := url.Parse(urlStr) 54 | if err != nil { 55 | return "", err 56 | } 57 | 58 | uri := "/" + u.Scheme + "/" + u.Host + u.Path 59 | expire := snapExpire(uri, time.Now().Unix(), int64(c.cfg.Expires)) 60 | expireStr := strconv.FormatInt(expire, 10) 61 | h := md5.Sum([]byte(expireStr + c.cfg.HashPrefix + uri + c.cfg.Secret)) 62 | sig := base64.RawURLEncoding.EncodeToString(h[:]) 63 | 64 | q := make(url.Values) 65 | q.Set("e", expireStr) 66 | q.Set("s", sig) 67 | 68 | return c.cfg.Prefix + uri + "?" + q.Encode(), nil 69 | } 70 | 71 | func snapExpire(key string, now, min int64) int64 { 72 | h := fnv.New32() 73 | _, _ = h.Write([]byte(key)) 74 | expire := (now+min+0xFFFF)&^0xFFFF + int64(h.Sum32()&0xFFFF) 75 | return expire 76 | } 77 | -------------------------------------------------------------------------------- /gate/gate.go: -------------------------------------------------------------------------------- 1 | // Package gate provides building blocks to limit concurrency. 2 | package gate 3 | 4 | import ( 5 | "sync" 6 | ) 7 | 8 | // Gate provides concurrency limitation. 9 | type Gate struct { 10 | maxInflight int 11 | maxWait int 12 | 13 | mu sync.Mutex 14 | num int 15 | queue chan *Reservation 16 | } 17 | 18 | // New creates a Gate. 19 | func New(maxInflight, maxWait int) *Gate { 20 | return &Gate{ 21 | maxInflight: maxInflight, 22 | maxWait: maxWait, 23 | queue: make(chan *Reservation, maxWait), 24 | } 25 | } 26 | 27 | // Reserve attempts to obtain a reservation. If the wait queue is too long, it 28 | // returns false. 29 | func (g *Gate) Reserve() (*Reservation, bool) { 30 | g.mu.Lock() 31 | defer g.mu.Unlock() 32 | 33 | if g.num >= g.maxInflight+g.maxWait { 34 | return nil, false 35 | } 36 | g.num++ 37 | 38 | // Grant immediately. 39 | if g.num <= g.maxInflight { 40 | return &Reservation{ 41 | g: g, 42 | }, true 43 | } 44 | 45 | // Grant later. 46 | r := &Reservation{ 47 | g: g, 48 | granted: make(chan struct{}), 49 | } 50 | g.queue <- r 51 | return r, true 52 | } 53 | 54 | func (g *Gate) release() { 55 | g.mu.Lock() 56 | defer g.mu.Unlock() 57 | 58 | g.num-- 59 | 60 | select { 61 | case r := <-g.queue: 62 | r.grant() 63 | default: 64 | } 65 | } 66 | 67 | // Reservation represents a reservation. 68 | type Reservation struct { 69 | g *Gate 70 | granted chan struct{} 71 | } 72 | 73 | // Wait blocks until number of inflight requests is lower than the maximum. 74 | func (r *Reservation) Wait() { 75 | if r.granted != nil { 76 | <-r.granted 77 | } 78 | } 79 | 80 | // Release returns the reservation. It must be called when the reservation is 81 | // no longer needed. It doesn't matter if Wait() was called or not. 82 | func (r *Reservation) Release() { 83 | r.g.release() 84 | } 85 | 86 | func (r *Reservation) grant() { 87 | close(r.granted) 88 | } 89 | 90 | func (r *Reservation) isGranted() bool { 91 | select { 92 | case <-r.granted: 93 | return true 94 | default: 95 | return r.granted == nil 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /gate/gate_test.go: -------------------------------------------------------------------------------- 1 | package gate 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | "sync/atomic" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestGate(t *testing.T) { 12 | g := New(4, 2) 13 | 14 | var rs [6]*Reservation 15 | for i := range rs { 16 | var ok bool 17 | rs[i], ok = g.Reserve() 18 | if !ok { 19 | t.Fatalf("#%v Reserve() = _, %v; want _, true", i, ok) 20 | } 21 | 22 | // First 4 reservation should be in already-granted state. 23 | granted := i < 4 24 | if got := rs[i].isGranted(); got != granted { 25 | t.Errorf("#%v isGranted() = %v, want %v", i, got, granted) 26 | } 27 | } 28 | 29 | // Reserved at maximum. Should start failing now. 30 | for i := len(rs); i < len(rs)*2; i++ { 31 | if _, ok := g.Reserve(); ok { 32 | t.Fatalf("#%v Reserve() = _, %v; want _, false", i, ok) 33 | } 34 | } 35 | 36 | // Wait should return immediately for the first 4 already granted 37 | // reservations. 38 | rs[0].Wait() 39 | rs[1].Wait() 40 | rs[2].Wait() 41 | rs[3].Wait() 42 | 43 | // Release 1 and Wait 1. 44 | rs[0].Release() 45 | rs[4].Wait() 46 | 47 | // Should be able to reserve another one now. 48 | if r, ok := g.Reserve(); !ok { 49 | t.Errorf("Reserve() = _, %v; want _, true", ok) 50 | } else { 51 | defer r.Release() 52 | if r.isGranted() { 53 | t.Errorf("isGranted() = true, want false") 54 | } 55 | } 56 | 57 | // Release 1 and Wait 1. 58 | rs[1].Release() 59 | rs[5].Wait() 60 | 61 | // Release the rest of reservations. 62 | for i := 2; i < len(rs); i++ { 63 | rs[i].Release() 64 | } 65 | } 66 | 67 | func concurrentWorker(t *testing.T, wg *sync.WaitGroup, g *Gate, num *int32, max int32) { 68 | defer wg.Done() 69 | for j := 0; j < 1000; j++ { 70 | us := rand.Intn(200) 71 | r, ok := g.Reserve() 72 | if !ok { 73 | time.Sleep(time.Duration(us) * time.Microsecond) 74 | j-- 75 | continue 76 | } 77 | 78 | r.Wait() 79 | if atomic.AddInt32(num, 1) > max { 80 | t.Fatal("num > max") 81 | } 82 | time.Sleep(time.Duration(us) * time.Microsecond) 83 | atomic.AddInt32(num, -1) 84 | r.Release() 85 | } 86 | } 87 | 88 | func TestConcurrent(t *testing.T) { 89 | const max = 10 90 | var num int32 91 | var wg sync.WaitGroup 92 | defer wg.Wait() 93 | 94 | g := New(max, max) 95 | for i := 0; i < 32; i++ { 96 | wg.Add(1) 97 | go concurrentWorker(t, &wg, g, &num, max) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ptt/pttweb 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b 7 | github.com/go-redis/redis v6.15.9+incompatible 8 | github.com/golang/protobuf v1.4.2 9 | github.com/gorilla/mux v1.8.0 10 | github.com/onsi/ginkgo v1.14.1 // indirect 11 | github.com/onsi/gomega v1.10.2 // indirect 12 | github.com/rvelhote/go-recaptcha v0.0.0-20170215232712-e143c6ea64e5 13 | golang.org/x/net v0.0.0-20200904194848-62affa334b73 14 | golang.org/x/tools v0.0.0-20200904185747-39188db58858 15 | google.golang.org/grpc v1.31.1 16 | google.golang.org/protobuf v1.23.0 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= 4 | github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= 5 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 6 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 7 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 8 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 9 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 10 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 11 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 12 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 13 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 14 | github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= 15 | github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 16 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 17 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 18 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 19 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 20 | github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= 21 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 22 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 23 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 24 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 25 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 26 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 27 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 28 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 29 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 30 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 31 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 32 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 33 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 34 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 35 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 36 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 37 | github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= 38 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 39 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 40 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 41 | github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= 42 | github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= 43 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 44 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 45 | github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= 46 | github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 47 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 48 | github.com/rvelhote/go-recaptcha v0.0.0-20170215232712-e143c6ea64e5 h1:Fl1oDyVOTLhv1+3C6G0X+FFGlIR1YqO699Fk+lbTw7g= 49 | github.com/rvelhote/go-recaptcha v0.0.0-20170215232712-e143c6ea64e5/go.mod h1:ksHT5JeCauMSS9Npj9jXZERdy+qpxDellQAA54dCYbY= 50 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 51 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 52 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 53 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 54 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 55 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 56 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 57 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 58 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 59 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 60 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 61 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 62 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 63 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 64 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 65 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 66 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 67 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 68 | golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= 69 | golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 70 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 71 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 72 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 73 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 74 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 75 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 76 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 77 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 78 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 79 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 80 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 81 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 82 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= 83 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 84 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= 85 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 86 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 87 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 88 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 89 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 90 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 91 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 92 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 93 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 94 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 95 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 96 | golang.org/x/tools v0.0.0-20200904185747-39188db58858 h1:xLt+iB5ksWcZVxqc+g9K41ZHy+6MKWfXCDsjSThnsPA= 97 | golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= 98 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 99 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 100 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 101 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 102 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 103 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 104 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 105 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 106 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= 107 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 108 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 109 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 110 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 111 | google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs= 112 | google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 113 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 114 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 115 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 116 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 117 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 118 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= 119 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 120 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 121 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 122 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 123 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 124 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 125 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 126 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 127 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 128 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 129 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 130 | -------------------------------------------------------------------------------- /page/ajax.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | ) 7 | 8 | type ArticlePollResp struct { 9 | ContentHtml string `json:"contentHtml"` 10 | PollUrl string `json:"pollUrl"` 11 | Success bool `json:"success"` 12 | } 13 | 14 | func WriteAjaxResp(w http.ResponseWriter, obj interface{}) error { 15 | w.Header().Set("Content-Type", "application/json") 16 | return json.NewEncoder(w).Encode(obj) 17 | } 18 | -------------------------------------------------------------------------------- /page/context.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | type Context interface { 8 | Request() *http.Request 9 | } 10 | 11 | type context struct { 12 | req *http.Request 13 | } 14 | 15 | func newContext(req *http.Request) (*context, error) { 16 | return &context{ 17 | req: req, 18 | }, nil 19 | } 20 | 21 | func (c *context) Request() *http.Request { 22 | return c.req 23 | } 24 | -------------------------------------------------------------------------------- /page/names.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | const ( 4 | TnameError = `error.html` 5 | TnameNotFound = `notfound.html` 6 | TnameClasslist = `classlist.html` 7 | TnameBbsIndex = `bbsindex.html` 8 | TnameBbsArticle = `bbsarticle.html` 9 | TnameAskOver18 = `askover18.html` 10 | TnameManIndex = `manindex.html` 11 | TnameManArticle = `manarticle.html` 12 | TnameCaptcha = `captcha.html` 13 | 14 | TnameLayout = `layout.html` 15 | TnameCommon = `common.html` 16 | ) 17 | -------------------------------------------------------------------------------- /page/pages.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | import ( 4 | "html/template" 5 | "net/http" 6 | 7 | manpb "github.com/ptt/pttweb/proto/man" 8 | "github.com/ptt/pttweb/pttbbs" 9 | ) 10 | 11 | type Page interface { 12 | TemplateName() string 13 | } 14 | 15 | type NoContent struct{} 16 | 17 | func (NoContent) TemplateName() string { return "" } 18 | 19 | type Redirect struct { 20 | NoContent 21 | To string 22 | } 23 | 24 | func (p *Redirect) WriteHeaders(w http.ResponseWriter) error { 25 | w.Header().Set("Location", p.To) 26 | w.WriteHeader(http.StatusFound) 27 | return nil 28 | } 29 | 30 | func NewRedirect(to string) *Redirect { 31 | return &Redirect{ 32 | To: to, 33 | } 34 | } 35 | 36 | type NotFound struct{} 37 | 38 | func (NotFound) TemplateName() string { return TnameNotFound } 39 | 40 | func (p *NotFound) WriteHeaders(w http.ResponseWriter) error { 41 | w.WriteHeader(http.StatusNotFound) 42 | return nil 43 | } 44 | 45 | type Error struct { 46 | Title string 47 | ContentHtml string 48 | } 49 | 50 | func (Error) TemplateName() string { return TnameError } 51 | 52 | type AskOver18 struct { 53 | From string 54 | } 55 | 56 | func (AskOver18) TemplateName() string { return TnameAskOver18 } 57 | 58 | type Classlist struct { 59 | Boards []pttbbs.Board 60 | IsHotboardList bool 61 | } 62 | 63 | func (Classlist) TemplateName() string { return TnameClasslist } 64 | 65 | type BbsIndex struct { 66 | Board pttbbs.Board 67 | Query string 68 | 69 | FirstPage string 70 | PrevPage string 71 | NextPage string 72 | LastPage string 73 | 74 | Articles []pttbbs.Article 75 | Bottoms []pttbbs.Article 76 | 77 | IsValid bool 78 | } 79 | 80 | func (BbsIndex) TemplateName() string { return TnameBbsIndex } 81 | 82 | type BbsArticle struct { 83 | Title string 84 | Description string 85 | Board *pttbbs.Board 86 | FileName string 87 | Content template.HTML 88 | ContentTail template.HTML 89 | ContentTruncated bool 90 | PollUrl string 91 | LongPollUrl string 92 | CurrOffset int 93 | } 94 | 95 | func (BbsArticle) TemplateName() string { return TnameBbsArticle } 96 | 97 | type ManIndex struct { 98 | Board pttbbs.Board 99 | Path string 100 | Entries []*manpb.Entry 101 | } 102 | 103 | func (ManIndex) TemplateName() string { return TnameManIndex } 104 | 105 | type ManArticle struct { 106 | Title string 107 | Description string 108 | Board *pttbbs.Board 109 | Path string 110 | Content template.HTML 111 | ContentTail template.HTML 112 | ContentTruncated bool 113 | } 114 | 115 | func (ManArticle) TemplateName() string { return TnameManArticle } 116 | 117 | type Captcha struct { 118 | // Handle is the opaque handle for getting the verification key from 119 | // database. 120 | Handle string 121 | 122 | // VerificationKey will be non-empty if the page should display the 123 | // verification key. Otherwise, it should display the captcha. 124 | VerificationKey string 125 | 126 | // InternalErrMessage signals an error, and some error texts should be 127 | // shown to user. 128 | InternalErrMessage string 129 | 130 | // CaptchaErr signals types of errors. 131 | CaptchaErr CaptchaErr 132 | 133 | // RecaptchaSiteKey is the recaptcha site key. 134 | RecaptchaSiteKey string 135 | 136 | // PostAction is the url to post response to. 137 | PostAction string 138 | } 139 | 140 | func (Captcha) TemplateName() string { return TnameCaptcha } 141 | 142 | type CaptchaErr struct { 143 | IsVerifyFailed bool 144 | IsNotFound bool 145 | } 146 | -------------------------------------------------------------------------------- /page/template.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | import ( 4 | "errors" 5 | "html/template" 6 | "net/http" 7 | "path/filepath" 8 | ) 9 | 10 | type TemplateMap map[string]*template.Template 11 | 12 | var ( 13 | templateFiles = [][]string{ 14 | {TnameError, TnameLayout, TnameCommon}, 15 | {TnameNotFound, TnameLayout, TnameCommon}, 16 | {TnameClasslist, TnameLayout, TnameCommon}, 17 | {TnameBbsIndex, TnameLayout, TnameCommon}, 18 | {TnameBbsArticle, TnameLayout, TnameCommon}, 19 | {TnameAskOver18, TnameLayout, TnameCommon}, 20 | {TnameManIndex, TnameLayout, TnameCommon}, 21 | {TnameManArticle, TnameLayout, TnameCommon}, 22 | {TnameCaptcha, TnameLayout, TnameCommon}, 23 | } 24 | 25 | tmpl TemplateMap 26 | ) 27 | 28 | func loadTemplates(dir string, filenames [][]string, funcMap template.FuncMap) (TemplateMap, error) { 29 | new_tmpl := make(TemplateMap) 30 | 31 | for _, fns := range filenames { 32 | t := template.New("") 33 | t.Funcs(funcMap) 34 | 35 | paths := make([]string, len(fns), len(fns)) 36 | for i, fn := range fns { 37 | paths[i] = filepath.Join(dir, fn) 38 | } 39 | 40 | if _, err := t.ParseFiles(paths...); err != nil { 41 | return nil, err 42 | } 43 | 44 | name := fns[0] 45 | root := t.Lookup("ROOT") 46 | if root == nil { 47 | return nil, errors.New("No ROOT template defined") 48 | } 49 | new_tmpl[name] = root 50 | } 51 | 52 | return new_tmpl, nil 53 | } 54 | 55 | func LoadTemplates(dir string, funcMap template.FuncMap) error { 56 | new_tmpl, err := loadTemplates(dir, templateFiles, funcMap) 57 | if err != nil { 58 | return err 59 | } 60 | tmpl = new_tmpl 61 | return nil 62 | } 63 | 64 | type NeedToWriteHeaders interface { 65 | WriteHeaders(w http.ResponseWriter) error 66 | } 67 | 68 | func ExecuteTemplate(w http.ResponseWriter, name string, arg interface{}) error { 69 | if p, ok := arg.(NeedToWriteHeaders); ok { 70 | if err := p.WriteHeaders(w); err != nil { 71 | return err 72 | } 73 | } 74 | if name == "" { 75 | return nil 76 | } 77 | return tmpl[name].Execute(w, arg) 78 | } 79 | 80 | func ExecutePage(w http.ResponseWriter, p Page) error { 81 | return ExecuteTemplate(w, p.TemplateName(), p) 82 | } 83 | -------------------------------------------------------------------------------- /page/wrapper.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/ptt/pttweb/pttbbs" 9 | "google.golang.org/grpc" 10 | "google.golang.org/grpc/codes" 11 | ) 12 | 13 | func setCommonResponseHeaders(w http.ResponseWriter) { 14 | h := w.Header() 15 | h.Set("Server", "Cryophoenix") 16 | h.Set("Content-Type", "text/html; charset=utf-8") 17 | } 18 | 19 | type ErrorWrapper func(Context, http.ResponseWriter) error 20 | 21 | func (fn ErrorWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) { 22 | setCommonResponseHeaders(w) 23 | 24 | if err := clarifyRemoteError(handleRequest(w, r, fn)); err != nil { 25 | if pg, ok := err.(Page); ok { 26 | if err = ExecutePage(w, pg); err != nil { 27 | log.Println("Failed to emit error page:", err) 28 | } 29 | return 30 | } 31 | internalError(w, err) 32 | } 33 | } 34 | 35 | func clarifyRemoteError(err error) error { 36 | if err == pttbbs.ErrNotFound { 37 | return newNotFoundError(err) 38 | } 39 | 40 | switch grpc.Code(err) { 41 | case codes.NotFound, codes.PermissionDenied: 42 | return newNotFoundError(err) 43 | } 44 | 45 | return err 46 | } 47 | 48 | func internalError(w http.ResponseWriter, err error) { 49 | log.Println(err) 50 | w.WriteHeader(http.StatusInternalServerError) 51 | ExecutePage(w, &Error{ 52 | Title: `500 - Internal Server Error`, 53 | ContentHtml: `500 - Internal Server Error / Server Too Busy.`, 54 | }) 55 | } 56 | 57 | func handleRequest(w http.ResponseWriter, r *http.Request, f func(Context, http.ResponseWriter) error) error { 58 | ctx, err := newContext(r) 59 | if err != nil { 60 | return err 61 | } 62 | return f(ctx, w) 63 | } 64 | 65 | type NotFoundError struct { 66 | NotFound 67 | UnderlyingErr error 68 | } 69 | 70 | func (e *NotFoundError) Error() string { 71 | return fmt.Sprintf("not found error page: %v", e.UnderlyingErr) 72 | } 73 | 74 | func newNotFoundError(err error) *NotFoundError { 75 | return &NotFoundError{ 76 | UnderlyingErr: err, 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /paging.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | ErrCursorOutOfRange = errors.New("Paging cursor out of range") 9 | ErrPageNoOutOfRange = errors.New("Page number out of range") 10 | ) 11 | 12 | type Paging struct { 13 | nrPerPage, nrEntries, nrPages, cursor, curPage int 14 | } 15 | 16 | func NewPaging(nrPerPage, nrEntries int) *Paging { 17 | npages := nrEntries / nrPerPage 18 | if nrEntries%nrPerPage > 0 { 19 | npages++ 20 | } 21 | if npages == 0 { 22 | npages++ 23 | } 24 | return &Paging{ 25 | nrPerPage: nrPerPage, 26 | nrEntries: nrEntries, 27 | nrPages: npages, 28 | cursor: 0, 29 | curPage: 1, 30 | } 31 | } 32 | 33 | func (p *Paging) HasPrev() bool { 34 | return p.curPage > 1 35 | } 36 | 37 | func (p *Paging) HasNext() bool { 38 | return p.curPage < p.LastPageNo() 39 | } 40 | 41 | func (p *Paging) PrevPageNo() int { 42 | return p.curPage - 1 43 | } 44 | 45 | func (p *Paging) NextPageNo() int { 46 | return p.curPage + 1 47 | } 48 | 49 | func (p *Paging) FirstPageNo() int { 50 | return 1 51 | } 52 | 53 | func (p *Paging) LastPageNo() int { 54 | return p.nrPages 55 | } 56 | 57 | func (p *Paging) SetCursor(i int) error { 58 | if i < 0 || i >= p.nrEntries { 59 | return ErrCursorOutOfRange 60 | } 61 | p.cursor = i 62 | return nil 63 | } 64 | 65 | func (p *Paging) SetPageNo(no int) error { 66 | if no < 1 || no > p.LastPageNo() { 67 | return ErrPageNoOutOfRange 68 | } 69 | p.cursor = (no - 1) * p.nrPerPage 70 | return nil 71 | } 72 | 73 | func (p *Paging) Cursor() int { 74 | return p.cursor 75 | } 76 | -------------------------------------------------------------------------------- /proto/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | 3 | all: 4 | make -C man 5 | make -C api 6 | -------------------------------------------------------------------------------- /proto/api/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | 3 | all: 4 | protoc --go_out=. --go-grpc_out=. \ 5 | --go_opt=paths=source_relative \ 6 | --go-grpc_opt=paths=source_relative \ 7 | --go_opt=Mboard.proto=github.com/ptt/pttweb/proto/api board.proto \ 8 | --go-grpc_opt=Mboard.proto=github.com/ptt/pttweb/proto/api board.proto 9 | -------------------------------------------------------------------------------- /proto/api/board.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | // Copyright (c) 2017, Robert Wang 4 | // All rights reserved. 5 | // 6 | // The MIT License. 7 | 8 | package pttbbs.api; 9 | 10 | service BoardService { 11 | rpc Board (BoardRequest) returns (BoardReply) {} 12 | rpc List (ListRequest) returns (ListReply) {} 13 | rpc Hotboard (HotboardRequest) returns (HotboardReply) {} 14 | rpc Content (ContentRequest) returns (ContentReply) {} 15 | rpc Search (SearchRequest) returns (SearchReply) {} 16 | } 17 | 18 | message BoardRequest { 19 | repeated BoardRef ref = 1; 20 | } 21 | 22 | message BoardReply { 23 | repeated Board boards = 1; 24 | } 25 | 26 | message HotboardRequest { 27 | } 28 | 29 | message HotboardReply { 30 | repeated Board boards = 1; 31 | } 32 | 33 | message BoardRef { 34 | oneof ref { 35 | uint32 bid = 1; 36 | string name = 2; 37 | } 38 | } 39 | 40 | message Board { 41 | uint32 bid = 1; 42 | string name = 2; 43 | string title = 3; 44 | string bclass = 4; 45 | string raw_moderators = 5; 46 | uint32 parent = 6; 47 | uint32 num_users = 7; 48 | uint32 num_posts = 8; 49 | uint32 attributes = 9; 50 | repeated uint32 children = 10; 51 | } 52 | 53 | message ListRequest { 54 | BoardRef ref = 1; 55 | bool include_posts = 2; 56 | int32 offset = 3; 57 | int32 length = 4; 58 | bool include_bottoms = 5; 59 | } 60 | 61 | message ListReply { 62 | repeated Post posts = 1; 63 | repeated Post bottoms = 2; 64 | } 65 | 66 | message Post { 67 | uint32 index = 1; 68 | string filename = 2; 69 | string raw_date = 3; 70 | int32 num_recommends = 4; 71 | int32 filemode = 5; 72 | string owner = 6; 73 | string title = 7; 74 | int64 modified_nsec = 8; 75 | } 76 | 77 | message ContentRequest { 78 | BoardRef board_ref = 1; 79 | string filename = 2; 80 | string consistency_token = 3; 81 | PartialOptions partial_options = 4; 82 | } 83 | 84 | message PartialOptions { 85 | enum SelectType { 86 | SELECT_FULL = 0; 87 | SELECT_HEAD = 1; 88 | SELECT_TAIL = 2; 89 | SELECT_PART = 3; 90 | } 91 | SelectType select_type = 1; 92 | int64 offset = 2; 93 | int64 max_length = 3; 94 | } 95 | 96 | message ContentReply { 97 | Content content = 1; 98 | } 99 | 100 | message Content { 101 | bytes content = 1; 102 | string consistency_token = 2; 103 | int64 offset = 3; 104 | int64 length = 4; 105 | int64 total_length = 5; 106 | } 107 | 108 | message SearchFilter { 109 | enum Type { 110 | TYPE_UNKNOWN = 0; 111 | TYPE_EXACT_TITLE = 1; 112 | TYPE_TITLE = 2; 113 | TYPE_AUTHOR = 3; 114 | TYPE_RECOMMEND = 4; 115 | TYPE_MONEY = 5; 116 | TYPE_MARK = 6; 117 | TYPE_SOLVED = 7; 118 | } 119 | Type type = 1; 120 | int64 number_data = 2; 121 | string string_data = 3; 122 | } 123 | 124 | message SearchRequest { 125 | BoardRef ref = 1; 126 | repeated SearchFilter filter = 2; 127 | int32 offset = 3; 128 | int32 length = 4; 129 | } 130 | 131 | message SearchReply { 132 | repeated Post posts = 1; 133 | int32 total_posts = 2; 134 | } 135 | -------------------------------------------------------------------------------- /proto/man/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | 3 | all: 4 | protoc --go_out=. --go-grpc_out=. \ 5 | --go_opt=paths=source_relative \ 6 | --go-grpc_opt=paths=source_relative \ 7 | --go_opt=Mman.proto=github.com/ptt/pttweb/proto/man man.proto \ 8 | --go-grpc_opt=Mman.proto=github.com/ptt/pttweb/proto/man man.proto 9 | -------------------------------------------------------------------------------- /proto/man/man.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | // Copyright (c) 2016, Robert Wang 4 | // All rights reserved. 5 | // 6 | // The MIT License. 7 | 8 | package pttbbs.man; 9 | 10 | service ManService { 11 | rpc List (ListRequest) returns (ListReply) {} 12 | rpc Article (ArticleRequest) returns (ArticleReply) {} 13 | } 14 | 15 | message ListRequest { 16 | string board_name = 1; 17 | string path = 2; 18 | } 19 | 20 | message ListReply { 21 | bool is_success = 1; 22 | repeated Entry entries = 2; 23 | } 24 | 25 | message Entry { 26 | string board_name = 1; 27 | string path = 2; 28 | string title = 3; 29 | bool is_dir = 4; 30 | } 31 | 32 | message ArticleRequest { 33 | string board_name = 1; 34 | string path = 2; 35 | 36 | enum SelectType { 37 | SELECT_FULL = 0; 38 | SELECT_HEAD = 1; 39 | SELECT_TAIL = 2; 40 | } 41 | SelectType select_type = 3; 42 | 43 | string cache_key = 4; 44 | int64 offset = 5; 45 | int64 max_length = 6; 46 | } 47 | 48 | message ArticleReply { 49 | bytes content = 1; 50 | string cache_key = 2; 51 | int64 file_size = 3; 52 | int64 selected_offset = 4; 53 | int64 selected_size = 5; 54 | } 55 | -------------------------------------------------------------------------------- /pttbbs/aid.go: -------------------------------------------------------------------------------- 1 | package pttbbs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | var ( 9 | ErrInvalidAid = errors.New("invalid aid") 10 | ) 11 | 12 | const ( 13 | aidTable = `0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_` 14 | ) 15 | 16 | type Aid uint64 17 | 18 | func ParseAid(s string) (Aid, error) { 19 | // Not to overflow 20 | if len(s) > 10 { 21 | return Aid(0), ErrInvalidAid 22 | } 23 | 24 | var aid Aid 25 | parseLoop: 26 | for _, c := range s { 27 | aid <<= 6 28 | switch { 29 | case c >= '0' && c <= '9': 30 | aid += Aid(c - '0') 31 | case c >= 'A' && c <= 'Z': 32 | aid += Aid(c - 'A' + 10) 33 | case c >= 'a' && c <= 'z': 34 | aid += Aid(c - 'a' + 36) 35 | case c == '-': 36 | aid += 62 37 | case c == '_': 38 | aid += 63 39 | case c == '@': 40 | break parseLoop 41 | default: 42 | return Aid(0), ErrInvalidAid 43 | } 44 | } 45 | return aid, nil 46 | } 47 | 48 | func (aid Aid) String() string { 49 | s := make([]rune, 10) 50 | var i int 51 | for i = len(s) - 1; aid > 0 && i >= 0; i-- { 52 | s[i] = rune(aidTable[aid%64]) 53 | aid /= 64 54 | } 55 | return string(s[i+1:]) 56 | } 57 | 58 | func (aid Aid) Filename() string { 59 | tp := uint64((aid >> 44) & 0xF) 60 | v1 := uint64((aid >> 12) & 0xFFFFFFFF) 61 | v2 := uint64(aid & 0xFFF) 62 | 63 | var tpc rune 64 | if tp == 0 { 65 | tpc = 'M' 66 | } else { 67 | tpc = 'G' 68 | } 69 | 70 | return fmt.Sprintf("%c.%v.A.%03X", tpc, v1, v2) 71 | } 72 | -------------------------------------------------------------------------------- /pttbbs/aid_test.go: -------------------------------------------------------------------------------- 1 | package pttbbs 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestAidConvert(t *testing.T) { 8 | checkMatch(t, "", "M.0.A.000") 9 | checkMatch(t, "1HNXB7zo", "M.1365119687.A.F72") 10 | checkMatch(t, "53HvTzpa", "G.1128765309.A.CE4") 11 | checkMatch(t, "1KLschD-", "M.1415014827.A.37E") 12 | checkMatch(t, "1KL8iv_2", "M.1414826809.A.FC2") 13 | } 14 | 15 | func checkMatch(t *testing.T, aidc, fn string) { 16 | aid, err := ParseAid(aidc) 17 | if err != nil { 18 | t.Error(err) 19 | } 20 | if tfn := aid.Filename(); tfn != fn { 21 | t.Error(aidc, "expected", fn, "got", tfn) 22 | } 23 | if str := aid.String(); str != aidc { 24 | t.Error(aidc, "convert back", "got", str) 25 | } 26 | } 27 | 28 | func TestAidTooLong(t *testing.T) { 29 | _, err := ParseAid("1234567890a") 30 | if err == nil { 31 | t.Fail() 32 | } 33 | } 34 | 35 | func TestAidInvalid(t *testing.T) { 36 | _, err := ParseAid("!@#$%^") 37 | if err == nil { 38 | t.Fail() 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pttbbs/board_ref.go: -------------------------------------------------------------------------------- 1 | package pttbbs 2 | 3 | import apipb "github.com/ptt/pttweb/proto/api" 4 | 5 | type boardRefByBid BoardID 6 | 7 | func (r boardRefByBid) boardRef() *apipb.BoardRef { 8 | return &apipb.BoardRef{Ref: &apipb.BoardRef_Bid{uint32(r)}} 9 | } 10 | 11 | func BoardRefByBid(bid BoardID) BoardRef { 12 | return boardRefByBid(bid) 13 | } 14 | 15 | func BoardRefsByBid(bids []BoardID) []BoardRef { 16 | refs := make([]BoardRef, len(bids)) 17 | for i := range bids { 18 | refs[i] = BoardRefByBid(bids[i]) 19 | } 20 | return refs 21 | } 22 | 23 | type boardRefByName string 24 | 25 | func (r boardRefByName) boardRef() *apipb.BoardRef { 26 | return &apipb.BoardRef{Ref: &apipb.BoardRef_Name{string(r)}} 27 | } 28 | 29 | func BoardRefByName(name string) BoardRef { 30 | return boardRefByName(name) 31 | } 32 | -------------------------------------------------------------------------------- /pttbbs/grpc.go: -------------------------------------------------------------------------------- 1 | package pttbbs 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "google.golang.org/grpc" 8 | 9 | apipb "github.com/ptt/pttweb/proto/api" 10 | ) 11 | 12 | var grpcCallOpts = []grpc.CallOption{grpc.FailFast(true)} 13 | 14 | type GrpcRemotePtt struct { 15 | service apipb.BoardServiceClient 16 | } 17 | 18 | func NewGrpcRemotePtt(boarddAddr string) (*GrpcRemotePtt, error) { 19 | conn, err := grpc.Dial(boarddAddr, grpc.WithInsecure(), grpc.WithBackoffMaxDelay(time.Second*5)) 20 | if err != nil { 21 | return nil, err 22 | } 23 | return &GrpcRemotePtt{ 24 | service: apipb.NewBoardServiceClient(conn), 25 | }, nil 26 | } 27 | 28 | func (p *GrpcRemotePtt) GetBoards(brefs ...BoardRef) ([]Board, error) { 29 | refs := make([]*apipb.BoardRef, len(brefs)) 30 | for i, ref := range brefs { 31 | refs[i] = ref.boardRef() 32 | } 33 | rep, err := p.service.Board(context.TODO(), &apipb.BoardRequest{ 34 | Ref: refs, 35 | }, grpcCallOpts...) 36 | if err != nil { 37 | return nil, err 38 | } 39 | boards := make([]Board, len(rep.Boards)) 40 | for i, b := range rep.Boards { 41 | boards[i] = toBoard(b) 42 | } 43 | return boards, nil 44 | } 45 | 46 | func toBoard(b *apipb.Board) Board { 47 | return Board{ 48 | Bid: BoardID(b.Bid), 49 | IsBoard: !hasFlag(b.Attributes, BoardGroup), 50 | Over18: hasFlag(b.Attributes, BoardOver18), 51 | Hidden: false, // All returned boards are public. 52 | BrdName: b.Name, 53 | Title: b.Title, 54 | Class: b.Bclass, 55 | BM: b.RawModerators, 56 | Parent: int(b.Parent), 57 | Nuser: int(b.NumUsers), 58 | NumPosts: int(b.NumPosts), 59 | Children: toBoardIDs(b.Children), 60 | } 61 | } 62 | 63 | func toBoardIDs(bids []uint32) []BoardID { 64 | out := make([]BoardID, len(bids)) 65 | for i := range bids { 66 | out[i] = BoardID(bids[i]) 67 | } 68 | return out 69 | } 70 | 71 | func hasFlag(bits, mask uint32) bool { 72 | return (bits & mask) == mask 73 | } 74 | 75 | func (p *GrpcRemotePtt) GetArticleList(ref BoardRef, offset, length int) ([]Article, error) { 76 | return p.doList(&apipb.ListRequest{ 77 | Ref: ref.boardRef(), 78 | IncludePosts: true, 79 | Offset: int32(offset), 80 | Length: int32(length), 81 | }, func(rep *apipb.ListReply) []*apipb.Post { return rep.Posts }) 82 | } 83 | 84 | func (p *GrpcRemotePtt) GetBottomList(ref BoardRef) ([]Article, error) { 85 | return p.doList(&apipb.ListRequest{ 86 | Ref: ref.boardRef(), 87 | IncludeBottoms: true, 88 | }, func(rep *apipb.ListReply) []*apipb.Post { return rep.Bottoms }) 89 | } 90 | 91 | func (p *GrpcRemotePtt) doList(req *apipb.ListRequest, extractArticles func(*apipb.ListReply) []*apipb.Post) ([]Article, error) { 92 | rep, err := p.service.List(context.TODO(), req, grpcCallOpts...) 93 | if err != nil { 94 | return nil, err 95 | } 96 | var articles []Article 97 | for _, a := range extractArticles(rep) { 98 | articles = append(articles, toArticle(a)) 99 | } 100 | return articles, nil 101 | } 102 | 103 | func toArticle(p *apipb.Post) Article { 104 | return Article{ 105 | Offset: int(p.Index), 106 | FileName: p.Filename, 107 | Date: p.RawDate, 108 | Recommend: int(p.NumRecommends), 109 | FileMode: int(p.Filemode), 110 | Owner: p.Owner, 111 | Title: p.Title, 112 | Modified: time.Unix(0, p.ModifiedNsec), 113 | } 114 | } 115 | 116 | func (p *GrpcRemotePtt) GetArticleSelect(ref BoardRef, meth SelectMethod, filename, cacheKey string, offset, maxlen int) (*ArticlePart, error) { 117 | rep, err := p.service.Content(context.TODO(), &apipb.ContentRequest{ 118 | BoardRef: ref.boardRef(), 119 | Filename: filename, 120 | ConsistencyToken: cacheKey, 121 | PartialOptions: &apipb.PartialOptions{ 122 | SelectType: toSelectType(meth), 123 | Offset: int64(offset), 124 | MaxLength: int64(maxlen), 125 | }, 126 | }, grpcCallOpts...) 127 | if err != nil { 128 | return nil, err 129 | } 130 | return toArticlePart(rep.Content), nil 131 | } 132 | 133 | func toSelectType(m SelectMethod) apipb.PartialOptions_SelectType { 134 | switch m { 135 | case SelectPart: 136 | return apipb.PartialOptions_SELECT_PART 137 | case SelectHead: 138 | return apipb.PartialOptions_SELECT_HEAD 139 | case SelectTail: 140 | return apipb.PartialOptions_SELECT_TAIL 141 | default: 142 | panic("unhandled select type") 143 | } 144 | } 145 | 146 | func toArticlePart(c *apipb.Content) *ArticlePart { 147 | return &ArticlePart{ 148 | CacheKey: c.ConsistencyToken, 149 | FileSize: int(c.TotalLength), 150 | Offset: int(c.Offset), 151 | Length: int(c.Length), 152 | Content: c.Content, 153 | } 154 | } 155 | 156 | func (p *GrpcRemotePtt) Hotboards() ([]Board, error) { 157 | rep, err := p.service.Hotboard(context.TODO(), &apipb.HotboardRequest{}, grpcCallOpts...) 158 | if err != nil { 159 | return nil, err 160 | } 161 | var boards []Board 162 | for _, b := range rep.Boards { 163 | boards = append(boards, toBoard(b)) 164 | } 165 | return boards, nil 166 | } 167 | 168 | func (p *GrpcRemotePtt) Search(ref BoardRef, preds []SearchPredicate, offset, length int) ([]Article, int, error) { 169 | var filters []*apipb.SearchFilter 170 | for _, pred := range preds { 171 | filters = append(filters, pred.toSearchFilter()) 172 | } 173 | rep, err := p.service.Search(context.TODO(), &apipb.SearchRequest{ 174 | Ref: ref.boardRef(), 175 | Filter: filters, 176 | Offset: int32(offset), 177 | Length: int32(length), 178 | }) 179 | if err != nil { 180 | return nil, 0, err 181 | } 182 | var articles []Article 183 | for _, p := range rep.Posts { 184 | articles = append(articles, toArticle(p)) 185 | } 186 | return articles, int(rep.TotalPosts), nil 187 | } 188 | -------------------------------------------------------------------------------- /pttbbs/search_predicate.go: -------------------------------------------------------------------------------- 1 | package pttbbs 2 | 3 | import apipb "github.com/ptt/pttweb/proto/api" 4 | 5 | type SearchPredicate interface { 6 | toSearchFilter() *apipb.SearchFilter 7 | } 8 | 9 | func WithTitle(title string) SearchPredicate { 10 | return &searchPredicate{&apipb.SearchFilter{ 11 | Type: apipb.SearchFilter_TYPE_TITLE, 12 | StringData: title, 13 | }} 14 | } 15 | 16 | func WithExactTitle(title string) SearchPredicate { 17 | return &searchPredicate{&apipb.SearchFilter{ 18 | Type: apipb.SearchFilter_TYPE_EXACT_TITLE, 19 | StringData: title, 20 | }} 21 | } 22 | 23 | func WithAuthor(author string) SearchPredicate { 24 | return &searchPredicate{&apipb.SearchFilter{ 25 | Type: apipb.SearchFilter_TYPE_AUTHOR, 26 | StringData: author, 27 | }} 28 | } 29 | 30 | func WithRecommend(n int) SearchPredicate { 31 | if n < -100 { 32 | n = -100 33 | } 34 | if n > 100 { 35 | n = 100 36 | } 37 | return &searchPredicate{&apipb.SearchFilter{ 38 | Type: apipb.SearchFilter_TYPE_RECOMMEND, 39 | NumberData: int64(n), 40 | }} 41 | } 42 | 43 | type searchPredicate struct { 44 | *apipb.SearchFilter 45 | } 46 | 47 | func (p *searchPredicate) toSearchFilter() *apipb.SearchFilter { 48 | return p.SearchFilter 49 | } 50 | -------------------------------------------------------------------------------- /pttbbs/string.go: -------------------------------------------------------------------------------- 1 | package pttbbs 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "regexp" 7 | "strconv" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | var ( 13 | ArticleFirstLineRegexp = regexp.MustCompile(`^(.+?): (.+) (.+?): (.+?)\n$`) 14 | ArticleMetaLineRegexp = regexp.MustCompile(`^(.+?): (.+)\n$`) 15 | 16 | QuotePrefixStrings = []string{": ", "> "} 17 | SignaturePrefixStrings = []string{"※", "==>"} 18 | 19 | ArticlePushPrefixStrings = []string{"推 ", "噓 ", "→ "} 20 | 21 | subjectPrefixStrings = []string{"re:", "fw:", "[轉錄]"} 22 | ) 23 | 24 | const ( 25 | ArticleAuthor = "作者" 26 | ArticleTitle = "標題" 27 | 28 | AllPostBrdName = "ALLPOST" 29 | ) 30 | 31 | var ( 32 | validBrdNameRegexp = regexp.MustCompile(`^[0-9a-zA-Z][0-9a-zA-Z_\.\-]+$`) 33 | validFileNameRegexp = regexp.MustCompile(`^[MG]\.\d+\.A(\.[0-9A-F]+)?$`) 34 | validUserIDRegexp = regexp.MustCompile(`^[a-zA-Z][0-9a-zA-Z]{1,11}$`) 35 | fileNameTimeRegexp = regexp.MustCompile(`^[MG]\.(\d+)\.A(\.[0-9A-F]+)?$`) 36 | allPostBrdNameTitleRegexp = regexp.MustCompile(`\(([0-9a-zA-Z][0-9a-zA-Z_\.\-]+)\)$`) 37 | ) 38 | 39 | func IsValidBrdName(brdname string) bool { 40 | return validBrdNameRegexp.MatchString(brdname) 41 | } 42 | 43 | func IsValidArticleFileName(filename string) bool { 44 | return validFileNameRegexp.MatchString(filename) 45 | } 46 | 47 | func IsValidUserID(userID string) bool { 48 | return validUserIDRegexp.MatchString(userID) 49 | } 50 | 51 | func ParseArticleFirstLine(line []byte) (tag1, val1, tag2, val2 []byte, ok bool) { 52 | m := ArticleFirstLineRegexp.FindSubmatch(line) 53 | if m == nil { 54 | ok = false 55 | } else { 56 | tag1, val1, tag2, val2 = m[1], m[2], m[3], m[4] 57 | ok = true 58 | } 59 | return 60 | } 61 | 62 | func ParseArticleMetaLine(line []byte) (tag, val []byte, ok bool) { 63 | m := ArticleMetaLineRegexp.FindSubmatch(line) 64 | if m == nil { 65 | ok = false 66 | } else { 67 | tag, val = m[1], m[2] 68 | ok = true 69 | } 70 | return 71 | } 72 | 73 | func MatchPrefixBytesToStrings(str []byte, patts []string) bool { 74 | for _, s := range patts { 75 | if bytes.HasPrefix(str, []byte(s)) { 76 | return true 77 | } 78 | } 79 | return false 80 | } 81 | 82 | func ParseFileNameTime(filename string) (time.Time, error) { 83 | m := fileNameTimeRegexp.FindStringSubmatch(filename) 84 | if len(m) == 0 { 85 | return time.Time{}, errors.New("invalid filename pattern") 86 | } 87 | unix, err := strconv.ParseUint(m[1], 10, 64) 88 | if err != nil { 89 | return time.Time{}, err 90 | } 91 | return time.Unix(int64(unix), 0), nil 92 | } 93 | 94 | func Subject(subject string) string { 95 | lower := strings.ToLower(subject) 96 | off := 0 97 | for _, p := range subjectPrefixStrings { 98 | for strings.HasPrefix(lower[off:], p) { 99 | off += len(p) 100 | off += countPrefixSpaces(lower[off:]) 101 | } 102 | off += countPrefixSpaces(lower[off:]) 103 | } 104 | return subject[off:] 105 | } 106 | 107 | func countPrefixSpaces(s string) int { 108 | for i, c := range s { 109 | if c != ' ' { 110 | return i 111 | } 112 | } 113 | return 0 114 | } 115 | 116 | // BrdNameFromAllPostTitle returns the board name parsed from the post title in 117 | // ALLPOST board. 118 | func BrdNameFromAllPostTitle(title string) (string, bool) { 119 | m := allPostBrdNameTitleRegexp.FindStringSubmatch(title) 120 | if len(m) == 0 { 121 | return "", false 122 | } 123 | return m[1], true 124 | } 125 | -------------------------------------------------------------------------------- /pttbbs/string_test.go: -------------------------------------------------------------------------------- 1 | package pttbbs 2 | 3 | import "testing" 4 | 5 | func TestBrdNameFromAllPostTitle(t *testing.T) { 6 | for _, test := range []struct { 7 | desc string 8 | title string 9 | want string 10 | }{ 11 | { 12 | desc: "normal title with board name", 13 | title: "測試 (Test)", 14 | want: "Test", 15 | }, 16 | { 17 | desc: "empty title with board name", 18 | title: "(SYSOP)", 19 | want: "SYSOP", 20 | }, 21 | { 22 | desc: "normal title but bad board name", 23 | title: "測試 (SYS%%) ", 24 | }, 25 | { 26 | desc: "bad title", 27 | title: "(Bad) ", 28 | }, 29 | { 30 | desc: "empty title", 31 | title: "", 32 | }, 33 | } { 34 | t.Run(test.desc, func(t *testing.T) { 35 | wantOk := test.want != "" 36 | 37 | got, gotOk := BrdNameFromAllPostTitle(test.title) 38 | if got != test.want || gotOk != wantOk { 39 | t.Errorf("BrdNameFromAllPostTitle(%q) = (%q, %t); want (%q, %t)", 40 | test.title, got, gotOk, test.want, wantOk) 41 | } 42 | }) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pttbbs/struct.go: -------------------------------------------------------------------------------- 1 | package pttbbs 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | apipb "github.com/ptt/pttweb/proto/api" 8 | ) 9 | 10 | var ( 11 | ErrNotFound = errors.New("not found") 12 | ) 13 | 14 | type BoardID uint32 15 | 16 | type BoardRef interface { 17 | boardRef() *apipb.BoardRef 18 | } 19 | 20 | type Pttbbs interface { 21 | GetBoards(refs ...BoardRef) ([]Board, error) 22 | GetArticleList(ref BoardRef, offset, length int) ([]Article, error) 23 | GetBottomList(ref BoardRef) ([]Article, error) 24 | GetArticleSelect(ref BoardRef, meth SelectMethod, filename, cacheKey string, offset, maxlen int) (*ArticlePart, error) 25 | Hotboards() ([]Board, error) 26 | Search(ref BoardRef, preds []SearchPredicate, offset, length int) (articles []Article, totalPosts int, err error) 27 | } 28 | 29 | func OneBoard(boards []Board, err error) (Board, error) { 30 | if err != nil { 31 | return Board{}, err 32 | } 33 | if len(boards) != 1 { 34 | return Board{}, errors.New("expect one board") 35 | } 36 | return boards[0], nil 37 | } 38 | 39 | type Board struct { 40 | Bid BoardID 41 | IsBoard bool 42 | Over18 bool 43 | Hidden bool 44 | BrdName string 45 | Title string 46 | Class string 47 | BM string 48 | Parent int 49 | Nuser int 50 | NumPosts int 51 | Children []BoardID 52 | } 53 | 54 | func (b Board) Ref() BoardRef { 55 | return BoardRefByBid(b.Bid) 56 | } 57 | 58 | type Article struct { 59 | Offset int 60 | FileName string 61 | Date string 62 | Recommend int 63 | FileMode int 64 | Owner string 65 | Title string 66 | Modified time.Time 67 | } 68 | 69 | type ArticlePart struct { 70 | CacheKey string 71 | FileSize int 72 | Offset int 73 | Length int 74 | Content []byte 75 | } 76 | 77 | // Non-mail file modes 78 | const ( 79 | FileLocal = 1 << iota 80 | FileMarked 81 | FileDigest 82 | FileBottom 83 | FileSolved 84 | ) 85 | 86 | // Mail file modes 87 | const ( 88 | FileRead = 1 << iota 89 | _ // FileMarked 90 | FileReplied 91 | FileMulti 92 | ) 93 | 94 | type SelectMethod string 95 | 96 | const ( 97 | SelectPart SelectMethod = `articlepart` 98 | SelectHead = `articlehead` 99 | SelectTail = `articletail` 100 | ) 101 | 102 | const ( 103 | BoardGroup uint32 = 0x00000008 104 | BoardOver18 = 0x01000000 105 | ) 106 | -------------------------------------------------------------------------------- /pttweb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "encoding/xml" 6 | "errors" 7 | "flag" 8 | "fmt" 9 | "hash/fnv" 10 | "html/template" 11 | "log" 12 | "net" 13 | "net/http" 14 | _ "net/http/pprof" 15 | "net/url" 16 | "os" 17 | "os/signal" 18 | "path" 19 | "path/filepath" 20 | "strconv" 21 | "strings" 22 | "time" 23 | 24 | "google.golang.org/grpc" 25 | "google.golang.org/grpc/codes" 26 | 27 | "golang.org/x/net/context" 28 | 29 | "github.com/ptt/pttweb/atomfeed" 30 | "github.com/ptt/pttweb/cache" 31 | "github.com/ptt/pttweb/captcha" 32 | "github.com/ptt/pttweb/extcache" 33 | "github.com/ptt/pttweb/page" 34 | manpb "github.com/ptt/pttweb/proto/man" 35 | "github.com/ptt/pttweb/pttbbs" 36 | "github.com/ptt/pttweb/pushstream" 37 | 38 | "github.com/gorilla/mux" 39 | ) 40 | 41 | const ( 42 | ArticleCacheTimeout = time.Minute * 10 43 | BbsIndexCacheTimeout = time.Minute * 5 44 | BbsIndexLastPageCacheTimeout = time.Minute * 1 45 | BbsSearchCacheTimeout = time.Minute * 10 46 | BbsSearchLastPageCacheTimeout = time.Minute * 3 47 | ) 48 | 49 | var ( 50 | ErrOver18CookieNotEnabled = errors.New("board is over18 but cookie not enabled") 51 | ErrSigMismatch = errors.New("push stream signature mismatch") 52 | ) 53 | 54 | var ptt pttbbs.Pttbbs 55 | var pttSearch pttbbs.Pttbbs 56 | var mand manpb.ManServiceClient 57 | var router *mux.Router 58 | var cacheMgr *cache.CacheManager 59 | var extCache extcache.ExtCache 60 | var atomConverter *atomfeed.Converter 61 | 62 | var configPath string 63 | var config PttwebConfig 64 | 65 | func init() { 66 | flag.StringVar(&configPath, "conf", "config.json", "config file") 67 | } 68 | 69 | func loadConfig() error { 70 | f, err := os.Open(configPath) 71 | if err != nil { 72 | return err 73 | } 74 | defer f.Close() 75 | 76 | if err := json.NewDecoder(f).Decode(&config); err != nil { 77 | return err 78 | } 79 | 80 | return config.CheckAndFillDefaults() 81 | } 82 | 83 | func main() { 84 | log.SetFlags(log.LstdFlags | log.Lshortfile) 85 | 86 | flag.Parse() 87 | 88 | if err := loadConfig(); err != nil { 89 | log.Fatal("loadConfig:", err) 90 | } 91 | 92 | // Init RemotePtt 93 | var err error 94 | ptt, err = pttbbs.NewGrpcRemotePtt(config.BoarddAddress) 95 | if err != nil { 96 | log.Fatal("cannot connect to boardd:", config.BoarddAddress, err) 97 | } 98 | 99 | if config.SearchAddress != "" { 100 | pttSearch, err = pttbbs.NewGrpcRemotePtt(config.SearchAddress) 101 | if err != nil { 102 | log.Fatal("cannot connect to boardd:", config.SearchAddress, err) 103 | } 104 | } else { 105 | pttSearch = ptt 106 | } 107 | 108 | // Init mand connection 109 | if conn, err := grpc.Dial(config.MandAddress, grpc.WithInsecure(), grpc.WithBackoffMaxDelay(time.Second*5)); err != nil { 110 | log.Fatal("cannot connect to mand:", config.MandAddress, err) 111 | } else { 112 | mand = manpb.NewManServiceClient(conn) 113 | } 114 | 115 | // Init cache manager 116 | cacheMgr = cache.NewCacheManager(config.MemcachedAddress, config.MemcachedMaxConn) 117 | 118 | // Init extcache module if configured 119 | extCache = extcache.New(config.ExtCacheConfig) 120 | 121 | // Init atom converter. 122 | atomConverter = &atomfeed.Converter{ 123 | FeedTitleTemplate: template.Must(template.New("").Parse(config.AtomFeedTitleTemplate)), 124 | LinkFeed: func(brdname string) (string, error) { 125 | return config.FeedPrefix + "/" + brdname + ".xml", nil 126 | }, 127 | LinkArticle: func(brdname, filename string) (string, error) { 128 | u, err := router.Get("bbsarticle").URLPath("brdname", brdname, "filename", filename) 129 | if err != nil { 130 | return "", err 131 | } 132 | return config.SitePrefix + u.String(), nil 133 | }, 134 | } 135 | 136 | // Load templates 137 | if err := page.LoadTemplates(config.TemplateDirectory, templateFuncMap()); err != nil { 138 | log.Fatal("cannot load templates:", err) 139 | } 140 | 141 | // Init router 142 | router = createRouter() 143 | http.Handle("/", router) 144 | 145 | if len(config.Bind) == 0 { 146 | log.Fatal("No bind addresses specified in config") 147 | } 148 | for _, addr := range config.Bind { 149 | part := strings.SplitN(addr, ":", 2) 150 | if len(part) != 2 { 151 | log.Fatal("Invalid bind address: ", addr) 152 | } 153 | if listener, err := net.Listen(part[0], part[1]); err != nil { 154 | log.Fatal("Listen failed for address: ", addr, " error: ", err) 155 | } else { 156 | if part[0] == "unix" { 157 | os.Chmod(part[1], 0777) 158 | // Ignores errors, we can't do anything to those. 159 | } 160 | svr := &http.Server{ 161 | MaxHeaderBytes: 64 * 1024, 162 | } 163 | go svr.Serve(listener) 164 | } 165 | } 166 | 167 | progExit := make(chan os.Signal) 168 | signal.Notify(progExit, os.Interrupt) 169 | <-progExit 170 | } 171 | 172 | func ReplaceVars(p string) string { 173 | var subs = [][]string{ 174 | {`aidc`, `[0-9A-Za-z\-_]+`}, 175 | {`brdname`, `[0-9A-Za-z][0-9A-Za-z_\.\-]+`}, 176 | {`filename`, `[MG]\.\d+\.A(?:\.[0-9A-F]+)?`}, 177 | {`fullpath`, `[0-9A-Za-z_\.\-\/]+`}, 178 | {`page`, `\d+`}, 179 | } 180 | for _, s := range subs { 181 | p = strings.Replace(p, fmt.Sprintf(`{%v}`, s[0]), fmt.Sprintf(`{%v:%v}`, s[0], s[1]), -1) 182 | } 183 | return p 184 | } 185 | 186 | func createRouter() *mux.Router { 187 | r := mux.NewRouter() 188 | 189 | staticFileServer := http.FileServer(http.Dir(filepath.Join(config.TemplateDirectory, `static`))) 190 | r.PathPrefix(`/static/`). 191 | Handler(http.StripPrefix(`/static/`, staticFileServer)) 192 | 193 | // Classlist 194 | r.Path(ReplaceVars(`/cls/{bid:[0-9]+}`)). 195 | Handler(ErrorWrapper(handleCls)). 196 | Name("classlist") 197 | r.Path(ReplaceVars(`/bbs/{x:|index\.html|hotboards\.html}`)). 198 | Handler(ErrorWrapper(handleHotboards)) 199 | 200 | // Board 201 | r.Path(ReplaceVars(`/bbs/{brdname}{x:/?}`)). 202 | Handler(ErrorWrapper(handleBbsIndexRedirect)) 203 | r.Path(ReplaceVars(`/bbs/{brdname}/index.html`)). 204 | Handler(ErrorWrapper(handleBbs)). 205 | Name("bbsindex") 206 | r.Path(ReplaceVars(`/bbs/{brdname}/index{page}.html`)). 207 | Handler(ErrorWrapper(handleBbs)). 208 | Name("bbsindex_page") 209 | r.Path(ReplaceVars(`/bbs/{brdname}/search`)). 210 | Handler(ErrorWrapper(handleBbsSearch)). 211 | Name("bbssearch") 212 | 213 | // Feed 214 | r.Path(ReplaceVars(`/atom/{brdname}.xml`)). 215 | Handler(ErrorWrapper(handleBoardAtomFeed)). 216 | Name("atomfeed") 217 | 218 | // Post 219 | r.Path(ReplaceVars(`/bbs/{brdname}/{filename}.html`)). 220 | Handler(ErrorWrapper(handleArticle)). 221 | Name("bbsarticle") 222 | r.Path(ReplaceVars(`/b/{brdname}/{aidc}`)). 223 | Handler(ErrorWrapper(handleAidc)). 224 | Name("bbsaidc") 225 | 226 | if config.EnablePushStream { 227 | r.Path(ReplaceVars(`/poll/{brdname}/{filename}.html`)). 228 | Handler(ErrorWrapper(handleArticlePoll)). 229 | Name("bbsarticlepoll") 230 | } 231 | 232 | r.Path(ReplaceVars(`/ask/over18`)). 233 | Handler(ErrorWrapper(handleAskOver18)). 234 | Name("askover18") 235 | 236 | // Man 237 | r.Path(ReplaceVars(`/man/{fullpath}.html`)). 238 | Handler(ErrorWrapper(handleMan)). 239 | Name("manentry") 240 | 241 | // Captcha 242 | if cfg := config.captchaConfig(); cfg.Enabled { 243 | if err := captcha.Install(cfg, r); err != nil { 244 | log.Fatal("captcha.Install:", err) 245 | } 246 | } 247 | 248 | return r 249 | } 250 | 251 | func templateFuncMap() template.FuncMap { 252 | return template.FuncMap{ 253 | "route_bbsindex": func(b pttbbs.Board) (*url.URL, error) { 254 | return router.Get("bbsindex").URLPath("brdname", b.BrdName) 255 | }, 256 | "route_bbsindex_page": func(b pttbbs.Board, pg int) (*url.URL, error) { 257 | return router.Get("bbsindex_page").URLPath("brdname", b.BrdName, "page", strconv.Itoa(pg)) 258 | }, 259 | "route_classlist_bid": func(bid int) (*url.URL, error) { 260 | return router.Get("classlist").URLPath("bid", strconv.Itoa(bid)) 261 | }, 262 | "route_classlist": func(b pttbbs.Board) (*url.URL, error) { 263 | return router.Get("classlist").URLPath("bid", strconv.FormatUint(uint64(b.Bid), 10)) 264 | }, 265 | "valid_article": func(a pttbbs.Article) bool { 266 | return pttbbs.IsValidArticleFileName(a.FileName) 267 | }, 268 | "route_bbsarticle": func(brdname, filename, title string) (*url.URL, error) { 269 | if config.EnableLinkOriginalInAllPost && brdname == pttbbs.AllPostBrdName { 270 | if origBrdName, ok := pttbbs.BrdNameFromAllPostTitle(title); ok { 271 | brdname = origBrdName 272 | } 273 | } 274 | return router.Get("bbsarticle").URLPath("brdname", brdname, "filename", filename) 275 | }, 276 | "route_askover18": func() (*url.URL, error) { 277 | return router.Get("askover18").URLPath() 278 | }, 279 | "route_manentry": func(e *manpb.Entry) (*url.URL, error) { 280 | var index string 281 | if e.IsDir { 282 | index = "index" 283 | } 284 | return router.Get("manentry").URLPath("fullpath", path.Join(e.BoardName, e.Path, index)) 285 | }, 286 | "route_manarticle": func(brdname, p string) (*url.URL, error) { 287 | return router.Get("manentry").URLPath("fullpath", path.Join(brdname, p)) 288 | }, 289 | "route_manindex": func(brdname, p string) (*url.URL, error) { 290 | return router.Get("manentry").URLPath("fullpath", path.Join(brdname, p, "index")) 291 | }, 292 | "route_manparent": func(brdname, p string) (*url.URL, error) { 293 | dir := path.Join(brdname, path.Dir(p)) 294 | return router.Get("manentry").URLPath("fullpath", path.Join(dir, "index")) 295 | }, 296 | "route": func(where string, attrs ...string) (*url.URL, error) { 297 | return router.Get(where).URLPath(attrs...) 298 | }, 299 | "route_search_author": func(b pttbbs.Board, author string) (*url.URL, error) { 300 | if !pttbbs.IsValidUserID(author) { 301 | return nil, nil 302 | } 303 | return bbsSearchURL(b, "author:"+author) 304 | }, 305 | "route_search_thread": func(b pttbbs.Board, title string) (*url.URL, error) { 306 | return bbsSearchURL(b, "thread:"+pttbbs.Subject(title)) 307 | }, 308 | "static_prefix": func() string { 309 | return config.StaticPrefix 310 | }, 311 | "colored_counter": colored_counter, 312 | "decorate_board_nuser": decorate_board_nuser, 313 | "post_mark": post_mark, 314 | "ga_account": func() string { 315 | return config.GAAccount 316 | }, 317 | "ga_domain": func() string { 318 | return config.GADomain 319 | }, 320 | "slice": func(args ...interface{}) []interface{} { 321 | return args 322 | }, 323 | } 324 | } 325 | 326 | func setCommonResponseHeaders(w http.ResponseWriter) { 327 | h := w.Header() 328 | h.Set("Server", "Cryophoenix") 329 | h.Set("Content-Type", "text/html; charset=utf-8") 330 | } 331 | 332 | type ErrorWrapper func(*Context, http.ResponseWriter) error 333 | 334 | func (fn ErrorWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) { 335 | setCommonResponseHeaders(w) 336 | 337 | if err := clarifyRemoteError(handleRequest(w, r, fn)); err != nil { 338 | if pg, ok := err.(page.Page); ok { 339 | if err = page.ExecutePage(w, pg); err != nil { 340 | log.Println("Failed to emit error page:", err) 341 | } 342 | return 343 | } 344 | internalError(w, err) 345 | } 346 | } 347 | 348 | func internalError(w http.ResponseWriter, err error) { 349 | log.Println(err) 350 | w.WriteHeader(http.StatusInternalServerError) 351 | page.ExecutePage(w, &page.Error{ 352 | Title: `500 - Internal Server Error`, 353 | ContentHtml: `500 - Internal Server Error / Server Too Busy.`, 354 | }) 355 | } 356 | 357 | func handleRequest(w http.ResponseWriter, r *http.Request, f func(*Context, http.ResponseWriter) error) error { 358 | c := new(Context) 359 | if err := c.MergeFromRequest(r); err != nil { 360 | return err 361 | } 362 | 363 | if err := f(c, w); err != nil { 364 | return err 365 | } 366 | 367 | return nil 368 | } 369 | 370 | func handleAskOver18(c *Context, w http.ResponseWriter) error { 371 | from := c.R.FormValue("from") 372 | if from == "" || !isSafeRedirectURI(from) { 373 | from = "/" 374 | } 375 | 376 | if c.R.Method == "POST" { 377 | if c.R.PostFormValue("yes") != "" { 378 | setOver18Cookie(w) 379 | w.Header().Set("Location", from) 380 | } else { 381 | w.Header().Set("Location", "/") 382 | } 383 | w.WriteHeader(http.StatusFound) 384 | return nil 385 | } else { 386 | return page.ExecutePage(w, &page.AskOver18{ 387 | From: from, 388 | }) 389 | } 390 | } 391 | 392 | func handleClsRoot(c *Context, w http.ResponseWriter) error { 393 | return handleClsWithBid(c, w, pttbbs.BoardID(1)) 394 | } 395 | 396 | func handleCls(c *Context, w http.ResponseWriter) error { 397 | vars := mux.Vars(c.R) 398 | bid, err := strconv.Atoi(vars["bid"]) 399 | if err != nil { 400 | return err 401 | } 402 | return handleClsWithBid(c, w, pttbbs.BoardID(bid)) 403 | } 404 | 405 | func handleClsWithBid(c *Context, w http.ResponseWriter, bid pttbbs.BoardID) error { 406 | if bid < 1 { 407 | return NewNotFoundError(fmt.Errorf("invalid bid: %v", bid)) 408 | } 409 | 410 | board, err := pttbbs.OneBoard(ptt.GetBoards(pttbbs.BoardRefByBid(bid))) 411 | if err != nil { 412 | return err 413 | } 414 | children, err := ptt.GetBoards(pttbbs.BoardRefsByBid(board.Children)...) 415 | if err != nil { 416 | return err 417 | } 418 | return page.ExecutePage(w, &page.Classlist{ 419 | Boards: validBoards(children), 420 | }) 421 | } 422 | 423 | func handleHotboards(c *Context, w http.ResponseWriter) error { 424 | boards, err := ptt.Hotboards() 425 | if err != nil { 426 | return err 427 | } 428 | return page.ExecutePage(w, &page.Classlist{ 429 | Boards: validBoards(boards), 430 | IsHotboardList: true, 431 | }) 432 | } 433 | 434 | func validBoards(boards []pttbbs.Board) []pttbbs.Board { 435 | var valids []pttbbs.Board 436 | for _, b := range boards { 437 | if pttbbs.IsValidBrdName(b.BrdName) && !b.Hidden { 438 | valids = append(valids, b) 439 | } 440 | } 441 | return valids 442 | } 443 | 444 | func handleBbsIndexRedirect(c *Context, w http.ResponseWriter) error { 445 | vars := mux.Vars(c.R) 446 | if url, err := router.Get("bbsindex").URLPath("brdname", vars["brdname"]); err != nil { 447 | return err 448 | } else { 449 | w.Header().Set("Location", url.String()) 450 | } 451 | w.WriteHeader(http.StatusFound) 452 | return nil 453 | } 454 | 455 | func handleBbs(c *Context, w http.ResponseWriter) error { 456 | vars := mux.Vars(c.R) 457 | brdname := vars["brdname"] 458 | 459 | // Note: TODO move timeout into the generating function. 460 | // We don't know if it is the last page without entry count. 461 | pageNo := 0 462 | timeout := BbsIndexLastPageCacheTimeout 463 | 464 | if pg, err := strconv.Atoi(vars["page"]); err == nil { 465 | pageNo = pg 466 | timeout = BbsIndexCacheTimeout 467 | } 468 | 469 | brd, err := getBoardByName(c, brdname) 470 | if err != nil { 471 | return err 472 | } 473 | 474 | obj, err := cacheMgr.Get(&BbsIndexRequest{ 475 | Brd: *brd, 476 | Page: pageNo, 477 | }, ZeroBbsIndex, timeout, generateBbsIndex) 478 | if err != nil { 479 | return err 480 | } 481 | bbsindex := obj.(*BbsIndex) 482 | 483 | if !bbsindex.IsValid { 484 | return NewNotFoundError(fmt.Errorf("not a valid cache.BbsIndex: %v/%v", brd.BrdName, pageNo)) 485 | } 486 | 487 | return page.ExecutePage(w, (*page.BbsIndex)(bbsindex)) 488 | } 489 | 490 | func bbsSearchURL(b pttbbs.Board, query string) (*url.URL, error) { 491 | u, err := router.Get("bbssearch").URLPath("brdname", b.BrdName) 492 | if err != nil { 493 | return nil, err 494 | } 495 | q := url.Values{} 496 | q.Set("q", query) 497 | u.RawQuery = q.Encode() 498 | return u, nil 499 | } 500 | 501 | func parseKeyValueTerm(term string) (pttbbs.SearchPredicate, bool) { 502 | kv := strings.SplitN(term, ":", 2) 503 | if len(kv) != 2 { 504 | return nil, false 505 | } 506 | k, v := strings.ToLower(kv[0]), kv[1] 507 | if len(v) == 0 { 508 | return nil, false 509 | } 510 | switch k { 511 | case "author": 512 | return pttbbs.WithAuthor(v), true 513 | case "recommend": 514 | n, err := strconv.Atoi(v) 515 | if err != nil { 516 | return nil, false 517 | } 518 | return pttbbs.WithRecommend(n), true 519 | } 520 | return nil, false 521 | } 522 | 523 | func parseQuery(query string) ([]pttbbs.SearchPredicate, error) { 524 | // Special case, thread takes up all the query. 525 | if strings.HasPrefix(query, "thread:") { 526 | return []pttbbs.SearchPredicate{ 527 | pttbbs.WithExactTitle(strings.TrimSpace(strings.TrimPrefix(query, "thread:"))), 528 | }, nil 529 | } 530 | 531 | segs := strings.Split(query, " ") 532 | var titleSegs []string 533 | var preds []pttbbs.SearchPredicate 534 | for _, s := range segs { 535 | if p, ok := parseKeyValueTerm(s); ok { 536 | preds = append(preds, p) 537 | } else { 538 | titleSegs = append(titleSegs, s) 539 | } 540 | } 541 | title := strings.TrimSpace(strings.Join(titleSegs, " ")) 542 | if title != "" { 543 | // Put title first. 544 | preds = append([]pttbbs.SearchPredicate{ 545 | pttbbs.WithTitle(title), 546 | }, preds...) 547 | } 548 | return preds, nil 549 | } 550 | 551 | func handleBbsSearch(c *Context, w http.ResponseWriter) error { 552 | vars := mux.Vars(c.R) 553 | brdname := vars["brdname"] 554 | 555 | if c.R.ParseForm() != nil { 556 | w.WriteHeader(http.StatusBadRequest) 557 | return nil 558 | } 559 | form := c.R.Form 560 | query := strings.TrimSpace(form.Get("q")) 561 | 562 | pageNo := 1 563 | // Note: TODO move timeout into the generating function. 564 | timeout := BbsSearchLastPageCacheTimeout 565 | 566 | if pageStr := form.Get("page"); pageStr != "" { 567 | pg, err := strconv.Atoi(pageStr) 568 | if err != nil || pg <= 0 { 569 | w.WriteHeader(http.StatusBadRequest) 570 | return nil 571 | } 572 | pageNo = pg 573 | timeout = BbsSearchCacheTimeout 574 | } 575 | 576 | preds, err := parseQuery(query) 577 | if err != nil { 578 | return err 579 | } 580 | 581 | brd, err := getBoardByName(c, brdname) 582 | if err != nil { 583 | return err 584 | } 585 | 586 | obj, err := cacheMgr.Get(&BbsSearchRequest{ 587 | Brd: *brd, 588 | Page: pageNo, 589 | Query: query, 590 | Preds: preds, 591 | }, ZeroBbsIndex, timeout, generateBbsSearch) 592 | if err != nil { 593 | return err 594 | } 595 | bbsindex := obj.(*BbsIndex) 596 | 597 | if !bbsindex.IsValid { 598 | return NewNotFoundError(fmt.Errorf("not a valid cache.BbsIndex: %v/%v", brd.BrdName, pageNo)) 599 | } 600 | 601 | return page.ExecutePage(w, (*page.BbsIndex)(bbsindex)) 602 | } 603 | 604 | func handleBoardAtomFeed(c *Context, w http.ResponseWriter) error { 605 | vars := mux.Vars(c.R) 606 | brdname := vars["brdname"] 607 | 608 | timeout := BbsIndexLastPageCacheTimeout 609 | 610 | c.SetSkipOver18() 611 | brd, err := getBoardByName(c, brdname) 612 | if err != nil { 613 | return err 614 | } 615 | 616 | obj, err := cacheMgr.Get(&BoardAtomFeedRequest{ 617 | Brd: *brd, 618 | }, ZeroBoardAtomFeed, timeout, generateBoardAtomFeed) 619 | if err != nil { 620 | return err 621 | } 622 | baf := obj.(*BoardAtomFeed) 623 | 624 | if !baf.IsValid { 625 | return NewNotFoundError(fmt.Errorf("not a valid cache.BoardAtomFeed: %v", brd.BrdName)) 626 | } 627 | 628 | w.Header().Set("Content-Type", "application/xml") 629 | if _, err = w.Write([]byte(xml.Header)); err != nil { 630 | return err 631 | } 632 | return xml.NewEncoder(w).Encode(baf.Feed) 633 | } 634 | 635 | func handleArticle(c *Context, w http.ResponseWriter) error { 636 | vars := mux.Vars(c.R) 637 | brdname := vars["brdname"] 638 | filename := vars["filename"] 639 | return handleArticleCommon(c, w, brdname, filename) 640 | } 641 | 642 | func handleAidc(c *Context, w http.ResponseWriter) error { 643 | vars := mux.Vars(c.R) 644 | brdname := vars["brdname"] 645 | aid, err := pttbbs.ParseAid(vars["aidc"]) 646 | if err != nil { 647 | return NewNotFoundError(fmt.Errorf("board %v, invalid aid: %v", brdname, err)) 648 | } 649 | return handleArticleCommon(c, w, brdname, aid.Filename()) 650 | } 651 | 652 | func handleArticleCommon(c *Context, w http.ResponseWriter, brdname, filename string) error { 653 | var err error 654 | 655 | brd, err := getBoardByName(c, brdname) 656 | if err != nil { 657 | return err 658 | } 659 | 660 | // Render content 661 | obj, err := cacheMgr.Get(&ArticleRequest{ 662 | Namespace: "bbs", 663 | Brd: *brd, 664 | Filename: filename, 665 | Select: func(m pttbbs.SelectMethod, offset, maxlen int) (*pttbbs.ArticlePart, error) { 666 | return ptt.GetArticleSelect(brd.Ref(), m, filename, "", offset, maxlen) 667 | }, 668 | }, ZeroArticle, ArticleCacheTimeout, generateArticle) 669 | // Try older filename when not found. 670 | if err == pttbbs.ErrNotFound { 671 | if name, ok := oldFilename(filename); ok { 672 | if handleArticleCommon(c, w, brdname, name) == nil { 673 | return nil 674 | } 675 | } 676 | } 677 | if err != nil { 678 | return err 679 | } 680 | ar := obj.(*Article) 681 | 682 | if !ar.IsValid { 683 | return NewNotFoundError(nil) 684 | } 685 | 686 | if len(ar.ContentHtml) > TruncateSize { 687 | log.Println("Large rendered article:", brd.BrdName, filename, len(ar.ContentHtml)) 688 | } 689 | 690 | pollUrl, longPollUrl, err := uriForPolling(brd.BrdName, filename, ar.CacheKey, ar.NextOffset) 691 | if err != nil { 692 | return err 693 | } 694 | 695 | return page.ExecutePage(w, &page.BbsArticle{ 696 | Title: ar.ParsedTitle, 697 | Description: ar.PreviewContent, 698 | Board: brd, 699 | FileName: filename, 700 | Content: template.HTML(string(ar.ContentHtml)), 701 | ContentTail: template.HTML(string(ar.ContentTailHtml)), 702 | ContentTruncated: ar.IsTruncated, 703 | PollUrl: pollUrl, 704 | LongPollUrl: longPollUrl, 705 | CurrOffset: ar.NextOffset, 706 | }) 707 | } 708 | 709 | // oldFilename returns the old filename of an article if any. Older articles 710 | // have no random suffix. This will result into ".000" suffix when converted 711 | // from AID. 712 | func oldFilename(filename string) (string, bool) { 713 | if !strings.HasSuffix(filename, ".000") { 714 | return "", false 715 | } 716 | return filename[:len(filename)-4], true 717 | } 718 | 719 | func verifySignature(brdname, filename string, size int64, sig string) bool { 720 | return (&pushstream.PushNotification{ 721 | Brdname: brdname, 722 | Filename: filename, 723 | Size: size, 724 | Signature: sig, 725 | }).CheckSignature(config.PushStreamSharedSecret) 726 | } 727 | 728 | func handleArticlePoll(c *Context, w http.ResponseWriter) error { 729 | vars := mux.Vars(c.R) 730 | brdname := vars["brdname"] 731 | filename := vars["filename"] 732 | cacheKey := c.R.FormValue("cacheKey") 733 | offset, err := strconv.Atoi(c.R.FormValue("offset")) 734 | if err != nil { 735 | return err 736 | } 737 | size, err := strconv.Atoi(c.R.FormValue("size")) 738 | if err != nil { 739 | return err 740 | } 741 | 742 | if !verifySignature(brdname, filename, int64(offset), c.R.FormValue("offset-sig")) || 743 | !verifySignature(brdname, filename, int64(size), c.R.FormValue("size-sig")) { 744 | return ErrSigMismatch 745 | } 746 | 747 | brd, err := getBoardByName(c, brdname) 748 | if err != nil { 749 | return err 750 | } 751 | 752 | obj, err := cacheMgr.Get(&ArticlePartRequest{ 753 | Brd: *brd, 754 | Filename: filename, 755 | CacheKey: cacheKey, 756 | Offset: offset, 757 | }, ZeroArticlePart, time.Minute, generateArticlePart) 758 | if err != nil { 759 | return err 760 | } 761 | ap := obj.(*ArticlePart) 762 | 763 | res := new(page.ArticlePollResp) 764 | res.Success = ap.IsValid 765 | if ap.IsValid { 766 | res.ContentHtml = ap.ContentHtml 767 | res.PollUrl, _, err = uriForPolling(brdname, filename, ap.CacheKey, ap.NextOffset) 768 | if err != nil { 769 | return err 770 | } 771 | } 772 | return page.WriteAjaxResp(w, res) 773 | } 774 | 775 | func uriForPolling(brdname, filename, cacheKey string, offset int) (poll, longPoll string, err error) { 776 | if !config.EnablePushStream { 777 | return 778 | } 779 | 780 | pn := pushstream.PushNotification{ 781 | Brdname: brdname, 782 | Filename: filename, 783 | Size: int64(offset), 784 | } 785 | pn.Sign(config.PushStreamSharedSecret) 786 | 787 | next, err := router.Get("bbsarticlepoll").URLPath("brdname", brdname, "filename", filename) 788 | if err != nil { 789 | return 790 | } 791 | args := make(url.Values) 792 | args.Set("cacheKey", cacheKey) 793 | args.Set("offset", strconv.FormatInt(pn.Size, 10)) 794 | args.Set("offset-sig", pn.Signature) 795 | poll = next.String() + "?" + args.Encode() 796 | 797 | lpArgs := make(url.Values) 798 | lpArgs.Set("id", pushstream.GetPushChannel(&pn, config.PushStreamSharedSecret)) 799 | longPoll = config.PushStreamSubscribeLocation + "?" + lpArgs.Encode() 800 | return 801 | } 802 | 803 | func getBoardByName(c *Context, brdname string) (*pttbbs.Board, error) { 804 | if !pttbbs.IsValidBrdName(brdname) { 805 | return nil, NewNotFoundError(fmt.Errorf("invalid board name: %s", brdname)) 806 | } 807 | 808 | brd, err := getBoardByNameCached(brdname) 809 | if err != nil { 810 | return nil, err 811 | } 812 | 813 | err = hasPermViewBoard(c, brd) 814 | if err != nil { 815 | return nil, err 816 | } 817 | 818 | return brd, nil 819 | } 820 | 821 | func hasPermViewBoard(c *Context, brd *pttbbs.Board) error { 822 | if !pttbbs.IsValidBrdName(brd.BrdName) || brd.Hidden { 823 | return NewNotFoundError(fmt.Errorf("no permission: %s", brd.BrdName)) 824 | } 825 | if brd.Over18 { 826 | if !config.EnableOver18Cookie { 827 | return NewNotFoundError(ErrOver18CookieNotEnabled) 828 | } 829 | if !c.IsCrawler() && !c.IsOver18() { 830 | return shouldAskOver18Error(c) 831 | } 832 | // Ok 833 | } 834 | return nil 835 | } 836 | 837 | func shouldAskOver18Error(c *Context) error { 838 | q := make(url.Values) 839 | q.Set("from", c.R.URL.String()) 840 | 841 | u, _ := router.Get("askover18").URLPath() 842 | 843 | err := new(ShouldAskOver18Error) 844 | err.To = u.String() + "?" + q.Encode() 845 | return err 846 | } 847 | 848 | func clarifyRemoteError(err error) error { 849 | if err == pttbbs.ErrNotFound { 850 | return NewNotFoundError(err) 851 | } 852 | return translateGrpcError(err) 853 | } 854 | 855 | func translateGrpcError(err error) error { 856 | switch grpc.Code(err) { 857 | case codes.NotFound, codes.PermissionDenied: 858 | return NewNotFoundError(err) 859 | } 860 | return err 861 | } 862 | 863 | func handleMan(c *Context, w http.ResponseWriter) error { 864 | vars := mux.Vars(c.R) 865 | fullpath := strings.Split(vars["fullpath"], "/") 866 | if len(fullpath) < 2 { 867 | return NewNotFoundError(fmt.Errorf("invalid path: %v", fullpath)) 868 | } 869 | brdname := fullpath[0] 870 | 871 | brd, err := getBoardByName(c, brdname) 872 | if err != nil { 873 | return err 874 | } 875 | 876 | if fullpath[len(fullpath)-1] == "index" { 877 | return handleManIndex(c, w, brd, strings.Join(fullpath[1:len(fullpath)-1], "/")) 878 | } 879 | return handleManArticle(c, w, brd, strings.Join(fullpath[1:], "/")) 880 | } 881 | 882 | func handleManIndex(c *Context, w http.ResponseWriter, brd *pttbbs.Board, path string) error { 883 | res, err := mand.List(context.TODO(), &manpb.ListRequest{ 884 | BoardName: brd.BrdName, 885 | Path: path, 886 | }, grpc.FailFast(true)) 887 | if err != nil { 888 | return err 889 | } 890 | return page.ExecutePage(w, &page.ManIndex{ 891 | Board: *brd, 892 | Path: path, 893 | Entries: res.Entries, 894 | }) 895 | } 896 | 897 | func handleManArticle(c *Context, w http.ResponseWriter, brd *pttbbs.Board, path string) error { 898 | obj, err := cacheMgr.Get(&ArticleRequest{ 899 | Namespace: "man", 900 | Brd: *brd, 901 | Filename: path, 902 | Select: func(m pttbbs.SelectMethod, offset, maxlen int) (*pttbbs.ArticlePart, error) { 903 | res, err := mand.Article(context.TODO(), &manpb.ArticleRequest{ 904 | BoardName: brd.BrdName, 905 | Path: path, 906 | SelectType: manSelectType(m), 907 | Offset: int64(offset), 908 | MaxLength: int64(maxlen), 909 | }, grpc.FailFast(true)) 910 | if err != nil { 911 | return nil, err 912 | } 913 | return &pttbbs.ArticlePart{ 914 | CacheKey: res.CacheKey, 915 | FileSize: int(res.FileSize), 916 | Offset: int(res.SelectedOffset), 917 | Length: int(res.SelectedSize), 918 | Content: res.Content, 919 | }, nil 920 | }, 921 | }, ZeroArticle, ArticleCacheTimeout, generateArticle) 922 | if err != nil { 923 | return err 924 | } 925 | ar := obj.(*Article) 926 | 927 | if !ar.IsValid { 928 | return NewNotFoundError(nil) 929 | } 930 | 931 | if len(ar.ContentHtml) > TruncateSize { 932 | log.Println("Large rendered article:", brd.BrdName, path, len(ar.ContentHtml)) 933 | } 934 | 935 | return page.ExecutePage(w, &page.ManArticle{ 936 | Title: ar.ParsedTitle, 937 | Description: ar.PreviewContent, 938 | Board: brd, 939 | Path: path, 940 | Content: template.HTML(string(ar.ContentHtml)), 941 | ContentTail: template.HTML(string(ar.ContentTailHtml)), 942 | ContentTruncated: ar.IsTruncated, 943 | }) 944 | } 945 | 946 | func manSelectType(m pttbbs.SelectMethod) manpb.ArticleRequest_SelectType { 947 | switch m { 948 | case pttbbs.SelectHead: 949 | return manpb.ArticleRequest_SELECT_HEAD 950 | case pttbbs.SelectTail: 951 | return manpb.ArticleRequest_SELECT_TAIL 952 | case pttbbs.SelectPart: 953 | return manpb.ArticleRequest_SELECT_FULL 954 | default: 955 | panic("unknown select type") 956 | } 957 | } 958 | 959 | func fastStrHash64(s string) uint64 { 960 | h := fnv.New64() 961 | _, _ = h.Write([]byte(s)) 962 | return h.Sum64() 963 | } 964 | -------------------------------------------------------------------------------- /pushstream/stream.go: -------------------------------------------------------------------------------- 1 | package pushstream 2 | 3 | import ( 4 | "crypto/sha1" 5 | "fmt" 6 | ) 7 | 8 | type PushNotification struct { 9 | Brdname string `json:"brdname"` 10 | Filename string `json:"filename"` 11 | Size int64 `json:"size"` 12 | Signature string `json:"sig"` 13 | CacheKey string `json:"cacheKey"` 14 | } 15 | 16 | func (p *PushNotification) Sign(secret string) { 17 | p.Signature = p.calcSig(secret) 18 | } 19 | 20 | func (p *PushNotification) CheckSignature(secret string) bool { 21 | return p.Signature == p.calcSig(secret) 22 | } 23 | 24 | func (p *PushNotification) calcSig(secret string) string { 25 | return sha1hex(fmt.Sprintf("%v/%v/%v/%v", p.Brdname, p.Filename, p.Size, secret)) 26 | } 27 | 28 | func GetPushChannel(p *PushNotification, secret string) string { 29 | return sha1hex(fmt.Sprintf("%v/%v/%v", p.Brdname, p.Filename, secret)) 30 | } 31 | 32 | func sha1hex(s string) string { 33 | return fmt.Sprintf("%x", sha1.Sum([]byte(s))) 34 | } 35 | -------------------------------------------------------------------------------- /richcontent/rich_content.go: -------------------------------------------------------------------------------- 1 | package richcontent 2 | 3 | import ( 4 | "sort" 5 | 6 | "golang.org/x/net/context" 7 | ) 8 | 9 | type Component interface { 10 | HTML() string 11 | } 12 | 13 | type RichContent interface { 14 | Pos() (int, int) 15 | URLString() string 16 | Components() []Component 17 | } 18 | 19 | type Finder func(ctx context.Context, input []byte) ([]RichContent, error) 20 | 21 | var defaultFinders = []Finder{ 22 | FindUrl, 23 | } 24 | 25 | func RegisterFinder(finder Finder) { 26 | defaultFinders = append(defaultFinders, finder) 27 | } 28 | 29 | func Find(ctx context.Context, input []byte) ([]RichContent, error) { 30 | var rcs []RichContent 31 | for _, f := range defaultFinders { 32 | found, err := f(ctx, input) 33 | if err != nil { 34 | return nil, err 35 | } 36 | rcs = append(rcs, found...) 37 | } 38 | sort.Sort(RichContentByBeginThenLongest(rcs)) 39 | 40 | var filtered []RichContent 41 | left := 0 42 | for _, rc := range rcs { 43 | l, r := rc.Pos() 44 | if left <= l { 45 | left = r 46 | filtered = append(filtered, rc) 47 | } 48 | } 49 | return filtered, nil 50 | } 51 | 52 | type RichContentByBeginThenLongest []RichContent 53 | 54 | func (c RichContentByBeginThenLongest) Len() int { return len(c) } 55 | func (c RichContentByBeginThenLongest) Swap(i, j int) { c[i], c[j] = c[j], c[i] } 56 | func (c RichContentByBeginThenLongest) Less(i, j int) bool { 57 | ib, ie := c[i].Pos() 58 | jb, je := c[j].Pos() 59 | switch { 60 | case ib < jb: 61 | return true 62 | case ib > jb: 63 | return false 64 | default: 65 | return je < ie 66 | } 67 | } 68 | 69 | type simpleRichComponent struct { 70 | begin, end int 71 | urlString string 72 | components []Component 73 | } 74 | 75 | func (c *simpleRichComponent) Pos() (int, int) { return c.begin, c.end } 76 | func (c *simpleRichComponent) URLString() string { return c.urlString } 77 | func (c *simpleRichComponent) Components() []Component { return c.components } 78 | 79 | func MakeRichContent(begin, end int, urlString string, components []Component) RichContent { 80 | return &simpleRichComponent{ 81 | begin: begin, 82 | end: end, 83 | urlString: urlString, 84 | components: components, 85 | } 86 | } 87 | 88 | type simpleComponent struct { 89 | html string 90 | } 91 | 92 | func (c *simpleComponent) HTML() string { return c.html } 93 | 94 | func MakeComponent(html string) Component { 95 | return &simpleComponent{html: html} 96 | } 97 | 98 | type MatchIndices []int 99 | 100 | func (m MatchIndices) Len() int { return len(m) / 2 } 101 | func (m MatchIndices) At(i int) (int, int) { return m[2*i], m[2*i+1] } 102 | func (m MatchIndices) ByteSliceOf(b []byte, i int) []byte { return b[m[2*i]:m[2*i+1]] } 103 | -------------------------------------------------------------------------------- /richcontent/url_based_finder.go: -------------------------------------------------------------------------------- 1 | package richcontent 2 | 3 | import ( 4 | "fmt" 5 | "html" 6 | "net/url" 7 | "regexp" 8 | 9 | "github.com/ptt/pttweb/extcache" 10 | "golang.org/x/net/context" 11 | ) 12 | 13 | func FindUrl(ctx context.Context, input []byte) ([]RichContent, error) { 14 | rcs := make([]RichContent, 0, 4) 15 | for _, u := range FindAllUrlsIndex(input) { 16 | urlBytes := input[u[0]:u[1]] 17 | var components []Component 18 | for _, p := range defaultUrlPatterns { 19 | if match := p.Pattern.FindSubmatchIndex(urlBytes); match != nil { 20 | if c, err := p.Handler(ctx, urlBytes, MatchIndices(match)); err == nil { 21 | components = c 22 | } 23 | break 24 | } 25 | } 26 | rcs = append(rcs, MakeRichContent(u[0], u[1], string(urlBytes), components)) 27 | } 28 | return rcs, nil 29 | } 30 | 31 | type UrlPatternHandler func(ctx context.Context, urlBytes []byte, match MatchIndices) ([]Component, error) 32 | 33 | type UrlPattern struct { 34 | Pattern *regexp.Regexp 35 | Handler UrlPatternHandler 36 | } 37 | 38 | var defaultUrlPatterns = []*UrlPattern{ 39 | newUrlPattern(`^https?://(?:www\.youtube\.com/watch\?(?:.+&)*v=|youtu\.be/)([\w\-]+)`, handleYoutube), 40 | newUrlPattern(`^https?://i\.imgur\.com/([\w]+)\.((?i)png|jpeg|jpg|gif)$`, handleImgurSingle), // Note: cuz some users use http 41 | newUrlPattern(`^http://picmoe\.net/d\.php\?id=(\d+)`, handlePicmoe), 42 | newUrlPattern(`\.(?i:png|jpeg|jpg|gif)$`, handleGenericImage), 43 | } 44 | 45 | func newUrlPattern(pattern string, handler UrlPatternHandler) *UrlPattern { 46 | return &UrlPattern{ 47 | Pattern: regexp.MustCompile(pattern), 48 | Handler: handler, 49 | } 50 | } 51 | 52 | func imageHtmlTag(urlString string) string { 53 | return fmt.Sprintf(``, html.EscapeString(urlString)) 54 | } 55 | 56 | // Handlers 57 | 58 | func handleYoutube(ctx context.Context, urlBytes []byte, match MatchIndices) ([]Component, error) { 59 | id := url.PathEscape(string(match.ByteSliceOf(urlBytes, 1))) 60 | return []Component{ 61 | MakeComponent(fmt.Sprintf( 62 | `
`, 63 | id)), 64 | }, nil 65 | } 66 | 67 | func handleSameSchemeImage(ctx context.Context, urlBytes []byte, match MatchIndices) ([]Component, error) { 68 | return []Component{MakeComponent(imageHtmlTag(string(match.ByteSliceOf(urlBytes, 1))))}, nil 69 | } 70 | 71 | func handleImgurSingle(ctx context.Context, urlBytes []byte, match MatchIndices) ([]Component, error) { 72 | cache, _ := extcache.FromContext(ctx) 73 | if cache == nil { 74 | return nil, nil 75 | } 76 | id := string(match.ByteSliceOf(urlBytes, 1)) 77 | ext := string(match.ByteSliceOf(urlBytes, 2)) 78 | // Use at biggest large image. 79 | if n := len(id); n > 0 && id[n-1] == 'h' { 80 | id = id[:n-1] + "l" 81 | } else { 82 | id += "l" 83 | } 84 | escapedId := url.PathEscape(id + "." + ext) 85 | src, err := cache.Generate("https://i.imgur.com/" + escapedId) 86 | if err != nil { 87 | return nil, nil // Silently ignore 88 | } 89 | return []Component{MakeComponent(imageHtmlTag(src))}, nil 90 | } 91 | 92 | func handlePicmoe(ctx context.Context, urlBytes []byte, match MatchIndices) ([]Component, error) { 93 | link := fmt.Sprintf(`http://picmoe.net/src/%ss.jpg`, string(match.ByteSliceOf(urlBytes, 1))) 94 | return []Component{MakeComponent(imageHtmlTag(link))}, nil 95 | } 96 | 97 | func handleGenericImage(ctx context.Context, urlBytes []byte, match MatchIndices) ([]Component, error) { 98 | return []Component{MakeComponent(imageHtmlTag(string(urlBytes)))}, nil 99 | } 100 | -------------------------------------------------------------------------------- /richcontent/url_detect.go: -------------------------------------------------------------------------------- 1 | package richcontent 2 | 3 | import ( 4 | "bytes" 5 | "regexp" 6 | ) 7 | 8 | var ( 9 | urlPattern = regexp.MustCompile(`(?:^|[^a-zA-Z0-9\-_])(https?://[!-~]+)`) 10 | httpLiteral = []byte("http") 11 | ) 12 | 13 | const ( 14 | urlPatternSubmatchLeadIndex = 0 15 | urlPatternSubmatchContentIndex = 1 16 | ) 17 | 18 | func FindAllUrls(input []byte) [][]byte { 19 | // Fast path. 20 | if !bytes.Contains(input, httpLiteral) { 21 | return nil 22 | } 23 | 24 | matches := urlPattern.FindAllSubmatch(input, -1) 25 | if len(matches) == 0 { 26 | return nil 27 | } 28 | 29 | urls := make([][]byte, len(matches)) 30 | for i := range matches { 31 | urls[i] = matches[i][urlPatternSubmatchContentIndex] 32 | } 33 | return urls 34 | } 35 | 36 | // [[start, end], [start, end], ...] 37 | func FindAllUrlsIndex(input []byte) [][]int { 38 | // Fast path. 39 | if !bytes.Contains(input, httpLiteral) { 40 | return nil 41 | } 42 | 43 | matches := urlPattern.FindAllSubmatchIndex(input, -1) 44 | for i := range matches { 45 | matches[i] = matches[i][2*urlPatternSubmatchContentIndex : 2*urlPatternSubmatchContentIndex+2] 46 | } 47 | return matches 48 | } 49 | -------------------------------------------------------------------------------- /richcontent/url_detect_test.go: -------------------------------------------------------------------------------- 1 | package richcontent 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestUrlMatches(t *testing.T) { 8 | check(t, "Xhttp://example.com") 9 | check(t, "http://example.com/安安", "http://example.com/") 10 | check(t, "http://example.com", "http://example.com") 11 | check(t, "http://example.com/index?y=1#yea", "http://example.com/index?y=1#yea") 12 | check(t, "http://foo.com/? http://example.com", "http://foo.com/?", "http://example.com") 13 | check(t, "[http://example.com/[({hmm})]", "http://example.com/[({hmm})]") 14 | check(t, "(http://example.com/(Hi))", "http://example.com/(Hi))") 15 | // Uppercase http doesn't match. 16 | check(t, "HTTP://example.com") 17 | } 18 | 19 | func check(t *testing.T, input string, truths ...string) { 20 | outputs := FindAllUrls([]byte(input)) 21 | for i, truth := range truths { 22 | if i >= len(outputs) { 23 | t.Error("Not matched: No output. Truth:", truth) 24 | } else if string(outputs[i]) != truth { 25 | t.Error("Not matched: Output:", string(outputs[i]), "Truth:", truth) 26 | } 27 | } 28 | for i := len(truths); i < len(outputs); i++ { 29 | t.Error("Too many outputs:", string(outputs[i])) 30 | } 31 | 32 | indices := FindAllUrlsIndex([]byte(input)) 33 | for i, truth := range truths { 34 | if i >= len(indices) { 35 | t.Error("Not matched: No output. Truth:", truth) 36 | } else if output := string(input[indices[i][0]:indices[i][1]]); output != truth { 37 | t.Error("Not matched: Output:", output, "Truth:", truth) 38 | } 39 | } 40 | for i := len(truths); i < len(indices); i++ { 41 | output := string(input[indices[i][0]:indices[i][1]]) 42 | t.Error("Too many outputs:", output) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /struct.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | 7 | "golang.org/x/tools/blog/atom" 8 | 9 | "github.com/ptt/pttweb/cache" 10 | "github.com/ptt/pttweb/page" 11 | ) 12 | 13 | // Useful when calling |NewFromBytes| 14 | var ( 15 | ZeroArticle *Article 16 | ZeroArticlePart *ArticlePart 17 | ZeroBbsIndex *BbsIndex 18 | ZeroBoardAtomFeed *BoardAtomFeed 19 | ) 20 | 21 | func gobEncodeBytes(obj interface{}) ([]byte, error) { 22 | var buf bytes.Buffer 23 | if err := gob.NewEncoder(&buf).Encode(obj); err != nil { 24 | return nil, err 25 | } 26 | return buf.Bytes(), nil 27 | } 28 | 29 | func gobDecode(in []byte, out interface{}) error { 30 | buf := bytes.NewBuffer(in) 31 | return gob.NewDecoder(buf).Decode(out) 32 | } 33 | 34 | func gobDecodeCacheable(data []byte, obj cache.Cacheable) (cache.Cacheable, error) { 35 | if err := gobDecode(data, obj); err != nil { 36 | return nil, err 37 | } 38 | return obj, nil 39 | } 40 | 41 | type Article struct { 42 | ParsedTitle string 43 | PreviewContent string 44 | ContentHtml []byte 45 | ContentTailHtml []byte 46 | IsPartial bool 47 | IsTruncated bool 48 | 49 | CacheKey string 50 | NextOffset int 51 | 52 | IsValid bool 53 | } 54 | 55 | func (_ *Article) NewFromBytes(data []byte) (cache.Cacheable, error) { 56 | return gobDecodeCacheable(data, new(Article)) 57 | } 58 | 59 | func (a *Article) EncodeToBytes() ([]byte, error) { 60 | return gobEncodeBytes(a) 61 | } 62 | 63 | type ArticlePart struct { 64 | ContentHtml string 65 | CacheKey string 66 | NextOffset int 67 | IsValid bool 68 | } 69 | 70 | func (_ *ArticlePart) NewFromBytes(data []byte) (cache.Cacheable, error) { 71 | return gobDecodeCacheable(data, new(ArticlePart)) 72 | } 73 | 74 | func (a *ArticlePart) EncodeToBytes() ([]byte, error) { 75 | return gobEncodeBytes(a) 76 | } 77 | 78 | type BbsIndex page.BbsIndex 79 | 80 | func (_ *BbsIndex) NewFromBytes(data []byte) (cache.Cacheable, error) { 81 | return gobDecodeCacheable(data, new(BbsIndex)) 82 | } 83 | 84 | func (bi *BbsIndex) EncodeToBytes() ([]byte, error) { 85 | return gobEncodeBytes(bi) 86 | } 87 | 88 | type BoardAtomFeed struct { 89 | Feed *atom.Feed 90 | IsValid bool 91 | } 92 | 93 | func (_ *BoardAtomFeed) NewFromBytes(data []byte) (cache.Cacheable, error) { 94 | return gobDecodeCacheable(data, new(BoardAtomFeed)) 95 | } 96 | 97 | func (bi *BoardAtomFeed) EncodeToBytes() ([]byte, error) { 98 | return gobEncodeBytes(bi) 99 | } 100 | 101 | func init() { 102 | gob.Register(Article{}) 103 | gob.Register(ArticlePart{}) 104 | gob.Register(BbsIndex{}) 105 | gob.Register(BoardAtomFeed{}) 106 | 107 | // Make sure they are |Cacheable| 108 | checkCacheable(new(Article)) 109 | checkCacheable(new(ArticlePart)) 110 | checkCacheable(new(BbsIndex)) 111 | checkCacheable(new(BoardAtomFeed)) 112 | } 113 | 114 | func checkCacheable(c cache.Cacheable) { 115 | // Empty 116 | } 117 | -------------------------------------------------------------------------------- /tmpl_funcs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "html/template" 5 | "strconv" 6 | 7 | "github.com/ptt/pttweb/pttbbs" 8 | ) 9 | 10 | func colored_counter(num int) template.HTML { 11 | switch { 12 | case num >= 100: 13 | return template.HTML(`爆`) 14 | case num >= 10: 15 | return template.HTML(`` + strconv.Itoa(num) + ``) 16 | case num > 0: 17 | return template.HTML(`` + strconv.Itoa(num) + ``) 18 | case num > -10: 19 | return "" 20 | case num > -100: 21 | return template.HTML(`X` + strconv.Itoa(-num/10) + ``) 22 | default: 23 | return template.HTML(`XX`) 24 | } 25 | } 26 | 27 | func decorate_board_nuser(num int) template.HTML { 28 | switch { 29 | case num < 1: 30 | return "" 31 | case num <= 10: 32 | return template.HTML(strconv.Itoa(num)) 33 | case num <= 50: 34 | return template.HTML(`` + strconv.Itoa(num) + ``) 35 | case num < 2000: 36 | return template.HTML(`` + strconv.Itoa(num) + ``) 37 | case num < 5000: 38 | return template.HTML(`` + strconv.Itoa(num) + ``) 39 | case num < 10000: 40 | return template.HTML(`` + strconv.Itoa(num) + ``) 41 | case num < 30000: 42 | return template.HTML(`` + strconv.Itoa(num) + ``) 43 | case num < 60000: 44 | return template.HTML(`` + strconv.Itoa(num) + ``) 45 | case num < 100000: 46 | return template.HTML(`` + strconv.Itoa(num) + ``) 47 | default: 48 | return template.HTML(`` + strconv.Itoa(num) + ``) 49 | } 50 | } 51 | 52 | func post_mark(mode int) string { 53 | if mode&(pttbbs.FileMarked|pttbbs.FileSolved) == pttbbs.FileMarked|pttbbs.FileSolved { 54 | return `!` 55 | } else if mode&pttbbs.FileMarked == pttbbs.FileMarked { 56 | return `M` 57 | } else if mode&pttbbs.FileSolved == pttbbs.FileSolved { 58 | return `S` 59 | } 60 | return `` 61 | } 62 | -------------------------------------------------------------------------------- /useragent.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | var ( 9 | // Whitelist crawlers here 10 | crawlerPatterns = [...]string{ 11 | "Google (+https://developers.google.com/+/web/snippet/)", 12 | "Googlebot", 13 | "bingbot", 14 | "MSNbot", 15 | "facebookexternalhit", 16 | "PlurkBot", 17 | "Twitterbot", 18 | "TelegramBot", 19 | "CloudFlare-AlwaysOnline", 20 | } 21 | ) 22 | 23 | func isCrawlerUserAgent(r *http.Request) bool { 24 | ua := r.UserAgent() 25 | 26 | for _, pattern := range crawlerPatterns { 27 | if strings.Contains(ua, pattern) { 28 | return true 29 | } 30 | } 31 | 32 | return false 33 | } 34 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | 7 | "github.com/ptt/pttweb/page" 8 | ) 9 | 10 | type ShouldAskOver18Error struct { 11 | page.Redirect 12 | } 13 | 14 | func (e *ShouldAskOver18Error) Error() string { 15 | return fmt.Sprintf("should ask over18, redirect: %v", e.To) 16 | } 17 | 18 | type NotFoundError struct { 19 | page.NotFound 20 | UnderlyingErr error 21 | } 22 | 23 | func (e *NotFoundError) Error() string { 24 | return fmt.Sprintf("not found error page: %v", e.UnderlyingErr) 25 | } 26 | 27 | func NewNotFoundError(err error) *NotFoundError { 28 | return &NotFoundError{ 29 | UnderlyingErr: err, 30 | } 31 | } 32 | 33 | func isSafeRedirectURI(uri string) bool { 34 | if len(uri) < 1 || uri[0] != '/' { 35 | return false 36 | } 37 | u, err := url.Parse(uri) 38 | return err == nil && u.Scheme == "" && u.User == nil && u.Host == "" 39 | } 40 | --------------------------------------------------------------------------------