├── .drone.yml ├── .envrc.gpg ├── .gcloudignore ├── .gitignore ├── README.md ├── app.yaml.gpg ├── credential.json.gpg ├── cron.yaml ├── db ├── config.go ├── db.go └── db_test.go ├── docker-compose.yml ├── go.mod ├── go.sum ├── index.html ├── jwt └── jwt.go ├── mail ├── config.go ├── mail.go └── mail.html ├── main.go ├── model ├── beauty.go ├── post.go └── post_test.go └── ptt ├── api ├── api.go ├── api_test.go └── parser.go ├── ptt.go └── time.go /.drone.yml: -------------------------------------------------------------------------------- 1 | kind: pipeline 2 | name: default 3 | 4 | steps: 5 | - name: test 6 | image: golang:1.13.1 7 | environment: 8 | MONGO_HOST: 9 | from_secret: MONGO_HOST 10 | MONGO_PORT: 11 | from_secret: MONGO_PORT 12 | MONGO_USER: 13 | from_secret: MONGO_USER 14 | MONGO_PASS: 15 | from_secret: MONGO_PASS 16 | SMTP_USER: 17 | from_secret: SMTP_USER 18 | SMTP_PWD: 19 | from_secret: SMTP_PWD 20 | SMTP_HOST: 21 | from_secret: SMTP_HOST 22 | SMTP_PORT: 23 | from_secret: SMTP_PORT 24 | commands: 25 | - go test ./... 26 | 27 | - name: decrypt 28 | image: vladgh/gpg 29 | environment: 30 | GPG_PASSPHASE: 31 | from_secret: GPG_PASSPHASE 32 | commands: 33 | - echo $GPG_PASSPHASE | gpg --batch --yes --passphrase-fd 0 app.yaml.gpg 34 | - echo $GPG_PASSPHASE | gpg --batch --yes --passphrase-fd 0 credential.json.gpg 35 | 36 | - name: deploy 37 | image: google/cloud-sdk:alpine 38 | commands: 39 | - gcloud components install app-engine-go -q # install component "app-engine-go" 40 | - gcloud auth activate-service-account --key-file credential.json 41 | - gcloud config set project daily-beauty-209105 42 | - gcloud app deploy -q 43 | - gcloud app deploy cron.yaml -q 44 | -------------------------------------------------------------------------------- /.envrc.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LarryLuTW/ptt-daily-beauty/d595f6b1562fd24393e13479c81cfd4365d323ee/.envrc.gpg -------------------------------------------------------------------------------- /.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | # Binaries for programs and plugins 17 | *.exe 18 | *.exe~ 19 | *.dll 20 | *.so 21 | *.dylib 22 | # Test binary, build with `go test -c` 23 | *.test 24 | # Output of the go coverage tool, specifically when used with LiteIDE 25 | *.out -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | app.yaml 3 | credential.json 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## [Daily Beauty - 表特日報](https://daily-beauty.xyz) 2 | 3 | Daily Beauty 每晚十一點會自動蒐集 PTT 表特版前三名 4 | 5 | 這些都是 PTT 鄉民篩選過的，想要看的話就快點到[網站](https://daily-beauty.xyz)上訂閱吧～～～ 6 | 7 | ![](https://i.imgur.com/RdNBuie.png) 8 | 9 | ## Deployment 10 | 11 | ### Setup GCP App Engine 12 | 13 | put your app engine credential here 14 | 15 | ```js 16 | // deploy-en/credential.json 17 | { 18 | "type": "service_account", 19 | "project_id": "YOUR_PROJECT_ID", 20 | "private_key_id": "YOUR_PRIVATE_KEY_ID", 21 | "private_key": "YOUR_PRIVATE_KEY", 22 | // ... 23 | } 24 | ``` 25 | -------------------------------------------------------------------------------- /app.yaml.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LarryLuTW/ptt-daily-beauty/d595f6b1562fd24393e13479c81cfd4365d323ee/app.yaml.gpg -------------------------------------------------------------------------------- /credential.json.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LarryLuTW/ptt-daily-beauty/d595f6b1562fd24393e13479c81cfd4365d323ee/credential.json.gpg -------------------------------------------------------------------------------- /cron.yaml: -------------------------------------------------------------------------------- 1 | # https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml 2 | cron: 3 | - description: 'send daily beauty email' 4 | url: /publish 5 | timezone: Asia/Taipei 6 | schedule: every day 23:00 7 | retry_parameters: 8 | job_retry_limit: 5 9 | min_backoff_seconds: 2 10 | max_doublings: 5 11 | -------------------------------------------------------------------------------- /db/config.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import "os" 4 | 5 | var ( 6 | host = os.Getenv("MONGO_HOST") 7 | port = os.Getenv("MONGO_PORT") 8 | user = os.Getenv("MONGO_USER") 9 | pass = os.Getenv("MONGO_PASS") 10 | ) 11 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/mongodb/mongo-go-driver/mongo" 8 | ) 9 | 10 | const ( 11 | dbName = "daily-beauty" 12 | collName = "emails" 13 | ) 14 | 15 | var ctx = context.Background() 16 | var emailsCol *mongo.Collection 17 | 18 | // init mongodb connection 19 | func init() { 20 | url := fmt.Sprintf("mongodb://%s:%s@%s:%s/%s", user, pass, host, port, dbName) 21 | client, err := mongo.NewClient(url) 22 | if err != nil { 23 | panic(err) 24 | } 25 | err = client.Connect(ctx) 26 | if err != nil { 27 | panic(err) 28 | } 29 | emailsCol = client.Database(dbName).Collection(collName) 30 | } 31 | 32 | func checkExist(email string) (bool, error) { 33 | filter := map[string]string{"email": email} 34 | n, err := emailsCol.Count(ctx, filter) 35 | return n != 0, err 36 | } 37 | 38 | // InsertAEmail save a email into the database 39 | func InsertAEmail(email string) error { 40 | // check email duplicate 41 | isExist, err := checkExist(email) 42 | if err != nil { 43 | return err 44 | } 45 | if isExist { 46 | return nil 47 | } 48 | 49 | data := map[string]string{"email": email} 50 | _, err = emailsCol.InsertOne(ctx, data) 51 | // if success, err will be nil 52 | return err 53 | } 54 | 55 | // RemoveAEmail removes a email from database 56 | func RemoveAEmail(email string) error { 57 | filter := map[string]string{"email": email} 58 | _, err := emailsCol.DeleteOne(ctx, filter) 59 | return err 60 | } 61 | 62 | // GetEmails get all emails in the database 63 | func GetEmails() ([]string, error) { 64 | cur, err := emailsCol.Find(ctx, nil) 65 | if err != nil { 66 | return nil, err 67 | } 68 | defer cur.Close(ctx) 69 | 70 | emails := []string{} 71 | for cur.Next(ctx) { 72 | raw, err := cur.DecodeBytes() 73 | if err != nil { 74 | return nil, err 75 | } 76 | email := raw.Lookup("email").StringValue() 77 | emails = append(emails, email) 78 | } 79 | return emails, nil 80 | } 81 | -------------------------------------------------------------------------------- /db/db_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func countEmailInSlice(s []string, email string) (count int) { 8 | for _, e := range s { 9 | if e == email { 10 | count++ 11 | } 12 | } 13 | return count 14 | } 15 | 16 | func TestInsertANewEmail(t *testing.T) { 17 | newEmail := "pudding850806+100@gmail.com" 18 | err := InsertAEmail(newEmail) 19 | if err != nil { 20 | t.Error(err) 21 | } 22 | emails, err := GetEmails() 23 | if err != nil { 24 | t.Error(err) 25 | } 26 | if countEmailInSlice(emails, newEmail) == 0 { 27 | t.Error("new email should be in db") 28 | } 29 | RemoveAEmail(newEmail) 30 | } 31 | 32 | func TestInsertADuplicateEmail(t *testing.T) { 33 | newEmail := "pudding850806+101@gmail.com" 34 | err := InsertAEmail(newEmail) 35 | if err != nil { 36 | t.Error(err) 37 | } 38 | err = InsertAEmail(newEmail) 39 | if err != nil { 40 | t.Error(err) 41 | } 42 | emails, err := GetEmails() 43 | if err != nil { 44 | t.Error(err) 45 | } 46 | if countEmailInSlice(emails, newEmail) != 1 { 47 | t.Errorf("there should be only a %s in db", newEmail) 48 | } 49 | RemoveAEmail(newEmail) 50 | } 51 | 52 | func TestRemoveAEmail(t *testing.T) { 53 | newEmail := "pudding850806+102@gmail.com" 54 | err := InsertAEmail(newEmail) 55 | if err != nil { 56 | t.Error(err) 57 | } 58 | err = RemoveAEmail(newEmail) 59 | if err != nil { 60 | t.Error(err) 61 | } 62 | emails, err := GetEmails() 63 | if err != nil { 64 | t.Error(err) 65 | } 66 | if countEmailInSlice(emails, newEmail) != 0 { 67 | t.Errorf("there should be NO %s in db", newEmail) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | deploy_en: 5 | container_name: PTTDB_deploy_en 6 | build: ./deploy-en 7 | tty: true 8 | volumes: 9 | - '${PWD}:/src' 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module main 2 | 3 | require ( 4 | github.com/PuerkitoBio/goquery v1.5.0 5 | github.com/dgrijalva/jwt-go v3.2.0+incompatible 6 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect 7 | github.com/gin-gonic/gin v1.3.0 8 | github.com/go-stack/stack v1.8.0 // indirect 9 | github.com/golang/protobuf v1.2.0 // indirect 10 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect 11 | github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e 12 | github.com/mattn/go-isatty v0.0.4 // indirect 13 | github.com/mongodb/mongo-go-driver v0.0.18 14 | github.com/ugorji/go/codec v0.0.0-20181125142609-66da5d561eb7 // indirect 15 | github.com/vjeantet/jodaTime v0.0.0-20170816150230-be924ce213fb 16 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect 17 | github.com/xdg/stringprep v1.0.0 // indirect 18 | golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869 // indirect 19 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a 20 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect 21 | golang.org/x/text v0.3.0 // indirect 22 | gopkg.in/go-playground/validator.v8 v8.18.2 // indirect 23 | gopkg.in/yaml.v2 v2.2.1 // indirect 24 | ) 25 | 26 | go 1.13 27 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= 2 | github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= 3 | github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= 4 | github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 5 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 6 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 7 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 h1:AzN37oI0cOS+cougNAV9szl6CVoj2RYwzS3DpUQNtlY= 8 | github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 9 | github.com/gin-gonic/gin v1.3.0 h1:kCmZyPklC0gVdL728E6Aj20uYBJV93nj/TkwBTKhFbs= 10 | github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= 11 | github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= 12 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 13 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 14 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 15 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 16 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 17 | github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e h1:ZZCvgaRDZg1gC9/1xrsgaJzQUCQgniKtw0xjWywWAOE= 18 | github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e/go.mod h1:+rHyWac2R9oAZwFe1wGY2HBzFJJy++RHBg1cU23NkD8= 19 | github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= 20 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 21 | github.com/mongodb/mongo-go-driver v0.0.18 h1:VdItX4LNE6/vXaKPeQtAs7K7zp6N7RyeN3fEeWEt7+Y= 22 | github.com/mongodb/mongo-go-driver v0.0.18/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= 23 | github.com/ugorji/go/codec v0.0.0-20181125142609-66da5d561eb7 h1:1bUo5QMaUTAFAGnXBin93icq1hqP27fbWVZb6vddDec= 24 | github.com/ugorji/go/codec v0.0.0-20181125142609-66da5d561eb7/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 25 | github.com/vjeantet/jodaTime v0.0.0-20170816150230-be924ce213fb h1:9Cx/q/wd5p+BjCDBjY+rauPbwoS+chrnQ9MKMUtv/hs= 26 | github.com/vjeantet/jodaTime v0.0.0-20170816150230-be924ce213fb/go.mod h1:XK4iy/zfkdRGe+lWQYwmebWh0IIMIe6+wi3APUAiCJ0= 27 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= 28 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 29 | github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= 30 | github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 31 | golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869 h1:kkXA53yGe04D0adEYJwEVQjeBppL01Exg+fnMjfUraU= 32 | golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 33 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 34 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= 35 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 36 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= 37 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 38 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 39 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 40 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 41 | gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= 42 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 43 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 44 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 45 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Daily Beauty 表特日報 7 | 8 | 13 | 17 | 18 | 71 | 72 | 73 |
74 |
75 |
76 |

77 | Daily Beauty
78 | 表特日報 79 |

80 |
81 |
82 |

Daily Beauty 就跟它的名字一樣，是一個有關表特的日報

83 |

每晚十一點會自動蒐集 PTT 表特版前三名送到你的信箱

84 |

這些都是經過鄉民篩選的，可以說是群眾智慧的展現

85 |

想要的話就快點訂閱吧～～～

86 |
87 | 88 | 96 |
97 |
98 | 99 |
100 |
101 |
102 | 103 | 104 | 124 | 125 | 126 | -------------------------------------------------------------------------------- /jwt/jwt.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "fmt" 5 | 6 | jwtGo "github.com/dgrijalva/jwt-go" 7 | ) 8 | 9 | var secret = []byte("I am Larry Lu, a Gopher.") 10 | 11 | // ParseToken parse a token string and get email 12 | func ParseToken(tokenStr string) (email string, err error) { 13 | // ref: https://godoc.org/github.com/dgrijalva/jwt-go#example-Parse--Hmac 14 | keyFunc := func(token *jwtGo.Token) (interface{}, error) { 15 | if _, ok := token.Method.(*jwtGo.SigningMethodHMAC); !ok { 16 | return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) 17 | } 18 | return secret, nil 19 | } 20 | 21 | token, err := jwtGo.Parse(tokenStr, keyFunc) 22 | if claims, ok := token.Claims.(jwtGo.MapClaims); ok && token.Valid { 23 | email := claims["email"].(string) 24 | return email, nil 25 | } else { 26 | return "", err 27 | } 28 | } 29 | 30 | // NewToken accept a email and generate a token 31 | func NewToken(email string) (tokenStr string) { 32 | // ref: https://godoc.org/github.com/dgrijalva/jwt-go#example-New--Hmac 33 | payload := jwtGo.MapClaims{"email": email} 34 | token := jwtGo.NewWithClaims(jwtGo.SigningMethodHS256, payload) 35 | tokenStr, _ = token.SignedString(secret) 36 | return tokenStr 37 | } 38 | -------------------------------------------------------------------------------- /mail/config.go: -------------------------------------------------------------------------------- 1 | package mail 2 | 3 | import "os" 4 | 5 | var ( 6 | host = os.Getenv("SMTP_HOST") 7 | port = os.Getenv("SMTP_PORT") 8 | user = os.Getenv("SMTP_USER") 9 | pwd = os.Getenv("SMTP_PWD") 10 | ) 11 | -------------------------------------------------------------------------------- /mail/mail.go: -------------------------------------------------------------------------------- 1 | package mail 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "html/template" 7 | "net/smtp" 8 | 9 | "main/model" 10 | ) 11 | 12 | const ( 13 | name = "Daily Beauty" 14 | from = "service@daily-beauty.xyz" 15 | ) 16 | 17 | var auth = smtp.PlainAuth("", user, pwd, host) 18 | 19 | func createMsg(to, subject, html string) []byte { 20 | msg := "" 21 | msg += fmt.Sprintf("To: %s\r\n", to) 22 | msg += fmt.Sprintf("From: %s <%s>\r\n", name, from) 23 | msg += fmt.Sprintf("Subject: %s\r\n", subject) 24 | 25 | msg += "MIME-version: 1.0;\r\n" 26 | msg += `Content-Type: text/html; charset="UTF-8"` + "\r\n" 27 | 28 | msg += "\r\n" 29 | msg += fmt.Sprintf("%s\r\n", html) 30 | return []byte(msg) 31 | } 32 | 33 | // Send sends the html to the receiver 34 | func Send(to, subject, html string) { 35 | addr := fmt.Sprintf("%s:%s", host, port) 36 | msg := createMsg(to, subject, html) 37 | smtp.SendMail(addr, auth, from, []string{to}, msg) 38 | } 39 | 40 | func reverse(bs []model.Beauty) []model.Beauty { 41 | n := len(bs) 42 | reversedBs := make([]model.Beauty, n) 43 | for i, b := range bs { 44 | reversedBs[n-i-1] = b 45 | } 46 | return reversedBs 47 | } 48 | 49 | // GenerateHTML generates a html from beauties slice 50 | func GenerateHTML(beauties []model.Beauty, randomBeauty model.Beauty, token string) string { 51 | tmpl := template.Must(template.ParseFiles("./mail/mail.html")) 52 | 53 | data := map[string]interface{}{ 54 | "Beauties": reverse(beauties), 55 | "RandomBeauty": randomBeauty, 56 | "Token": token, 57 | } 58 | 59 | var b bytes.Buffer 60 | tmpl.Execute(&b, data) 61 | html := b.String() 62 | 63 | return html 64 | } 65 | -------------------------------------------------------------------------------- /mail/mail.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 72 | 73 | 74 |

Daily Beauty 表特日報

75 | 
 76 |       前陣子在忙鐵人賽，剛好 PTT 又改了一些東西，所以大概壞了兩個多月，如果這兩個月以來你已經洗心革面不想再看妹了，那可以到下方按取消訂閱～
 77 |
78 |
本日精選
79 | {{ range $.Beauties }} 80 | 81 |

{{ .Title }}({{ .NImage }}圖)

82 | 83 | 84 | {{ end }} 85 | 86 |
隨機推薦
87 | 88 |

{{ $.RandomBeauty.Title }}({{ $.RandomBeauty.NImage }}圖)

89 | 90 | 91 | 92 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "math/rand" 8 | "net/http" 9 | "os" 10 | "strconv" 11 | "time" 12 | 13 | "github.com/gin-gonic/gin" 14 | "github.com/jmoiron/jsonq" 15 | "github.com/vjeantet/jodaTime" 16 | 17 | "main/db" 18 | "main/jwt" 19 | "main/mail" 20 | "main/ptt" 21 | ) 22 | 23 | func sendDailyBeauty(subscribers []string, isTest bool) { 24 | log.Println("getting daily beauty...") 25 | // TODO: do parallelly 26 | beauties, err := ptt.FetchBeauties() 27 | if err != nil { 28 | panic(err) 29 | } 30 | 31 | randomBeauty, err := ptt.FetchRandomBeauty() 32 | if err != nil { 33 | panic(err) 34 | } 35 | 36 | loc, _ := time.LoadLocation("Asia/Taipei") 37 | date := jodaTime.Format("YYYY-MM-dd", time.Now().In(loc)) 38 | subject := fmt.Sprintf("表特日報-%s", date) 39 | 40 | if isTest { 41 | subject += " " + strconv.Itoa(rand.Int()) 42 | } 43 | 44 | log.Println("sending...") 45 | for _, to := range subscribers { 46 | token := jwt.NewToken(to) 47 | html := mail.GenerateHTML(beauties, randomBeauty, token) 48 | mail.Send(to, subject, html) 49 | log.Printf("Send to '%s' success", to) 50 | time.Sleep(200 * time.Millisecond) 51 | } 52 | 53 | log.Println("Finish") 54 | } 55 | 56 | func testHandler(c *gin.Context) { 57 | toMails := []string{"flaviogptdb@gmail.com"} 58 | sendDailyBeauty(toMails, true) 59 | log.Printf("Test successfully\n") 60 | c.String(200, "Test successfully") 61 | } 62 | 63 | func publishHandler(c *gin.Context) { 64 | toMails, err := db.GetEmails() 65 | if err != nil { 66 | panic(err) 67 | } 68 | 69 | sendDailyBeauty(toMails, false) 70 | log.Printf("Publish to %d users successfully\n", len(toMails)) 71 | c.String(200, "Publish successfully") 72 | } 73 | 74 | func subscribeHandler(c *gin.Context) { 75 | data := map[string]interface{}{} 76 | dec := json.NewDecoder(c.Request.Body) 77 | dec.Decode(&data) 78 | jq := jsonq.NewQuery(data) 79 | email, err := jq.String("email") 80 | if err != nil { 81 | panic(err) 82 | } 83 | db.InsertAEmail(email) 84 | } 85 | 86 | // api/unsubscribe?token={jwt_token} 87 | func unsubscribeHandler(c *gin.Context) { 88 | tokenStr := c.Query("token") 89 | email, err := jwt.ParseToken(tokenStr) 90 | 91 | if err != nil { 92 | c.AbortWithError(400, err) 93 | // TODO: render error to frontend 94 | return 95 | } 96 | 97 | db.RemoveAEmail(email) 98 | log.Printf("%s unsubscribe", email) 99 | c.String(200, "you(%s) have been unsubscribed from our mailing list", email) 100 | } 101 | 102 | func emailsHandler(c *gin.Context) { 103 | emails, err := db.GetEmails() 104 | if err != nil { 105 | panic(err) 106 | } 107 | c.JSON(200, emails) 108 | } 109 | 110 | func homePageHandler(c *gin.Context) { 111 | c.HTML(http.StatusOK, "index.html", nil) 112 | } 113 | 114 | // Redirect /ptt/redirect/M.1543991133.A.1A1 115 | // to https://www.ptt.cc/bbs/Beauty/M.1543991133.A.1A1.html 116 | func pttRedirectHandler(c *gin.Context) { 117 | baseURL := "https://www.ptt.cc/bbs/Beauty/" 118 | articleID := c.Param("articleID") 119 | location := fmt.Sprintf("%s%s.html", baseURL, articleID) 120 | c.Redirect(302, location) 121 | } 122 | 123 | func main() { 124 | r := gin.Default() 125 | r.GET("/test", testHandler) 126 | r.GET("/publish", publishHandler) 127 | 128 | r.POST("/api/subscribe", subscribeHandler) 129 | r.GET("/api/unsubscribe", unsubscribeHandler) 130 | r.GET("/api/emails", emailsHandler) 131 | 132 | r.GET("/ptt/redirect/:articleID", pttRedirectHandler) 133 | 134 | r.LoadHTMLFiles("index.html") 135 | r.GET("/", homePageHandler) 136 | 137 | port := os.Getenv("PORT") 138 | if port == "" { 139 | port = "8080" 140 | } 141 | 142 | log.Printf("listen on port %s", port) 143 | err := r.Run(":" + port) 144 | panic(err) 145 | } 146 | 147 | // TODO: analysis 轉網址 148 | // TODO: 禮拜幾標題變化 149 | // TODO: 下載所有圖片 150 | // TODO: 防止手動觸發 cron 151 | -------------------------------------------------------------------------------- /model/beauty.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | // Beauty is a struct from getDailyBeauties api 4 | type Beauty struct { 5 | NVote int 6 | NImage int 7 | Title string 8 | Href string 9 | PreviewImg string 10 | } 11 | -------------------------------------------------------------------------------- /model/post.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | "time" 8 | 9 | "github.com/PuerkitoBio/goquery" 10 | ) 11 | 12 | // Post is a corresponding to a post on ptt 13 | type Post struct { 14 | Title string 15 | Href string 16 | NVote int 17 | Date time.Time 18 | } 19 | 20 | // fetchPreviewImg get the preview image of a post 21 | func fetchPreviewImg(p *Post) string { 22 | // TODO: handle error 23 | client := http.DefaultClient 24 | req, _ := http.NewRequest("GET", p.Href, nil) 25 | req.Header.Set("Cookie", "over18=1") 26 | 27 | res, _ := client.Do(req) 28 | doc, _ := goquery.NewDocumentFromResponse(res) 29 | 30 | imgSelector := `#main-content a[href$=".jpg"],a[href$=".png"],a[href$=".gif"]` 31 | imgURL, _ := doc.Find(imgSelector).Attr("href") 32 | return imgURL 33 | } 34 | 35 | // fetchImageAmount get the amount of images in a post 36 | func fetchImageAmount(p *Post) int { 37 | // TODO: handle error 38 | client := http.DefaultClient 39 | req, _ := http.NewRequest("GET", p.Href, nil) 40 | req.Header.Set("Cookie", "over18=1") 41 | 42 | res, _ := client.Do(req) 43 | doc, _ := goquery.NewDocumentFromResponse(res) 44 | 45 | doc.Find("div.push").Each(func(i int, s *goquery.Selection) { 46 | // remove push comment 47 | s.Remove() 48 | }) 49 | imgSelector := `#main-content a[href$=".jpg"],a[href$=".png"],a[href$=".gif"]` 50 | nImage := doc.Find(imgSelector).Size() 51 | return nImage 52 | } 53 | 54 | // "[正妹] 大橋未久" -> "大橋未久" 55 | func trimTitlePrefix(title string) string { 56 | return strings.TrimPrefix(title, "[正妹] ") 57 | } 58 | 59 | // transform https://www.ptt.cc/bbs/Beauty/M.1543991133.A.1A1.html 60 | // to https://daily-beauty.xyz/ptt/redirect/M.1543991133.A.1A1 61 | func transformURL(pttURL string) string { 62 | var articleID string 63 | fmt.Sscanf(pttURL, "https://www.ptt.cc/bbs/Beauty/%18s.html", &articleID) 64 | return fmt.Sprintf("https://daily-beauty.xyz/ptt/redirect/%s", articleID) 65 | } 66 | 67 | // ToBeauty transform a Post to a Beauty 68 | func (p *Post) ToBeauty() Beauty { 69 | previewImg := fetchPreviewImg(p) 70 | nImage := fetchImageAmount(p) 71 | return Beauty{ 72 | NVote: p.NVote, 73 | NImage: nImage, 74 | Title: trimTitlePrefix(p.Title), 75 | Href: transformURL(p.Href), 76 | PreviewImg: previewImg, 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /model/post_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestPostToBeauty(t *testing.T) { 9 | { 10 | p := Post{ 11 | Title: "[正妹] 覺得還不錯", 12 | Href: "https://www.ptt.cc/bbs/Beauty/M.1543280871.A.39A.html", 13 | NVote: 50, 14 | Date: time.Now(), 15 | } 16 | b := p.ToBeauty() 17 | if b.PreviewImg != "https://imgur.com/30XW9qD.jpg" { 18 | t.Error("preview image error") 19 | } 20 | if b.Title != "覺得還不錯" { 21 | t.Error("trim title error") 22 | } 23 | if b.Href != "https://daily-beauty.xyz/ptt/redirect/M.1543280871.A.39A" { 24 | t.Errorf("transform url error %s", b.Href) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /ptt/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | 8 | "main/model" 9 | 10 | "github.com/PuerkitoBio/goquery" 11 | ) 12 | 13 | // FetchPageAmount get latest page number 14 | func FetchPageAmount() (int, error) { 15 | url := "https://www.ptt.cc/bbs/Beauty/index.html" 16 | 17 | client := http.DefaultClient 18 | req, _ := http.NewRequest("GET", url, nil) 19 | req.Header.Set("Cookie", "over18=1") 20 | 21 | res, _ := client.Do(req) 22 | doc, _ := goquery.NewDocumentFromResponse(res) 23 | 24 | prevPageSelector := "div.btn-group.btn-group-paging a:nth-child(2)" 25 | href, _ := doc.Find(prevPageSelector).Attr("href") 26 | 27 | var n int 28 | fmt.Sscanf(href, "/bbs/Beauty/index%d.html", &n) 29 | 30 | if n == 0 { 31 | return 0, errors.New("Cannot connect to PTT") 32 | } 33 | return n + 1, nil 34 | } 35 | 36 | // FetchPage get all posts in a page 37 | func FetchPage(prefix string, page int) ([]model.Post, error) { 38 | baseURL := "https://www.ptt.cc/bbs/Beauty/" 39 | url := fmt.Sprintf("%sindex%d.html", baseURL, page) 40 | 41 | // TODO: refactor HTTP client 42 | client := http.DefaultClient 43 | req, _ := http.NewRequest("GET", url, nil) 44 | req.Header.Set("Cookie", "over18=1") 45 | 46 | res, _ := client.Do(req) 47 | doc, err := goquery.NewDocumentFromResponse(res) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | posts := parseDoc2Posts(doc, prefix) 53 | return posts, nil 54 | } 55 | 56 | // Search use PTT search to get search result 57 | // sometimes PTT cache search result 58 | func Search(prefix string, page, recommend int) ([]model.Post, error) { 59 | // page from 1, 2, ... 60 | baseURL := "https://www.ptt.cc/bbs/Beauty/search" 61 | url := fmt.Sprintf("%s?page=%d&q=%s+recommend:%d", baseURL, page, prefix, recommend) 62 | 63 | client := http.DefaultClient 64 | req, _ := http.NewRequest("GET", url, nil) 65 | req.Header.Set("Cookie", "over18=1") 66 | 67 | res, _ := client.Do(req) 68 | doc, err := goquery.NewDocumentFromResponse(res) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | posts := parseDoc2Posts(doc, prefix) 74 | return posts, nil 75 | } 76 | -------------------------------------------------------------------------------- /ptt/api/api_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestFetchPageAmount(t *testing.T) { 9 | nPage, err := FetchPageAmount() 10 | if nPage < 2733 { 11 | t.Errorf("nPage is wrong") 12 | } 13 | if err != nil { 14 | t.Error(nil) 15 | } 16 | } 17 | 18 | func TestFetchPage(t *testing.T) { 19 | prefix := "[" 20 | posts, err := FetchPage(prefix, 2733) 21 | if err != nil { 22 | t.Error(err) 23 | } 24 | if len(posts) == 0 { 25 | t.Errorf("posts should NOT be empty") 26 | } 27 | if !strings.HasPrefix(posts[0].Title, prefix) { 28 | t.Errorf("posts should be prefixed with %s", prefix) 29 | } 30 | } 31 | 32 | func TestSearch(t *testing.T) { 33 | prefix := "[" 34 | posts, err := Search(prefix, 1, 10) 35 | if err != nil { 36 | t.Error(err) 37 | } 38 | if len(posts) == 0 { 39 | t.Errorf("len(posts) should be 20") 40 | } 41 | if posts[0].NVote < 10 { 42 | t.Errorf("post.NVote should >= 10") 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /ptt/api/parser.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | "time" 8 | 9 | "main/model" 10 | 11 | "github.com/PuerkitoBio/goquery" 12 | "github.com/vjeantet/jodaTime" 13 | ) 14 | 15 | // parseNVote parses vote text to int 16 | // "50" => 50, "爆" => 100 17 | // "" => 0 18 | // "X7" => -1 19 | // there is no need to handle nVote <= 0 20 | // because they are filterer out when searching 21 | func parseNVote(nVoteText string) int { 22 | if nVoteText == "爆" { 23 | return 100 24 | } 25 | if nVoteText == "" { 26 | return 0 27 | } 28 | if strings.HasPrefix(nVoteText, "X") { 29 | return -1 30 | } 31 | nVote, _ := strconv.Atoi(nVoteText) 32 | return nVote 33 | } 34 | 35 | func parseDoc2Posts(doc *goquery.Document, prefix string) []model.Post { 36 | // TODO: remove 置頂文 37 | posts := make([]model.Post, 0, 20) 38 | doc.Find(".r-ent").Each(func(i int, el *goquery.Selection) { 39 | nVoteText := el.Find(".hl").Text() 40 | nVote := parseNVote(nVoteText) 41 | 42 | titleEl := el.Find(".title > a") 43 | title := titleEl.Text() 44 | 45 | if !strings.HasPrefix(title, prefix) { 46 | return 47 | } 48 | 49 | hrefText, _ := titleEl.Attr("href") 50 | href := "https://www.ptt.cc" + hrefText 51 | 52 | currentYear := time.Now().Year() 53 | mmdd := strings.TrimSpace(el.Find(".meta .date").Text()) 54 | dateText := fmt.Sprintf("%d/%s", currentYear, mmdd) 55 | date, _ := jodaTime.ParseInLocation("YYYY/M/dd", dateText, "Asia/Taipei") 56 | 57 | p := model.Post{ 58 | Title: title, 59 | Href: href, 60 | NVote: nVote, 61 | Date: date, 62 | } 63 | 64 | posts = append(posts, p) 65 | }) 66 | return posts 67 | } 68 | -------------------------------------------------------------------------------- /ptt/ptt.go: -------------------------------------------------------------------------------- 1 | package ptt 2 | 3 | import ( 4 | "main/model" 5 | "main/ptt/api" 6 | "math/rand" 7 | "sort" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | // TODO: split ptt api layer and utils layer 13 | func init() { 14 | rand.Seed(time.Now().UnixNano()) 15 | } 16 | 17 | func fetchYesterdayPosts() ([]model.Post, error) { 18 | prefix := "[正妹]" 19 | recentPosts := make([]model.Post, 0, 20) 20 | 21 | // get recent posts 22 | page, err := api.FetchPageAmount() 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | for ; ; page-- { 28 | posts, err := api.FetchPage(prefix, page) 29 | 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | recentPosts = append(recentPosts, posts...) 35 | oldestDate := recentPosts[len(recentPosts)-1].Date 36 | if isBeforeYesterday(oldestDate) { 37 | break 38 | } 39 | } 40 | 41 | // filter yesterday post 42 | yesterdayPosts := make([]model.Post, 0, 10) 43 | for _, p := range recentPosts { 44 | if isYesterday(p.Date) { 45 | yesterdayPosts = append(yesterdayPosts, p) 46 | } 47 | } 48 | 49 | return yesterdayPosts, nil 50 | } 51 | 52 | // FetchRandomBeauty randomly fetch a model.Beauty 53 | func FetchRandomBeauty() (model.Beauty, error) { 54 | prefix := "[正妹]" 55 | page := rand.Intn(40) + 11 // 11 ~ 50 56 | posts, err := api.Search(prefix, page, 99) 57 | 58 | if err != nil { 59 | return model.Beauty{}, err 60 | } 61 | 62 | idx := rand.Intn(len(posts)) // 0 ~ len(posts)-1 63 | p := posts[idx] 64 | b := p.ToBeauty() 65 | return b, nil 66 | } 67 | 68 | func getBestBeauties(posts []model.Post) []model.Beauty { 69 | sort.SliceStable(posts, func(i, j int) bool { 70 | return posts[i].NVote > posts[j].NVote 71 | }) 72 | 73 | nBeauty := 2 74 | champions := posts[:nBeauty] 75 | beauties := make([]model.Beauty, nBeauty) 76 | 77 | var wg sync.WaitGroup 78 | wg.Add(nBeauty) 79 | for i, p := range champions { 80 | go func(i int, p model.Post) { 81 | beauties[i] = p.ToBeauty() 82 | wg.Done() 83 | }(i, p) 84 | } 85 | wg.Wait() 86 | 87 | return beauties 88 | } 89 | 90 | // FetchBeauties send a request to get beauties from getDailyBeauties api 91 | func FetchBeauties() ([]model.Beauty, error) { 92 | posts, err := fetchYesterdayPosts() 93 | if err != nil { 94 | return nil, err 95 | } 96 | beauties := getBestBeauties(posts) 97 | return beauties, nil 98 | } 99 | -------------------------------------------------------------------------------- /ptt/time.go: -------------------------------------------------------------------------------- 1 | package ptt 2 | 3 | import "time" 4 | 5 | func isToday(t time.Time) bool { 6 | loc, _ := time.LoadLocation("Asia/Taipei") 7 | current := time.Now().In(loc) 8 | return t.YearDay() == current.YearDay() 9 | } 10 | 11 | func isYesterday(t time.Time) bool { 12 | // FIXME: 跨年 1/1 < 12/31 13 | loc, _ := time.LoadLocation("Asia/Taipei") 14 | current := time.Now().In(loc) 15 | return t.YearDay() == current.YearDay()-1 16 | } 17 | 18 | func isBeforeYesterday(t time.Time) bool { 19 | // FIXME: 跨年 1/1 < 12/31 20 | loc, _ := time.LoadLocation("Asia/Taipei") 21 | current := time.Now().In(loc) 22 | return t.YearDay() < current.YearDay()-1 23 | } 24 | --------------------------------------------------------------------------------