├── README.md ├── scripts ├── remove_subscribers.json └── add_subscribers.json ├── package.json ├── pkg ├── common │ ├── endpoint.go │ ├── jsontime_test.go │ ├── token_test.go │ ├── subscriber_test.go │ ├── token.go │ ├── store.go │ ├── jsontime.go │ ├── sesmessage_test.go │ ├── subscriber.go │ └── sesmessage.go ├── db │ ├── notifications.go │ └── subscribers.go ├── email │ ├── email.go │ └── confirm_html.go └── api │ ├── api.go │ └── api_test.go ├── .travis.yml ├── .gitignore ├── cmd ├── listing-cli │ ├── filter.go │ ├── complaints.go │ ├── unsubscribe.go │ ├── delete.go │ ├── subscribe.go │ ├── import.go │ ├── client.go │ ├── export.go │ ├── printer.go │ ├── main.go │ └── client_test.go ├── listing-send │ ├── sender.go │ ├── campaign.go │ └── main.go ├── ladmin │ └── main.go ├── sesnotify │ └── main.go └── listing │ └── main.go ├── .codecov.yml ├── secrets.json.example ├── appveyor.yml ├── Gopkg.toml ├── docs ├── ENDPOINTS.md ├── SEND.md ├── CLI.md ├── TESTING.md └── DEPLOYMENT.md ├── LICENSE ├── Makefile ├── serverless-admin.yml ├── serverless-db.yml ├── serverless-api.yml └── Gopkg.lock /README.md: -------------------------------------------------------------------------------- 1 | # Moved to [GitLab](https://gitlab.com/ribtoks/listing/) 2 | -------------------------------------------------------------------------------- /scripts/remove_subscribers.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"newsletter":"Listing1","email":"email1@domain.com"}, 3 | {"newsletter":"Listing1","email":"email9@domain.com"}, 4 | {"newsletter":"Listing1","email":"ema1l1@domain.com"}, 5 | {"newsletter":"Listing1","email":"ema5l1@domain.com"} 6 | ] 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "listing", 3 | "description": "", 4 | "version": "0.1.0", 5 | "dependencies": { 6 | "serverless-domain-manager": "^3.3.2" 7 | }, 8 | "devDependencies": { 9 | "serverless-iam-roles-per-function": "^2.0.2", 10 | "serverless-localstack": "^0.4.19" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /pkg/common/endpoint.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const ( 4 | SubscribersEndpoint = "/subscribers" 5 | SubscribeEndpoint = "/subscribe" 6 | UnsubscribeEndpoint = "/unsubscribe" 7 | ComplaintsEndpoint = "/complaints" 8 | ConfirmEndpoint = "/confirm" 9 | ParamNewsletter = "newsletter" 10 | ParamToken = "token" 11 | ParamEmail = "email" 12 | ParamName = "name" 13 | ) 14 | -------------------------------------------------------------------------------- /pkg/common/jsontime_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "testing" 4 | 5 | func TestJsonTimeMarshal(t *testing.T) { 6 | jt := JsonTimeNow() 7 | b, err := jt.MarshalJSON() 8 | if err != nil { 9 | t.Fatal(err) 10 | } 11 | var jt2 JSONTime 12 | err = jt2.UnmarshalJSON(b) 13 | if err != nil { 14 | t.Fatal(err) 15 | } 16 | if jt.String() != jt2.String() { 17 | t.Errorf("Times are not equal. jt=%v jt2=%v", jt.Time(), jt2.Time()) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pkg/common/token_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "testing" 4 | 5 | func TestTokenSignUnsign(t *testing.T) { 6 | secret := "abcd" 7 | value := "email@domain.com" 8 | 9 | signed := Sign(secret, value) 10 | unsigned, success := Unsign(secret, signed) 11 | 12 | if !success { 13 | t.Errorf("Failed to unsign") 14 | } 15 | 16 | if unsigned != value { 17 | t.Errorf("Values do not match. unsigned=%v value=%v", unsigned, value) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: true 2 | dist: trusty 3 | 4 | language: go 5 | 6 | os: 7 | - linux 8 | 9 | git: 10 | depth: 3 11 | 12 | before_script: 13 | - go version 14 | - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 15 | - dep ensure 16 | 17 | script: 18 | - make build 19 | - go test -v -covermode=atomic -coverprofile=coverage.out ./... 20 | - bash <(curl -s https://codecov.io/bash) || echo 'Codecov failed to upload'; 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | vendor/ 16 | node_modules/ 17 | .serverless/ 18 | 19 | secrets.json 20 | 21 | *.swp 22 | *.swo 23 | 24 | bin/ 25 | listing-cli/listing-cli 26 | listing-cli.log 27 | -------------------------------------------------------------------------------- /cmd/listing-cli/filter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "log" 4 | 5 | func (c *listingClient) filter(data []byte) error { 6 | ss, err := c.parseSubscribers(data) 7 | if err != nil { 8 | return err 9 | } 10 | skipped := 0 11 | for _, s := range ss { 12 | if c.isSubscriberOK(s) { 13 | c.printer.Append(s) 14 | } else { 15 | skipped += 1 16 | } 17 | } 18 | c.printer.Render() 19 | log.Printf("Filtered subscribers. count=%v skipped=%v", len(ss), skipped) 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: yes 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "70...100" 8 | 9 | status: 10 | project: 11 | default: 12 | target: 80% 13 | threshold: 2% 14 | patch: off 15 | changes: no 16 | 17 | parsers: 18 | gcov: 19 | branch_detection: 20 | conditional: yes 21 | loop: yes 22 | method: no 23 | macro: no 24 | 25 | comment: 26 | layout: "header,diff" 27 | behavior: default 28 | require_changes: no 29 | -------------------------------------------------------------------------------- /secrets.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "tokenSecret": "85e0c0d3d4f0837c7f3d9201bf", 3 | "apiToken": "996558b4f0837c7f3d9201bfd23391dd7", 4 | "subscribeRedirectUrl": "http://localhost:1313/", 5 | "unsubscribeRedirectUrl": "http://localhost:1313/", 6 | "confirmRedirectUrl": "http://localhost:1313/", 7 | "confirmUrl": "http://localhost:1313/", 8 | "supportedNewsletters": "Listing1;Listing2", 9 | "emailFrom": "no-reply@test.test", 10 | "devDomain": "dev.domain.com", 11 | "prodDomain": "prod.domain.com" 12 | } 13 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | # version format 2 | version: 0.1.{build}-{branch} 3 | 4 | skip_tags: false 5 | 6 | skip_commits: 7 | message: /.*\[ci skip\]/ # Regex for matching commit message 8 | 9 | # clone directory 10 | clone_folder: c:\gopath\src\github.com\ribtoks\listing 11 | 12 | environment: 13 | GOPATH: c:\gopath 14 | 15 | clone_depth: 3 # clone entire repository history if not defined 16 | 17 | before_build: 18 | - go version 19 | - choco install dep 20 | - dep ensure 21 | 22 | build_script: 23 | - go build ./... 24 | 25 | test_script: 26 | - go test -v ./... 27 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | 22 | 23 | [[constraint]] 24 | name = "github.com/aws/aws-lambda-go" 25 | version = "1.x" 26 | -------------------------------------------------------------------------------- /pkg/common/subscriber_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestConfirmed(t *testing.T) { 9 | s := &Subscriber{ 10 | CreatedAt: JsonTimeNow(), 11 | ConfirmedAt: JSONTime(time.Unix(1, 1)), 12 | } 13 | if s.Confirmed() { 14 | t.Errorf("Subscriber is confirmed with incorrect time") 15 | } 16 | s.ConfirmedAt = JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 17 | if !s.Confirmed() { 18 | t.Errorf("Subscriber is not confirmed with correct time") 19 | } 20 | } 21 | 22 | func TestSubscribed(t *testing.T) { 23 | s := &Subscriber{ 24 | CreatedAt: JsonTimeNow(), 25 | UnsubscribedAt: JSONTime(time.Unix(1, 1)), 26 | } 27 | if s.Unsubscribed() { 28 | t.Errorf("Subscriber is unsubscribed with incorrect time") 29 | } 30 | s.UnsubscribedAt = JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 31 | if !s.Unsubscribed() { 32 | t.Errorf("Subscriber is not unsubscribed with correct time") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /docs/ENDPOINTS.md: -------------------------------------------------------------------------------- 1 | Here are endpoints created by *Listing* 2 | 3 | Endpoint | Method | Parameters | Description 4 | --- | --- | --- | --- 5 | `/subscribe` | POST | `newsletter`, `email`, `name`? | Subscribe form on your website 6 | `/confirm` | GET | `newsletter`, `token` | "Confirm Email" button in the confirmation email 7 | `/unsubscribe` | GET | `newsletter`, `token` | "Unsubscribe" link in the newsletter emails 8 | `/subscribers` | GET | `newsletter` | Protected API to retrieve all subscribers for a newsletter 9 | `/subscribers` | PUT | JSON with Subscribers array | Protected API to import subscribers 10 | `/subscribers` | DELETE | JSON with Subscriber Keys array | Protected API to delete subscribers 11 | `/complaints` | GET | none | Protected API to retrieve all bounces and complaints from AWS SES 12 | 13 | `token` parameter is a salted hash of the email used to uniquely identify every user. It is a security measure to protect from unauthorized unsubscribes/confirmations. 14 | 15 | `name` parameter in `/subscribe` endpoint is optional. 16 | -------------------------------------------------------------------------------- /cmd/listing-send/sender.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | "net/url" 8 | "strconv" 9 | 10 | "github.com/go-gomail/gomail" 11 | ) 12 | 13 | func smtpDialer(smtpUrl, user, pass string) (*gomail.Dialer, error) { 14 | surl, err := url.Parse(smtpUrl) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | // Port 20 | var port int 21 | if i, err := strconv.Atoi(surl.Port()); err == nil { 22 | port = i 23 | } else if surl.Scheme == "smtp" { 24 | port = 25 25 | } else { 26 | port = 465 27 | } 28 | 29 | d := gomail.NewPlainDialer(surl.Hostname(), port, user, pass) 30 | d.SSL = (surl.Scheme == "smtps") 31 | return d, nil 32 | } 33 | 34 | type dryRunSender struct { 35 | out string 36 | } 37 | 38 | func (s *dryRunSender) Send(from string, to []string, msg io.WriterTo) error { 39 | var buf bytes.Buffer 40 | msg.WriteTo(&buf) 41 | ioutil.WriteFile(s.out+to[0]+".eml", buf.Bytes(), 0644) 42 | return nil 43 | } 44 | 45 | func (s *dryRunSender) Close() error { 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/common/token.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha256" 6 | "encoding/base64" 7 | "strings" 8 | ) 9 | 10 | // helpers. 11 | var ( 12 | encode = base64.URLEncoding.EncodeToString 13 | decode = base64.URLEncoding.DecodeString 14 | ) 15 | 16 | // Sign a value. 17 | func Sign(secret, value string) string { 18 | m := hmac.New(sha256.New, []byte(secret)) 19 | m.Write([]byte(value)) 20 | return encode(m.Sum(nil)) + "." + encode([]byte(value)) 21 | } 22 | 23 | // Unsign a value. 24 | func Unsign(secret, msg string) (string, bool) { 25 | p := strings.Split(msg, ".") 26 | if len(p) != 2 { 27 | return "", false 28 | } 29 | 30 | signature, err := decode(p[0]) 31 | if err != nil { 32 | return "", false 33 | } 34 | 35 | payload, err := decode(p[1]) 36 | if err != nil { 37 | return "", false 38 | } 39 | 40 | m := hmac.New(sha256.New, []byte(secret)) 41 | m.Write(payload) 42 | expected := m.Sum(nil) 43 | 44 | if !hmac.Equal(signature, expected) { 45 | return "", false 46 | } 47 | 48 | return string(payload), true 49 | } 50 | -------------------------------------------------------------------------------- /pkg/common/store.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // SubscribersStore is an interface used to manage subscribers DB from the main API 4 | type SubscribersStore interface { 5 | AddSubscriber(newsletter, email, name string) error 6 | RemoveSubscriber(newsletter, email string) error 7 | Subscribers(newsletter string) (subscribers []*Subscriber, err error) 8 | AddSubscribers(subscribers []*Subscriber) error 9 | DeleteSubscribers(keys []*SubscriberKey) error 10 | ConfirmSubscriber(newsletter, email string) error 11 | GetSubscriber(newsletter, email string) (*Subscriber, error) 12 | } 13 | 14 | // Mailer is an interface for sending confirmation emails for subscriptions 15 | type Mailer interface { 16 | SendConfirmation(newsletter, email, name, confirmURL string) error 17 | } 18 | 19 | // NotificationsStore is an interface used to manage SES bounce and complaint 20 | // notifications from sesnotify API 21 | type NotificationsStore interface { 22 | AddBounce(email, from string, isTransient bool) error 23 | AddComplaint(email, from string) error 24 | Notifications() (notifications []*SesNotification, err error) 25 | } 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Taras Kushnir 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pkg/common/jsontime.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | const jsonTimeLayout = time.RFC3339 10 | 11 | // JSONTime is the time.Time with JSON marshal and unmarshal capability 12 | type JSONTime time.Time 13 | 14 | // JsonTimeNow() is an alias to time.Now() casted to JSONTime 15 | func JsonTimeNow() JSONTime { 16 | return JSONTime(time.Now().UTC()) 17 | } 18 | 19 | // UnmarshalJSON will unmarshal using 2006-01-02T15:04:05+07:00 layout 20 | func (t *JSONTime) UnmarshalJSON(b []byte) error { 21 | s := strings.Trim(string(b), `"`) 22 | nt, err := time.Parse(jsonTimeLayout, s) 23 | if err != nil { 24 | return err 25 | } 26 | *t = JSONTime(nt) 27 | return nil 28 | } 29 | 30 | // Time returns builtin time.Time for current JSONTime 31 | func (t JSONTime) Time() time.Time { 32 | return time.Time(t) 33 | } 34 | 35 | // MarshalJSON will marshal using 2006-01-02T15:04:05+07:00 layout 36 | func (t *JSONTime) MarshalJSON() ([]byte, error) { 37 | return []byte(t.String()), nil 38 | } 39 | 40 | // String returns the time in the custom format 41 | func (t JSONTime) String() string { 42 | ct := time.Time(t) 43 | return fmt.Sprintf("%q", ct.Format(jsonTimeLayout)) 44 | } 45 | -------------------------------------------------------------------------------- /cmd/listing-cli/complaints.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/ribtoks/listing/pkg/common" 9 | ) 10 | 11 | var ( 12 | emptyComplaints []*common.SesNotification 13 | ) 14 | 15 | func (c *listingClient) fetchComplaints(url string) ([]*common.SesNotification, error) { 16 | log.Printf("About to fetch complaints. url=%v", url) 17 | if c.dryRun { 18 | log.Printf("Dry run mode. Exiting...") 19 | return emptyComplaints, nil 20 | } 21 | 22 | req, err := http.NewRequest("GET", url, nil) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | req.SetBasicAuth("any", c.authToken) 28 | resp, err := c.client.Do(req) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | log.Printf("Received complaints response. status=%v", resp.StatusCode) 34 | 35 | defer resp.Body.Close() 36 | ss := make([]*common.SesNotification, 0) 37 | err = json.NewDecoder(resp.Body).Decode(&ss) 38 | return ss, nil 39 | } 40 | 41 | func (c *listingClient) updateComplaints() error { 42 | endpoint, err := c.complaintsURL() 43 | if err != nil { 44 | return err 45 | } 46 | 47 | complaints, err := c.fetchComplaints(endpoint) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | for _, ct := range complaints { 53 | if ct.Notification == common.SoftBounceType { 54 | continue 55 | } 56 | 57 | c.complaints[ct.Email] = true 58 | } 59 | 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /cmd/listing-cli/unsubscribe.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | 9 | "github.com/ribtoks/listing/pkg/common" 10 | ) 11 | 12 | func (c *listingClient) sendUnsubscribeRequest(email, newsletter, endpoint string) error { 13 | log.Printf("About to send unsubscribe request. email=%v newsletter=%v url=%v", email, newsletter, endpoint) 14 | if c.dryRun { 15 | log.Println("Dry run mode. Exiting...") 16 | return nil 17 | } 18 | 19 | req, err := http.NewRequest("GET", endpoint, nil) 20 | q := req.URL.Query() 21 | q.Add(common.ParamNewsletter, newsletter) 22 | q.Add(common.ParamToken, common.Sign(c.secret, email)) 23 | req.URL.RawQuery = q.Encode() 24 | 25 | resp, err := c.client.Do(req) 26 | if err != nil { 27 | return err 28 | } 29 | log.Printf("Received unsubscribe response. status=%v", resp.StatusCode) 30 | if resp.StatusCode != http.StatusFound { 31 | body, _ := ioutil.ReadAll(resp.Body) 32 | return fmt.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 33 | } 34 | return nil 35 | } 36 | 37 | func (c *listingClient) unsubscribe(email, newsletter string) error { 38 | if email == "" { 39 | return errMissingEmail 40 | } 41 | 42 | if newsletter == "" { 43 | return errMissingNewsletter 44 | } 45 | 46 | url, err := c.unsubscribeURL() 47 | if err != nil { 48 | return err 49 | } 50 | 51 | return c.sendUnsubscribeRequest(email, newsletter, url) 52 | } 53 | -------------------------------------------------------------------------------- /pkg/common/sesmessage_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | ) 7 | 8 | const sesNotificationJson = `{ 9 | "notificationType":"Complaint", 10 | "complaint":{ 11 | "userAgent":"Comcast Feedback Loop (V0.01)", 12 | "complainedRecipients":[ 13 | { 14 | "emailAddress":"recipient1@example.com" 15 | } 16 | ], 17 | "complaintFeedbackType":"abuse", 18 | "arrivalDate":"2009-12-03T04:24:21.000-05:00", 19 | "timestamp":"2012-05-25T14:59:38.623-07:00", 20 | "feedbackId":"000001378603177f-18c07c78-fa81-4a58-9dd1-fedc3cb8f49a-000000" 21 | }, 22 | "mail":{ 23 | "timestamp":"2012-05-25T14:59:38.623-07:00", 24 | "messageId":"000001378603177f-7a5433e7-8edb-42ae-af10-f0181f34d6ee-000000", 25 | "source":"email_1337983178623@amazon.com", 26 | "sourceArn": "arn:aws:sns:us-east-1:XXXXXXXXXXXX:ses-notifications", 27 | "sendingAccountId":"XXXXXXXXXXXX", 28 | "destination":[ 29 | "recipient1@example.com", 30 | "recipient2@example.com", 31 | "recipient3@example.com", 32 | "recipient4@example.com" 33 | ] 34 | } 35 | }` 36 | 37 | func TestParseSesNotification(t *testing.T) { 38 | var sesMessage SesMessage 39 | err := json.Unmarshal([]byte(sesNotificationJson), &sesMessage) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build clean deploy 2 | 3 | STAGE ?= dev 4 | REGION ?= eu-west-1 5 | 6 | test: 7 | go test ./... 8 | 9 | build: 10 | dep ensure -v 11 | go build -o cmd/listing-cli/listing-cli cmd/listing-cli/*.go 12 | go build -o cmd/listing-send/listing-send cmd/listing-send/*.go 13 | env GOOS=linux go build -ldflags="-s -w" -o bin/listing cmd/listing/*.go 14 | env GOOS=linux go build -ldflags="-s -w" -o bin/sesnotify cmd/sesnotify/*.go 15 | env GOOS=linux go build -ldflags="-s -w" -o bin/ladmin cmd/ladmin/*.go 16 | 17 | clean: 18 | rm -rf ./bin ./vendor ./.serverless Gopkg.lock 19 | 20 | domain: 21 | sls create_domain --stage '${STAGE}' --region '$(REGION)' --config serverless-api.yml 22 | 23 | deploy-db: 24 | sls deploy --config serverless-db.yml --stage '$(STAGE)' --region '$(REGION)' --verbose 25 | 26 | remove-db: 27 | sls remove --config serverless-db.yml --stage '$(STAGE)' --region '$(REGION)' --verbose 28 | 29 | deploy-api: 30 | sls deploy --config serverless-api.yml --stage '$(STAGE)' --region '$(REGION)' --verbose 31 | 32 | remove-api: 33 | sls remove --config serverless-api.yml --stage '$(STAGE)' --region '$(REGION)' --verbose 34 | 35 | deploy-admin: 36 | sls deploy --config serverless-admin.yml --stage '$(STAGE)' --region '$(REGION)' --verbose 37 | 38 | remove-admin: 39 | sls remove --config serverless-admin.yml --stage '$(STAGE)' --region '$(REGION)' --verbose 40 | 41 | deploy-all: clean build deploy-db deploy-api deploy-admin 42 | echo "Done for stage=${STAGE} region=${REGION}" 43 | 44 | remove-all: remove-api remove-admin remove-db 45 | echo "Done for stage=${STAGE} region=${REGION}" 46 | -------------------------------------------------------------------------------- /cmd/listing-cli/delete.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | 11 | "github.com/ribtoks/listing/pkg/common" 12 | ) 13 | 14 | func (c *listingClient) prepareDeletePayload(data []byte) ([]byte, error) { 15 | subscribers, err := c.parseSubscribers(data) 16 | if err != nil { 17 | return nil, err 18 | } 19 | keys := make([]*common.SubscriberKey, 0) 20 | for _, s := range subscribers { 21 | keys = append(keys, &common.SubscriberKey{ 22 | Email: s.Email, 23 | Newsletter: s.Newsletter, 24 | }) 25 | } 26 | return json.Marshal(keys) 27 | } 28 | 29 | func (c *listingClient) sendDeleteRequest(endpoint string, payload []byte) error { 30 | req, err := http.NewRequest("DELETE", endpoint, bytes.NewBuffer(payload)) 31 | if err != nil { 32 | return err 33 | } 34 | req.Header.Set("Content-Type", "application/json") 35 | req.SetBasicAuth("any", c.authToken) 36 | 37 | resp, err := c.client.Do(req) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | if resp.StatusCode != http.StatusOK { 43 | body, _ := ioutil.ReadAll(resp.Body) 44 | return fmt.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 45 | 46 | } 47 | return nil 48 | } 49 | 50 | func (c *listingClient) deleteSubscribers(data []byte) error { 51 | endpoint, err := c.importURL() 52 | if err != nil { 53 | return err 54 | } 55 | payload, err := c.prepareDeletePayload(data) 56 | if err != nil { 57 | return err 58 | } 59 | log.Printf("About to send delete request. bytes=%v", len(payload)) 60 | if c.dryRun { 61 | log.Println("Dry run mode. Exiting...") 62 | return nil 63 | } 64 | return c.sendDeleteRequest(endpoint, payload) 65 | } 66 | -------------------------------------------------------------------------------- /cmd/ladmin/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strings" 9 | 10 | "github.com/aws/aws-lambda-go/events" 11 | "github.com/aws/aws-lambda-go/lambda" 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/awslabs/aws-lambda-go-api-proxy/httpadapter" 15 | "github.com/ribtoks/listing/pkg/api" 16 | "github.com/ribtoks/listing/pkg/db" 17 | ) 18 | 19 | var ( 20 | handlerLambda *httpadapter.HandlerAdapter 21 | ) 22 | 23 | // Handler is the main entry point to this lambda 24 | func Handler(ctx context.Context, req events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { 25 | return handlerLambda.ProxyWithContext(ctx, req) 26 | } 27 | 28 | func main() { 29 | apiToken := os.Getenv("API_TOKEN") 30 | subscribersTableName := os.Getenv("SUBSCRIBERS_TABLE") 31 | notificationsTableName := os.Getenv("NOTIFICATIONS_TABLE") 32 | supportedNewsletters := os.Getenv("SUPPORTED_NEWSLETTERS") 33 | 34 | sess, err := session.NewSession(&aws.Config{ 35 | Region: aws.String(os.Getenv("AWS_REGION")), 36 | }) 37 | 38 | if err != nil { 39 | log.Fatalf("Failed to create AWS session. err=%v", err) 40 | } 41 | 42 | subscribers := db.NewSubscribersStore(subscribersTableName, sess) 43 | notifications := db.NewNotificationsStore(notificationsTableName, sess) 44 | 45 | router := http.NewServeMux() 46 | newsletter := &api.AdminResource{ 47 | APIToken: apiToken, 48 | Subscribers: subscribers, 49 | Notifications: notifications, 50 | Newsletters: make(map[string]bool), 51 | } 52 | 53 | sn := strings.Split(supportedNewsletters, ";") 54 | newsletter.AddNewsletters(sn) 55 | 56 | newsletter.Setup(router) 57 | handlerLambda = httpadapter.New(router) 58 | 59 | lambda.Start(Handler) 60 | } 61 | -------------------------------------------------------------------------------- /cmd/listing-cli/subscribe.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/ribtoks/listing/pkg/common" 14 | ) 15 | 16 | var ( 17 | errMissingNewsletter = errors.New("Newsletter parameter is empty") 18 | errMissingEmail = errors.New("Email parameter is empty") 19 | ) 20 | 21 | func (c *listingClient) sendSubscribeRequest(email, newsletter, name, endpoint string) error { 22 | log.Printf("About to send subscribe request. email=%v newsletter=%v name=%v url=%v", email, newsletter, name, endpoint) 23 | if c.dryRun { 24 | log.Println("Dry run mode. Exiting...") 25 | return nil 26 | } 27 | data := url.Values{} 28 | data.Set(common.ParamNewsletter, newsletter) 29 | data.Set(common.ParamEmail, email) 30 | if name != "" { 31 | data.Set(common.ParamName, name) 32 | } 33 | encoded := data.Encode() 34 | req, err := http.NewRequest("POST", endpoint, strings.NewReader(encoded)) 35 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 36 | req.Header.Add("Content-Length", strconv.Itoa(len(encoded))) 37 | 38 | resp, err := c.client.Do(req) 39 | if err != nil { 40 | return err 41 | } 42 | log.Printf("Received subscribe response. status=%v", resp.StatusCode) 43 | if resp.StatusCode != http.StatusFound { 44 | body, _ := ioutil.ReadAll(resp.Body) 45 | return fmt.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 46 | } 47 | return nil 48 | } 49 | 50 | func (c *listingClient) subscribe(email, newsletter, name string) error { 51 | if email == "" { 52 | return errMissingEmail 53 | } 54 | 55 | if newsletter == "" { 56 | return errMissingNewsletter 57 | } 58 | 59 | url, err := c.subscribeURL() 60 | if err != nil { 61 | return err 62 | } 63 | 64 | return c.sendSubscribeRequest(email, newsletter, name, url) 65 | } 66 | -------------------------------------------------------------------------------- /cmd/listing-cli/import.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | 11 | "github.com/ribtoks/listing/pkg/common" 12 | ) 13 | 14 | func (c *listingClient) parseSubscribers(data []byte) ([]*common.Subscriber, error) { 15 | dec := json.NewDecoder(bytes.NewBuffer(data)) 16 | dec.DisallowUnknownFields() 17 | 18 | var subscribers []*common.Subscriber 19 | err := dec.Decode(&subscribers) 20 | if err != nil { 21 | return nil, err 22 | } 23 | log.Printf("Parsed subscribers. count=%v", len(subscribers)) 24 | return subscribers, nil 25 | } 26 | 27 | func (c *listingClient) prepareImportPayload(data []byte) ([]byte, error) { 28 | subscribers, err := c.parseSubscribers(data) 29 | if err != nil { 30 | return nil, err 31 | } 32 | return json.Marshal(subscribers) 33 | } 34 | 35 | func (c *listingClient) sendImportRequest(endpoint string, payload []byte) error { 36 | req, err := http.NewRequest("PUT", endpoint, bytes.NewBuffer(payload)) 37 | if err != nil { 38 | return err 39 | } 40 | req.Header.Set("Content-Type", "application/json") 41 | req.SetBasicAuth("any", c.authToken) 42 | 43 | resp, err := c.client.Do(req) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | if resp.StatusCode != http.StatusOK { 49 | body, _ := ioutil.ReadAll(resp.Body) 50 | return fmt.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 51 | 52 | } 53 | return nil 54 | } 55 | 56 | func (c *listingClient) importSubscribers(data []byte) error { 57 | endpoint, err := c.importURL() 58 | if err != nil { 59 | return err 60 | } 61 | payload, err := c.prepareImportPayload(data) 62 | if err != nil { 63 | return err 64 | } 65 | log.Printf("About to send import request. bytes=%v", len(payload)) 66 | if c.dryRun { 67 | log.Println("Dry run mode. Exiting...") 68 | return nil 69 | } 70 | return c.sendImportRequest(endpoint, payload) 71 | } 72 | -------------------------------------------------------------------------------- /cmd/listing-cli/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "strings" 7 | 8 | "github.com/ribtoks/listing/pkg/common" 9 | ) 10 | 11 | type listingClient struct { 12 | client *http.Client 13 | printer Printer 14 | url string 15 | authToken string 16 | secret string 17 | complaints map[string]bool 18 | dryRun bool 19 | noUnconfirmed bool 20 | noConfirmed bool 21 | noUnsubscribed bool 22 | ignoreComplaints bool 23 | } 24 | 25 | func (c *listingClient) endpoint(e string) string { 26 | baseURL := c.url 27 | if strings.HasSuffix(baseURL, "/") { 28 | baseURL = strings.TrimRight(baseURL, "/") 29 | } 30 | baseURL = baseURL + e 31 | return baseURL 32 | } 33 | 34 | func (c *listingClient) importURL() (string, error) { 35 | u, err := url.Parse(c.endpoint(common.SubscribersEndpoint)) 36 | if err != nil { 37 | return "", err 38 | } 39 | return u.String(), nil 40 | } 41 | 42 | func (c *listingClient) subscribersURL(newsletter string) (string, error) { 43 | u, err := url.Parse(c.endpoint(common.SubscribersEndpoint)) 44 | if err != nil { 45 | return "", err 46 | } 47 | q := u.Query() 48 | q.Set(common.ParamNewsletter, newsletter) 49 | u.RawQuery = q.Encode() 50 | return u.String(), nil 51 | } 52 | 53 | func (c *listingClient) subscribeURL() (string, error) { 54 | u, err := url.Parse(c.endpoint(common.SubscribeEndpoint)) 55 | if err != nil { 56 | return "", err 57 | } 58 | return u.String(), nil 59 | } 60 | 61 | func (c *listingClient) unsubscribeURL() (string, error) { 62 | u, err := url.Parse(c.endpoint(common.UnsubscribeEndpoint)) 63 | if err != nil { 64 | return "", err 65 | } 66 | return u.String(), nil 67 | } 68 | 69 | func (c *listingClient) complaintsURL() (string, error) { 70 | u, err := url.Parse(c.endpoint(common.ComplaintsEndpoint)) 71 | if err != nil { 72 | return "", err 73 | } 74 | return u.String(), nil 75 | } 76 | -------------------------------------------------------------------------------- /docs/SEND.md: -------------------------------------------------------------------------------- 1 | `listing-send` is a simple application that allows to send Go-templated emails to subscribers through SMTP server (AWS SES). 2 | 3 | `listing-send` requires path to html and text templates. These templates are executed using common parameters (`-params` option, available in template as `{{.Params.xxx}}`) and recepient params from the subscription list (`-list` option, available in template as `{{.Recepient.xxx}}`). 4 | 5 | After execution, rendered emails can be saved locally using `-dry-run` option (they are saved to directory from parameter `-out`) or sent to SMTP server. 6 | 7 | ## Options 8 | 9 | ``` 10 | > ./listing-send -help 11 | -dry-run 12 | Simulate selected action 13 | -from-email string 14 | Sender address 15 | -from-name string 16 | Sender name 17 | -help 18 | Print help 19 | -html-template string 20 | Path to html email template 21 | -l string 22 | Absolute path to log file (default "listing-send.log") 23 | -list string 24 | Path to file with email list (default "list.json") 25 | -out string 26 | Path to directory for dry run results (default "./") 27 | -params string 28 | Path to file with common params (default "params.json") 29 | -pass string 30 | SMTP password flag 31 | -rate int 32 | Emails per second sending rate (default 25) 33 | -stdout 34 | Log to stdout and to logfile 35 | -subject string 36 | Html campaign subject 37 | -txt-template string 38 | Path to text email template 39 | -url string 40 | SMTP server url 41 | -user string 42 | SMTP username flag 43 | -workers int 44 | Number of workers to send emails (default 2) 45 | ``` 46 | 47 | ## Example 48 | 49 | ``` 50 | listing-send -url "smtp://email-smtp.us-east-1.amazonaws.com:587" -user "" -pass "" -subject "Newsletter digest January 2020" -from-email "email@example.com" -from-name "Email from Example" -html-template layouts/layout-1-2-3.html -txt-template layouts/_default.text -params content/newsletters/jan-2020-params.json -list lists/test.json 51 | ``` 52 | -------------------------------------------------------------------------------- /cmd/sesnotify/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "log" 7 | "os" 8 | 9 | "github.com/aws/aws-lambda-go/events" 10 | "github.com/aws/aws-lambda-go/lambda" 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/session" 13 | "github.com/ribtoks/listing/pkg/common" 14 | "github.com/ribtoks/listing/pkg/db" 15 | ) 16 | 17 | var ( 18 | store common.NotificationsStore 19 | ) 20 | 21 | func handler(ctx context.Context, snsEvent events.SNSEvent) { 22 | log.Printf("Processing %v records", len(snsEvent.Records)) 23 | 24 | for _, record := range snsEvent.Records { 25 | snsRecord := record.SNS 26 | var sesMessage common.SesMessage 27 | err := json.Unmarshal([]byte(snsRecord.Message), &sesMessage) 28 | if err != nil { 29 | log.Printf("Error parsing message: %v", err) 30 | continue 31 | } 32 | 33 | switch sesMessage.NotificationType { 34 | case "Bounce": 35 | { 36 | isTransient := sesMessage.Bounce.BounceType == "Transient" 37 | for _, r := range sesMessage.Bounce.BouncedRecipients { 38 | err = store.AddBounce(r.EmailAddress, sesMessage.Mail.Source, isTransient) 39 | if err != nil { 40 | log.Printf("Failed to add bounce: %v", err) 41 | } 42 | } 43 | } 44 | case "Complaint": 45 | { 46 | for _, r := range sesMessage.Bounce.BouncedRecipients { 47 | err = store.AddComplaint(r.EmailAddress, sesMessage.Mail.Source) 48 | if err != nil { 49 | log.Printf("Failed to add complaint: %v", err) 50 | } 51 | } 52 | } 53 | default: 54 | { 55 | log.Printf("Unexpected message type: %v", sesMessage.NotificationType) 56 | } 57 | } 58 | } 59 | } 60 | 61 | func main() { 62 | tableName := os.Getenv("NOTIFICATIONS_TABLE") 63 | 64 | sess, err := session.NewSession(&aws.Config{ 65 | Region: aws.String(os.Getenv("AWS_REGION")), 66 | }) 67 | 68 | if err != nil { 69 | log.Fatalf("Failed to create AWS session. err=%v", err) 70 | } 71 | 72 | store = db.NewNotificationsStore(tableName, sess) 73 | 74 | lambda.Start(handler) 75 | } 76 | -------------------------------------------------------------------------------- /pkg/common/subscriber.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/rs/xid" 4 | 5 | // Subscriber incapsulates newsletter subscriber information 6 | // stored in the DynamoDB table 7 | type Subscriber struct { 8 | Name string `json:"name,omitempty"` 9 | Newsletter string `json:"newsletter"` 10 | Email string `json:"email"` 11 | CreatedAt JSONTime `json:"created_at"` 12 | UnsubscribedAt JSONTime `json:"unsubscribed_at"` 13 | ConfirmedAt JSONTime `json:"confirmed_at"` 14 | UserID string `json:"user_id,omitempty"` 15 | } 16 | 17 | // Confirmed checks if subscriber has confirmed the email via link 18 | func (s *Subscriber) Confirmed() bool { 19 | return s.ConfirmedAt.Time().After(s.CreatedAt.Time()) 20 | } 21 | 22 | // Unsubscribed checks if subscriber pressed "Unsubscribe" link 23 | func (s *Subscriber) Unsubscribed() bool { 24 | return s.UnsubscribedAt.Time().After(s.CreatedAt.Time()) 25 | } 26 | 27 | func (s *Subscriber) Validate() { 28 | if len(s.UserID) > 0 { 29 | return 30 | } 31 | 32 | guid := xid.New() 33 | s.UserID = guid.String() 34 | } 35 | 36 | // SubscriberKey is used for deletion of subscribers 37 | type SubscriberKey struct { 38 | Newsletter string `json:"newsletter"` 39 | Email string `json:"email"` 40 | } 41 | 42 | type SubscriberEx struct { 43 | Name string `json:"name" yaml:"name"` 44 | Newsletter string `json:"newsletter" yaml:"newsletter"` 45 | Email string `json:"email" yaml:"email"` 46 | Token string `json:"token" yaml:"token"` 47 | Confirmed bool `json:"confirmed" yaml:"confirmed"` 48 | Unsubscribed bool `json:"unsubscribed" yaml:"unsubscribed"` 49 | UserID string `json:"user_id" yaml:"user_id"` 50 | } 51 | 52 | func NewSubscriberEx(s *Subscriber, secret string) *SubscriberEx { 53 | return &SubscriberEx{ 54 | Name: s.Name, 55 | Newsletter: s.Newsletter, 56 | Email: s.Email, 57 | Confirmed: s.Confirmed(), 58 | Unsubscribed: s.Unsubscribed(), 59 | Token: Sign(secret, s.Email), 60 | UserID: s.UserID, 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pkg/common/sesmessage.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const ( 4 | SoftBounceType = "sb" 5 | HardBounceType = "hb" 6 | ComplaintType = "ct" 7 | ) 8 | 9 | type SesNotification struct { 10 | Email string `json:"email"` 11 | From string `json:"from"` 12 | ReceivedAt JSONTime `json:"received_at"` 13 | Notification string `json:"notification"` 14 | } 15 | 16 | // types used for deserializing of SES notifications 17 | 18 | type SesMessage struct { 19 | NotificationType string `json:"notificationType"` 20 | Content string `json:"content"` 21 | Mail Mail `json:"mail"` 22 | Bounce Bounce `json:"bounce"` 23 | Complaint Complaint `json:"complaint"` 24 | Receipt map[string]interface{} `json:"receipt"` 25 | } 26 | 27 | type Bounce struct { 28 | BounceType string `json:"bounceType"` 29 | BounceSubType string `json:"bounceSubType"` 30 | BouncedRecipients []MailRecipient `json:"bouncedRecipients"` 31 | Timestamp string `json:"timestamp"` 32 | FeedbackID string `json:"feedbackId"` 33 | RemoteMtaIp string `json:"remoteMtaIp"` 34 | } 35 | 36 | type MailRecipient struct { 37 | EmailAddress string `json:"emailAddress"` 38 | } 39 | 40 | type Complaint struct { 41 | UserAgent string `json:"userAgent"` 42 | ComplainedRecipients []MailRecipient `json:"complainedRecipients"` 43 | ComplaintFeedbackType string `json:"complaintFeedbackType"` 44 | ArrivalDate string `json:"arrivalDate"` 45 | Timestamp string `json:"timestamp"` 46 | FeedbackID string `json:"feedbackId"` 47 | } 48 | 49 | type Mail struct { 50 | Timestamp string `json:"timestamp"` 51 | Source string `json:"source"` 52 | MessageId string `json:"messageId"` 53 | HeadersTruncated bool `json:"headersTruncated"` 54 | Destination []string `json:"destination"` 55 | Headers []map[string]string `json:"headers"` 56 | CommonHeaders map[string]interface{} `json:"commonHeaders"` 57 | } 58 | -------------------------------------------------------------------------------- /docs/CLI.md: -------------------------------------------------------------------------------- 1 | *Listing* serves couple of ordinary http endpoints and you can do all actions using `curl`, however, we provide `listing-cli` command line application that simplifies many actions and automates others. 2 | 3 | ## Options 4 | 5 | Here are the supported options. 6 | 7 | ``` 8 | > ./listing-cli -help 9 | 10 | -auth-token string 11 | Auth token for admin access 12 | -dry-run 13 | Simulate selected action 14 | -email string 15 | Email for subscribe|unsubscribe 16 | -format string 17 | Ouput format of subscribers: csv|tsv|table|raw|yaml (default "table") 18 | -help 19 | Print help 20 | -ignore-complaints 21 | Ignore bounces and complaints for export 22 | -l string 23 | Absolute path to log file (default "listing-cli.log") 24 | -mode string 25 | Execution mode: subscribe|unsubscribe|export|import|delete 26 | -name string 27 | (optional) Name for subscribe 28 | -newsletter string 29 | Newsletter for subscribe|unsubscribe 30 | -no-unconfirmed 31 | Do not export unconfirmed emails 32 | -no-unsubscribed 33 | Do not export unsubscribed emails 34 | -secret string 35 | Secret for email salt 36 | -stdout 37 | Log to stdout and to logfile 38 | -url string 39 | Base URL to the listing API 40 | ``` 41 | 42 | `secret` and `auth-token` are the parameters from `secrets.json` that you use when deploying lambda functions. 43 | 44 | `listing-cli` supports `dry-run` parameter that allows to see what the specific chosen action or mode will do without actually doing it. 45 | 46 | `url` parameter should be a "root" of your deployed API. 47 | 48 | Use `-format yaml` in order to use *Listing* with [paperboy](https://github.com/rykov/paperboy) and `-format raw` for dump of the subscribers DB. 49 | 50 | Use `-format json` in order to use *Listing* with `listing-send`. 51 | 52 | Use `-format raw` to export subscribers for backup or further import. 53 | 54 | ## Examples 55 | 56 | ``` 57 | # subscribing to a newsletter 58 | ./listing-cli -mode subscribe -email foo@bar.com -newsletter NewsletterName -url https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev 59 | 60 | # listing all subscribers 61 | ./listing-cli -secret secret-here -auth-token your-token-here -url "https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev" -mode export -newsletter Listing1 62 | 63 | # importing subscribers from file 64 | cat raw_export.json | ./listing-cli -secret secret-here -auth-token your-token-here -url "https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev" -mode import 65 | ``` 66 | -------------------------------------------------------------------------------- /cmd/listing/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strings" 9 | 10 | "github.com/aws/aws-lambda-go/events" 11 | "github.com/aws/aws-lambda-go/lambda" 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/aws/aws-sdk-go/service/ses" 15 | "github.com/awslabs/aws-lambda-go-api-proxy/httpadapter" 16 | "github.com/ribtoks/listing/pkg/api" 17 | "github.com/ribtoks/listing/pkg/db" 18 | "github.com/ribtoks/listing/pkg/email" 19 | ) 20 | 21 | var ( 22 | handlerLambda *httpadapter.HandlerAdapter 23 | ) 24 | 25 | // Handler is the main entry point to this lambda 26 | func Handler(ctx context.Context, req events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { 27 | return handlerLambda.ProxyWithContext(ctx, req) 28 | } 29 | 30 | func main() { 31 | secret := os.Getenv("TOKEN_SECRET") 32 | subscribeRedirectURL := os.Getenv("SUBSCRIBE_REDIRECT_URL") 33 | unsubscribeRedirectURL := os.Getenv("UNSUBSCRIBE_REDIRECT_URL") 34 | confirmRedirectURL := os.Getenv("CONFIRM_REDIRECT_URL") 35 | confirmURL := os.Getenv("CONFIRM_URL") 36 | subscribersTableName := os.Getenv("SUBSCRIBERS_TABLE") 37 | notificationsTableName := os.Getenv("NOTIFICATIONS_TABLE") 38 | supportedNewsletters := os.Getenv("SUPPORTED_NEWSLETTERS") 39 | emailFrom := os.Getenv("EMAIL_FROM") 40 | 41 | sess, err := session.NewSession(&aws.Config{ 42 | Region: aws.String(os.Getenv("AWS_REGION")), 43 | }) 44 | 45 | if err != nil { 46 | log.Fatalf("Failed to create AWS session. err=%v", err) 47 | } 48 | 49 | subscribers := db.NewSubscribersStore(subscribersTableName, sess) 50 | notifications := db.NewNotificationsStore(notificationsTableName, sess) 51 | mailer := &email.SESMailer{ 52 | Svc: ses.New(sess), 53 | Sender: emailFrom, 54 | Secret: secret, 55 | } 56 | 57 | router := http.NewServeMux() 58 | newsletter := &api.NewsletterResource{ 59 | Secret: secret, 60 | SubscribeRedirectURL: subscribeRedirectURL, 61 | UnsubscribeRedirectURL: unsubscribeRedirectURL, 62 | ConfirmRedirectURL: confirmRedirectURL, 63 | ConfirmURL: confirmURL, 64 | Subscribers: subscribers, 65 | Notifications: notifications, 66 | Mailer: mailer, 67 | Newsletters: make(map[string]bool), 68 | } 69 | 70 | sn := strings.Split(supportedNewsletters, ";") 71 | newsletter.AddNewsletters(sn) 72 | 73 | newsletter.Setup(router) 74 | handlerLambda = httpadapter.New(router) 75 | 76 | lambda.Start(Handler) 77 | } 78 | -------------------------------------------------------------------------------- /serverless-admin.yml: -------------------------------------------------------------------------------- 1 | service: listing-admin 2 | frameworkVersion: '>=1.28.0 <2.0.0' 3 | 4 | provider: 5 | name: aws 6 | runtime: go1.x 7 | stage: ${opt:stage, 'dev'} 8 | region: ${opt:region, 'us-east-1'} 9 | logRetentionInDays: ${self:custom.logRetentionInDays.${self:provider.stage}, 7} 10 | logs: 11 | restApi: ${self:custom.apiGatewayLogs.${self:provider.stage}} 12 | memorySize: 128 13 | usagePlan: 14 | throttle: 15 | burstLimit: 10 16 | rateLimit: 2 17 | 18 | plugins: 19 | - serverless-iam-roles-per-function 20 | 21 | package: 22 | individually: true 23 | exclude: 24 | - ./** 25 | 26 | functions: 27 | handler: 28 | handler: bin/ladmin 29 | package: 30 | include: 31 | - ./bin/ladmin 32 | events: 33 | - http: 34 | path: subscribers 35 | method: GET 36 | cors: true 37 | - http: 38 | path: subscribers 39 | method: PUT 40 | cors: true 41 | - http: 42 | path: subscribers 43 | method: DELETE 44 | cors: true 45 | - http: 46 | path: complaints 47 | method: GET 48 | cors: true 49 | iamRoleStatements: 50 | - Effect: Allow 51 | Action: 52 | - "dynamodb:DescribeTable" 53 | - "dynamodb:Query" 54 | - "dynamodb:Scan" 55 | - "dynamodb:GetItem" 56 | - "dynamodb:PutItem" 57 | - "dynamodb:UpdateItem" 58 | - "dynamodb:DeleteItem" 59 | - "dynamodb:BatchWriteItem" 60 | Resource: 61 | - { 'Fn::ImportValue': '${self:provider.stage}-ListingSubscriptionsTableArn' } 62 | - Effect: Allow 63 | Action: 64 | - "dynamodb:DescribeTable" 65 | - "dynamodb:Query" 66 | - "dynamodb:Scan" 67 | - "dynamodb:GetItem" 68 | Resource: 69 | - { 'Fn::ImportValue': '${self:provider.stage}-ListingNotificationsTableArn' } 70 | environment: 71 | API_TOKEN: ${self:custom.secrets.apiToken} 72 | SUBSCRIBERS_TABLE: ${self:custom.subscribersTableName} 73 | NOTIFICATIONS_TABLE: ${self:custom.snsTableName} 74 | SUPPORTED_NEWSLETTERS: ${self:custom.secrets.supportedNewsletters} 75 | 76 | custom: 77 | secrets: ${file(secrets.json)} 78 | subscribersTableName: ${self:provider.stage}-listing-subscribers 79 | snsTableName: ${self:provider.stage}-listing-sesnotify 80 | snsTopicName: ${self:provider.stage}-listing-ses-notifications 81 | stages: 82 | - local 83 | - dev 84 | apiGatewayLogs: 85 | dev: true 86 | prod: true 87 | logRetentionInDays: 88 | prod: 14 89 | dev: 7 90 | 91 | -------------------------------------------------------------------------------- /docs/TESTING.md: -------------------------------------------------------------------------------- 1 | Testing doc describes how to invoke different endpoints of *Listing*. You may want to do that after deployment. 2 | 3 | ## Subscribing to a newsletter 4 | 5 | `./listing-cli -mode subscribe -email foo@bar.com -newsletter NewsletterName -url https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev` 6 | 7 | OR 8 | 9 | `curl -v --data "newsletter=NewsletterName&email=foo@bar.com" https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev/subscribe` 10 | 11 | ## Email confirmation 12 | 13 | Run previous "subscription" command to the real email address and click "Confirm email" button. Go to DynamoDB tables in AWS Console UI and check `listing-subscribers` table. 14 | 15 | ## Unsubscribing from a newsletter 16 | 17 | `./listing-cli -mode unsubscribe -email foo@bar.com -secret your-secret -newsletter NewsletterName -url https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev` 18 | 19 | Unsubscribing with curl might be a little bit tricky since you need to generate a token which is handled in `listing-cli` for you. 20 | 21 | ## Listing all subscribers 22 | 23 | `./listing-cli -auth-token your-token-here -url "https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev" -mode export -newsletter Listing1` 24 | 25 | OR 26 | 27 | `curl -v -u :your-api-token https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev/subscribers?newsletter=NewsletterName` 28 | 29 | ## Importing subscribers 30 | 31 | `cat scripts/add_subscribers.json | ./listing-cli/listing-cli -auth-token your-api-token -mode import -url https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev` 32 | 33 | OR 34 | 35 | `curl -v -u :your-api-token -X PUT https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev/subscribers -H "Content-Type: application/json" --data-binary "@scripts/add_subscribers.json"` 36 | 37 | ## Deleting subscribers 38 | 39 | `cat scripts/remove_subscribers.json | ./listing-cli/listing-cli -auth-token your-api-token -mode delete -url https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev` 40 | 41 | OR 42 | 43 | `curl -v -u :your-api-token -X DELETE https://qwerty12345.execute-api.us-east-1.amazonaws.com/dev/subscribers -H "Content-Type: application/json" --data-binary "@scripts/remove_subscribers.json"` 44 | 45 | ## Handling bounce and complaint 46 | 47 | Send email to `bounce@simulator.amazonses.com` from SES UI in AWS Console and check DynamoDB table `listing-sesnotify` if it contains the bounce notification. 48 | 49 | Send email to `complaint@simulator.amazonses.com` from SES UI in AWS Console and check DynamoDB table `listing-sesnotify` if it contains the complaint notification. (this might not work as expected since it is dependent on your ISP as well) 50 | -------------------------------------------------------------------------------- /serverless-db.yml: -------------------------------------------------------------------------------- 1 | service: listing-db 2 | frameworkVersion: '>=1.28.0 <2.0.0' 3 | 4 | provider: 5 | name: aws 6 | stage: ${opt:stage, 'dev'} 7 | region: ${opt:region, 'us-east-1'} 8 | 9 | resources: 10 | Resources: 11 | # main table that stores subscriptions for the newsletters 12 | SubscriptionsDynamoDBTable: 13 | Type: 'AWS::DynamoDB::Table' 14 | Properties: 15 | TableName: ${self:custom.subscribersTableName} 16 | AttributeDefinitions: 17 | - AttributeName: newsletter 18 | AttributeType: S 19 | - AttributeName: email 20 | AttributeType: S 21 | KeySchema: 22 | - AttributeName: newsletter 23 | KeyType: HASH 24 | - AttributeName: email 25 | KeyType: RANGE 26 | BillingMode: PAY_PER_REQUEST 27 | # table that will store complains and bounces from AWS SES 28 | # received through SNS notifications topic SESNotificationsTopic 29 | SesNotificationsDynamoDBTable: 30 | Type: 'AWS::DynamoDB::Table' 31 | Properties: 32 | TableName: ${self:custom.snsTableName} 33 | AttributeDefinitions: 34 | - AttributeName: email 35 | AttributeType: S 36 | - AttributeName: notification 37 | AttributeType: S 38 | KeySchema: 39 | - AttributeName: email 40 | KeyType: HASH 41 | - AttributeName: notification 42 | KeyType: RANGE 43 | BillingMode: PAY_PER_REQUEST 44 | # SNS topic that will receive notifications from AWS SES 45 | SESNotificationsTopic: 46 | Type: 'AWS::SNS::Topic' 47 | Properties: 48 | TopicName: ${self:custom.snsTopicName} 49 | # Outputs contain ARNs of tables and sns topics used by lambdas 50 | # in the serverless-api.yml file. This allows to deploy them separately 51 | # since DB and SNS resources will almost never change unlike API code 52 | Outputs: 53 | SubscriptionsTableArn: 54 | Description: The ARN of the subscription table 55 | Value: 56 | Fn::GetAtt: 57 | - SubscriptionsDynamoDBTable 58 | - Arn 59 | Export: 60 | Name: ${self:provider.stage}-ListingSubscriptionsTableArn 61 | NotificationsTableArn: 62 | Description: The ARN of the notifications table 63 | Value: 64 | Fn::GetAtt: 65 | - SesNotificationsDynamoDBTable 66 | - Arn 67 | Export: 68 | Name: ${self:provider.stage}-ListingNotificationsTableArn 69 | NotificationsTopicArn: 70 | Description: The ARN of the SNS topic 71 | Value: 72 | Ref: SESNotificationsTopic 73 | Export: 74 | Name: ${self:provider.stage}-ListingNotificationsTopicArn 75 | 76 | custom: 77 | subscribersTableName: ${opt:stage, 'dev'}-listing-subscribers 78 | snsTableName: ${opt:stage, 'dev'}-listing-sesnotify 79 | snsTopicName: ${opt:stage, 'dev'}-listing-ses-notifications 80 | 81 | -------------------------------------------------------------------------------- /cmd/listing-cli/export.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "log" 7 | "net/http" 8 | "sync" 9 | 10 | "github.com/ribtoks/listing/pkg/common" 11 | ) 12 | 13 | var ( 14 | errInvalidNewsletter = errors.New("Invalid newsletter parameter") 15 | emptySubscribers []*common.Subscriber 16 | ) 17 | 18 | func (c *listingClient) fetchSubscribers(url string) ([]*common.Subscriber, error) { 19 | log.Printf("About to fetch subscribers. url=%v", url) 20 | if c.dryRun { 21 | log.Println("Dry run mode. Exiting...") 22 | return emptySubscribers, nil 23 | } 24 | 25 | req, err := http.NewRequest("GET", url, nil) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | req.SetBasicAuth("any", c.authToken) 31 | resp, err := c.client.Do(req) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | log.Printf("Received subscribers response. status=%v", resp.StatusCode) 37 | 38 | defer resp.Body.Close() 39 | ss := make([]*common.Subscriber, 0) 40 | err = json.NewDecoder(resp.Body).Decode(&ss) 41 | return ss, nil 42 | } 43 | 44 | func (c *listingClient) isSubscriberOK(s *common.Subscriber) bool { 45 | if c.noUnconfirmed && !s.Confirmed() { 46 | log.Printf("Skipping unconfirmed subscriber. created_at=%v confirmed_at=%v confirmed=%v", s.CreatedAt, s.ConfirmedAt, s.Confirmed()) 47 | return false 48 | } 49 | 50 | if c.noUnsubscribed && s.Unsubscribed() { 51 | log.Printf("Skipping unsubscribed subscriber. created_at=%v unsubscribed_at=%v unsubscribed=%v", s.CreatedAt, s.UnsubscribedAt, s.Unsubscribed()) 52 | return false 53 | } 54 | 55 | if c.noConfirmed && s.Confirmed() { 56 | log.Printf("Skipping confirmed subscriber. created_at=%v confirmed_at=%v confirmed=%v", s.CreatedAt, s.ConfirmedAt, s.Confirmed()) 57 | return false 58 | } 59 | 60 | if _, ok := c.complaints[s.Email]; ok { 61 | log.Printf("Skipping bounced or complained subscriber. email=%v", s.Email) 62 | return false 63 | } 64 | 65 | return true 66 | } 67 | 68 | func (c *listingClient) export(newsletter string) error { 69 | if newsletter == "" { 70 | return errInvalidNewsletter 71 | } 72 | endpoint, err := c.subscribersURL(newsletter) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | var wg sync.WaitGroup 78 | 79 | if !c.ignoreComplaints { 80 | wg.Add(1) 81 | go func() { 82 | defer wg.Done() 83 | err := c.updateComplaints() 84 | if err != nil { 85 | log.Printf("Failed to update complaints. err=%v", err) 86 | } 87 | }() 88 | } 89 | 90 | ss, err := c.fetchSubscribers(endpoint) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | wg.Wait() 96 | 97 | skipped := 0 98 | for _, s := range ss { 99 | if c.isSubscriberOK(s) { 100 | c.printer.Append(s) 101 | } else { 102 | skipped += 1 103 | } 104 | } 105 | c.printer.Render() 106 | log.Printf("Exported subscribers. count=%v skipped=%v", len(ss), skipped) 107 | return nil 108 | } 109 | -------------------------------------------------------------------------------- /cmd/listing-send/campaign.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "html/template" 7 | "log" 8 | "sync" 9 | "time" 10 | 11 | "github.com/go-gomail/gomail" 12 | "github.com/ribtoks/listing/pkg/common" 13 | ) 14 | 15 | type campaign struct { 16 | htmlTemplate *template.Template 17 | textTemplate *template.Template 18 | params map[string]interface{} 19 | subscribers []*common.SubscriberEx 20 | subject string 21 | fromEmail string 22 | fromName string 23 | rate int 24 | workersCount int 25 | dryRun bool 26 | waiter *sync.WaitGroup 27 | messages chan *gomail.Message 28 | } 29 | 30 | const xMailer = "listing/0.1 (https://github.com/ribtoks/listing)" 31 | 32 | func (c *campaign) send() { 33 | log.Printf("Starting to send messages. from=%v dry-run=%v", c.fromEmail, c.dryRun) 34 | c.waiter.Add(1) 35 | go c.generateMessages() 36 | log.Printf("Starting workers. count=%v", c.workersCount) 37 | for i := 0; i < c.workersCount; i++ { 38 | go c.sendMessages(i) 39 | } 40 | log.Printf("Waiting for workers. count=%v", c.workersCount) 41 | c.waiter.Wait() 42 | close(c.messages) 43 | log.Println("Finished sending messages") 44 | } 45 | 46 | func (c *campaign) generateMessages() { 47 | defer c.waiter.Done() 48 | rate := time.Second / time.Duration(c.rate) 49 | throttle := time.Tick(rate) 50 | 51 | for _, s := range c.subscribers { 52 | m := gomail.NewMessage() 53 | if err := c.renderMessage(m, s); err != nil { 54 | log.Printf("Failed to render message. err=%s", err) 55 | return 56 | } 57 | <-throttle // rate limit 58 | c.waiter.Add(1) 59 | c.messages <- m 60 | } 61 | } 62 | 63 | func (c *campaign) renderMessage(m *gomail.Message, s *common.SubscriberEx) error { 64 | data, err := json.Marshal(s) 65 | if err != nil { 66 | return err 67 | } 68 | recepient := make(map[string]interface{}) 69 | err = json.Unmarshal(data, &recepient) 70 | if err != nil { 71 | return err 72 | } 73 | ctx := make(map[string]interface{}) 74 | ctx["Params"] = c.params 75 | ctx["Recepient"] = recepient 76 | 77 | var htmlBodyTpl bytes.Buffer 78 | if err := c.htmlTemplate.Execute(&htmlBodyTpl, ctx); err != nil { 79 | return err 80 | } 81 | 82 | var textBodyTpl bytes.Buffer 83 | if err := c.textTemplate.Execute(&textBodyTpl, ctx); err != nil { 84 | return err 85 | } 86 | 87 | m.Reset() // Return to NewMessage state 88 | m.SetAddressHeader("To", s.Email, s.Name) 89 | m.SetAddressHeader("From", c.fromEmail, c.fromName) 90 | m.SetHeader("Subject", c.subject) 91 | m.SetHeader("X-Mailer", xMailer) 92 | m.SetBody("text/plain", textBodyTpl.String()) 93 | m.AddAlternative("text/html", htmlBodyTpl.String()) 94 | log.Printf("Rendered email message. recepient=%v", s.Email) 95 | return nil 96 | } 97 | 98 | func (c *campaign) sendMessages(id int) { 99 | log.Printf("Started sending messages worker. id=%v", id) 100 | sender, err := createSender() 101 | if err != nil { 102 | log.Fatal(err) 103 | } 104 | for m := range c.messages { 105 | if err := gomail.Send(sender, m); err != nil { 106 | log.Printf("Error sending message. err=%s id=%v to=%v", err, id, m.GetHeader("To")) 107 | sender.Close() 108 | sender, err = createSender() 109 | if err != nil { 110 | log.Fatal(err) 111 | } 112 | } else { 113 | log.Printf("Sent email. id=%v to=%v", id, m.GetHeader("To")) 114 | } 115 | c.waiter.Done() 116 | } 117 | sender.Close() 118 | } 119 | -------------------------------------------------------------------------------- /pkg/db/notifications.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/aws/aws-sdk-go/aws/session" 7 | "github.com/aws/aws-sdk-go/service/dynamodb" 8 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" 9 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" 10 | "github.com/ribtoks/listing/pkg/common" 11 | ) 12 | 13 | // NotificationsDynamoDB is an implementation of Store interface 14 | // that is capable of working with AWS DynamoDB 15 | type NotificationsDynamoDB struct { 16 | TableName string 17 | Client dynamodbiface.DynamoDBAPI 18 | } 19 | 20 | var _ common.NotificationsStore = (*NotificationsDynamoDB)(nil) 21 | 22 | // NewNotificationsStore returns new instance of NotificationsDynamoDB 23 | func NewNotificationsStore(table string, sess *session.Session) *NotificationsDynamoDB { 24 | return &NotificationsDynamoDB{ 25 | Client: dynamodb.New(sess), 26 | TableName: table, 27 | } 28 | } 29 | 30 | func (s *NotificationsDynamoDB) StoreNotification(email, from string, t string) error { 31 | i, err := dynamodbattribute.MarshalMap(common.SesNotification{ 32 | Email: email, 33 | ReceivedAt: common.JsonTimeNow(), 34 | Notification: t, 35 | From: from, 36 | }) 37 | 38 | if err != nil { 39 | return err 40 | } 41 | 42 | _, err = s.Client.PutItem(&dynamodb.PutItemInput{ 43 | TableName: &s.TableName, 44 | Item: i, 45 | }) 46 | 47 | if err != nil { 48 | return err 49 | } 50 | 51 | log.Printf("Stored notification email=%v type=%v", email, t) 52 | return nil 53 | } 54 | 55 | func (s *NotificationsDynamoDB) AddBounce(email, from string, isTransient bool) error { 56 | bounceType := common.SoftBounceType 57 | if !isTransient { 58 | bounceType = common.HardBounceType 59 | } 60 | return s.StoreNotification(email, from, bounceType) 61 | } 62 | 63 | func (s *NotificationsDynamoDB) AddComplaint(email, from string) error { 64 | return s.StoreNotification(email, from, common.ComplaintType) 65 | } 66 | 67 | func (s *NotificationsDynamoDB) Notifications() (notifications []*common.SesNotification, err error) { 68 | query := &dynamodb.QueryInput{ 69 | TableName: &s.TableName, 70 | } 71 | 72 | err = s.Client.QueryPages(query, func(page *dynamodb.QueryOutput, more bool) bool { 73 | var items []*common.SesNotification 74 | err := dynamodbattribute.UnmarshalListOfMaps(page.Items, &items) 75 | if err != nil { 76 | // print the error and continue receiving pages 77 | log.Printf("Could not unmarshal AWS data. err=%v", err) 78 | return true 79 | } 80 | 81 | notifications = append(notifications, items...) 82 | // continue receiving pages (can be used to limit the number of pages) 83 | return true 84 | }) 85 | 86 | return 87 | } 88 | 89 | type NotificationsMapStore struct { 90 | items []*common.SesNotification 91 | } 92 | 93 | var _ common.NotificationsStore = (*NotificationsMapStore)(nil) 94 | 95 | func (s *NotificationsMapStore) AddBounce(email, from string, isTransient bool) error { 96 | t := common.SoftBounceType 97 | if !isTransient { 98 | t = common.HardBounceType 99 | } 100 | s.items = append(s.items, &common.SesNotification{ 101 | Email: email, 102 | ReceivedAt: common.JsonTimeNow(), 103 | Notification: t, 104 | From: from, 105 | }) 106 | return nil 107 | } 108 | 109 | func (s *NotificationsMapStore) AddComplaint(email, from string) error { 110 | s.items = append(s.items, &common.SesNotification{ 111 | Email: email, 112 | ReceivedAt: common.JsonTimeNow(), 113 | Notification: common.ComplaintType, 114 | From: from, 115 | }) 116 | return nil 117 | } 118 | 119 | func (s *NotificationsMapStore) Notifications() (notifications []*common.SesNotification, err error) { 120 | return s.items, nil 121 | } 122 | 123 | func NewSubscribersMapStore() *SubscribersMapStore { 124 | return &SubscribersMapStore{ 125 | items: make(map[string]*common.Subscriber), 126 | } 127 | } 128 | 129 | func NewNotificationsMapStore() *NotificationsMapStore { 130 | return &NotificationsMapStore{ 131 | items: make([]*common.SesNotification, 0), 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /serverless-api.yml: -------------------------------------------------------------------------------- 1 | service: listing 2 | frameworkVersion: '>=1.28.0 <2.0.0' 3 | 4 | provider: 5 | name: aws 6 | runtime: go1.x 7 | stage: ${opt:stage, 'dev'} 8 | region: ${opt:region, 'us-east-1'} 9 | logRetentionInDays: ${self:custom.logRetentionInDays.${self:provider.stage}, 7} 10 | logs: 11 | restApi: ${self:custom.apiGatewayLogs.${self:provider.stage}} 12 | memorySize: 128 13 | usagePlan: 14 | throttle: 15 | burstLimit: 10 16 | rateLimit: 2 17 | 18 | plugins: 19 | - serverless-iam-roles-per-function 20 | - serverless-domain-manager 21 | 22 | package: 23 | individually: true 24 | exclude: 25 | - ./** 26 | 27 | functions: 28 | # "main" entry point into this application responsible for 29 | # collecting subscribers, storing them in the table and sending 30 | # confirmation emails to them 31 | handler: 32 | handler: bin/listing 33 | package: 34 | include: 35 | - ./bin/listing 36 | events: 37 | - http: 38 | path: subscribe 39 | method: POST 40 | cors: true 41 | - http: 42 | path: unsubscribe 43 | method: GET 44 | cors: true 45 | - http: 46 | path: confirm 47 | method: GET 48 | cors: true 49 | iamRoleStatements: 50 | - Effect: Allow 51 | Action: 52 | - "dynamodb:DescribeTable" 53 | - "dynamodb:Query" 54 | - "dynamodb:Scan" 55 | - "dynamodb:GetItem" 56 | - "dynamodb:PutItem" 57 | - "dynamodb:UpdateItem" 58 | - "dynamodb:DeleteItem" 59 | - "dynamodb:BatchWriteItem" 60 | Resource: 61 | - { 'Fn::ImportValue': '${self:provider.stage}-ListingSubscriptionsTableArn' } 62 | - Effect: Allow 63 | Action: 64 | - "ses:SendEmail" 65 | - "ses:SendRawEmail" 66 | Resource: "arn:aws:ses:${self:provider.region}:*:identity/*" 67 | - Effect: Allow 68 | Action: 69 | - "dynamodb:DescribeTable" 70 | - "dynamodb:Query" 71 | - "dynamodb:Scan" 72 | - "dynamodb:GetItem" 73 | Resource: 74 | - { 'Fn::ImportValue': '${self:provider.stage}-ListingNotificationsTableArn' } 75 | environment: 76 | CONFIRM_URL: ${self:custom.secrets.confirmUrl} 77 | EMAIL_FROM: ${self:custom.secrets.emailFrom} 78 | TOKEN_SECRET: ${self:custom.secrets.tokenSecret} 79 | SUBSCRIBE_REDIRECT_URL: ${self:custom.secrets.subscribeRedirectUrl} 80 | CONFIRM_REDIRECT_URL: ${self:custom.secrets.confirmRedirectUrl} 81 | UNSUBSCRIBE_REDIRECT_URL: ${self:custom.secrets.unsubscribeRedirectUrl} 82 | SUBSCRIBERS_TABLE: ${self:custom.subscribersTableName} 83 | NOTIFICATIONS_TABLE: ${self:custom.snsTableName} 84 | SUPPORTED_NEWSLETTERS: ${self:custom.secrets.supportedNewsletters} 85 | # lambda used to handle bounce and complaint notifications from SES 86 | sesnotify: 87 | handler: bin/sesnotify 88 | package: 89 | include: 90 | - ./bin/sesnotify 91 | events: 92 | - sns: 93 | topicName: ${self:custom.snsTopicName} 94 | arn: { 'Fn::ImportValue': '${self:provider.stage}-ListingNotificationsTopicArn' } 95 | environment: 96 | NOTIFICATIONS_TABLE: ${self:custom.snsTableName} 97 | iamRoleStatements: 98 | - Effect: Allow 99 | Action: 100 | - "dynamodb:DescribeTable" 101 | - "dynamodb:PutItem" 102 | Resource: 103 | - { 'Fn::ImportValue': '${self:provider.stage}-ListingNotificationsTableArn' } 104 | 105 | custom: 106 | secrets: ${file(secrets.json)} 107 | subscribersTableName: ${self:provider.stage}-listing-subscribers 108 | snsTableName: ${self:provider.stage}-listing-sesnotify 109 | snsTopicName: ${self:provider.stage}-listing-ses-notifications 110 | apiGatewayLogs: 111 | dev: true 112 | prod: true 113 | stages: 114 | - local 115 | - dev 116 | domains: 117 | prod: ${self:custom.secrets.prodDomain} 118 | dev: ${self:custom.secrets.devDomain} 119 | customDomain: 120 | basePath: '' 121 | domainName: ${self:custom.domains.${self:provider.stage}} 122 | stage: ${self:provider.stage} 123 | createRoute53Record: false 124 | endpointType: edge 125 | logRetentionInDays: 126 | prod: 14 127 | dev: 7 128 | 129 | -------------------------------------------------------------------------------- /scripts/add_subscribers.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"newsletter":"Listing1","email":"email1@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 3 | {"newsletter":"Listing1","email":"email2@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 4 | {"newsletter":"Listing1","email":"email3@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 5 | {"newsletter":"Listing1","email":"email4@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 6 | {"newsletter":"Listing1","email":"email5@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 7 | {"newsletter":"Listing1","email":"email6@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 8 | {"newsletter":"Listing1","email":"email7@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 9 | {"newsletter":"Listing1","email":"email8@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 10 | {"newsletter":"Listing1","email":"email9@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 11 | {"newsletter":"Listing1","email":"email0@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 12 | {"newsletter":"Listing1","email":"emai11@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 13 | {"newsletter":"Listing1","email":"emai21@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 14 | {"newsletter":"Listing1","email":"emai31@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 15 | {"newsletter":"Listing1","email":"emai41@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 16 | {"newsletter":"Listing1","email":"emai51@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 17 | {"newsletter":"Listing1","email":"emai61@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 18 | {"newsletter":"Listing1","email":"emai71@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 19 | {"newsletter":"Listing1","email":"emai81@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 20 | {"newsletter":"Listing1","email":"emai91@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 21 | {"newsletter":"Listing1","email":"emai01@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 22 | {"newsletter":"Listing1","email":"ema1l1@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 23 | {"newsletter":"Listing1","email":"ema2l1@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 24 | {"newsletter":"Listing1","email":"ema3l1@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 25 | {"newsletter":"Listing1","email":"ema4l1@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 26 | {"newsletter":"Listing1","email":"ema5l1@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"}, 27 | {"newsletter":"Listing1","email":"ema6l1@domain.com","created_at":"2019-12-26T18:48:04Z","unsubscribed_at":"1970-01-01T00:00:01Z","confirmed_at":"2019-12-26T18:50:12Z"} 28 | ] 29 | -------------------------------------------------------------------------------- /docs/DEPLOYMENT.md: -------------------------------------------------------------------------------- 1 | Except of quite obvious prerequisites, the typical deployment procedure is as easy as editing `secrets.json` and running `serverless deploy` in the root of the repository. See detailed steps below. 2 | 3 | ## Generic prerequisites 4 | 5 | * Buy/Own a custom domain 6 | * [Create an AWS account](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/) 7 | * Configure and [verify your domain in AWS SES](https://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-domain-procedure.html) (Simple Email Service) 8 | * [Exit "sandbox" mode in SES](https://docs.aws.amazon.com/ses/latest/DeveloperGuide/request-production-access.html) (you will have to contact support) 9 | * Create "confirm"/"unsubscribed" pages on your website 10 | 11 | ## Install prerequisites 12 | 13 | * [Go](https://golang.org/dl/) 14 | * [Node.js](https://nodejs.org/en/download/) (for npm and serverless) 15 | * [Serverless](https://serverless.com/framework/docs/getting-started/) (`npm install -g serverless`) 16 | * Go dep (run `go get -u github.com/golang/dep/cmd/dep` anywhere) 17 | * Serverless plugins (run `npm install` in repository root) 18 | * [Configure AWS credentials](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/setup-credentials.html) 19 | 20 | ## Edit secrets 21 | 22 | Copy example file to `secrets.json` 23 | 24 | `cp secrets.json.example secrets.json` 25 | 26 | and edit it. 27 | 28 | Most of the properties are self-descriptive. Redirect URLs are urls where user will be redirected to after pressing "Confirm", "Subscribe" or "Unsubscribe" buttons. `confirmUrl` is an url of one of the lambda functions used for email confirmation (can be arbitrary since it's edited after deployment). `emailFrom` is an email that will be used to send this confirmation email. `supportedNewsletters` is semicolon-separated list of newsletter names. *Listing* will ignore all subscribe/unsubscribe requests for newsletters that are not in this list. 29 | 30 | ## Configure custom domain 31 | 32 | If you want to deploy _listing_ as `listing.yourdomain.com` you will need to do couple of things: 33 | 34 | * Request custom certificate in AWS Certificate manager for region `us-east-1` (even if you plan to deploy to other regions) for your domain 35 | * Make sure `secrets.json` from previous step has correct domain(s) configured 36 | * Run `STAGE=dev REGION=eu-west-1 make create_domain` (replace stage and region with your preferences) - this has to be run only once per "lifetime" 37 | 38 | You can see those domains in API Gateway section of AWS Console. Currently _listing_ assumes you are using `dev-listing.yourdomain.com` for `dev` and `listing.yourdomain.com` for `prod`. You can change this logic in `serverless-api.yml` file in `customDomains` section. 39 | 40 | ## Deploy listing 41 | 42 | `STAGE=dev REGION=eu-west-1 make deploy-all` 43 | 44 | This command will compile and deploy whole _listing_ to AWS in the said stage and region. In case you are making changes to API or to Admin parts, you can redeploy them separately from DB. Use commands `make deploy-db`, `make deploy-api` and `make deploy-admin` and their `make remove-xxx` counterparts to deploy and remove separate pieces. 45 | 46 | ## Configure confirm and redirect URLs 47 | 48 | Go to AWS Console UI and in Lambda section find `listing-subscribe` function. Set `CONFIRM_URL` in it's environmental variables to point to the API Gateway address for `listing-confirm` function's address. 49 | 50 | (optional - you can do that in the end) Configure `SUBSCRIBE_REDIRECT_URL`, `UNSUBSCRIBE_REDIRECT_URL`, `CONFIRM_REDIRECT_URL` in the appropriate lambda function to point to the pages on your website. 51 | 52 | ## Configure SNS topic for bounces and complaints 53 | 54 | In order to exit "sandbox" mode in AWS SES you need to have a procedure for handling bounces and complaints. *Listing* provides this functionality, but you have to do 1 manual action. 55 | 56 | Go to [AWS Console UI and set Bounce and Complaint](https://docs.aws.amazon.com/ses/latest/DeveloperGuide/configure-sns-notifications.html) SNS topic's ARN for your SES domain to the `listing-ses-notifications` topic. You can find it in `SES -> Domains -> (select your domain) -> Notifications`. Arn will be an output of `serverless deploy` command for `serverless-db.yml` config. Example of such ARN: `arn:aws:sns:eu-west-1:1234567890:dev-listing-ses-notifications`. 57 | 58 | ## Configure redirect URLs to your website 59 | 60 | Open lambda function properties in the AWS management console and set all `_REDIRECT_URL` variables to the appropriate values that point to your website. You will need to have pages for email confirmation, unsubscribe confirmation etc. 61 | 62 | -------------------------------------------------------------------------------- /pkg/email/email.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "net/url" 7 | "strings" 8 | "text/template" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/awserr" 12 | "github.com/aws/aws-sdk-go/service/ses" 13 | "github.com/ribtoks/listing/pkg/common" 14 | ) 15 | 16 | const ( 17 | // Specify a configuration set. To use a configuration 18 | // set, comment the next line and line 92. 19 | //ConfigurationSet = "ConfigSet" 20 | 21 | // Subject line for the email 22 | Subject = "Confirm your email" 23 | 24 | // CharSet is the character encoding for the email 25 | CharSet = "UTF-8" 26 | 27 | // TextBody is a plain text copy of HTMLBody 28 | TextBody = ` 29 | Hello, 30 | 31 | Thank you for subscribing to {{.Newsletter}} newsletter! Please confirm your email by clicking link below. 32 | 33 | {{.ConfirmURL}} 34 | 35 | You are receiveing this email because somebody, hopefully you, subscribed to {{.Newsletter}} newsletter. If it was not you, you can safely ignore this email. 36 | 37 | {{.Newsletter}} team 38 | ` 39 | ) 40 | 41 | var ( 42 | HtmlTemplate *template.Template 43 | TextTemplate *template.Template 44 | ) 45 | 46 | // SESMailer is an implementation of Mailer interface that works with AWS SES 47 | type SESMailer struct { 48 | Sender string 49 | Secret string 50 | Svc *ses.SES 51 | } 52 | 53 | var _ common.Mailer = (*SESMailer)(nil) 54 | 55 | func (sm *SESMailer) confirmURL(newsletter, email string, confirmBaseURL string) (string, error) { 56 | token := common.Sign(sm.Secret, email) 57 | baseUrl, err := url.Parse(confirmBaseURL) 58 | if err != nil { 59 | log.Println("Malformed URL: ", err.Error()) 60 | return "", err 61 | } 62 | params := url.Values{} 63 | params.Add(common.ParamNewsletter, newsletter) 64 | params.Add(common.ParamToken, token) 65 | baseUrl.RawQuery = params.Encode() 66 | return baseUrl.String(), nil 67 | } 68 | 69 | func (sm *SESMailer) sendEmail(email, htmlBody, textBody string) error { 70 | // Assemble the email. 71 | input := &ses.SendEmailInput{ 72 | Destination: &ses.Destination{ 73 | CcAddresses: []*string{}, 74 | ToAddresses: []*string{ 75 | aws.String(email), 76 | }, 77 | }, 78 | Message: &ses.Message{ 79 | Body: &ses.Body{ 80 | Html: &ses.Content{ 81 | Charset: aws.String(CharSet), 82 | Data: aws.String(htmlBody), 83 | }, 84 | Text: &ses.Content{ 85 | Charset: aws.String(CharSet), 86 | Data: aws.String(textBody), 87 | }, 88 | }, 89 | Subject: &ses.Content{ 90 | Charset: aws.String(CharSet), 91 | Data: aws.String(Subject), 92 | }, 93 | }, 94 | Source: aws.String(sm.Sender), 95 | // Uncomment to use a configuration set 96 | //ConfigurationSetName: aws.String(ConfigurationSet), 97 | } 98 | 99 | // Attempt to send the email. 100 | result, err := sm.Svc.SendEmail(input) 101 | log.Printf("Email send result=%v", result) 102 | 103 | // Display error messages if they occur. 104 | if err != nil { 105 | if aerr, ok := err.(awserr.Error); ok { 106 | switch aerr.Code() { 107 | case ses.ErrCodeMessageRejected: 108 | log.Println(ses.ErrCodeMessageRejected, aerr.Error()) 109 | case ses.ErrCodeMailFromDomainNotVerifiedException: 110 | log.Println(ses.ErrCodeMailFromDomainNotVerifiedException, aerr.Error()) 111 | case ses.ErrCodeConfigurationSetDoesNotExistException: 112 | log.Println(ses.ErrCodeConfigurationSetDoesNotExistException, aerr.Error()) 113 | default: 114 | log.Println(aerr.Error()) 115 | } 116 | } else { 117 | // Print the error, cast err to awserr.Error to get the Code and 118 | // Message from an error. 119 | log.Println(err.Error()) 120 | } 121 | 122 | return err 123 | } 124 | 125 | return nil 126 | 127 | } 128 | 129 | func (sm *SESMailer) SendConfirmation(newsletter, email, name, confirmBaseURL string) error { 130 | confirmURL, err := sm.confirmURL(newsletter, email, confirmBaseURL) 131 | if err != nil { 132 | return err 133 | } 134 | 135 | nameParts := strings.Split(strings.TrimSpace(name), " ") 136 | 137 | data := struct { 138 | Newsletter string 139 | ConfirmURL string 140 | FirstName string 141 | }{ 142 | Newsletter: newsletter, 143 | ConfirmURL: confirmURL, 144 | FirstName: strings.TrimSpace(nameParts[0]), 145 | } 146 | 147 | var htmlBodyTpl bytes.Buffer 148 | if err := HtmlTemplate.Execute(&htmlBodyTpl, data); err != nil { 149 | return err 150 | } 151 | 152 | var textBodyTpl bytes.Buffer 153 | if err := TextTemplate.Execute(&textBodyTpl, data); err != nil { 154 | return err 155 | } 156 | 157 | return sm.sendEmail(email, htmlBodyTpl.String(), textBodyTpl.String()) 158 | } 159 | 160 | func init() { 161 | HtmlTemplate = template.Must(template.New("HtmlBody").Parse(HTMLBody)) 162 | TextTemplate = template.Must(template.New("TextBody").Parse(TextBody)) 163 | } 164 | -------------------------------------------------------------------------------- /cmd/listing-send/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "html/template" 9 | "io" 10 | "io/ioutil" 11 | "log" 12 | "os" 13 | "sync" 14 | "time" 15 | 16 | "github.com/go-gomail/gomail" 17 | "github.com/ribtoks/listing/pkg/common" 18 | ) 19 | 20 | var ( 21 | smtpServerFlag = flag.String("url", "", "SMTP server url") 22 | smtpUsernameFlag = flag.String("user", "", "SMTP username flag") 23 | smtpPassFlag = flag.String("pass", "", "SMTP password flag") 24 | subjectFlag = flag.String("subject", "", "Html campaign subject") 25 | fromEmailFlag = flag.String("from-email", "", "Sender address") 26 | fromNameFlag = flag.String("from-name", "", "Sender name") 27 | htmlTemplateFlag = flag.String("html-template", "", "Path to html email template") 28 | txtTemplateFlag = flag.String("txt-template", "", "Path to text email template") 29 | paramsFlag = flag.String("params", "params.json", "Path to file with common params") 30 | workersFlag = flag.Int("workers", 2, "Number of workers to send emails") 31 | listFlag = flag.String("list", "list.json", "Path to file with email list") 32 | rateFlag = flag.Int("rate", 25, "Emails per second sending rate") 33 | dryRunFlag = flag.Bool("dry-run", false, "Simulate selected action") 34 | outFlag = flag.String("out", "./", "Path to directory for dry run results") 35 | helpFlag = flag.Bool("help", false, "Print help") 36 | logPathFlag = flag.String("l", "listing-send.log", "Absolute path to log file") 37 | stdoutFlag = flag.Bool("stdout", false, "Log to stdout and to logfile") 38 | ) 39 | 40 | const ( 41 | appName = "listing-send" 42 | smtpRetryAttempts = 3 43 | smtpRetrySleep = 1 * time.Second 44 | ) 45 | 46 | func main() { 47 | err := parseFlags() 48 | if err != nil { 49 | flag.PrintDefaults() 50 | log.Fatal(err.Error()) 51 | } 52 | 53 | logfile, err := setupLogging() 54 | if err != nil { 55 | defer logfile.Close() 56 | } 57 | 58 | htmlTemplate, err := template.ParseFiles(*htmlTemplateFlag) 59 | if err != nil { 60 | log.Fatal(err) 61 | } 62 | 63 | textTemplate, err := template.ParseFiles(*txtTemplateFlag) 64 | if err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | params, err := readParams(*paramsFlag) 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | 73 | subscribers, err := readSubscribers(*listFlag) 74 | if err != nil { 75 | log.Fatal(err) 76 | } 77 | 78 | c := &campaign{ 79 | htmlTemplate: htmlTemplate, 80 | textTemplate: textTemplate, 81 | params: params, 82 | subscribers: subscribers, 83 | subject: *subjectFlag, 84 | fromEmail: *fromEmailFlag, 85 | fromName: *fromNameFlag, 86 | rate: *rateFlag, 87 | dryRun: *dryRunFlag, 88 | messages: make(chan *gomail.Message, 10), 89 | waiter: &sync.WaitGroup{}, 90 | workersCount: *workersFlag, 91 | } 92 | 93 | c.send() 94 | } 95 | 96 | func readSubscribers(filepath string) ([]*common.SubscriberEx, error) { 97 | f, err := os.Open(filepath) 98 | if err != nil { 99 | return nil, err 100 | } 101 | defer f.Close() 102 | 103 | data, err := ioutil.ReadAll(f) 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | dec := json.NewDecoder(bytes.NewBuffer(data)) 109 | dec.DisallowUnknownFields() 110 | 111 | var subscribers []*common.SubscriberEx 112 | err = dec.Decode(&subscribers) 113 | if err != nil { 114 | return nil, err 115 | } 116 | log.Printf("Parsed subscribers. count=%v", len(subscribers)) 117 | 118 | return subscribers, nil 119 | } 120 | 121 | func readParams(filepath string) (map[string]interface{}, error) { 122 | f, err := os.Open(filepath) 123 | if err != nil { 124 | return nil, err 125 | } 126 | defer f.Close() 127 | 128 | data, err := ioutil.ReadAll(f) 129 | if err != nil { 130 | return nil, err 131 | } 132 | 133 | params := make(map[string]interface{}) 134 | err = json.Unmarshal(data, ¶ms) 135 | return params, err 136 | } 137 | 138 | func setupLogging() (f *os.File, err error) { 139 | f, err = os.OpenFile(*logPathFlag, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 140 | if err != nil { 141 | fmt.Printf("error opening file: %v", *logPathFlag) 142 | return nil, err 143 | } 144 | 145 | if *stdoutFlag || *dryRunFlag { 146 | mw := io.MultiWriter(os.Stdout, f) 147 | log.SetOutput(mw) 148 | } else { 149 | log.SetOutput(f) 150 | } 151 | 152 | log.Println("------------------------------") 153 | log.Println(appName + " log started") 154 | 155 | return f, err 156 | } 157 | 158 | func parseFlags() (err error) { 159 | flag.Parse() 160 | return nil 161 | } 162 | 163 | func createSender() (gomail.SendCloser, error) { 164 | if *dryRunFlag { 165 | return &dryRunSender{out: *outFlag}, nil 166 | } 167 | 168 | dialer, err := smtpDialer(*smtpServerFlag, *smtpUsernameFlag, *smtpPassFlag) 169 | if err != nil { 170 | return nil, err 171 | } 172 | 173 | var sender gomail.SendCloser 174 | for i := 0; i < smtpRetryAttempts; i++ { 175 | sender, err = dialer.Dial() 176 | if err == nil { 177 | log.Printf("Dialed to SMTP. server=%v", *smtpServerFlag) 178 | break 179 | } else { 180 | log.Printf("Failed to dial SMTP. err=%v attempt=%v", err, i) 181 | log.Printf("Sleeping before retry. interval=%v", smtpRetrySleep) 182 | time.Sleep(smtpRetrySleep) 183 | } 184 | } 185 | return sender, nil 186 | } 187 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | digest = "1:ad3a958de5100511e795bc6abd52a039b9538b7e42d7b393af962238aaa584ef" 6 | name = "github.com/aws/aws-lambda-go" 7 | packages = [ 8 | "events", 9 | "lambda", 10 | "lambda/handlertrace", 11 | "lambda/messages", 12 | "lambdacontext", 13 | ] 14 | pruneopts = "" 15 | revision = "18b606949c45bc923e421d7ea891d6d02a34f7dd" 16 | version = "v1.16.0" 17 | 18 | [[projects]] 19 | digest = "1:6f6819a0ff6795ffee5f6b5e17540cd279c9fc636930fe7ee2f2313c823b4eda" 20 | name = "github.com/aws/aws-sdk-go" 21 | packages = [ 22 | "aws", 23 | "aws/awserr", 24 | "aws/awsutil", 25 | "aws/client", 26 | "aws/client/metadata", 27 | "aws/corehandlers", 28 | "aws/credentials", 29 | "aws/credentials/ec2rolecreds", 30 | "aws/credentials/endpointcreds", 31 | "aws/credentials/processcreds", 32 | "aws/credentials/stscreds", 33 | "aws/crr", 34 | "aws/csm", 35 | "aws/defaults", 36 | "aws/ec2metadata", 37 | "aws/endpoints", 38 | "aws/request", 39 | "aws/session", 40 | "aws/signer/v4", 41 | "internal/context", 42 | "internal/ini", 43 | "internal/sdkio", 44 | "internal/sdkmath", 45 | "internal/sdkrand", 46 | "internal/sdkuri", 47 | "internal/shareddefaults", 48 | "internal/strings", 49 | "internal/sync/singleflight", 50 | "private/protocol", 51 | "private/protocol/json/jsonutil", 52 | "private/protocol/jsonrpc", 53 | "private/protocol/query", 54 | "private/protocol/query/queryutil", 55 | "private/protocol/rest", 56 | "private/protocol/xml/xmlutil", 57 | "service/dynamodb", 58 | "service/dynamodb/dynamodbattribute", 59 | "service/dynamodb/dynamodbiface", 60 | "service/ses", 61 | "service/sts", 62 | "service/sts/stsiface", 63 | ] 64 | pruneopts = "" 65 | revision = "013e43dff3a1cd1c7778d7b131988a66a543598c" 66 | version = "v1.30.17" 67 | 68 | [[projects]] 69 | digest = "1:553343cc3a8c970e995572c6a5445fa6fd0babbb43b6411b43a0daab816b4466" 70 | name = "github.com/awslabs/aws-lambda-go-api-proxy" 71 | packages = [ 72 | "core", 73 | "httpadapter", 74 | ] 75 | pruneopts = "" 76 | revision = "9f6f9d2a6cbc7f3d441488125b2811b6ffeeaff0" 77 | version = "v0.6.0" 78 | 79 | [[projects]] 80 | digest = "1:cba75ccf02d7b1d508f860e1f579a986e6b94ccc26841d6dd67d18cd1196e132" 81 | name = "github.com/go-gomail/gomail" 82 | packages = ["."] 83 | pruneopts = "" 84 | revision = "41f3572897373c5538c50a2402db15db079fa4fd" 85 | version = "2.0.0" 86 | 87 | [[projects]] 88 | digest = "1:13fe471d0ed891e8544eddfeeb0471fd3c9f2015609a1c000aefdedf52a19d40" 89 | name = "github.com/jmespath/go-jmespath" 90 | packages = ["."] 91 | pruneopts = "" 92 | revision = "c2b33e84" 93 | 94 | [[projects]] 95 | digest = "1:59794624db141f0f0a893d110111b962984c9844cf565efa91c35bedb882c9ff" 96 | name = "github.com/mattn/go-runewidth" 97 | packages = ["."] 98 | pruneopts = "" 99 | revision = "14e809f6d78fcf9f48ff9b70981472b64c05f754" 100 | version = "v0.0.9" 101 | 102 | [[projects]] 103 | digest = "1:be76fb5b006046835859bbfd38adfc01574c672685fd5d4c6f0bce073ac16bfb" 104 | name = "github.com/olekukonko/tablewriter" 105 | packages = ["."] 106 | pruneopts = "" 107 | revision = "876dd0e0227ec99c0243b639b92139915b65331a" 108 | version = "v0.0.4" 109 | 110 | [[projects]] 111 | digest = "1:ef0f9731bc6c3c59396adc50f36da36ca98c704812c449794f9326f7bc64b5f1" 112 | name = "github.com/ribtoks/backoff" 113 | packages = ["."] 114 | pruneopts = "" 115 | revision = "8eab2debe79d12b7bd3d10653910df25fa9552ba" 116 | version = "1.0.0" 117 | 118 | [[projects]] 119 | digest = "1:9f4b6e3942dda8d2136792d1b22b3c289be1109ae12da0c88a1befad438acdfb" 120 | name = "github.com/ribtoks/checkmail" 121 | packages = ["."] 122 | pruneopts = "" 123 | revision = "9661bd69e9ad6fce1f7579022bdab0440807722a" 124 | version = "v1.1" 125 | 126 | [[projects]] 127 | digest = "1:a9ac823421527eaf6566301956cc5ae6dc312f4c0746afc0aa2f23b96f6458bd" 128 | name = "github.com/rs/xid" 129 | packages = ["."] 130 | pruneopts = "" 131 | revision = "15d26544def341f036c5f8dca987a4cbe575032c" 132 | version = "v1.2.1" 133 | 134 | [[projects]] 135 | branch = "v3" 136 | digest = "1:fa33d1dde8ce5b0b37c38c767c250a7684ecd385f9dbabe594ea8543e4d55807" 137 | name = "gopkg.in/alexcesaro/quotedprintable.v3" 138 | packages = ["."] 139 | pruneopts = "" 140 | revision = "2caba252f4dc53eaf6b553000885530023f54623" 141 | 142 | [[projects]] 143 | digest = "1:2efc9662a6a1ff28c65c84fc2f9030f13d3afecdb2ecad445f3b0c80e75fc281" 144 | name = "gopkg.in/yaml.v2" 145 | packages = ["."] 146 | pruneopts = "" 147 | revision = "53403b58ad1b561927d19068c655246f2db79d48" 148 | version = "v2.2.8" 149 | 150 | [solve-meta] 151 | analyzer-name = "dep" 152 | analyzer-version = 1 153 | input-imports = [ 154 | "github.com/aws/aws-lambda-go/events", 155 | "github.com/aws/aws-lambda-go/lambda", 156 | "github.com/aws/aws-sdk-go/aws", 157 | "github.com/aws/aws-sdk-go/aws/awserr", 158 | "github.com/aws/aws-sdk-go/aws/session", 159 | "github.com/aws/aws-sdk-go/service/dynamodb", 160 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute", 161 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface", 162 | "github.com/aws/aws-sdk-go/service/ses", 163 | "github.com/awslabs/aws-lambda-go-api-proxy/httpadapter", 164 | "github.com/go-gomail/gomail", 165 | "github.com/olekukonko/tablewriter", 166 | "github.com/ribtoks/backoff", 167 | "github.com/ribtoks/checkmail", 168 | "github.com/rs/xid", 169 | "gopkg.in/yaml.v2", 170 | ] 171 | solver-name = "gps-cdcl" 172 | solver-version = 1 173 | -------------------------------------------------------------------------------- /cmd/listing-cli/printer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "reflect" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/olekukonko/tablewriter" 13 | "github.com/ribtoks/listing/pkg/common" 14 | "gopkg.in/yaml.v2" 15 | ) 16 | 17 | type Printer interface { 18 | Append(s *common.Subscriber) 19 | Render() error 20 | } 21 | 22 | func structToMap(cs *common.SubscriberEx) map[string]string { 23 | values := make(map[string]string) 24 | s := reflect.ValueOf(cs).Elem() 25 | typeOfT := s.Type() 26 | 27 | for i := 0; i < s.NumField(); i++ { 28 | f := s.Field(i) 29 | var v string 30 | switch f.Interface().(type) { 31 | case bool: 32 | v = fmt.Sprintf("%v", f.Bool()) 33 | case int, int8, int16, int32, int64: 34 | v = strconv.FormatInt(f.Int(), 10) 35 | case uint, uint8, uint16, uint32, uint64: 36 | v = strconv.FormatUint(f.Uint(), 10) 37 | case float32: 38 | v = strconv.FormatFloat(f.Float(), 'f', 4, 32) 39 | case float64: 40 | v = strconv.FormatFloat(f.Float(), 'f', 4, 64) 41 | case []byte: 42 | v = string(f.Bytes()) 43 | case string: 44 | v = f.String() 45 | case common.JSONTime: 46 | v = f.Interface().(common.JSONTime).String() 47 | v = strings.Trim(v, `"`) 48 | } 49 | values[typeOfT.Field(i).Name] = v 50 | } 51 | return values 52 | } 53 | 54 | func mapValues(m map[string]string, fields []string) []string { 55 | row := make([]string, 0, len(fields)) 56 | for _, k := range fields { 57 | if v, ok := m[k]; ok { 58 | row = append(row, v) 59 | } 60 | } 61 | return row 62 | } 63 | 64 | type TablePrinter struct { 65 | secret string 66 | table *tablewriter.Table 67 | fields []string 68 | } 69 | 70 | func SubscriberHeaders() []string { 71 | statType := reflect.TypeOf(common.SubscriberEx{}) 72 | header := make([]string, 0, statType.NumField()) 73 | for i := 0; i < statType.NumField(); i++ { 74 | field := statType.Field(i) 75 | header = append(header, field.Name) 76 | } 77 | return header 78 | } 79 | 80 | func NewTablePrinter(secret string) *TablePrinter { 81 | tr := &TablePrinter{ 82 | secret: secret, 83 | table: tablewriter.NewWriter(os.Stdout), 84 | fields: SubscriberHeaders(), 85 | } 86 | 87 | tr.table.SetHeader(tr.fields) 88 | return tr 89 | } 90 | 91 | func NewTSVPrinter(secret string) *TablePrinter { 92 | tr := NewTablePrinter(secret) 93 | 94 | tr.table.SetAutoWrapText(false) 95 | tr.table.SetAutoFormatHeaders(true) 96 | tr.table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) 97 | tr.table.SetAlignment(tablewriter.ALIGN_LEFT) 98 | tr.table.SetCenterSeparator("") 99 | tr.table.SetColumnSeparator("") 100 | tr.table.SetRowSeparator("") 101 | tr.table.SetHeaderLine(false) 102 | tr.table.SetBorder(false) 103 | tr.table.SetTablePadding("\t") // pad with tabs 104 | tr.table.SetNoWhiteSpace(true) 105 | 106 | return tr 107 | } 108 | 109 | func (tr *TablePrinter) Append(s *common.Subscriber) { 110 | se := common.NewSubscriberEx(s, tr.secret) 111 | m := structToMap(se) 112 | row := mapValues(m, tr.fields) 113 | tr.table.Append(row) 114 | } 115 | 116 | func (tr *TablePrinter) Render() error { 117 | tr.table.Render() 118 | return nil 119 | } 120 | 121 | type CSVPrinter struct { 122 | secret string 123 | w *csv.Writer 124 | fields []string 125 | } 126 | 127 | func NewCSVPrinter(secret string) *CSVPrinter { 128 | cr := &CSVPrinter{ 129 | secret: secret, 130 | w: csv.NewWriter(os.Stdout), 131 | fields: SubscriberHeaders(), 132 | } 133 | cr.w.Write(cr.fields) 134 | return cr 135 | } 136 | 137 | func (cr *CSVPrinter) Append(s *common.Subscriber) { 138 | se := common.NewSubscriberEx(s, cr.secret) 139 | m := structToMap(se) 140 | row := mapValues(m, cr.fields) 141 | cr.w.Write(row) 142 | } 143 | 144 | func (cr *CSVPrinter) Render() error { 145 | cr.w.Flush() 146 | return nil 147 | } 148 | 149 | type JsonPrinter struct { 150 | subscribers []*common.SubscriberEx 151 | secret string 152 | } 153 | 154 | func NewJsonPrinter(secret string) *JsonPrinter { 155 | rp := &JsonPrinter{ 156 | subscribers: make([]*common.SubscriberEx, 0), 157 | secret: secret, 158 | } 159 | return rp 160 | } 161 | 162 | func (rp *JsonPrinter) Append(s *common.Subscriber) { 163 | rp.subscribers = append(rp.subscribers, common.NewSubscriberEx(s, rp.secret)) 164 | } 165 | 166 | func (rp *JsonPrinter) Render() error { 167 | data, err := json.MarshalIndent(rp.subscribers, "", " ") 168 | if err != nil { 169 | return err 170 | } 171 | fmt.Println(string(data)) 172 | return nil 173 | } 174 | 175 | type RawPrinter struct { 176 | subscribers []*common.Subscriber 177 | } 178 | 179 | func NewRawPrinter() *RawPrinter { 180 | rp := &RawPrinter{ 181 | subscribers: make([]*common.Subscriber, 0), 182 | } 183 | return rp 184 | } 185 | 186 | func (rp *RawPrinter) Append(s *common.Subscriber) { 187 | rp.subscribers = append(rp.subscribers, s) 188 | } 189 | 190 | func (rp *RawPrinter) Render() error { 191 | data, err := json.MarshalIndent(rp.subscribers, "", " ") 192 | if err != nil { 193 | return err 194 | } 195 | fmt.Println(string(data)) 196 | return nil 197 | } 198 | 199 | type YamlPrinter struct { 200 | secret string 201 | subscribers []*common.SubscriberEx 202 | } 203 | 204 | func NewYamlPrinter(secret string) *YamlPrinter { 205 | yp := &YamlPrinter{ 206 | secret: secret, 207 | subscribers: make([]*common.SubscriberEx, 0), 208 | } 209 | return yp 210 | } 211 | 212 | func (yp *YamlPrinter) Append(s *common.Subscriber) { 213 | se := common.NewSubscriberEx(s, yp.secret) 214 | yp.subscribers = append(yp.subscribers, se) 215 | } 216 | 217 | func (yp *YamlPrinter) Render() error { 218 | data, err := yaml.Marshal(yp.subscribers) 219 | if err != nil { 220 | return err 221 | } 222 | fmt.Println(string(data)) 223 | return nil 224 | } 225 | -------------------------------------------------------------------------------- /cmd/listing-cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "net/url" 12 | "os" 13 | "time" 14 | ) 15 | 16 | var ( 17 | modeFlag = flag.String("mode", "", "Execution mode: subscribe|unsubscribe|export|import|delete") 18 | urlFlag = flag.String("url", "", "Base URL to the listing API") 19 | emailFlag = flag.String("email", "", "Email for subscribe|unsubscribe") 20 | authTokenFlag = flag.String("auth-token", "", "Auth token for admin access") 21 | secretFlag = flag.String("secret", "", "Secret for email salt") 22 | newsletterFlag = flag.String("newsletter", "", "Newsletter for subscribe|unsubscribe") 23 | formatFlag = flag.String("format", "table", "Ouput format of subscribers: csv|tsv|table|raw|yaml") 24 | nameFlag = flag.String("name", "", "(optional) Name for subscribe") 25 | logPathFlag = flag.String("l", "listing-cli.log", "Absolute path to log file") 26 | stdoutFlag = flag.Bool("stdout", false, "Log to stdout and to logfile") 27 | helpFlag = flag.Bool("help", false, "Print help") 28 | dryRunFlag = flag.Bool("dry-run", false, "Simulate selected action") 29 | noUnconfirmedFlag = flag.Bool("no-unconfirmed", false, "Do not export unconfirmed emails") 30 | noConfirmedFlag = flag.Bool("no-confirmed", false, "Do not export confirmed emails") 31 | noUnsubscribedFlag = flag.Bool("no-unsubscribed", false, "Do not export unsubscribed emails") 32 | ignoreComplaintsFlag = flag.Bool("ignore-complaints", false, "Ignore bounces and complaints for export") 33 | ) 34 | 35 | const ( 36 | appName = "listing-cli" 37 | modeSubscribe = "subscribe" 38 | modeUnsubscribe = "unsubscribe" 39 | modeExport = "export" 40 | modeImport = "import" 41 | modeDelete = "delete" 42 | modeFilter = "filter" 43 | ) 44 | 45 | func main() { 46 | err := parseFlags() 47 | if err != nil { 48 | flag.PrintDefaults() 49 | log.Fatal(err.Error()) 50 | } 51 | 52 | logfile, err := setupLogging() 53 | if err != nil { 54 | defer logfile.Close() 55 | } 56 | 57 | client := &listingClient{ 58 | client: &http.Client{ 59 | Timeout: 10 * time.Second, 60 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 61 | return http.ErrUseLastResponse 62 | }, 63 | }, 64 | printer: NewPrinter(), 65 | url: *urlFlag, 66 | authToken: *authTokenFlag, 67 | secret: *secretFlag, 68 | complaints: make(map[string]bool), 69 | dryRun: *dryRunFlag, 70 | noUnconfirmed: *noUnconfirmedFlag, 71 | noConfirmed: *noConfirmedFlag, 72 | noUnsubscribed: *noUnsubscribedFlag, 73 | ignoreComplaints: *ignoreComplaintsFlag, 74 | } 75 | 76 | switch *modeFlag { 77 | case modeExport: 78 | { 79 | err = client.export(*newsletterFlag) 80 | } 81 | case modeFilter: 82 | { 83 | bytes, _ := ioutil.ReadAll(os.Stdin) 84 | err = client.filter(bytes) 85 | } 86 | case modeSubscribe: 87 | { 88 | err = client.subscribe(*emailFlag, *newsletterFlag, *nameFlag) 89 | } 90 | case modeUnsubscribe: 91 | { 92 | err = client.unsubscribe(*emailFlag, *newsletterFlag) 93 | } 94 | case modeImport: 95 | { 96 | bytes, _ := ioutil.ReadAll(os.Stdin) 97 | err = client.importSubscribers(bytes) 98 | } 99 | case modeDelete: 100 | { 101 | bytes, _ := ioutil.ReadAll(os.Stdin) 102 | err = client.deleteSubscribers(bytes) 103 | } 104 | default: 105 | fmt.Printf("Mode %v is not supported yet", *modeFlag) 106 | } 107 | if err != nil { 108 | fmt.Printf("Error: %v", err) 109 | os.Exit(1) 110 | } 111 | } 112 | 113 | func setupLogging() (f *os.File, err error) { 114 | f, err = os.OpenFile(*logPathFlag, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 115 | if err != nil { 116 | fmt.Printf("error opening file: %v", *logPathFlag) 117 | return nil, err 118 | } 119 | 120 | if *stdoutFlag || *dryRunFlag { 121 | mw := io.MultiWriter(os.Stdout, f) 122 | log.SetOutput(mw) 123 | } else { 124 | log.SetOutput(f) 125 | } 126 | 127 | log.Println("------------------------------") 128 | log.Println(appName + " log started") 129 | 130 | return f, err 131 | } 132 | 133 | func parseFlags() (err error) { 134 | flag.Parse() 135 | 136 | switch *modeFlag { 137 | case "": 138 | err = errors.New("Mode is required") 139 | case modeDelete, modeExport, modeImport, modeSubscribe, modeUnsubscribe, modeFilter: 140 | err = nil 141 | default: 142 | err = fmt.Errorf("Mode %v is not supported", *modeFlag) 143 | } 144 | if err != nil { 145 | return 146 | } 147 | 148 | if *modeFlag != modeFilter { 149 | switch *urlFlag { 150 | case "": 151 | err = errors.New("Url is required") 152 | default: 153 | if _, e := url.Parse(*urlFlag); e != nil { 154 | err = fmt.Errorf("Failed to parse url. err=%v", e) 155 | } 156 | } 157 | } 158 | if err != nil { 159 | return 160 | } 161 | 162 | switch *modeFlag { 163 | case modeExport, modeUnsubscribe, modeFilter: 164 | if *secretFlag == "" { 165 | err = errors.New("Secret flag is required") 166 | } 167 | } 168 | if err != nil { 169 | return 170 | } 171 | 172 | switch *modeFlag { 173 | case modeExport, modeImport, modeDelete: 174 | if *authTokenFlag == "" { 175 | err = errors.New("Auth token is required") 176 | } 177 | } 178 | return 179 | } 180 | 181 | func NewPrinter() Printer { 182 | switch *formatFlag { 183 | case "table": 184 | return NewTablePrinter(*secretFlag) 185 | case "csv": 186 | return NewCSVPrinter(*secretFlag) 187 | case "tsv": 188 | return NewTSVPrinter(*secretFlag) 189 | case "raw": 190 | return NewRawPrinter() 191 | case "json": 192 | return NewJsonPrinter(*secretFlag) 193 | case "yaml": 194 | return NewYamlPrinter(*secretFlag) 195 | default: 196 | return NewTablePrinter(*secretFlag) 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /pkg/db/subscribers.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "strings" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | "github.com/aws/aws-sdk-go/service/dynamodb" 12 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" 13 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" 14 | "github.com/ribtoks/backoff" 15 | "github.com/ribtoks/listing/pkg/common" 16 | ) 17 | 18 | var ( 19 | incorrectTime = common.JSONTime(time.Unix(1, 1)) 20 | errChunkTooBig = errors.New("Chunk of data contains more than allowed 25 items") 21 | errResultIsNil = errors.New("Result is nil") 22 | errSubscriberDoesNotExist = errors.New("Subscriber does not exist") 23 | ) 24 | 25 | const ( 26 | dynamoDBChunkSize = 25 27 | ) 28 | 29 | // NewSubscribersStore creates an instance of SubscribersDynamoDB struct 30 | func NewSubscribersStore(table string, sess *session.Session) *SubscribersDynamoDB { 31 | return &SubscribersDynamoDB{ 32 | Client: dynamodb.New(sess), 33 | TableName: table, 34 | } 35 | } 36 | 37 | type SubscribersDynamoDB struct { 38 | TableName string 39 | Client dynamodbiface.DynamoDBAPI 40 | } 41 | 42 | // make sure SubscribersDynamoDB implements interface 43 | var _ common.SubscribersStore = (*SubscribersDynamoDB)(nil) 44 | 45 | func (s *SubscribersDynamoDB) GetSubscriber(newsletter, email string) (*common.Subscriber, error) { 46 | input := &dynamodb.GetItemInput{ 47 | TableName: aws.String(s.TableName), 48 | Key: map[string]*dynamodb.AttributeValue{ 49 | "newsletter": &dynamodb.AttributeValue{ 50 | S: &newsletter, 51 | }, 52 | "email": &dynamodb.AttributeValue{ 53 | S: &email, 54 | }, 55 | }, 56 | } 57 | 58 | result, err := s.Client.GetItem(input) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | if result.Item == nil { 64 | return nil, errResultIsNil 65 | } 66 | 67 | cs := new(common.Subscriber) 68 | err = dynamodbattribute.UnmarshalMap(result.Item, cs) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | return cs, nil 74 | } 75 | 76 | func (s *SubscribersDynamoDB) AddSubscriber(newsletter, email, name string) error { 77 | sr := &common.Subscriber{ 78 | Name: name, 79 | Newsletter: newsletter, 80 | Email: email, 81 | CreatedAt: common.JsonTimeNow(), 82 | UnsubscribedAt: incorrectTime, 83 | ConfirmedAt: incorrectTime, 84 | } 85 | sr.Validate() 86 | 87 | i, err := dynamodbattribute.MarshalMap(sr) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | _, err = s.Client.PutItem(&dynamodb.PutItemInput{ 93 | TableName: &s.TableName, 94 | Item: i, 95 | }) 96 | 97 | if err != nil { 98 | return err 99 | } 100 | 101 | return nil 102 | } 103 | 104 | func (s *SubscribersDynamoDB) RemoveSubscriber(newsletter, email string) error { 105 | updateVal := struct { 106 | UnsubscribedAt common.JSONTime `json:":unsubscribed_at"` 107 | }{ 108 | UnsubscribedAt: common.JsonTimeNow(), 109 | } 110 | 111 | update, err := dynamodbattribute.MarshalMap(updateVal) 112 | if err != nil { 113 | return err 114 | } 115 | input := &dynamodb.UpdateItemInput{ 116 | ExpressionAttributeValues: update, 117 | UpdateExpression: aws.String("set unsubscribed_at = :unsubscribed_at"), 118 | TableName: &s.TableName, 119 | Key: map[string]*dynamodb.AttributeValue{ 120 | "newsletter": &dynamodb.AttributeValue{ 121 | S: &newsletter, 122 | }, 123 | "email": &dynamodb.AttributeValue{ 124 | S: &email, 125 | }, 126 | }, 127 | ReturnValues: aws.String("UPDATED_NEW"), 128 | } 129 | _, err = s.Client.UpdateItem(input) 130 | return err 131 | } 132 | 133 | func (s *SubscribersDynamoDB) Subscribers(newsletter string) (subscribers []*common.Subscriber, err error) { 134 | query := &dynamodb.QueryInput{ 135 | TableName: &s.TableName, 136 | KeyConditionExpression: aws.String(`newsletter = :newsletter`), 137 | ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ 138 | ":newsletter": &dynamodb.AttributeValue{ 139 | S: &newsletter, 140 | }, 141 | }, 142 | } 143 | 144 | err = s.Client.QueryPages(query, func(page *dynamodb.QueryOutput, more bool) bool { 145 | var items []*common.Subscriber 146 | err := dynamodbattribute.UnmarshalListOfMaps(page.Items, &items) 147 | if err != nil { 148 | // print the error and continue receiving pages 149 | log.Printf("Could not unmarshal AWS data. err=%v", err) 150 | return true 151 | } 152 | 153 | subscribers = append(subscribers, items...) 154 | // continue receiving pages (can be used to limit the number of pages) 155 | return true 156 | }) 157 | 158 | return 159 | } 160 | 161 | func (s *SubscribersDynamoDB) AddSubscribersChunk(subscribers []*common.Subscriber) error { 162 | // AWS DynamoDB restriction 163 | if len(subscribers) > dynamoDBChunkSize { 164 | return errChunkTooBig 165 | } 166 | 167 | requests := make([]*dynamodb.WriteRequest, 0, len(subscribers)) 168 | for _, i := range subscribers { 169 | i.Validate() 170 | 171 | attr, err := dynamodbattribute.MarshalMap(i) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | requests = append(requests, &dynamodb.WriteRequest{ 177 | PutRequest: &dynamodb.PutRequest{ 178 | Item: attr, 179 | }, 180 | }) 181 | } 182 | 183 | b := &backoff.Backoff{ 184 | Min: 100 * time.Millisecond, 185 | Max: 1 * time.Second, 186 | Factor: 2, 187 | Jitter: false, 188 | } 189 | 190 | for len(requests) > 0 { 191 | input := &dynamodb.BatchWriteItemInput{ 192 | RequestItems: map[string][]*dynamodb.WriteRequest{ 193 | s.TableName: requests, 194 | }, 195 | } 196 | res, err := s.Client.BatchWriteItem(input) 197 | if err != nil { 198 | return err 199 | } 200 | if unprocessed, ok := res.UnprocessedItems[s.TableName]; ok { 201 | log.Printf("Found unprocessed items. count=%v", len(unprocessed)) 202 | requests = unprocessed 203 | } else { 204 | break 205 | } 206 | time.Sleep(b.Duration()) 207 | } 208 | return nil 209 | } 210 | 211 | func (s *SubscribersDynamoDB) AddSubscribers(subscribers []*common.Subscriber) error { 212 | for i := 0; i < len(subscribers); i += dynamoDBChunkSize { 213 | end := i + dynamoDBChunkSize 214 | 215 | if end > len(subscribers) { 216 | end = len(subscribers) 217 | } 218 | 219 | err := s.AddSubscribersChunk(subscribers[i:end]) 220 | if err != nil { 221 | return err 222 | } 223 | } 224 | return nil 225 | } 226 | 227 | func (s *SubscribersDynamoDB) ConfirmSubscriber(newsletter, email string) error { 228 | updateVal := struct { 229 | ConfirmedAt common.JSONTime `json:":confirmed_at"` 230 | }{ 231 | ConfirmedAt: common.JsonTimeNow(), 232 | } 233 | 234 | update, err := dynamodbattribute.MarshalMap(updateVal) 235 | if err != nil { 236 | return err 237 | } 238 | 239 | input := &dynamodb.UpdateItemInput{ 240 | ExpressionAttributeValues: update, 241 | UpdateExpression: aws.String("set confirmed_at = :confirmed_at"), 242 | TableName: &s.TableName, 243 | Key: map[string]*dynamodb.AttributeValue{ 244 | "newsletter": &dynamodb.AttributeValue{ 245 | S: &newsletter, 246 | }, 247 | "email": &dynamodb.AttributeValue{ 248 | S: &email, 249 | }, 250 | }, 251 | ReturnValues: aws.String("UPDATED_NEW"), 252 | } 253 | _, err = s.Client.UpdateItem(input) 254 | return err 255 | } 256 | 257 | func (s *SubscribersDynamoDB) DeleteSubscribersChunk(keys []*common.SubscriberKey) error { 258 | // AWS DynamoDB restriction 259 | if len(keys) > dynamoDBChunkSize { 260 | return errChunkTooBig 261 | } 262 | 263 | requests := make([]*dynamodb.WriteRequest, 0, len(keys)) 264 | for _, k := range keys { 265 | attr, err := dynamodbattribute.MarshalMap(k) 266 | if err != nil { 267 | return err 268 | } 269 | 270 | requests = append(requests, &dynamodb.WriteRequest{ 271 | DeleteRequest: &dynamodb.DeleteRequest{ 272 | Key: attr, 273 | }, 274 | }) 275 | } 276 | 277 | b := &backoff.Backoff{ 278 | Min: 100 * time.Millisecond, 279 | Max: 1 * time.Second, 280 | Factor: 2, 281 | Jitter: false, 282 | } 283 | 284 | for len(requests) > 0 { 285 | input := &dynamodb.BatchWriteItemInput{ 286 | RequestItems: map[string][]*dynamodb.WriteRequest{ 287 | s.TableName: requests, 288 | }, 289 | } 290 | res, err := s.Client.BatchWriteItem(input) 291 | if err != nil { 292 | return err 293 | } 294 | if unprocessed, ok := res.UnprocessedItems[s.TableName]; ok { 295 | log.Printf("Found unprocessed items. count=%v", len(unprocessed)) 296 | requests = unprocessed 297 | } else { 298 | break 299 | } 300 | time.Sleep(b.Duration()) 301 | } 302 | 303 | return nil 304 | } 305 | 306 | func (s *SubscribersDynamoDB) DeleteSubscribers(keys []*common.SubscriberKey) error { 307 | for i := 0; i < len(keys); i += dynamoDBChunkSize { 308 | end := i + dynamoDBChunkSize 309 | 310 | if end > len(keys) { 311 | end = len(keys) 312 | } 313 | 314 | err := s.DeleteSubscribersChunk(keys[i:end]) 315 | if err != nil { 316 | return err 317 | } 318 | } 319 | return nil 320 | } 321 | 322 | type SubscribersMapStore struct { 323 | items map[string]*common.Subscriber 324 | } 325 | 326 | var _ common.SubscribersStore = (*SubscribersMapStore)(nil) 327 | 328 | func (s *SubscribersMapStore) key(newsletter, email string) string { 329 | return newsletter + email 330 | } 331 | 332 | func (s *SubscribersMapStore) contains(newsletter, email string) bool { 333 | _, ok := s.items[s.key(newsletter, email)] 334 | return ok 335 | } 336 | 337 | func (s *SubscribersMapStore) Count() int { 338 | return len(s.items) 339 | } 340 | 341 | func (s *SubscribersMapStore) GetSubscriber(newsletter, email string) (*common.Subscriber, error) { 342 | key := s.key(newsletter, email) 343 | sr, ok := s.items[key] 344 | if !ok { 345 | return nil, errSubscriberDoesNotExist 346 | } 347 | return sr, nil 348 | } 349 | 350 | func (s *SubscribersMapStore) AddSubscriber(newsletter, email, name string) error { 351 | key := s.key(newsletter, email) 352 | if _, ok := s.items[key]; ok { 353 | log.Printf("Subscriber already exists. email=%v newsletter=%v", email, newsletter) 354 | } 355 | 356 | sr := &common.Subscriber{ 357 | Newsletter: newsletter, 358 | Email: email, 359 | CreatedAt: common.JsonTimeNow(), 360 | ConfirmedAt: incorrectTime, 361 | UnsubscribedAt: incorrectTime, 362 | } 363 | sr.Validate() 364 | 365 | s.items[key] = sr 366 | return nil 367 | } 368 | 369 | func (s *SubscribersMapStore) RemoveSubscriber(newsletter, email string) error { 370 | key := s.key(newsletter, email) 371 | if i, ok := s.items[key]; ok { 372 | i.UnsubscribedAt = common.JsonTimeNow() 373 | return nil 374 | } 375 | return errSubscriberDoesNotExist 376 | } 377 | 378 | func (s *SubscribersMapStore) DeleteSubscribers(keys []*common.SubscriberKey) error { 379 | for _, k := range keys { 380 | key := s.key(k.Newsletter, k.Email) 381 | delete(s.items, key) 382 | } 383 | return nil 384 | } 385 | 386 | func (s *SubscribersMapStore) Subscribers(newsletter string) (subscribers []*common.Subscriber, err error) { 387 | for key, value := range s.items { 388 | if strings.HasPrefix(key, newsletter) { 389 | subscribers = append(subscribers, value) 390 | } 391 | } 392 | return subscribers, nil 393 | } 394 | 395 | func (s *SubscribersMapStore) AddSubscribers(subscribers []*common.Subscriber) error { 396 | for _, i := range subscribers { 397 | s.items[s.key(i.Newsletter, i.Email)] = i 398 | } 399 | return nil 400 | } 401 | 402 | func (s *SubscribersMapStore) ConfirmSubscriber(newsletter, email string) error { 403 | key := s.key(newsletter, email) 404 | if i, ok := s.items[key]; ok { 405 | i.ConfirmedAt = common.JsonTimeNow() 406 | return nil 407 | } 408 | return errSubscriberDoesNotExist 409 | } 410 | -------------------------------------------------------------------------------- /pkg/email/confirm_html.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | const ( 4 | // HTMLBody is a template of the email used for 5 | // confirmation of the subscription emails 6 | HTMLBody = ` 7 | 8 | 9 | 10 | 11 | Simple Transactional Email 12 | 339 | 340 | 341 | Confirm your email for {{.Newsletter}} newsletter subscription. 342 | 343 | 344 | 345 | 399 | 400 | 401 | 402 | 403 | ` 404 | ) 405 | -------------------------------------------------------------------------------- /pkg/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "strings" 8 | 9 | "github.com/ribtoks/checkmail" 10 | "github.com/ribtoks/listing/pkg/common" 11 | ) 12 | 13 | type ListingResource interface { 14 | Setup(router *http.ServeMux) 15 | AddNewsletters(n []string) 16 | } 17 | 18 | // NewsletterResource manages http requests and data storage 19 | // for newsletter subscriptions 20 | type NewsletterResource struct { 21 | Secret string 22 | SubscribeRedirectURL string 23 | UnsubscribeRedirectURL string 24 | ConfirmRedirectURL string 25 | ConfirmURL string 26 | Newsletters map[string]bool 27 | Subscribers common.SubscribersStore 28 | Notifications common.NotificationsStore 29 | Mailer common.Mailer 30 | } 31 | 32 | var _ ListingResource = (*NewsletterResource)(nil) 33 | 34 | type AdminResource struct { 35 | APIToken string 36 | Newsletters map[string]bool 37 | Subscribers common.SubscribersStore 38 | Notifications common.NotificationsStore 39 | } 40 | 41 | var _ ListingResource = (*AdminResource)(nil) 42 | 43 | const ( 44 | // assume there cannot be such a huge http requests for subscription 45 | kilobyte = 1024 46 | megabyte = 1024 * kilobyte 47 | maxSubscribeBodySize = kilobyte / 2 48 | maxImportBodySize = 25 * megabyte 49 | maxDeleteBodySize = 5 * megabyte 50 | ) 51 | 52 | func (ar *AdminResource) Setup(router *http.ServeMux) { 53 | router.HandleFunc(common.SubscribersEndpoint, ar.auth(ar.serveSubscribers)) 54 | router.HandleFunc(common.ComplaintsEndpoint, ar.auth(ar.complaints)) 55 | } 56 | 57 | func (nr *NewsletterResource) Setup(router *http.ServeMux) { 58 | router.HandleFunc(common.SubscribeEndpoint, nr.method("POST", nr.subscribe)) 59 | router.HandleFunc(common.UnsubscribeEndpoint, nr.method("GET", nr.unsubscribe)) 60 | router.HandleFunc(common.ConfirmEndpoint, nr.method("GET", nr.confirm)) 61 | } 62 | 63 | func (nr *NewsletterResource) AddNewsletters(n []string) { 64 | for _, i := range n { 65 | nr.Newsletters[i] = true 66 | } 67 | } 68 | 69 | func (nr *NewsletterResource) method(m string, next http.HandlerFunc) http.HandlerFunc { 70 | return func(w http.ResponseWriter, r *http.Request) { 71 | if r.Method != m { 72 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 73 | return 74 | } 75 | 76 | next.ServeHTTP(w, r) 77 | } 78 | } 79 | 80 | func (nr *NewsletterResource) isValidNewsletter(n string) bool { 81 | if n == "" { 82 | return false 83 | } 84 | 85 | _, ok := nr.Newsletters[n] 86 | 87 | return ok 88 | } 89 | 90 | func (nr *NewsletterResource) validate(newsletter, email string) bool { 91 | err := checkmail.ValidateFormat(email) 92 | if err != nil { 93 | log.Printf("Failed to validate email. value=%q err=%q", email, err) 94 | return false 95 | } 96 | 97 | if !nr.isValidNewsletter(newsletter) { 98 | log.Printf("Invalid newsletter. value=%v", newsletter) 99 | return false 100 | } 101 | 102 | return true 103 | } 104 | 105 | func (nr *NewsletterResource) subscribe(w http.ResponseWriter, r *http.Request) { 106 | r.Body = http.MaxBytesReader(w, r.Body, maxSubscribeBodySize) 107 | err := r.ParseForm() 108 | 109 | if err != nil { 110 | log.Printf("Failed to parse form. err=%v", err) 111 | } 112 | 113 | newsletter := r.FormValue(common.ParamNewsletter) 114 | email := r.FormValue(common.ParamEmail) 115 | 116 | if ok := nr.validate(newsletter, email); !ok { 117 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 118 | return 119 | } 120 | 121 | if s, err := nr.Subscribers.GetSubscriber(newsletter, email); err == nil { 122 | log.Printf("Subscriber already exists. email=%v newsletter=%v", email, newsletter) 123 | 124 | if s.Confirmed() && !s.Unsubscribed() { 125 | log.Printf("Email is already confirmed. email=%v newsletter=%v confirmed_at=%v", 126 | email, newsletter, s.ConfirmedAt.Time()) 127 | w.Header().Set("Location", nr.ConfirmRedirectURL) 128 | http.Redirect(w, r, nr.ConfirmRedirectURL, http.StatusFound) 129 | 130 | return 131 | } 132 | } 133 | 134 | // name is optional 135 | name := strings.TrimSpace(r.FormValue(common.ParamName)) 136 | 137 | err = nr.Subscribers.AddSubscriber(newsletter, email, name) 138 | if err != nil { 139 | log.Printf("Failed to add subscription. email=%q newsletter=%q name=%v err=%v", email, newsletter, name, err) 140 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 141 | 142 | return 143 | } 144 | 145 | log.Printf("Added subscription email=%q newsletter=%q name=%v", email, newsletter, name) 146 | 147 | _ = nr.Mailer.SendConfirmation(newsletter, email, name, nr.ConfirmURL) 148 | 149 | w.Header().Set("Location", nr.SubscribeRedirectURL) 150 | http.Redirect(w, r, nr.SubscribeRedirectURL, http.StatusFound) 151 | } 152 | 153 | // unsubscribe route. 154 | func (nr *NewsletterResource) unsubscribe(w http.ResponseWriter, r *http.Request) { 155 | newsletter := r.URL.Query().Get(common.ParamNewsletter) 156 | unsubscribeToken := r.URL.Query().Get(common.ParamToken) 157 | 158 | if newsletter == "" { 159 | http.Error(w, "The newsletter query-string parameter is required", http.StatusBadRequest) 160 | return 161 | } 162 | 163 | if !nr.isValidNewsletter(newsletter) { 164 | http.Error(w, "Invalid newsletter param", http.StatusBadRequest) 165 | return 166 | } 167 | 168 | email, ok := common.Unsign(nr.Secret, unsubscribeToken) 169 | if !ok { 170 | log.Printf("Failed to unsign token. value=%q", unsubscribeToken) 171 | http.Error(w, "Invalid unsubscribe token", http.StatusBadRequest) 172 | 173 | return 174 | } 175 | 176 | err := nr.Subscribers.RemoveSubscriber(newsletter, email) 177 | if err != nil { 178 | log.Printf("Failed to unsubscribe. email=%q err=%v", email, err) 179 | http.Error(w, "Error unsubscribing from newsletter", http.StatusInternalServerError) 180 | 181 | return 182 | } 183 | 184 | log.Printf("Unsubscribed. email=%q newsletter=%q", email, newsletter) 185 | w.Header().Set("Location", nr.UnsubscribeRedirectURL) 186 | http.Redirect(w, r, nr.UnsubscribeRedirectURL, http.StatusFound) 187 | } 188 | 189 | func (nr *NewsletterResource) confirm(w http.ResponseWriter, r *http.Request) { 190 | newsletter := r.URL.Query().Get(common.ParamNewsletter) 191 | subscribeToken := r.URL.Query().Get(common.ParamToken) 192 | 193 | if !nr.isValidNewsletter(newsletter) { 194 | http.Error(w, "Invalid newsletter param", http.StatusBadRequest) 195 | return 196 | } 197 | 198 | email, ok := common.Unsign(nr.Secret, subscribeToken) 199 | if !ok { 200 | log.Printf("Failed to unsign token. value=%q", subscribeToken) 201 | http.Error(w, "Invalid subscribe token", http.StatusBadRequest) 202 | 203 | return 204 | } 205 | 206 | if s, err := nr.Subscribers.GetSubscriber(newsletter, email); err == nil { 207 | if s.Unsubscribed() { 208 | log.Printf("Subscriber has already unsubscribed. newsletter=%v email=%v", newsletter, email) 209 | w.Header().Set("Location", nr.UnsubscribeRedirectURL) 210 | http.Redirect(w, r, nr.UnsubscribeRedirectURL, http.StatusFound) 211 | 212 | return 213 | } 214 | } else { 215 | log.Printf("Subscriber cannot be found. newsletter=%v email=%v err=%v", newsletter, email, err) 216 | http.Error(w, "Error confirming subscription", http.StatusInternalServerError) 217 | 218 | return 219 | } 220 | 221 | err := nr.Subscribers.ConfirmSubscriber(newsletter, email) 222 | if err != nil { 223 | log.Printf("Failed to confirm subscription. email=%q err=%v", email, err) 224 | http.Error(w, "Error confirming subscription", http.StatusInternalServerError) 225 | 226 | return 227 | } 228 | 229 | log.Printf("Confirmed subscription. email=%q newsletter=%q", email, newsletter) 230 | w.Header().Set("Location", nr.ConfirmRedirectURL) 231 | http.Redirect(w, r, nr.ConfirmRedirectURL, http.StatusFound) 232 | } 233 | 234 | func (ar *AdminResource) complaints(w http.ResponseWriter, r *http.Request) { 235 | notifications, err := ar.Notifications.Notifications() 236 | if err != nil { 237 | log.Printf("Failed to fetch notifications. err=%v", err) 238 | http.Error(w, err.Error(), http.StatusInternalServerError) 239 | 240 | return 241 | } 242 | 243 | w.Header().Set("Content-Type", "application/json") 244 | w.WriteHeader(http.StatusOK) 245 | 246 | err = json.NewEncoder(w).Encode(notifications) 247 | if err != nil { 248 | http.Error(w, err.Error(), http.StatusInternalServerError) 249 | } 250 | } 251 | 252 | // auth middleware. 253 | func (ar *AdminResource) auth(next http.HandlerFunc) http.HandlerFunc { 254 | return func(w http.ResponseWriter, r *http.Request) { 255 | _, pass, ok := r.BasicAuth() 256 | if !ok { 257 | http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) 258 | return 259 | } 260 | 261 | if pass != ar.APIToken { 262 | http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 263 | return 264 | } 265 | 266 | next.ServeHTTP(w, r) 267 | } 268 | } 269 | 270 | func (ar *AdminResource) serveSubscribers(w http.ResponseWriter, r *http.Request) { 271 | switch r.Method { 272 | case "GET": 273 | { 274 | ar.getSubscribers(w, r) 275 | } 276 | case "PUT": 277 | { 278 | ar.putSubscribers(w, r) 279 | } 280 | case "DELETE": 281 | { 282 | ar.deleteSubscribers(w, r) 283 | } 284 | default: 285 | { 286 | log.Printf("Unsupported method for subscribers. method=%v", r.Method) 287 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 288 | return 289 | } 290 | } 291 | } 292 | 293 | func (ar *AdminResource) getSubscribers(w http.ResponseWriter, r *http.Request) { 294 | newsletter := r.URL.Query().Get(common.ParamNewsletter) 295 | 296 | if !ar.isValidNewsletter(newsletter) { 297 | http.Error(w, "The newsletter parameter is invalid", http.StatusBadRequest) 298 | return 299 | } 300 | 301 | emails, err := ar.Subscribers.Subscribers(newsletter) 302 | if err != nil { 303 | log.Printf("Failed to fetch subscribers. err=%v", err) 304 | http.Error(w, err.Error(), http.StatusInternalServerError) 305 | 306 | return 307 | } 308 | 309 | w.Header().Set("Content-Type", "application/json") 310 | w.WriteHeader(http.StatusOK) 311 | 312 | err = json.NewEncoder(w).Encode(emails) 313 | if err != nil { 314 | http.Error(w, err.Error(), http.StatusInternalServerError) 315 | } 316 | } 317 | 318 | func (ar *AdminResource) putSubscribers(w http.ResponseWriter, r *http.Request) { 319 | if r.Header.Get("Content-Type") != "application/json" { 320 | http.Error(w, "Content-Type header is not application/json", http.StatusUnsupportedMediaType) 321 | return 322 | } 323 | 324 | r.Body = http.MaxBytesReader(w, r.Body, maxImportBodySize) 325 | dec := json.NewDecoder(r.Body) 326 | dec.DisallowUnknownFields() 327 | 328 | var subscribers []*common.Subscriber 329 | 330 | err := dec.Decode(&subscribers) 331 | if err != nil { 332 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 333 | return 334 | } 335 | 336 | ss := make([]*common.Subscriber, 0, len(subscribers)) 337 | 338 | for _, s := range subscribers { 339 | if !ar.isValidNewsletter(s.Newsletter) { 340 | log.Printf("Skipping unsupported newsletter. value=%v", s.Newsletter) 341 | continue 342 | } 343 | 344 | if err = checkmail.ValidateFormat(s.Email); err != nil { 345 | log.Printf("Skipping invalid email. value=%v", s.Email) 346 | continue 347 | } 348 | 349 | if !s.Confirmed() && !s.Unsubscribed() { 350 | s.CreatedAt = common.JsonTimeNow() 351 | } 352 | 353 | ss = append(ss, s) 354 | } 355 | 356 | if len(ss) == 0 { 357 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 358 | return 359 | } 360 | 361 | err = ar.Subscribers.AddSubscribers(ss) 362 | if err != nil { 363 | log.Printf("Failed to import subscribers. err=%v", err) 364 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 365 | 366 | return 367 | } 368 | 369 | w.WriteHeader(http.StatusOK) 370 | } 371 | 372 | func (ar *AdminResource) deleteSubscribers(w http.ResponseWriter, r *http.Request) { 373 | if r.Header.Get("Content-Type") != "application/json" { 374 | http.Error(w, "Content-Type header is not application/json", http.StatusUnsupportedMediaType) 375 | return 376 | } 377 | 378 | r.Body = http.MaxBytesReader(w, r.Body, maxDeleteBodySize) 379 | dec := json.NewDecoder(r.Body) 380 | dec.DisallowUnknownFields() 381 | 382 | var keys []*common.SubscriberKey 383 | 384 | err := dec.Decode(&keys) 385 | if err != nil { 386 | log.Printf("Failed to decode keys. err=%v", err) 387 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 388 | 389 | return 390 | } 391 | 392 | err = ar.Subscribers.DeleteSubscribers(keys) 393 | if err != nil { 394 | log.Printf("Failed to delete subscribers. err=%v", err) 395 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 396 | 397 | return 398 | } 399 | 400 | w.WriteHeader(http.StatusOK) 401 | } 402 | 403 | func (ar *AdminResource) isValidNewsletter(n string) bool { 404 | if n == "" { 405 | return false 406 | } 407 | 408 | _, ok := ar.Newsletters[n] 409 | 410 | return ok 411 | } 412 | 413 | func (ar *AdminResource) AddNewsletters(n []string) { 414 | for _, i := range n { 415 | ar.Newsletters[i] = true 416 | } 417 | } 418 | -------------------------------------------------------------------------------- /cmd/listing-cli/client_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | "time" 11 | 12 | "github.com/ribtoks/listing/pkg/api" 13 | "github.com/ribtoks/listing/pkg/common" 14 | "github.com/ribtoks/listing/pkg/db" 15 | ) 16 | 17 | const ( 18 | secret = "secret123" 19 | apiToken = "qwerty123456" 20 | testName = "Foo Bar" 21 | testEmail = "foo@bar.com" 22 | testNewsletter = "testnewsletter" 23 | ) 24 | 25 | var ( 26 | incorrectTime = common.JSONTime(time.Unix(1, 1)) 27 | errFromFailingStore = errors.New("Error!") 28 | ) 29 | 30 | type FailingSubscriberStore struct{} 31 | 32 | var _ common.SubscribersStore = (*FailingSubscriberStore)(nil) 33 | 34 | func (s *FailingSubscriberStore) GetSubscriber(newsletter, email string) (*common.Subscriber, error) { 35 | return nil, errFromFailingStore 36 | } 37 | 38 | func (s *FailingSubscriberStore) AddSubscriber(newsletter, email, name string) error { 39 | return errFromFailingStore 40 | } 41 | 42 | func (s *FailingSubscriberStore) RemoveSubscriber(newsletter, email string) error { 43 | return errFromFailingStore 44 | } 45 | 46 | func (s *FailingSubscriberStore) Subscribers(newsletter string) (subscribers []*common.Subscriber, err error) { 47 | return nil, errFromFailingStore 48 | } 49 | 50 | func (s *FailingSubscriberStore) AddSubscribers(subscribers []*common.Subscriber) error { 51 | return errFromFailingStore 52 | } 53 | 54 | func (s *FailingSubscriberStore) DeleteSubscribers(keys []*common.SubscriberKey) error { 55 | return errFromFailingStore 56 | } 57 | 58 | func (s *FailingSubscriberStore) ConfirmSubscriber(newsletter, email string) error { 59 | return errFromFailingStore 60 | } 61 | 62 | func NewFailingStore() *FailingSubscriberStore { 63 | return &FailingSubscriberStore{} 64 | } 65 | 66 | type DevNullMailer struct{} 67 | 68 | func (m *DevNullMailer) SendConfirmation(newsletter, email, name, confirmUrl string) error { 69 | return nil 70 | } 71 | 72 | func alternateConfirm(ss []*common.Subscriber) { 73 | for i, v := range ss { 74 | if i%2 == 0 { 75 | v.ConfirmedAt = common.JSONTime(v.CreatedAt.Time().Add(1 * time.Second)) 76 | } 77 | } 78 | } 79 | 80 | func alternateUnsubscribe(ss []*common.Subscriber) { 81 | for i, v := range ss { 82 | if i%2 == 0 { 83 | v.UnsubscribedAt = common.JSONTime(v.CreatedAt.Time().Add(1 * time.Second)) 84 | } 85 | } 86 | } 87 | 88 | func NewTestNewsResource(subscribers common.SubscribersStore, notifications common.NotificationsStore) *api.NewsletterResource { 89 | newsletters := &api.NewsletterResource{ 90 | Subscribers: subscribers, 91 | Notifications: notifications, 92 | Secret: secret, 93 | Newsletters: make(map[string]bool), 94 | Mailer: &DevNullMailer{}, 95 | } 96 | return newsletters 97 | } 98 | 99 | func NewTestAdminResource(subscribers common.SubscribersStore, notifications common.NotificationsStore) *api.AdminResource { 100 | admins := &api.AdminResource{ 101 | Subscribers: subscribers, 102 | Notifications: notifications, 103 | APIToken: apiToken, 104 | Newsletters: make(map[string]bool), 105 | } 106 | return admins 107 | } 108 | 109 | func testingHttpClient(handler http.HandlerFunc) (*http.Client, func()) { 110 | s := httptest.NewServer(handler) 111 | 112 | cli := &http.Client{ 113 | Transport: &http.Transport{ 114 | DialContext: func(_ context.Context, network, _ string) (net.Conn, error) { 115 | return net.Dial(network, s.Listener.Addr().String()) 116 | }, 117 | }, 118 | } 119 | 120 | return cli, s.Close 121 | } 122 | 123 | type RawTestPrinter struct { 124 | subscribers []*common.Subscriber 125 | } 126 | 127 | func NewRawTestPrinter() *RawTestPrinter { 128 | rp := &RawTestPrinter{ 129 | subscribers: make([]*common.Subscriber, 0), 130 | } 131 | return rp 132 | } 133 | 134 | func (rp *RawTestPrinter) Append(s *common.Subscriber) { 135 | rp.subscribers = append(rp.subscribers, s) 136 | } 137 | 138 | func (rp *RawTestPrinter) Render() error { 139 | return nil 140 | } 141 | 142 | func NewTestClient(resource api.ListingResource, p Printer) (*httptest.Server, *listingClient) { 143 | mux := http.NewServeMux() 144 | server := httptest.NewServer(mux) 145 | resource.Setup(mux) 146 | 147 | client := &listingClient{ 148 | client: &http.Client{ 149 | Timeout: 10 * time.Second, 150 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 151 | return http.ErrUseLastResponse 152 | }, 153 | }, 154 | printer: p, 155 | url: server.URL, 156 | authToken: apiToken, 157 | secret: secret, 158 | complaints: make(map[string]bool), 159 | dryRun: false, 160 | noUnconfirmed: false, 161 | noUnsubscribed: false, 162 | ignoreComplaints: false, 163 | } 164 | 165 | return server, client 166 | } 167 | 168 | func TestExportSubscribedSubscribers(t *testing.T) { 169 | store := db.NewSubscribersMapStore() 170 | store.AddSubscriber(testNewsletter, "email1@domain.com", testName) 171 | store.AddSubscriber(testNewsletter, "email2@domain.com", testName) 172 | ss, _ := store.Subscribers(testNewsletter) 173 | alternateUnsubscribe(ss) 174 | 175 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 176 | nr.AddNewsletters([]string{testNewsletter}) 177 | 178 | p := NewRawTestPrinter() 179 | srv, cli := NewTestClient(nr, p) 180 | defer srv.Close() 181 | 182 | cli.noUnsubscribed = true 183 | err := cli.export(testNewsletter) 184 | if err != nil { 185 | t.Fatal(err) 186 | } 187 | 188 | if len(p.subscribers) != 1 { 189 | t.Errorf("Unexpected number of subscribers: %v", len(p.subscribers)) 190 | } 191 | } 192 | 193 | func TestExportConfirmedSubscribers(t *testing.T) { 194 | store := db.NewSubscribersMapStore() 195 | store.AddSubscriber(testNewsletter, "email1@domain.com", testName) 196 | store.AddSubscriber(testNewsletter, "email2@domain.com", testName) 197 | ss, _ := store.Subscribers(testNewsletter) 198 | alternateConfirm(ss) 199 | 200 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 201 | nr.AddNewsletters([]string{testNewsletter}) 202 | 203 | p := NewRawTestPrinter() 204 | srv, cli := NewTestClient(nr, p) 205 | defer srv.Close() 206 | 207 | cli.noUnconfirmed = true 208 | err := cli.export(testNewsletter) 209 | if err != nil { 210 | t.Fatal(err) 211 | } 212 | 213 | if len(p.subscribers) != 1 { 214 | t.Errorf("Unexpected number of subscribers: %v", len(p.subscribers)) 215 | } 216 | } 217 | 218 | func TestExportAllSubscribers(t *testing.T) { 219 | store := db.NewSubscribersMapStore() 220 | store.AddSubscriber(testNewsletter, "email1@domain.com", testName) 221 | store.AddSubscriber(testNewsletter, "email2@domain.com", testName) 222 | 223 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 224 | nr.AddNewsletters([]string{testNewsletter}) 225 | 226 | p := NewRawTestPrinter() 227 | srv, cli := NewTestClient(nr, p) 228 | defer srv.Close() 229 | 230 | err := cli.export(testNewsletter) 231 | if err != nil { 232 | t.Fatal(err) 233 | } 234 | 235 | if len(p.subscribers) != 2 { 236 | t.Errorf("Unexpected number of subscribers: %v", len(p.subscribers)) 237 | } 238 | } 239 | 240 | func SubscribeSuite(t *testing.T, store common.SubscribersStore, dryRun bool) { 241 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 242 | nr.AddNewsletters([]string{testNewsletter}) 243 | 244 | p := NewRawTestPrinter() 245 | srv, cli := NewTestClient(nr, p) 246 | defer srv.Close() 247 | 248 | cli.dryRun = dryRun 249 | err := cli.subscribe(testEmail, testNewsletter, testName) 250 | if err != nil { 251 | t.Fatal(err) 252 | } 253 | 254 | expectedCount := 1 255 | if dryRun { 256 | expectedCount = 0 257 | } 258 | 259 | ss, _ := store.Subscribers(testNewsletter) 260 | if len(ss) != expectedCount { 261 | t.Errorf("Unexpected number of subscribers. count=%v expected=%v", len(ss), expectedCount) 262 | } 263 | } 264 | 265 | func TestSubscribeDryRun(t *testing.T) { 266 | store := db.NewSubscribersMapStore() 267 | SubscribeSuite(t, store, true /*dry run*/) 268 | } 269 | 270 | func TestSubscribe(t *testing.T) { 271 | store := db.NewSubscribersMapStore() 272 | SubscribeSuite(t, store, false /*dry run*/) 273 | 274 | if _, err := store.GetSubscriber(testNewsletter, testEmail); err != nil { 275 | t.Errorf("Subscriber is not added to the store") 276 | } 277 | } 278 | 279 | func UnsubscribeSuite(t *testing.T, dryRun bool) { 280 | store := db.NewSubscribersMapStore() 281 | store.AddSubscriber(testNewsletter, testEmail, testName) 282 | 283 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 284 | nr.AddNewsletters([]string{testNewsletter}) 285 | 286 | p := NewRawTestPrinter() 287 | srv, cli := NewTestClient(nr, p) 288 | defer srv.Close() 289 | 290 | if !dryRun { 291 | time.Sleep(1 * time.Millisecond) 292 | } 293 | 294 | cli.dryRun = dryRun 295 | err := cli.unsubscribe(testEmail, testNewsletter) 296 | if err != nil { 297 | t.Fatal(err) 298 | } 299 | 300 | if store.Count() != 1 { 301 | t.Errorf("Unexpected number of subscribers: %v", store.Count()) 302 | } 303 | 304 | i, _ := store.GetSubscriber(testNewsletter, testEmail) 305 | // if dry run, should NOT be unsubscribed 306 | expected := !dryRun 307 | if i.Unsubscribed() != expected { 308 | t.Errorf("Unexpected unsubscribe state. created_at=%v unsubscribed_at=%v unsubscribed=%v expected=%v", i.CreatedAt, i.UnsubscribedAt, i.Unsubscribed(), expected) 309 | } 310 | } 311 | 312 | func TestUnsubscribeDryRun(t *testing.T) { 313 | UnsubscribeSuite(t, true /*dry run*/) 314 | } 315 | 316 | func TestUnsubscribe(t *testing.T) { 317 | UnsubscribeSuite(t, false /*dry run*/) 318 | } 319 | 320 | func TestExportEmptyNewsletter(t *testing.T) { 321 | store := db.NewSubscribersMapStore() 322 | store.AddSubscriber(testNewsletter, testEmail, testName) 323 | 324 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 325 | nr.AddNewsletters([]string{testNewsletter}) 326 | 327 | p := NewRawTestPrinter() 328 | srv, cli := NewTestClient(nr, p) 329 | defer srv.Close() 330 | 331 | err := cli.export("") 332 | if err == nil { 333 | t.Fatalf("Managed to export empty newsletter") 334 | } 335 | } 336 | 337 | func TestExportDryRun(t *testing.T) { 338 | store := db.NewSubscribersMapStore() 339 | store.AddSubscriber(testNewsletter, testEmail, testName) 340 | 341 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 342 | nr.AddNewsletters([]string{testNewsletter}) 343 | 344 | p := NewRawTestPrinter() 345 | srv, cli := NewTestClient(nr, p) 346 | defer srv.Close() 347 | 348 | cli.dryRun = true 349 | err := cli.export(testNewsletter) 350 | if err != nil { 351 | t.Fatal(err) 352 | } 353 | 354 | if len(p.subscribers) > 0 { 355 | t.Errorf("Dry run exported data") 356 | } 357 | } 358 | 359 | func ExportSubscribersComplaintsSuite(t *testing.T, p Printer, ignoreComplaints bool) { 360 | store := db.NewSubscribersMapStore() 361 | store.AddSubscriber(testNewsletter, "email1@domain.com", testName) 362 | store.AddSubscriber(testNewsletter, "email2@domain.com", testName) 363 | store.AddSubscriber(testNewsletter, "email3@domain.com", testName) 364 | 365 | complaints := db.NewNotificationsMapStore() 366 | complaints.AddBounce("email1@domain.com", "no-reply@newsletter.com", false /*is transient*/) 367 | complaints.AddBounce("email2@domain.com", "no-reply@newsletter.com", true /*is transient*/) 368 | complaints.AddComplaint("email3@domain.com", "no-reply@newsletter.com") 369 | 370 | nr := NewTestAdminResource(store, complaints) 371 | nr.AddNewsletters([]string{testNewsletter}) 372 | 373 | srv, cli := NewTestClient(nr, p) 374 | defer srv.Close() 375 | 376 | cli.ignoreComplaints = ignoreComplaints 377 | err := cli.export(testNewsletter) 378 | if err != nil { 379 | t.Fatal(err) 380 | } 381 | } 382 | 383 | func TestExportSubscribersWithComplaints(t *testing.T) { 384 | p := NewRawTestPrinter() 385 | ExportSubscribersComplaintsSuite(t, p, false /*ignore complaints*/) 386 | 387 | if len(p.subscribers) != 1 { 388 | t.Errorf("Unexpected number of subscribers. actual=%v", len(p.subscribers)) 389 | } 390 | 391 | if p.subscribers[0].Email != "email2@domain.com" { 392 | t.Errorf("Wrong subsciber has been exported") 393 | } 394 | } 395 | 396 | func TestExportSubscribersWithoutComplaints(t *testing.T) { 397 | p := NewRawTestPrinter() 398 | ExportSubscribersComplaintsSuite(t, p, true /*ignore complaints*/) 399 | if len(p.subscribers) != 3 { 400 | t.Errorf("Unexpected number of subscribers. actual=%v", len(p.subscribers)) 401 | } 402 | } 403 | 404 | func TestSubscribeErrorStore(t *testing.T) { 405 | nr := NewTestAdminResource(NewFailingStore(), db.NewNotificationsMapStore()) 406 | nr.AddNewsletters([]string{testNewsletter}) 407 | 408 | p := NewRawTestPrinter() 409 | srv, cli := NewTestClient(nr, p) 410 | defer srv.Close() 411 | 412 | err := cli.subscribe(testEmail, testNewsletter, testName) 413 | if err == nil { 414 | t.Fatal("Subscribed with failing store") 415 | } 416 | } 417 | 418 | func TestUnsubscribeErrorStore(t *testing.T) { 419 | nr := NewTestAdminResource(NewFailingStore(), db.NewNotificationsMapStore()) 420 | nr.AddNewsletters([]string{testNewsletter}) 421 | 422 | p := NewRawTestPrinter() 423 | srv, cli := NewTestClient(nr, p) 424 | defer srv.Close() 425 | 426 | err := cli.unsubscribe(testEmail, testNewsletter) 427 | if err == nil { 428 | t.Fatal("Unsubscribed with failing store") 429 | } 430 | } 431 | 432 | func TestSubscribeErrors(t *testing.T) { 433 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 434 | nr.AddNewsletters([]string{testNewsletter}) 435 | 436 | p := NewRawTestPrinter() 437 | srv, cli := NewTestClient(nr, p) 438 | defer srv.Close() 439 | 440 | err := cli.subscribe("", testNewsletter, testName) 441 | if err == nil { 442 | t.Fatal("Subscribed with empty email") 443 | } 444 | 445 | err = cli.subscribe(testEmail, "", testName) 446 | if err == nil { 447 | t.Fatal("Subscribed with empty newsletter") 448 | } 449 | } 450 | 451 | func TestUnsubscribeErrors(t *testing.T) { 452 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 453 | nr.AddNewsletters([]string{testNewsletter}) 454 | 455 | p := NewRawTestPrinter() 456 | srv, cli := NewTestClient(nr, p) 457 | defer srv.Close() 458 | 459 | err := cli.unsubscribe("", testNewsletter) 460 | if err == nil { 461 | t.Fatal("Unsubscribed with empty email") 462 | } 463 | 464 | err = cli.unsubscribe(testEmail, "") 465 | if err == nil { 466 | t.Fatal("Unsubscribed with empty newsletter") 467 | } 468 | } 469 | 470 | func ImportSubscribersSuite(t *testing.T, dryRun bool) { 471 | store := db.NewSubscribersMapStore() 472 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 473 | nr.AddNewsletters([]string{testNewsletter}) 474 | 475 | srv, cli := NewTestClient(nr, NewRawTestPrinter()) 476 | defer srv.Close() 477 | 478 | data := `[{ 479 | "name": "JohnSmith", 480 | "newsletter": "testnewsletter", 481 | "email": "email7@domain.com", 482 | "created_at": "2019-12-28T02:42:23Z", 483 | "unsubscribed_at": "1970-01-01T00:00:01Z", 484 | "confirmed_at": "2019-12-26T18:50:12Z" 485 | }, 486 | { 487 | "name": "", 488 | "newsletter": "testnewsletter", 489 | "email": "email8@domain.com", 490 | "created_at": "2019-12-28T02:42:23Z", 491 | "unsubscribed_at": "1970-01-01T00:00:01Z", 492 | "confirmed_at": "2019-12-26T18:50:12Z" 493 | }, 494 | { 495 | "name": "Foo Bar", 496 | "newsletter": "testnewsletter", 497 | "email": "foo@bar.com", 498 | "created_at": "2019-12-29T01:24:11Z", 499 | "unsubscribed_at": "1970-01-01T00:00:01Z", 500 | "confirmed_at": "2019-12-29T01:36:52Z" 501 | } 502 | ]` 503 | 504 | cli.dryRun = dryRun 505 | err := cli.importSubscribers([]byte(data)) 506 | if err != nil { 507 | t.Fatal(err) 508 | } 509 | 510 | expectedCount := 3 511 | if dryRun { 512 | expectedCount = 0 513 | } 514 | if store.Count() != expectedCount { 515 | t.Errorf("Wrong number of items in store. actual=%v expected=%v dry_run=%v", store.Count(), expectedCount, dryRun) 516 | } 517 | } 518 | 519 | func TestImportSubscribers(t *testing.T) { 520 | ImportSubscribersSuite(t, false /*dry run*/) 521 | } 522 | 523 | func TestImportSubscribersDryRun(t *testing.T) { 524 | ImportSubscribersSuite(t, true /*dry run*/) 525 | } 526 | 527 | func TestImportSubscribersMalformedJson(t *testing.T) { 528 | store := db.NewSubscribersMapStore() 529 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 530 | nr.AddNewsletters([]string{testNewsletter}) 531 | 532 | srv, cli := NewTestClient(nr, NewRawTestPrinter()) 533 | defer srv.Close() 534 | 535 | data := `[{ 536 | "name": "JohnSmith", 537 | "newsletter": "testnewsletter" 538 | "email": "email7@domain.com" 539 | "created_at": "2019-12-28T02:42:23Z", 540 | "unsubscribed_at": "1970-01-01T00:00:01Z", 541 | "confirmed_at": "2019-12-26T18:50:12Z" 542 | } { 543 | "name": "Foo Bar", 544 | "newsletter": "testnewsletter", 545 | "email": "foo@bar.com", 546 | "created_at": "2019-12-29T01:24:11Z", 547 | "unsubscribed_at": "1970-01-01T00:00:01Z", 548 | "confirmed_at": "2019-12-29T01:36:52Z", 549 | ` 550 | 551 | err := cli.importSubscribers([]byte(data)) 552 | if err == nil { 553 | t.Errorf("Import finished successfully") 554 | } 555 | } 556 | 557 | func DeleteSubscribersSuite(t *testing.T, dryRun bool) { 558 | store := db.NewSubscribersMapStore() 559 | store.AddSubscriber(testNewsletter, "email7@domain.com", testName) 560 | store.AddSubscriber(testNewsletter, "email8@domain.com", testName) 561 | store.AddSubscriber(testNewsletter, "foo@bar.com", testName) 562 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 563 | nr.AddNewsletters([]string{testNewsletter}) 564 | 565 | srv, cli := NewTestClient(nr, NewRawTestPrinter()) 566 | defer srv.Close() 567 | 568 | data := `[{ 569 | "name": "JohnSmith", 570 | "newsletter": "testnewsletter", 571 | "email": "email7@domain.com", 572 | "created_at": "2019-12-28T02:42:23Z", 573 | "unsubscribed_at": "1970-01-01T00:00:01Z", 574 | "confirmed_at": "2019-12-26T18:50:12Z" 575 | }, 576 | { 577 | "name": "", 578 | "newsletter": "testnewsletter", 579 | "email": "email8@domain.com", 580 | "created_at": "2019-12-28T02:42:23Z", 581 | "unsubscribed_at": "1970-01-01T00:00:01Z", 582 | "confirmed_at": "2019-12-26T18:50:12Z" 583 | }, 584 | { 585 | "name": "Foo Bar", 586 | "newsletter": "testnewsletter", 587 | "email": "foo@bar.com", 588 | "created_at": "2019-12-29T01:24:11Z", 589 | "unsubscribed_at": "1970-01-01T00:00:01Z", 590 | "confirmed_at": "2019-12-29T01:36:52Z" 591 | } 592 | ]` 593 | 594 | cli.dryRun = dryRun 595 | err := cli.deleteSubscribers([]byte(data)) 596 | if err != nil { 597 | t.Fatal(err) 598 | } 599 | 600 | expectedCount := 0 601 | if dryRun { 602 | expectedCount = 3 603 | } 604 | if store.Count() != expectedCount { 605 | t.Errorf("Wrong number of items in store. actual=%v expected=%v dry_run=%v", store.Count(), expectedCount, dryRun) 606 | } 607 | } 608 | 609 | func TestDeleteSubscribers(t *testing.T) { 610 | DeleteSubscribersSuite(t, false /*dry run*/) 611 | } 612 | 613 | func TestDeleteSubscribersDryRun(t *testing.T) { 614 | DeleteSubscribersSuite(t, true /*dry run*/) 615 | } 616 | -------------------------------------------------------------------------------- /pkg/api/api_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "net/http/httptest" 11 | "net/url" 12 | "strconv" 13 | "strings" 14 | "testing" 15 | "time" 16 | 17 | "github.com/ribtoks/listing/pkg/common" 18 | "github.com/ribtoks/listing/pkg/db" 19 | ) 20 | 21 | const ( 22 | secret = "secret123" 23 | apiToken = "qwerty123456" 24 | testName = "Foo Bar" 25 | testEmail = "foo@bar.com" 26 | testNewsletter = "testnewsletter" 27 | testUrl = "http://mysupertest.com/location" 28 | ) 29 | 30 | var incorrectTime = common.JSONTime(time.Unix(1, 1)) 31 | var errFromFailingStore = errors.New("Error!") 32 | 33 | type DevNullMailer struct{} 34 | 35 | func (m *DevNullMailer) SendConfirmation(newsletter, email, name, confirmUrl string) error { 36 | return nil 37 | } 38 | 39 | type FailingSubscriberStore struct { 40 | failGetSubscriber bool 41 | } 42 | 43 | var _ common.SubscribersStore = (*FailingSubscriberStore)(nil) 44 | 45 | func (s *FailingSubscriberStore) GetSubscriber(newsletter, email string) (*common.Subscriber, error) { 46 | if s.failGetSubscriber { 47 | return nil, errFromFailingStore 48 | } 49 | return &common.Subscriber{}, nil 50 | } 51 | 52 | func (s *FailingSubscriberStore) AddSubscriber(newsletter, email, name string) error { 53 | return errFromFailingStore 54 | } 55 | 56 | func (s *FailingSubscriberStore) RemoveSubscriber(newsletter, email string) error { 57 | return errFromFailingStore 58 | } 59 | 60 | func (s *FailingSubscriberStore) Subscribers(newsletter string) (subscribers []*common.Subscriber, err error) { 61 | return nil, errFromFailingStore 62 | } 63 | 64 | func (s *FailingSubscriberStore) AddSubscribers(subscribers []*common.Subscriber) error { 65 | return errFromFailingStore 66 | } 67 | 68 | func (s *FailingSubscriberStore) DeleteSubscribers(keys []*common.SubscriberKey) error { 69 | return errFromFailingStore 70 | } 71 | 72 | func (s *FailingSubscriberStore) ConfirmSubscriber(newsletter, email string) error { 73 | return errFromFailingStore 74 | } 75 | 76 | func NewFailingStore() *FailingSubscriberStore { 77 | return &FailingSubscriberStore{ 78 | failGetSubscriber: true, 79 | } 80 | } 81 | 82 | type FailingNotificationsStore struct{} 83 | 84 | func (s *FailingNotificationsStore) AddBounce(email, from string, isTransient bool) error { 85 | return errFromFailingStore 86 | } 87 | 88 | func (s *FailingNotificationsStore) AddComplaint(email, from string) error { 89 | return errFromFailingStore 90 | } 91 | 92 | func (s *FailingNotificationsStore) Notifications() (notifications []*common.SesNotification, err error) { 93 | return nil, errFromFailingStore 94 | } 95 | 96 | func NewTestNewsResource(subscribers common.SubscribersStore, notifications common.NotificationsStore) *NewsletterResource { 97 | newsletters := &NewsletterResource{ 98 | Subscribers: subscribers, 99 | Notifications: notifications, 100 | Secret: secret, 101 | Newsletters: make(map[string]bool), 102 | Mailer: &DevNullMailer{}, 103 | } 104 | return newsletters 105 | } 106 | 107 | func NewTestAdminResource(subscribers common.SubscribersStore, notifications common.NotificationsStore) *AdminResource { 108 | admins := &AdminResource{ 109 | Subscribers: subscribers, 110 | Notifications: notifications, 111 | APIToken: apiToken, 112 | Newsletters: make(map[string]bool), 113 | } 114 | return admins 115 | } 116 | 117 | func TestGetSubscribeMethodIsNotSupported(t *testing.T) { 118 | srv := http.NewServeMux() 119 | nr := NewTestNewsResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 120 | nr.Setup(srv) 121 | 122 | req, err := http.NewRequest("GET", common.SubscribeEndpoint, nil) 123 | if err != nil { 124 | t.Fatal(err) 125 | } 126 | 127 | w := httptest.NewRecorder() 128 | srv.ServeHTTP(w, req) 129 | 130 | resp := w.Result() 131 | 132 | if resp.StatusCode != http.StatusBadRequest { 133 | t.Errorf("Unexpected status code %d", resp.StatusCode) 134 | } 135 | } 136 | 137 | func TestSubscribeWithoutParams(t *testing.T) { 138 | srv := http.NewServeMux() 139 | nr := NewTestNewsResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 140 | nr.Setup(srv) 141 | 142 | req, err := http.NewRequest("POST", common.SubscribeEndpoint, nil) 143 | if err != nil { 144 | t.Fatal(err) 145 | } 146 | 147 | w := httptest.NewRecorder() 148 | srv.ServeHTTP(w, req) 149 | 150 | resp := w.Result() 151 | 152 | if resp.StatusCode != http.StatusBadRequest { 153 | t.Errorf("Unexpected status code %d", resp.StatusCode) 154 | } 155 | } 156 | 157 | func TestSubscribeWithBadEmail(t *testing.T) { 158 | srv := http.NewServeMux() 159 | nr := NewTestNewsResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 160 | nr.Setup(srv) 161 | 162 | data := url.Values{} 163 | data.Set(common.ParamNewsletter, "foo") 164 | data.Set(common.ParamEmail, "bar") 165 | 166 | req, err := http.NewRequest("POST", common.SubscribeEndpoint, strings.NewReader(data.Encode())) 167 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 168 | req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) 169 | if err != nil { 170 | t.Fatal(err) 171 | } 172 | 173 | w := httptest.NewRecorder() 174 | srv.ServeHTTP(w, req) 175 | 176 | resp := w.Result() 177 | 178 | if resp.StatusCode != http.StatusBadRequest { 179 | t.Errorf("Unexpected status code %d", resp.StatusCode) 180 | } 181 | } 182 | 183 | func TestSubscribeIncorrectNewsletter(t *testing.T) { 184 | srv := http.NewServeMux() 185 | store := db.NewSubscribersMapStore() 186 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 187 | nr.AddNewsletters([]string{testNewsletter}) 188 | nr.Setup(srv) 189 | 190 | data := url.Values{} 191 | data.Set(common.ParamNewsletter, "foo") 192 | data.Set(common.ParamEmail, "bar@foo.com") 193 | 194 | req, err := http.NewRequest("POST", common.SubscribeEndpoint, strings.NewReader(data.Encode())) 195 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 196 | req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) 197 | if err != nil { 198 | t.Fatal(err) 199 | } 200 | 201 | w := httptest.NewRecorder() 202 | srv.ServeHTTP(w, req) 203 | 204 | resp := w.Result() 205 | 206 | if resp.StatusCode != http.StatusBadRequest { 207 | t.Errorf("Unexpected status code %d", resp.StatusCode) 208 | } 209 | } 210 | 211 | func TestSubscribe(t *testing.T) { 212 | srv := http.NewServeMux() 213 | newsletter := "foo" 214 | store := db.NewSubscribersMapStore() 215 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 216 | nr.AddNewsletters([]string{newsletter}) 217 | nr.Setup(srv) 218 | nr.SubscribeRedirectURL = testUrl 219 | 220 | data := url.Values{} 221 | data.Set(common.ParamNewsletter, newsletter) 222 | data.Set(common.ParamEmail, "bar@foo.com") 223 | 224 | req, err := http.NewRequest("POST", common.SubscribeEndpoint, strings.NewReader(data.Encode())) 225 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 226 | req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) 227 | if err != nil { 228 | t.Fatal(err) 229 | } 230 | 231 | w := httptest.NewRecorder() 232 | srv.ServeHTTP(w, req) 233 | 234 | resp := w.Result() 235 | 236 | if resp.StatusCode != http.StatusFound { 237 | t.Errorf("Unexpected status code %d", resp.StatusCode) 238 | } 239 | 240 | l, err := resp.Location() 241 | if err != nil { 242 | t.Fatal(err) 243 | } 244 | 245 | if l.String() != testUrl { 246 | t.Errorf("Path does not match. expected=%v actual=%v", l.Path, testUrl) 247 | } 248 | 249 | ss, _ := store.Subscribers(newsletter) 250 | if len(ss) != 1 { 251 | t.Errorf("Wrong number of items in the store: %v", len(ss)) 252 | } 253 | } 254 | 255 | func TestSubscribeFailingStore(t *testing.T) { 256 | srv := http.NewServeMux() 257 | newsletter := "foo" 258 | nr := NewTestNewsResource(NewFailingStore(), db.NewNotificationsMapStore()) 259 | nr.AddNewsletters([]string{newsletter}) 260 | nr.Setup(srv) 261 | 262 | data := url.Values{} 263 | data.Set(common.ParamNewsletter, newsletter) 264 | data.Set(common.ParamEmail, "bar@foo.com") 265 | 266 | req, err := http.NewRequest("POST", common.SubscribeEndpoint, strings.NewReader(data.Encode())) 267 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 268 | req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) 269 | if err != nil { 270 | t.Fatal(err) 271 | } 272 | 273 | w := httptest.NewRecorder() 274 | srv.ServeHTTP(w, req) 275 | 276 | resp := w.Result() 277 | 278 | if resp.StatusCode != http.StatusInternalServerError { 279 | t.Errorf("Unexpected status code %d", resp.StatusCode) 280 | } 281 | } 282 | 283 | func TestConfirmSubscribeFailingStore(t *testing.T) { 284 | srv := http.NewServeMux() 285 | 286 | nr := NewTestNewsResource(NewFailingStore(), db.NewNotificationsMapStore()) 287 | nr.AddNewsletters([]string{testNewsletter}) 288 | nr.Setup(srv) 289 | 290 | data := url.Values{} 291 | data.Set(common.ParamNewsletter, testNewsletter) 292 | data.Set(common.ParamToken, common.Sign(secret, testEmail)) 293 | 294 | req, err := http.NewRequest("GET", common.ConfirmEndpoint, nil) 295 | if err != nil { 296 | t.Fatal(err) 297 | } 298 | 299 | q := req.URL.Query() 300 | q.Add(common.ParamNewsletter, testNewsletter) 301 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 302 | req.URL.RawQuery = q.Encode() 303 | 304 | w := httptest.NewRecorder() 305 | srv.ServeHTTP(w, req) 306 | 307 | resp := w.Result() 308 | 309 | if resp.StatusCode != http.StatusInternalServerError { 310 | body, _ := ioutil.ReadAll(resp.Body) 311 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 312 | } 313 | } 314 | 315 | func TestConfirmSubscribeFailingStore2(t *testing.T) { 316 | srv := http.NewServeMux() 317 | 318 | store := NewFailingStore() 319 | store.failGetSubscriber = false 320 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 321 | nr.AddNewsletters([]string{testNewsletter}) 322 | nr.Setup(srv) 323 | 324 | data := url.Values{} 325 | data.Set(common.ParamNewsletter, testNewsletter) 326 | data.Set(common.ParamToken, common.Sign(secret, testEmail)) 327 | 328 | req, err := http.NewRequest("GET", common.ConfirmEndpoint, nil) 329 | if err != nil { 330 | t.Fatal(err) 331 | } 332 | 333 | q := req.URL.Query() 334 | q.Add(common.ParamNewsletter, testNewsletter) 335 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 336 | req.URL.RawQuery = q.Encode() 337 | 338 | w := httptest.NewRecorder() 339 | srv.ServeHTTP(w, req) 340 | 341 | resp := w.Result() 342 | 343 | if resp.StatusCode != http.StatusInternalServerError { 344 | body, _ := ioutil.ReadAll(resp.Body) 345 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 346 | } 347 | } 348 | 349 | func TestConfirmSubscribeWithoutToken(t *testing.T) { 350 | srv := http.NewServeMux() 351 | 352 | store := db.NewSubscribersMapStore() 353 | store.AddSubscriber(testNewsletter, testEmail, testName) 354 | 355 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 356 | nr.AddNewsletters([]string{testNewsletter}) 357 | nr.Setup(srv) 358 | 359 | data := url.Values{} 360 | data.Set(common.ParamNewsletter, testNewsletter) 361 | 362 | req, err := http.NewRequest("GET", common.ConfirmEndpoint, nil) 363 | if err != nil { 364 | t.Fatal(err) 365 | } 366 | 367 | q := req.URL.Query() 368 | q.Add(common.ParamNewsletter, testNewsletter) 369 | req.URL.RawQuery = q.Encode() 370 | 371 | w := httptest.NewRecorder() 372 | srv.ServeHTTP(w, req) 373 | 374 | resp := w.Result() 375 | 376 | if resp.StatusCode != http.StatusBadRequest { 377 | body, _ := ioutil.ReadAll(resp.Body) 378 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 379 | } 380 | } 381 | 382 | func TestConfirmSubscribeIncorrectNewsletter(t *testing.T) { 383 | srv := http.NewServeMux() 384 | 385 | store := db.NewSubscribersMapStore() 386 | store.AddSubscriber(testNewsletter, testEmail, testName) 387 | 388 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 389 | nr.AddNewsletters([]string{testNewsletter}) 390 | nr.Setup(srv) 391 | 392 | req, err := http.NewRequest("GET", common.ConfirmEndpoint, nil) 393 | if err != nil { 394 | t.Fatal(err) 395 | } 396 | 397 | q := req.URL.Query() 398 | q.Add(common.ParamNewsletter, "foo") 399 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 400 | req.URL.RawQuery = q.Encode() 401 | 402 | w := httptest.NewRecorder() 403 | srv.ServeHTTP(w, req) 404 | 405 | resp := w.Result() 406 | 407 | if resp.StatusCode != http.StatusBadRequest { 408 | body, _ := ioutil.ReadAll(resp.Body) 409 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 410 | } 411 | } 412 | 413 | func TestConfirmSubscribe(t *testing.T) { 414 | srv := http.NewServeMux() 415 | 416 | store := db.NewSubscribersMapStore() 417 | store.AddSubscriber(testNewsletter, testEmail, testName) 418 | 419 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 420 | nr.AddNewsletters([]string{testNewsletter}) 421 | nr.Setup(srv) 422 | nr.ConfirmRedirectURL = testUrl 423 | 424 | req, err := http.NewRequest("GET", common.ConfirmEndpoint, nil) 425 | if err != nil { 426 | t.Fatal(err) 427 | } 428 | 429 | q := req.URL.Query() 430 | q.Add(common.ParamNewsletter, testNewsletter) 431 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 432 | req.URL.RawQuery = q.Encode() 433 | 434 | w := httptest.NewRecorder() 435 | time.Sleep(10 * time.Nanosecond) 436 | srv.ServeHTTP(w, req) 437 | 438 | resp := w.Result() 439 | 440 | if resp.StatusCode != http.StatusFound { 441 | body, _ := ioutil.ReadAll(resp.Body) 442 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 443 | } 444 | 445 | l, err := resp.Location() 446 | if err != nil { 447 | t.Fatal(err) 448 | } 449 | 450 | if l.String() != testUrl { 451 | t.Errorf("Path does not match. expected=%v actual=%v", l.Path, testUrl) 452 | } 453 | 454 | i, _ := store.GetSubscriber(testNewsletter, testEmail) 455 | if !i.Confirmed() { 456 | t.Errorf("Confirm time not updated. created=%v confirm=%v", i.CreatedAt, i.ConfirmedAt) 457 | } 458 | } 459 | 460 | func TestGetSubscribersUnauthorized(t *testing.T) { 461 | srv := http.NewServeMux() 462 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 463 | nr.Setup(srv) 464 | 465 | req, err := http.NewRequest("GET", common.SubscribersEndpoint, nil) 466 | if err != nil { 467 | t.Fatal(err) 468 | } 469 | 470 | w := httptest.NewRecorder() 471 | srv.ServeHTTP(w, req) 472 | 473 | resp := w.Result() 474 | 475 | if resp.StatusCode != http.StatusUnauthorized { 476 | t.Errorf("Unexpected status code %d", resp.StatusCode) 477 | } 478 | } 479 | 480 | func TestGetSubscribersWithWrongPassword(t *testing.T) { 481 | srv := http.NewServeMux() 482 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 483 | nr.Setup(srv) 484 | 485 | req, err := http.NewRequest("GET", common.SubscribersEndpoint, nil) 486 | if err != nil { 487 | t.Fatal(err) 488 | } 489 | 490 | req.SetBasicAuth("any username", "wrong password") 491 | w := httptest.NewRecorder() 492 | srv.ServeHTTP(w, req) 493 | 494 | resp := w.Result() 495 | 496 | if resp.StatusCode != http.StatusForbidden { 497 | t.Errorf("Unexpected status code %d", resp.StatusCode) 498 | } 499 | } 500 | 501 | func TestGetSubscribersWithoutParam(t *testing.T) { 502 | srv := http.NewServeMux() 503 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 504 | nr.Setup(srv) 505 | 506 | req, err := http.NewRequest("GET", common.SubscribersEndpoint, nil) 507 | if err != nil { 508 | t.Fatal(err) 509 | } 510 | 511 | req.SetBasicAuth("any username", apiToken) 512 | w := httptest.NewRecorder() 513 | srv.ServeHTTP(w, req) 514 | 515 | resp := w.Result() 516 | 517 | if resp.StatusCode != http.StatusBadRequest { 518 | t.Errorf("Unexpected status code %d", resp.StatusCode) 519 | } 520 | } 521 | 522 | func TestGetSubscribersWrongNewsletter(t *testing.T) { 523 | srv := http.NewServeMux() 524 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 525 | nr.Setup(srv) 526 | 527 | req, err := http.NewRequest("GET", common.SubscribersEndpoint, nil) 528 | if err != nil { 529 | t.Fatal(err) 530 | } 531 | q := req.URL.Query() 532 | q.Add(common.ParamNewsletter, "test") 533 | req.URL.RawQuery = q.Encode() 534 | req.SetBasicAuth("any username", apiToken) 535 | 536 | w := httptest.NewRecorder() 537 | srv.ServeHTTP(w, req) 538 | 539 | resp := w.Result() 540 | 541 | if resp.StatusCode != http.StatusBadRequest { 542 | body, _ := ioutil.ReadAll(resp.Body) 543 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 544 | } 545 | } 546 | 547 | func TestGetSubscribersFailingStore(t *testing.T) { 548 | srv := http.NewServeMux() 549 | 550 | nr := NewTestAdminResource(NewFailingStore(), db.NewNotificationsMapStore()) 551 | nr.Setup(srv) 552 | nr.AddNewsletters([]string{testNewsletter}) 553 | 554 | req, err := http.NewRequest("GET", common.SubscribersEndpoint, nil) 555 | if err != nil { 556 | t.Fatal(err) 557 | } 558 | q := req.URL.Query() 559 | q.Add(common.ParamNewsletter, testNewsletter) 560 | req.URL.RawQuery = q.Encode() 561 | 562 | req.SetBasicAuth("any username", apiToken) 563 | w := httptest.NewRecorder() 564 | srv.ServeHTTP(w, req) 565 | 566 | resp := w.Result() 567 | 568 | if resp.StatusCode != http.StatusInternalServerError { 569 | body, _ := ioutil.ReadAll(resp.Body) 570 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 571 | } 572 | } 573 | 574 | func TestGetSubscribersOK(t *testing.T) { 575 | srv := http.NewServeMux() 576 | 577 | store := db.NewSubscribersMapStore() 578 | store.AddSubscriber(testNewsletter, testEmail, testName) 579 | 580 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 581 | nr.Setup(srv) 582 | nr.AddNewsletters([]string{testNewsletter}) 583 | 584 | req, err := http.NewRequest("GET", common.SubscribersEndpoint, nil) 585 | if err != nil { 586 | t.Fatal(err) 587 | } 588 | q := req.URL.Query() 589 | q.Add(common.ParamNewsletter, testNewsletter) 590 | req.URL.RawQuery = q.Encode() 591 | 592 | req.SetBasicAuth("any username", apiToken) 593 | w := httptest.NewRecorder() 594 | srv.ServeHTTP(w, req) 595 | 596 | resp := w.Result() 597 | 598 | if resp.StatusCode != http.StatusOK { 599 | body, _ := ioutil.ReadAll(resp.Body) 600 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 601 | } 602 | 603 | body, err := ioutil.ReadAll(resp.Body) 604 | if err != nil { 605 | t.Fatal(err) 606 | } 607 | 608 | ss := make([]*common.Subscriber, 0) 609 | err = json.Unmarshal(body, &ss) 610 | if err != nil { 611 | t.Fatal(err) 612 | } 613 | 614 | if len(ss) != 1 { 615 | t.Errorf("Wrong number of items in response: %v", len(ss)) 616 | } 617 | 618 | if ss[0].Email != testEmail { 619 | t.Errorf("Wrong data received: %v", body) 620 | } 621 | } 622 | 623 | func TestUnsubscribeWrongMethod(t *testing.T) { 624 | srv := http.NewServeMux() 625 | nr := NewTestNewsResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 626 | nr.Setup(srv) 627 | 628 | req, err := http.NewRequest("POST", common.UnsubscribeEndpoint, nil) 629 | if err != nil { 630 | t.Fatal(err) 631 | } 632 | 633 | w := httptest.NewRecorder() 634 | srv.ServeHTTP(w, req) 635 | 636 | resp := w.Result() 637 | 638 | if resp.StatusCode != http.StatusBadRequest { 639 | t.Errorf("Unexpected status code %d", resp.StatusCode) 640 | } 641 | } 642 | 643 | func TestUnsubscribeWithoutNewsletter(t *testing.T) { 644 | srv := http.NewServeMux() 645 | nr := NewTestNewsResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 646 | nr.Setup(srv) 647 | 648 | req, err := http.NewRequest("GET", common.UnsubscribeEndpoint, nil) 649 | if err != nil { 650 | t.Fatal(err) 651 | } 652 | 653 | w := httptest.NewRecorder() 654 | srv.ServeHTTP(w, req) 655 | 656 | resp := w.Result() 657 | 658 | if resp.StatusCode != http.StatusBadRequest { 659 | t.Errorf("Unexpected status code %d", resp.StatusCode) 660 | } 661 | } 662 | 663 | func TestUnsubscribeWithoutToken(t *testing.T) { 664 | srv := http.NewServeMux() 665 | 666 | store := db.NewSubscribersMapStore() 667 | store.AddSubscriber(testNewsletter, testEmail, testName) 668 | 669 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 670 | nr.AddNewsletters([]string{testNewsletter}) 671 | nr.Setup(srv) 672 | 673 | req, err := http.NewRequest("GET", common.UnsubscribeEndpoint, nil) 674 | if err != nil { 675 | t.Fatal(err) 676 | } 677 | 678 | q := req.URL.Query() 679 | q.Add(common.ParamNewsletter, testNewsletter) 680 | req.URL.RawQuery = q.Encode() 681 | 682 | w := httptest.NewRecorder() 683 | srv.ServeHTTP(w, req) 684 | 685 | resp := w.Result() 686 | 687 | if resp.StatusCode != http.StatusBadRequest { 688 | t.Errorf("Unexpected status code %d", resp.StatusCode) 689 | } 690 | 691 | if store.Count() != 1 { 692 | t.Errorf("Wrong number of subscribers: %v", store.Count()) 693 | } 694 | } 695 | 696 | func TestUnsubscribeWithBadToken(t *testing.T) { 697 | srv := http.NewServeMux() 698 | 699 | store := db.NewSubscribersMapStore() 700 | store.AddSubscriber(testNewsletter, testEmail, testName) 701 | 702 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 703 | nr.Setup(srv) 704 | 705 | req, err := http.NewRequest("GET", common.UnsubscribeEndpoint, nil) 706 | if err != nil { 707 | t.Fatal(err) 708 | } 709 | 710 | q := req.URL.Query() 711 | q.Add(common.ParamNewsletter, "random value") 712 | q.Add(common.ParamToken, "abcde") 713 | req.URL.RawQuery = q.Encode() 714 | 715 | w := httptest.NewRecorder() 716 | srv.ServeHTTP(w, req) 717 | 718 | resp := w.Result() 719 | 720 | if resp.StatusCode != http.StatusBadRequest { 721 | t.Errorf("Unexpected status code %d", resp.StatusCode) 722 | } 723 | 724 | if store.Count() != 1 { 725 | t.Errorf("Wrong number of subscribers: %v", store.Count()) 726 | } 727 | } 728 | 729 | func TestUnsubscribeFailingStore(t *testing.T) { 730 | srv := http.NewServeMux() 731 | 732 | nr := NewTestNewsResource(NewFailingStore(), db.NewNotificationsMapStore()) 733 | nr.AddNewsletters([]string{testNewsletter}) 734 | nr.Setup(srv) 735 | 736 | req, err := http.NewRequest("GET", common.UnsubscribeEndpoint, nil) 737 | if err != nil { 738 | t.Fatal(err) 739 | } 740 | q := req.URL.Query() 741 | q.Add(common.ParamNewsletter, testNewsletter) 742 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 743 | req.URL.RawQuery = q.Encode() 744 | 745 | w := httptest.NewRecorder() 746 | srv.ServeHTTP(w, req) 747 | 748 | resp := w.Result() 749 | 750 | if resp.StatusCode != http.StatusInternalServerError { 751 | t.Errorf("Unexpected status code %d", resp.StatusCode) 752 | } 753 | } 754 | 755 | func TestUnsubscribe(t *testing.T) { 756 | srv := http.NewServeMux() 757 | 758 | store := db.NewSubscribersMapStore() 759 | store.AddSubscriber(testNewsletter, testEmail, testName) 760 | 761 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 762 | nr.AddNewsletters([]string{testNewsletter}) 763 | nr.Setup(srv) 764 | nr.UnsubscribeRedirectURL = testUrl 765 | 766 | req, err := http.NewRequest("GET", common.UnsubscribeEndpoint, nil) 767 | if err != nil { 768 | t.Fatal(err) 769 | } 770 | q := req.URL.Query() 771 | q.Add(common.ParamNewsletter, testNewsletter) 772 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 773 | req.URL.RawQuery = q.Encode() 774 | 775 | w := httptest.NewRecorder() 776 | time.Sleep(10 * time.Nanosecond) 777 | srv.ServeHTTP(w, req) 778 | resp := w.Result() 779 | 780 | if resp.StatusCode != http.StatusFound { 781 | t.Errorf("Unexpected status code %d", resp.StatusCode) 782 | } 783 | 784 | l, err := resp.Location() 785 | if err != nil { 786 | t.Fatal(err) 787 | } 788 | 789 | if l.String() != testUrl { 790 | t.Errorf("Path does not match. expected=%v actual=%v", l.Path, testUrl) 791 | } 792 | 793 | if store.Count() != 1 { 794 | t.Errorf("Wrong number of subscribers left: %d", store.Count()) 795 | } 796 | 797 | i, _ := store.GetSubscriber(testNewsletter, testEmail) 798 | if !i.Unsubscribed() { 799 | t.Errorf("Unsubscribe time not updated. created=%v unsubscribe=%v", i.CreatedAt, i.UnsubscribedAt) 800 | } 801 | } 802 | 803 | func TestPostSubscribers(t *testing.T) { 804 | srv := http.NewServeMux() 805 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 806 | nr.Setup(srv) 807 | 808 | req, err := http.NewRequest("POST", common.SubscribersEndpoint, nil) 809 | if err != nil { 810 | t.Fatal(err) 811 | } 812 | 813 | req.SetBasicAuth("any username", apiToken) 814 | w := httptest.NewRecorder() 815 | srv.ServeHTTP(w, req) 816 | 817 | resp := w.Result() 818 | 819 | if resp.StatusCode != http.StatusBadRequest { 820 | t.Errorf("Unexpected status code %d", resp.StatusCode) 821 | } 822 | } 823 | 824 | func TestPutSubscribersUnauthorized(t *testing.T) { 825 | srv := http.NewServeMux() 826 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 827 | nr.Setup(srv) 828 | 829 | req, err := http.NewRequest("PUT", common.SubscribersEndpoint, nil) 830 | if err != nil { 831 | t.Fatal(err) 832 | } 833 | 834 | w := httptest.NewRecorder() 835 | srv.ServeHTTP(w, req) 836 | 837 | resp := w.Result() 838 | 839 | if resp.StatusCode != http.StatusUnauthorized { 840 | t.Errorf("Unexpected status code %d", resp.StatusCode) 841 | } 842 | } 843 | 844 | func TestPutSubscribersInvalidMedia(t *testing.T) { 845 | newsletter := "TestNewsletter" 846 | 847 | srv := http.NewServeMux() 848 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 849 | nr.Setup(srv) 850 | 851 | var subscribers []*common.Subscriber 852 | for i := 0; i < 10; i++ { 853 | subscribers = append(subscribers, &common.Subscriber{ 854 | Newsletter: newsletter, 855 | Email: fmt.Sprintf("foo%v@bar.com", i), 856 | CreatedAt: common.JsonTimeNow(), 857 | UnsubscribedAt: incorrectTime, 858 | ConfirmedAt: incorrectTime, 859 | }) 860 | } 861 | data, err := json.Marshal(subscribers) 862 | if err != nil { 863 | t.Fatal(err) 864 | } 865 | 866 | req, err := http.NewRequest("PUT", common.SubscribersEndpoint, bytes.NewBuffer(data)) 867 | if err != nil { 868 | t.Fatal(err) 869 | } 870 | req.SetBasicAuth("any username", apiToken) 871 | 872 | w := httptest.NewRecorder() 873 | srv.ServeHTTP(w, req) 874 | 875 | resp := w.Result() 876 | 877 | if resp.StatusCode != http.StatusUnsupportedMediaType { 878 | body, _ := ioutil.ReadAll(resp.Body) 879 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 880 | } 881 | } 882 | 883 | func TestPutSubscribersWrongNewsletter(t *testing.T) { 884 | newsletter := "TestNewsletter" 885 | 886 | srv := http.NewServeMux() 887 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 888 | nr.Setup(srv) 889 | 890 | var subscribers []*common.Subscriber 891 | for i := 0; i < 10; i++ { 892 | subscribers = append(subscribers, &common.Subscriber{ 893 | Newsletter: newsletter, 894 | Email: fmt.Sprintf("foo%v@bar.com", i), 895 | CreatedAt: common.JsonTimeNow(), 896 | UnsubscribedAt: incorrectTime, 897 | ConfirmedAt: incorrectTime, 898 | }) 899 | } 900 | data, err := json.Marshal(subscribers) 901 | if err != nil { 902 | t.Fatal(err) 903 | } 904 | 905 | req, err := http.NewRequest("PUT", common.SubscribersEndpoint, bytes.NewBuffer(data)) 906 | if err != nil { 907 | t.Fatal(err) 908 | } 909 | req.Header.Set("Content-Type", "application/json") 910 | req.SetBasicAuth("any username", apiToken) 911 | 912 | w := httptest.NewRecorder() 913 | srv.ServeHTTP(w, req) 914 | 915 | resp := w.Result() 916 | 917 | if resp.StatusCode != http.StatusBadRequest { 918 | body, _ := ioutil.ReadAll(resp.Body) 919 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 920 | } 921 | } 922 | 923 | func TestPutSubscribersFailingStore(t *testing.T) { 924 | newsletter := "TestNewsletter" 925 | 926 | srv := http.NewServeMux() 927 | nr := NewTestAdminResource(NewFailingStore(), db.NewNotificationsMapStore()) 928 | nr.Setup(srv) 929 | nr.AddNewsletters([]string{newsletter}) 930 | 931 | var subscribers []*common.Subscriber 932 | for i := 0; i < 10; i++ { 933 | subscribers = append(subscribers, &common.Subscriber{ 934 | Newsletter: newsletter, 935 | Email: fmt.Sprintf("foo%v@bar.com", i), 936 | CreatedAt: common.JsonTimeNow(), 937 | UnsubscribedAt: incorrectTime, 938 | ConfirmedAt: incorrectTime, 939 | }) 940 | } 941 | data, err := json.Marshal(subscribers) 942 | if err != nil { 943 | t.Fatal(err) 944 | } 945 | 946 | req, err := http.NewRequest("PUT", common.SubscribersEndpoint, bytes.NewBuffer(data)) 947 | if err != nil { 948 | t.Fatal(err) 949 | } 950 | req.Header.Set("Content-Type", "application/json") 951 | req.SetBasicAuth("any username", apiToken) 952 | 953 | w := httptest.NewRecorder() 954 | srv.ServeHTTP(w, req) 955 | 956 | resp := w.Result() 957 | 958 | if resp.StatusCode != http.StatusInternalServerError { 959 | body, _ := ioutil.ReadAll(resp.Body) 960 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 961 | } 962 | } 963 | 964 | func PutSubscribersBaseSuite(subscribers []*common.Subscriber, store common.SubscribersStore) (*http.Response, error) { 965 | srv := http.NewServeMux() 966 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 967 | nr.Setup(srv) 968 | nr.AddNewsletters([]string{testNewsletter}) 969 | 970 | data, err := json.Marshal(subscribers) 971 | if err != nil { 972 | return nil, err 973 | } 974 | 975 | req, err := http.NewRequest("PUT", common.SubscribersEndpoint, bytes.NewBuffer(data)) 976 | if err != nil { 977 | return nil, err 978 | } 979 | req.Header.Set("Content-Type", "application/json") 980 | req.SetBasicAuth("any username", apiToken) 981 | 982 | w := httptest.NewRecorder() 983 | srv.ServeHTTP(w, req) 984 | 985 | resp := w.Result() 986 | return resp, nil 987 | } 988 | 989 | func PutSubscribersSuite(subscribers []*common.Subscriber, t *testing.T) { 990 | subscribersMap := make(map[string]*common.Subscriber) 991 | for _, s := range subscribers { 992 | subscribersMap[s.Email] = s 993 | } 994 | store := db.NewSubscribersMapStore() 995 | resp, err := PutSubscribersBaseSuite(subscribers, store) 996 | if err != nil { 997 | t.Fatal(err) 998 | } 999 | 1000 | if resp.StatusCode != http.StatusOK { 1001 | body, _ := ioutil.ReadAll(resp.Body) 1002 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 1003 | } 1004 | 1005 | for k, s := range subscribersMap { 1006 | es, err := store.GetSubscriber(testNewsletter, k) 1007 | if err != nil { 1008 | t.Errorf("Email not imported: %v", k) 1009 | } 1010 | if es.Confirmed() != s.Confirmed() { 1011 | t.Fatalf("Confirmed status does not match. email=%v", k) 1012 | } 1013 | if es.Unsubscribed() != s.Unsubscribed() { 1014 | t.Fatalf("Unsubscribed status does not match. email=%v", k) 1015 | } 1016 | } 1017 | } 1018 | 1019 | func TestPutUnconfirmedSubscribers(t *testing.T) { 1020 | var subscribers []*common.Subscriber 1021 | for i := 0; i < 10; i++ { 1022 | s := &common.Subscriber{ 1023 | Newsletter: testNewsletter, 1024 | Email: fmt.Sprintf("foo%v@bar.com", i), 1025 | CreatedAt: common.JsonTimeNow(), 1026 | UnsubscribedAt: incorrectTime, 1027 | ConfirmedAt: incorrectTime, 1028 | } 1029 | subscribers = append(subscribers, s) 1030 | } 1031 | PutSubscribersSuite(subscribers, t) 1032 | } 1033 | 1034 | func TestPutConfirmedSubscribers(t *testing.T) { 1035 | var subscribers []*common.Subscriber 1036 | for i := 0; i < 10; i++ { 1037 | jt := common.JSONTime(time.Now().UTC().Add(-1 * time.Second)) 1038 | s := &common.Subscriber{ 1039 | Newsletter: testNewsletter, 1040 | Email: fmt.Sprintf("foo%v@bar.com", i), 1041 | CreatedAt: jt, 1042 | UnsubscribedAt: incorrectTime, 1043 | ConfirmedAt: incorrectTime, 1044 | } 1045 | s.ConfirmedAt = common.JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 1046 | subscribers = append(subscribers, s) 1047 | } 1048 | PutSubscribersSuite(subscribers, t) 1049 | } 1050 | 1051 | func TestPutUnsubscribedSubscribers(t *testing.T) { 1052 | var subscribers []*common.Subscriber 1053 | for i := 0; i < 10; i++ { 1054 | jt := common.JSONTime(time.Now().UTC().Add(-1 * time.Second)) 1055 | s := &common.Subscriber{ 1056 | Newsletter: testNewsletter, 1057 | Email: fmt.Sprintf("foo%v@bar.com", i), 1058 | CreatedAt: jt, 1059 | UnsubscribedAt: incorrectTime, 1060 | ConfirmedAt: incorrectTime, 1061 | } 1062 | s.UnsubscribedAt = common.JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 1063 | subscribers = append(subscribers, s) 1064 | } 1065 | PutSubscribersSuite(subscribers, t) 1066 | } 1067 | 1068 | func TestGetComplaintsUnauthorized(t *testing.T) { 1069 | srv := http.NewServeMux() 1070 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 1071 | nr.Setup(srv) 1072 | 1073 | req, err := http.NewRequest("GET", common.ComplaintsEndpoint, nil) 1074 | if err != nil { 1075 | t.Fatal(err) 1076 | } 1077 | 1078 | w := httptest.NewRecorder() 1079 | srv.ServeHTTP(w, req) 1080 | 1081 | resp := w.Result() 1082 | 1083 | if resp.StatusCode != http.StatusUnauthorized { 1084 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1085 | } 1086 | } 1087 | 1088 | func TestGetComplaintsFailingStore(t *testing.T) { 1089 | srv := http.NewServeMux() 1090 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), &FailingNotificationsStore{}) 1091 | nr.Setup(srv) 1092 | 1093 | req, err := http.NewRequest("GET", common.ComplaintsEndpoint, nil) 1094 | if err != nil { 1095 | t.Fatal(err) 1096 | } 1097 | 1098 | req.SetBasicAuth("any username", apiToken) 1099 | w := httptest.NewRecorder() 1100 | srv.ServeHTTP(w, req) 1101 | 1102 | resp := w.Result() 1103 | 1104 | if resp.StatusCode != http.StatusInternalServerError { 1105 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1106 | } 1107 | } 1108 | 1109 | func TestGetComplaintsOK(t *testing.T) { 1110 | srv := http.NewServeMux() 1111 | store := db.NewNotificationsMapStore() 1112 | store.AddBounce(testEmail, "from@email.com", false /*is transient*/) 1113 | store.AddComplaint(testEmail, "from@email.com") 1114 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), store) 1115 | nr.Setup(srv) 1116 | 1117 | req, err := http.NewRequest("GET", common.ComplaintsEndpoint, nil) 1118 | if err != nil { 1119 | t.Fatal(err) 1120 | } 1121 | 1122 | req.SetBasicAuth("any username", apiToken) 1123 | w := httptest.NewRecorder() 1124 | srv.ServeHTTP(w, req) 1125 | 1126 | resp := w.Result() 1127 | 1128 | if resp.StatusCode != http.StatusOK { 1129 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1130 | } 1131 | 1132 | body, err := ioutil.ReadAll(resp.Body) 1133 | if err != nil { 1134 | t.Fatal(err) 1135 | } 1136 | 1137 | ss := make([]*common.SesNotification, 0) 1138 | err = json.Unmarshal(body, &ss) 1139 | if err != nil { 1140 | t.Fatal(err) 1141 | } 1142 | 1143 | if len(ss) != 2 { 1144 | t.Errorf("Wrong number of items in response: %v", len(ss)) 1145 | } 1146 | } 1147 | 1148 | func TestDeleteSubscribersUnauthorized(t *testing.T) { 1149 | srv := http.NewServeMux() 1150 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 1151 | nr.Setup(srv) 1152 | 1153 | req, err := http.NewRequest("DELETE", common.SubscribersEndpoint, nil) 1154 | if err != nil { 1155 | t.Fatal(err) 1156 | } 1157 | 1158 | w := httptest.NewRecorder() 1159 | srv.ServeHTTP(w, req) 1160 | 1161 | resp := w.Result() 1162 | 1163 | if resp.StatusCode != http.StatusUnauthorized { 1164 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1165 | } 1166 | } 1167 | 1168 | func TestDeleteSubscribersInvalidMedia(t *testing.T) { 1169 | srv := http.NewServeMux() 1170 | nr := NewTestAdminResource(db.NewSubscribersMapStore(), db.NewNotificationsMapStore()) 1171 | nr.Setup(srv) 1172 | 1173 | keys := []*common.SubscriberKey{ 1174 | &common.SubscriberKey{ 1175 | Newsletter: testNewsletter, 1176 | Email: "email1@email.com", 1177 | }, 1178 | } 1179 | 1180 | data, err := json.Marshal(keys) 1181 | if err != nil { 1182 | t.Fatal(err) 1183 | } 1184 | 1185 | req, err := http.NewRequest("DELETE", common.SubscribersEndpoint, bytes.NewBuffer(data)) 1186 | if err != nil { 1187 | t.Fatal(err) 1188 | } 1189 | 1190 | req.SetBasicAuth("any username", apiToken) 1191 | 1192 | w := httptest.NewRecorder() 1193 | srv.ServeHTTP(w, req) 1194 | 1195 | resp := w.Result() 1196 | 1197 | if resp.StatusCode != http.StatusUnsupportedMediaType { 1198 | t.Errorf("Unexpected status code: %d", resp.StatusCode) 1199 | } 1200 | } 1201 | 1202 | func TestDeleteSubscribers(t *testing.T) { 1203 | srv := http.NewServeMux() 1204 | store := db.NewSubscribersMapStore() 1205 | for i := 0; i < 10; i++ { 1206 | store.AddSubscriber(testNewsletter, fmt.Sprintf("email%v@email.com", i), testName) 1207 | } 1208 | 1209 | nr := NewTestAdminResource(store, db.NewNotificationsMapStore()) 1210 | nr.Setup(srv) 1211 | 1212 | keys := []*common.SubscriberKey{ 1213 | &common.SubscriberKey{ 1214 | Newsletter: testNewsletter, 1215 | Email: "email1@email.com", 1216 | }, 1217 | &common.SubscriberKey{ 1218 | Newsletter: testNewsletter, 1219 | Email: "email3@email.com", 1220 | }, 1221 | } 1222 | 1223 | data, err := json.Marshal(keys) 1224 | if err != nil { 1225 | t.Fatal(err) 1226 | } 1227 | 1228 | req, err := http.NewRequest("DELETE", common.SubscribersEndpoint, bytes.NewBuffer(data)) 1229 | if err != nil { 1230 | t.Fatal(err) 1231 | } 1232 | 1233 | req.Header.Set("Content-Type", "application/json") 1234 | req.SetBasicAuth("any username", apiToken) 1235 | 1236 | w := httptest.NewRecorder() 1237 | srv.ServeHTTP(w, req) 1238 | 1239 | resp := w.Result() 1240 | 1241 | if resp.StatusCode != http.StatusOK { 1242 | body, _ := ioutil.ReadAll(resp.Body) 1243 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 1244 | } 1245 | 1246 | if store.Count() != 8 { 1247 | t.Errorf("Items were not deleted") 1248 | } 1249 | } 1250 | 1251 | func TestDeleteSubscribersFailingStore(t *testing.T) { 1252 | srv := http.NewServeMux() 1253 | 1254 | nr := NewTestAdminResource(NewFailingStore(), db.NewNotificationsMapStore()) 1255 | nr.Setup(srv) 1256 | 1257 | keys := []*common.SubscriberKey{ 1258 | &common.SubscriberKey{ 1259 | Newsletter: testNewsletter, 1260 | Email: "email1@email.com", 1261 | }, 1262 | &common.SubscriberKey{ 1263 | Newsletter: testNewsletter, 1264 | Email: "email3@email.com", 1265 | }, 1266 | } 1267 | 1268 | data, err := json.Marshal(keys) 1269 | if err != nil { 1270 | t.Fatal(err) 1271 | } 1272 | 1273 | req, err := http.NewRequest("DELETE", common.SubscribersEndpoint, bytes.NewBuffer(data)) 1274 | if err != nil { 1275 | t.Fatal(err) 1276 | } 1277 | 1278 | req.Header.Set("Content-Type", "application/json") 1279 | req.SetBasicAuth("any username", apiToken) 1280 | 1281 | w := httptest.NewRecorder() 1282 | srv.ServeHTTP(w, req) 1283 | 1284 | resp := w.Result() 1285 | 1286 | if resp.StatusCode != http.StatusInternalServerError { 1287 | body, _ := ioutil.ReadAll(resp.Body) 1288 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 1289 | } 1290 | } 1291 | 1292 | func TestSubscribeAlreadyConfirmed(t *testing.T) { 1293 | srv := http.NewServeMux() 1294 | store := db.NewSubscribersMapStore() 1295 | store.AddSubscriber(testNewsletter, testEmail, testName) 1296 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 1297 | nr.AddNewsletters([]string{testNewsletter}) 1298 | nr.Setup(srv) 1299 | nr.ConfirmRedirectURL = testUrl 1300 | 1301 | s, _ := store.GetSubscriber(testNewsletter, testEmail) 1302 | s.ConfirmedAt = common.JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 1303 | if !s.Confirmed() { 1304 | t.Errorf("Confirmed() is not updated") 1305 | } 1306 | 1307 | data := url.Values{} 1308 | data.Set(common.ParamNewsletter, testNewsletter) 1309 | data.Set(common.ParamEmail, testEmail) 1310 | 1311 | req, err := http.NewRequest("POST", common.SubscribeEndpoint, strings.NewReader(data.Encode())) 1312 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 1313 | req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) 1314 | if err != nil { 1315 | t.Fatal(err) 1316 | } 1317 | 1318 | w := httptest.NewRecorder() 1319 | srv.ServeHTTP(w, req) 1320 | resp := w.Result() 1321 | 1322 | if resp.StatusCode != http.StatusFound { 1323 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1324 | } 1325 | 1326 | l, err := resp.Location() 1327 | if err != nil { 1328 | t.Fatal(err) 1329 | } 1330 | 1331 | if l.String() != testUrl { 1332 | t.Errorf("Path does not match. expected=%v actual=%v", l.Path, testUrl) 1333 | } 1334 | } 1335 | 1336 | func TestSubscribeAlreadyUnsubscribed(t *testing.T) { 1337 | srv := http.NewServeMux() 1338 | store := db.NewSubscribersMapStore() 1339 | store.AddSubscriber(testNewsletter, testEmail, testName) 1340 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 1341 | nr.AddNewsletters([]string{testNewsletter}) 1342 | nr.Setup(srv) 1343 | nr.SubscribeRedirectURL = testUrl 1344 | 1345 | s, _ := store.GetSubscriber(testNewsletter, testEmail) 1346 | s.UnsubscribedAt = common.JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 1347 | if !s.Unsubscribed() { 1348 | t.Errorf("Unsubscribed() is not updated") 1349 | } 1350 | 1351 | data := url.Values{} 1352 | data.Set(common.ParamNewsletter, testNewsletter) 1353 | data.Set(common.ParamEmail, testEmail) 1354 | 1355 | req, err := http.NewRequest("POST", common.SubscribeEndpoint, strings.NewReader(data.Encode())) 1356 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 1357 | req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) 1358 | if err != nil { 1359 | t.Fatal(err) 1360 | } 1361 | 1362 | w := httptest.NewRecorder() 1363 | srv.ServeHTTP(w, req) 1364 | resp := w.Result() 1365 | 1366 | if resp.StatusCode != http.StatusFound { 1367 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1368 | } 1369 | 1370 | l, err := resp.Location() 1371 | if err != nil { 1372 | t.Fatal(err) 1373 | } 1374 | 1375 | if l.String() != testUrl { 1376 | t.Errorf("Path does not match. expected=%v actual=%v", l.Path, testUrl) 1377 | } 1378 | } 1379 | 1380 | func TestSubscribeAlreadyUnsubscribedAndConfirmed(t *testing.T) { 1381 | srv := http.NewServeMux() 1382 | store := db.NewSubscribersMapStore() 1383 | store.AddSubscriber(testNewsletter, testEmail, testName) 1384 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 1385 | nr.AddNewsletters([]string{testNewsletter}) 1386 | nr.Setup(srv) 1387 | nr.SubscribeRedirectURL = testUrl 1388 | 1389 | s, _ := store.GetSubscriber(testNewsletter, testEmail) 1390 | s.UnsubscribedAt = common.JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 1391 | s.ConfirmedAt = common.JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 1392 | if !s.Unsubscribed() && !s.Confirmed() { 1393 | t.Errorf("Unsubscribed() and Confirmed() are not updated") 1394 | } 1395 | 1396 | data := url.Values{} 1397 | data.Set(common.ParamNewsletter, testNewsletter) 1398 | data.Set(common.ParamEmail, testEmail) 1399 | 1400 | req, err := http.NewRequest("POST", common.SubscribeEndpoint, strings.NewReader(data.Encode())) 1401 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 1402 | req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) 1403 | if err != nil { 1404 | t.Fatal(err) 1405 | } 1406 | 1407 | w := httptest.NewRecorder() 1408 | srv.ServeHTTP(w, req) 1409 | resp := w.Result() 1410 | 1411 | if resp.StatusCode != http.StatusFound { 1412 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1413 | } 1414 | 1415 | l, err := resp.Location() 1416 | if err != nil { 1417 | t.Fatal(err) 1418 | } 1419 | 1420 | if l.String() != testUrl { 1421 | t.Errorf("Path does not match. expected=%v actual=%v", l.Path, testUrl) 1422 | } 1423 | } 1424 | 1425 | func TestUnsubscribeNotSubscribedYet(t *testing.T) { 1426 | srv := http.NewServeMux() 1427 | 1428 | store := db.NewSubscribersMapStore() 1429 | 1430 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 1431 | nr.AddNewsletters([]string{testNewsletter}) 1432 | nr.Setup(srv) 1433 | nr.UnsubscribeRedirectURL = testUrl 1434 | 1435 | req, err := http.NewRequest("GET", common.UnsubscribeEndpoint, nil) 1436 | if err != nil { 1437 | t.Fatal(err) 1438 | } 1439 | q := req.URL.Query() 1440 | q.Add(common.ParamNewsletter, testNewsletter) 1441 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 1442 | req.URL.RawQuery = q.Encode() 1443 | 1444 | w := httptest.NewRecorder() 1445 | time.Sleep(10 * time.Nanosecond) 1446 | srv.ServeHTTP(w, req) 1447 | 1448 | resp := w.Result() 1449 | 1450 | if resp.StatusCode != http.StatusInternalServerError { 1451 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1452 | } 1453 | } 1454 | 1455 | func TestUnsubscribeUnsubscribed(t *testing.T) { 1456 | srv := http.NewServeMux() 1457 | 1458 | store := db.NewSubscribersMapStore() 1459 | store.AddSubscriber(testNewsletter, testEmail, testName) 1460 | 1461 | s, _ := store.GetSubscriber(testNewsletter, testEmail) 1462 | s.UnsubscribedAt = common.JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 1463 | if !s.Unsubscribed() { 1464 | t.Errorf("Unsubscribed() is not updated") 1465 | } 1466 | 1467 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 1468 | nr.AddNewsletters([]string{testNewsletter}) 1469 | nr.Setup(srv) 1470 | nr.UnsubscribeRedirectURL = testUrl 1471 | 1472 | req, err := http.NewRequest("GET", common.UnsubscribeEndpoint, nil) 1473 | if err != nil { 1474 | t.Fatal(err) 1475 | } 1476 | q := req.URL.Query() 1477 | q.Add(common.ParamNewsletter, testNewsletter) 1478 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 1479 | req.URL.RawQuery = q.Encode() 1480 | 1481 | w := httptest.NewRecorder() 1482 | time.Sleep(10 * time.Nanosecond) 1483 | srv.ServeHTTP(w, req) 1484 | resp := w.Result() 1485 | 1486 | if resp.StatusCode != http.StatusFound { 1487 | t.Errorf("Unexpected status code %d", resp.StatusCode) 1488 | } 1489 | 1490 | l, err := resp.Location() 1491 | if err != nil { 1492 | t.Fatal(err) 1493 | } 1494 | 1495 | if l.String() != testUrl { 1496 | t.Errorf("Path does not match. expected=%v actual=%v", l.Path, testUrl) 1497 | } 1498 | } 1499 | 1500 | func TestConfirmUnsubscribed(t *testing.T) { 1501 | srv := http.NewServeMux() 1502 | 1503 | store := db.NewSubscribersMapStore() 1504 | store.AddSubscriber(testNewsletter, testEmail, testName) 1505 | s, _ := store.GetSubscriber(testNewsletter, testEmail) 1506 | s.UnsubscribedAt = common.JSONTime(s.CreatedAt.Time().Add(1 * time.Second)) 1507 | if !s.Unsubscribed() { 1508 | t.Errorf("Unsubscribed() is not updated") 1509 | } 1510 | 1511 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 1512 | nr.AddNewsletters([]string{testNewsletter}) 1513 | nr.Setup(srv) 1514 | nr.UnsubscribeRedirectURL = testUrl 1515 | 1516 | req, err := http.NewRequest("GET", common.ConfirmEndpoint, nil) 1517 | if err != nil { 1518 | t.Fatal(err) 1519 | } 1520 | 1521 | q := req.URL.Query() 1522 | q.Add(common.ParamNewsletter, testNewsletter) 1523 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 1524 | req.URL.RawQuery = q.Encode() 1525 | 1526 | w := httptest.NewRecorder() 1527 | time.Sleep(10 * time.Nanosecond) 1528 | srv.ServeHTTP(w, req) 1529 | 1530 | resp := w.Result() 1531 | 1532 | if resp.StatusCode != http.StatusFound { 1533 | body, _ := ioutil.ReadAll(resp.Body) 1534 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 1535 | } 1536 | 1537 | l, err := resp.Location() 1538 | if err != nil { 1539 | t.Fatal(err) 1540 | } 1541 | 1542 | if l.String() != testUrl { 1543 | t.Errorf("Path does not match. expected=%v actual=%v", l.Path, testUrl) 1544 | } 1545 | 1546 | i, _ := store.GetSubscriber(testNewsletter, testEmail) 1547 | if i.Confirmed() { 1548 | t.Errorf("Confirm time was updated. created=%v confirm=%v", i.CreatedAt, i.ConfirmedAt) 1549 | } 1550 | } 1551 | 1552 | func TestConfirmMissing(t *testing.T) { 1553 | srv := http.NewServeMux() 1554 | store := db.NewSubscribersMapStore() 1555 | nr := NewTestNewsResource(store, db.NewNotificationsMapStore()) 1556 | nr.AddNewsletters([]string{testNewsletter}) 1557 | nr.Setup(srv) 1558 | nr.UnsubscribeRedirectURL = testUrl 1559 | 1560 | req, err := http.NewRequest("GET", common.ConfirmEndpoint, nil) 1561 | if err != nil { 1562 | t.Fatal(err) 1563 | } 1564 | 1565 | q := req.URL.Query() 1566 | q.Add(common.ParamNewsletter, testNewsletter) 1567 | q.Add(common.ParamToken, common.Sign(secret, testEmail)) 1568 | req.URL.RawQuery = q.Encode() 1569 | 1570 | w := httptest.NewRecorder() 1571 | time.Sleep(10 * time.Nanosecond) 1572 | srv.ServeHTTP(w, req) 1573 | 1574 | resp := w.Result() 1575 | 1576 | if resp.StatusCode != http.StatusInternalServerError { 1577 | body, _ := ioutil.ReadAll(resp.Body) 1578 | t.Errorf("Unexpected status code: %d, body: %v", resp.StatusCode, string(body)) 1579 | } 1580 | 1581 | if store.Count() != 0 { 1582 | t.Errorf("Wrong count in store: %v", store.Count()) 1583 | } 1584 | } 1585 | --------------------------------------------------------------------------------