├── LICENSE ├── README.md └── spoolgore.go /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 unbit 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Spoolgore 2 | ========= 3 | 4 | A simple mail "spool and send" daemon written in Go 5 | 6 | Building it 7 | =========== 8 | 9 | ```sh 10 | go build spoolgore.go 11 | ``` 12 | 13 | that's all... 14 | 15 | Using it 16 | ======== 17 | 18 | 19 | ```sh 20 | # send every mail spooled to /var/spool/yourapp via example.com smtp service 21 | spoolgore -smtpaddr example.com:25 /var/spool/yourapp 22 | ``` 23 | 24 | ```sh 25 | # send every mail spooled to /var/spool/yourapp via foobar.it smtp service using plain authentication 26 | spoolgore -smtpaddr foobar.it:25 -smtpuser kratos -smtppassword deimos /var/spool/yourapp 27 | ``` 28 | 29 | ```sh 30 | # send every mail spooled to /var/spool/yourapp via example.com smtp service, do not try for more than 30 times on smtp error 31 | spoolgore -smtpaddr example.com:25 -attempts 30 /var/spool/yourapp 32 | ``` 33 | 34 | JSON status file 35 | ================ 36 | 37 | During its lifecycle, Spoolgore constantly updates a json file with its internal status. You can simply view that file 38 | to understand what is going on. By default the file is stored as .spoolgore.js in your spool directory, but you can change its path with the -json option. 39 | 40 | Options 41 | ======= 42 | 43 | -smtpaddr 44 | 45 | -smtpuser 46 | 47 | -smtppassword 48 | 49 | -smtpmd5user 50 | 51 | -smtpmd5password 52 | 53 | -freq 54 | 55 | -attempts 56 | 57 | -json 58 | 59 | Signals 60 | ======= 61 | 62 | SIGURG -> suddenly re-scan the queue 63 | 64 | SIGHUP -> reload the json status file 65 | 66 | SIGTSTP -> block queue scan, useful for manually updating the json status 67 | 68 | Why ? 69 | ===== 70 | 71 | Sending e-mails from your app via remote smtp can be a huge problem: if your smtp service is slow, your app will be slow, if your service is blocked your app will be blocked. If you need to send a gazillion email in one round your app could be blocked for ages. 72 | 73 | "spool and send" daemons allow you to store the email as a simple file on a directory (the 'spool' directory), while a background daemon will send it as soon as possible, taking care to retry on any error. 74 | 75 | Projects like nullmailer (http://untroubled.org/nullmailer/) work well, but they are somewhat limited (in the nullmailer case having multiple instances running on the system requires patching). 76 | 77 | Spoolgore tries to address the problem in the easiest possible way. 78 | 79 | Why Go ? 80 | ======== 81 | 82 | MTAs (and similar) tend to spawn an additional process for each SMTP transaction. This is good (and holy) for privileges separation, but spoolgore is meant to be run by each user (or single application stack) so we do not need this kind of isolation in SMTP transactions. 83 | 84 | Go (thanks to goroutines) allows us to enqueue hundreds of SMTP transactions at the cost of few KB of memory. 85 | 86 | Obviously we could have written it in python/gevent or perl/coro::anyevent or whatever non-blocking coroutine/based technology you like, but we choose go as we wanted to give it a try (yes, no other reasons) 87 | 88 | TODO 89 | ==== 90 | 91 | implement rate limiter (lot of services limit the number of emails you can enqueue per minute/hour) 92 | 93 | Issues 94 | ====== 95 | 96 | windows is not supported, if you want to use Spoolgore on it, just patch the code to not use syscall.SIGURG and syscall.SIGTSTP 97 | -------------------------------------------------------------------------------- /spoolgore.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "time" 6 | "log" 7 | "io" 8 | "io/ioutil" 9 | "net/smtp" 10 | "net/mail" 11 | "strings" 12 | "bytes" 13 | "os" 14 | "path" 15 | "path/filepath" 16 | "encoding/json" 17 | "os/signal" 18 | "syscall" 19 | _ "crypto/sha512" 20 | ) 21 | 22 | type Config struct { 23 | SpoolDir string 24 | SmtpAddr string 25 | SmtpUser string 26 | SmtpPassword string 27 | PlainUser string 28 | PlainPassword string 29 | MD5User string 30 | MD5Password string 31 | SmtpAuth smtp.Auth 32 | Freq int 33 | MaxAttempts int 34 | JsonPath string 35 | } 36 | 37 | var config Config 38 | 39 | /* 40 | 0 -> just pushed 41 | 1 -> still in progress 42 | 2 -> sent 43 | */ 44 | 45 | type SentStatus struct { 46 | Address string 47 | Status int 48 | Attempts int 49 | NextAttempt time.Time 50 | Error string 51 | } 52 | 53 | type MailStatus struct { 54 | From string 55 | To []*SentStatus 56 | Cc []*SentStatus 57 | Bcc []*SentStatus 58 | Attempts int 59 | Enqueued time.Time 60 | } 61 | 62 | var status map[string]MailStatus 63 | 64 | func parse_options() { 65 | flag.StringVar(&config.SmtpAddr, "smtpaddr", "127.0.0.1:25", "address of the smtp address to use in the addr:port format") 66 | flag.StringVar(&config.PlainUser, "smtpuser", "", "username for smtp plain authentication") 67 | flag.StringVar(&config.PlainPassword, "smtppassword", "", "password for smtp plain authentication") 68 | flag.StringVar(&config.MD5User, "smtpmd5user", "", "username for smtp cram md5 authentication") 69 | flag.StringVar(&config.MD5Password, "smtpmd5password", "", "password for smtp cram md5 authentication") 70 | flag.IntVar(&config.Freq, "freq", 10, "frequency of spool directory scans") 71 | flag.IntVar(&config.MaxAttempts, "attempts", 100, "max attempts for failed SMTP transactions before giving up") 72 | flag.StringVar(&config.JsonPath, "json", "", "path of the json status file") 73 | flag.Parse() 74 | if flag.NArg() < 1 { 75 | log.Fatal("please specify a spool directory") 76 | } 77 | config.SpoolDir = flag.Args()[0] 78 | } 79 | 80 | func send_mail(ss *SentStatus, file string, from string, to string, msg *[]byte) { 81 | log.Println(file,"sending mail to", to, "attempt:", ss.Attempts) 82 | dest := []string{to} 83 | err := smtp.SendMail(config.SmtpAddr, config.SmtpAuth, from, dest, *msg) 84 | if err != nil { 85 | log.Println(file,"SMTP error, mail to", to, err) 86 | ss.Status = 0 87 | ss.Attempts++ 88 | ss.Error = err.Error() 89 | if ss.Attempts >= config.MaxAttempts { 90 | log.Println(file, "max SMTP attempts reached for",to, "... giving up") 91 | ss.Status = 2 92 | } 93 | if (ss.Attempts > 30) { 94 | ss.NextAttempt = time.Now().Add(time.Duration(30) * time.Duration(60) * time.Second) 95 | } else { 96 | ss.NextAttempt = time.Now().Add(time.Duration(ss.Attempts) * time.Duration(60) * time.Second) 97 | } 98 | return 99 | } 100 | ss.Status = 2 101 | ss.Error = "" 102 | log.Println(file, "successfully sent to", to) 103 | } 104 | 105 | func spool_flush() { 106 | num := 0 107 | now := time.Now() 108 | for key, _ := range status { 109 | for i, _ := range status[key].To { 110 | status[key].To[i].NextAttempt = now 111 | num += 1 112 | } 113 | for i, _ := range status[key].Cc { 114 | status[key].Cc[i].NextAttempt = now 115 | num += 1 116 | } 117 | for i, _ := range status[key].Bcc { 118 | status[key].Bcc[i].NextAttempt = now 119 | num += 1 120 | } 121 | } 122 | log.Printf("spool directory flushed (%d messages in the queue)", num) 123 | scan_spooldir(config.SpoolDir) 124 | } 125 | 126 | func read_json(file string) { 127 | f, err := os.Open(file) 128 | if err != nil { 129 | return 130 | } 131 | var buffer bytes.Buffer 132 | _, err = io.Copy(&buffer, f) 133 | f.Close() 134 | if err != nil { 135 | return 136 | } 137 | err = json.Unmarshal(buffer.Bytes(), &status) 138 | if err != nil { 139 | log.Println(err) 140 | } else { 141 | // reset transactions in progress 142 | for key, _ := range status { 143 | for i, ss := range status[key].To { 144 | if ss.Status == 1 { 145 | status[key].To[i].Status = 0 146 | } 147 | } 148 | for i, ss := range status[key].Cc { 149 | if ss.Status == 1 { 150 | status[key].Cc[i].Status = 0 151 | } 152 | } 153 | for i, ss := range status[key].Bcc { 154 | if ss.Status == 1 { 155 | status[key].Bcc[i].Status = 0 156 | } 157 | } 158 | } 159 | } 160 | } 161 | 162 | func write_json(file string, j []byte) { 163 | f, err := os.Create(file) 164 | 165 | if (err != nil) { 166 | log.Println(err) 167 | return 168 | } 169 | 170 | _, err = f.Write(j) 171 | if err != nil { 172 | log.Println(err) 173 | } 174 | f.Close() 175 | } 176 | 177 | func check_status() { 178 | changed := false 179 | for key, _ := range status { 180 | if _, err := os.Stat(key); os.IsNotExist(err) { 181 | delete(status, key) 182 | changed = true 183 | } 184 | } 185 | 186 | if changed == true { 187 | update_json() 188 | } 189 | } 190 | 191 | func update_json() { 192 | // update status 193 | js, err := json.MarshalIndent(status, "", "\t") 194 | if err != nil { 195 | log.Println(err) 196 | } else { 197 | // here we save the file 198 | write_json(config.JsonPath, js) 199 | } 200 | } 201 | 202 | func try_again(file string, msg *mail.Message) { 203 | 204 | update_json() 205 | 206 | in_progress := false 207 | 208 | mail_status := status[file] 209 | 210 | // rebuild message (strip Bcc) 211 | var buffer bytes.Buffer 212 | for key,_ := range msg.Header { 213 | if key == "Bcc" { 214 | continue 215 | } 216 | buffer.WriteString(key) 217 | buffer.WriteString(": ") 218 | header_line := strings.Join(msg.Header[key], ",") 219 | buffer.WriteString(header_line) 220 | buffer.WriteString("\r\n") 221 | } 222 | 223 | buffer.WriteString("\r\n") 224 | _, err := io.Copy(&buffer, msg.Body) 225 | if (err != nil) { 226 | log.Println(file,"unable to reassemble the mail message", err); 227 | return 228 | } 229 | 230 | b := buffer.Bytes() 231 | 232 | // manage To 233 | for i, send_status := range mail_status.To { 234 | s := send_status.Status 235 | switch s { 236 | case 0: 237 | in_progress = true 238 | if send_status.NextAttempt.Equal(time.Now()) == true || send_status.NextAttempt.Before(time.Now()) == true { 239 | // do not use send_status here !!! 240 | mail_status.To[i].Status = 1 241 | go send_mail(mail_status.To[i], file, mail_status.From, send_status.Address, &b) 242 | } 243 | case 1: 244 | in_progress = true 245 | } 246 | } 247 | 248 | // manage Cc 249 | for i, send_status := range mail_status.Cc { 250 | s := send_status.Status 251 | switch s { 252 | case 0: 253 | in_progress = true 254 | if send_status.NextAttempt.Equal(time.Now()) == true || send_status.NextAttempt.Before(time.Now()) == true { 255 | // do not use send_status here !!! 256 | mail_status.Cc[i].Status = 1 257 | go send_mail(mail_status.Cc[i], file, mail_status.From, send_status.Address, &b) 258 | } 259 | case 1: 260 | in_progress = true 261 | } 262 | } 263 | 264 | // manage Bcc 265 | for i, send_status := range mail_status.Bcc { 266 | s := send_status.Status 267 | switch s { 268 | case 0: 269 | in_progress = true 270 | if send_status.NextAttempt.Equal(time.Now()) == true || send_status.NextAttempt.Before(time.Now()) == true { 271 | // do not use send_status here !!! 272 | mail_status.Bcc[i].Status = 1 273 | go send_mail(mail_status.Bcc[i], file, mail_status.From, send_status.Address, &b) 274 | } 275 | case 1: 276 | in_progress = true 277 | } 278 | } 279 | 280 | 281 | 282 | if in_progress == false { 283 | // first we try to remove the file, on error we avoid to respool the file 284 | err := os.Remove(file) 285 | if err != nil { 286 | log.Println(file,"unable to remove mail file,", err) 287 | return 288 | } 289 | // ok we can now remove the item from the status 290 | delete(status, file) 291 | update_json() 292 | } 293 | } 294 | 295 | func parse_mail(file string) { 296 | f, err := os.Open(file) 297 | if err != nil { 298 | log.Println(file,"unable to open mail file,", err) 299 | return 300 | } 301 | defer f.Close() 302 | msg, err := mail.ReadMessage(f) 303 | if err != nil { 304 | log.Println(file,"unable to parse mail file,", err) 305 | return 306 | } 307 | 308 | mail_status := MailStatus{} 309 | mail_status.To = make([]*SentStatus, 0) 310 | mail_status.Cc = make([]*SentStatus, 0) 311 | mail_status.Bcc = make([]*SentStatus, 0) 312 | 313 | if _,ok := msg.Header["From"]; ok { 314 | mail_status.From = msg.Header["From"][0] 315 | } 316 | 317 | if _,ok := msg.Header["To"]; ok { 318 | to_addresses, err := msg.Header.AddressList("To") 319 | if err != nil { 320 | log.Println(file,"unable to parse mail \"To\" header,", err) 321 | return 322 | } 323 | for _,addr := range to_addresses { 324 | ss := SentStatus{Address: addr.Address, Status:0, NextAttempt: time.Now()} 325 | mail_status.To = append(mail_status.To, &ss) 326 | } 327 | } 328 | 329 | if _,ok := msg.Header["Cc"]; ok { 330 | cc_addresses, err := msg.Header.AddressList("Cc") 331 | if err != nil { 332 | log.Println(file,"unable to parse mail \"Cc\" header,", err) 333 | return 334 | } 335 | for _,addr := range cc_addresses { 336 | ss := SentStatus{Address: addr.Address, Status:0, NextAttempt: time.Now()} 337 | mail_status.Cc = append(mail_status.Cc, &ss) 338 | } 339 | } 340 | 341 | if _,ok := msg.Header["Bcc"]; ok { 342 | bcc_addresses, err := msg.Header.AddressList("Bcc") 343 | if err != nil { 344 | log.Println(file,"unable to parse mail \"Bcc\" header,", err) 345 | return 346 | } 347 | for _,addr := range bcc_addresses { 348 | ss := SentStatus{Address: addr.Address, Status:0, NextAttempt: time.Now()} 349 | mail_status.Bcc = append(mail_status.Bcc, &ss) 350 | } 351 | } 352 | 353 | // is the mail already collected ? 354 | if _,ok := status[file]; ok { 355 | try_again(file, msg) 356 | return 357 | } 358 | 359 | mail_status.Enqueued = time.Now() 360 | status[file] = mail_status 361 | try_again(file, msg) 362 | } 363 | 364 | func scan_spooldir(dir string) { 365 | d, err := ioutil.ReadDir(dir) 366 | if err != nil { 367 | log.Println("unable to access spool directory,", err) 368 | return 369 | } 370 | for _, entry := range d { 371 | if entry.IsDir() { 372 | continue 373 | } 374 | if strings.HasPrefix(entry.Name(), ".") { 375 | continue 376 | } 377 | abs,err := filepath.Abs(path.Join(config.SpoolDir, entry.Name())) 378 | if err != nil { 379 | log.Println("unable to get absolute path,", err) 380 | continue 381 | } 382 | parse_mail(path.Clean(abs)) 383 | } 384 | 385 | // cleanup the json often 386 | check_status() 387 | } 388 | 389 | func main() { 390 | parse_options() 391 | if config.PlainUser != "" { 392 | config.SmtpAuth = smtp.PlainAuth("", config.PlainUser, config.PlainPassword, strings.Split(config.SmtpAddr, ":")[0]) 393 | } else if config.MD5User != "" { 394 | config.SmtpAuth = smtp.CRAMMD5Auth(config.MD5User, config.MD5Password) 395 | } 396 | log.Printf("--- starting Spoolgore (pid: %d) on directory %s ---", os.Getpid(), config.SpoolDir) 397 | if config.JsonPath == "" { 398 | config.JsonPath = path.Join(config.SpoolDir, ".spoolgore.js") 399 | } 400 | status = make(map[string]MailStatus) 401 | read_json(config.JsonPath) 402 | timer := time.NewTimer(time.Second * time.Duration(config.Freq)) 403 | urg := make(chan os.Signal, 1) 404 | hup := make(chan os.Signal, 1) 405 | tstp := make(chan os.Signal, 1) 406 | signal.Notify(urg, syscall.SIGURG) 407 | signal.Notify(hup, syscall.SIGHUP) 408 | signal.Notify(tstp, syscall.SIGTSTP) 409 | blocked := false 410 | for { 411 | select { 412 | case <- timer.C: 413 | if blocked == false { 414 | scan_spooldir(config.SpoolDir) 415 | } 416 | timer.Reset(time.Second * time.Duration(config.Freq)) 417 | case <-urg: 418 | spool_flush() 419 | blocked = false 420 | case <-hup: 421 | read_json(config.JsonPath) 422 | blocked = false 423 | log.Println("status reloaded") 424 | case <-tstp: 425 | blocked = true 426 | log.Println("Spoolgore is suspended, send SIGHUP or SIGURG to unpause it") 427 | } 428 | } 429 | } 430 | --------------------------------------------------------------------------------