├── deploy ├── .gitignore └── db │ └── .gitignore ├── server ├── src │ ├── test │ │ ├── resources │ │ │ ├── META-INF │ │ │ │ ├── db │ │ │ │ │ └── .gitignore │ │ │ │ └── logs │ │ │ │ │ ├── arbiter │ │ │ │ │ ├── mail.log │ │ │ │ │ ├── mail.log.1 │ │ │ │ │ ├── mail.log.2 │ │ │ │ │ ├── mail.log.3 │ │ │ │ │ └── mail.log.4 │ │ │ │ │ ├── parser │ │ │ │ │ ├── tailed.log │ │ │ │ │ ├── backup-multiple-clients.log.2.gz │ │ │ │ │ └── backup-single-client.log.1 │ │ │ │ │ └── real │ │ │ │ │ ├── mail.log.2.gz │ │ │ │ │ ├── mail.log │ │ │ │ │ └── mail.log.1 │ │ │ ├── logback.xml │ │ │ └── application-test.conf │ │ └── scala │ │ │ └── com │ │ │ └── fg │ │ │ └── mail │ │ │ └── smtp │ │ │ ├── index │ │ │ ├── NoPersistence.scala │ │ │ ├── AutoCleanUpPersistence.scala │ │ │ └── IndexServiceSuite.scala │ │ │ ├── rest │ │ │ └── ControllerSuite.scala │ │ │ ├── db │ │ │ └── MapDbSuite.scala │ │ │ ├── TestSupport.scala │ │ │ ├── parser │ │ │ ├── BounceMapSuite.scala │ │ │ └── LogParserSuite.scala │ │ │ ├── tail │ │ │ └── TailSuite.scala │ │ │ ├── StressSuite.scala │ │ │ └── regexp │ │ │ └── RegexpSuite.scala │ └── main │ │ ├── resources │ │ ├── logback-production.xml │ │ ├── assembly.xml │ │ ├── bounce-regex-list.xml │ │ ├── application.conf │ │ └── control.sh │ │ └── scala │ │ └── com │ │ └── fg │ │ └── mail │ │ └── smtp │ │ ├── Agent.scala │ │ ├── util │ │ ├── CollectionImplicits.scala │ │ ├── Profilable.scala │ │ ├── Commons.scala │ │ └── ParsingUtils.scala │ │ ├── index │ │ ├── Queue.scala │ │ ├── IndexRecord.scala │ │ ├── DbManager.scala │ │ ├── Digestor.scala │ │ └── Index.scala │ │ ├── Supervisor.scala │ │ ├── stats │ │ ├── ProfilingCounter.scala │ │ └── LastIndexingStatus.scala │ │ ├── tail │ │ ├── Rotation.scala │ │ └── TailingInputStream.scala │ │ ├── rest │ │ ├── Server.scala │ │ ├── Controller.scala │ │ ├── Dispatcher.scala │ │ └── RestDSL.scala │ │ ├── notification │ │ └── MailClient.scala │ │ ├── Message.scala │ │ ├── parser │ │ └── BounceListParser.scala │ │ └── Settings.scala ├── .gitignore ├── common.properties └── common.sh ├── .gitignore ├── client ├── .gitignore ├── src │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── fg │ │ │ └── mail │ │ │ └── smtp │ │ │ └── client │ │ │ ├── request │ │ │ ├── Printable.java │ │ │ ├── filter │ │ │ │ ├── AgentUrlPath.java │ │ │ │ ├── UrlPathPart.java │ │ │ │ ├── AppendablePath.java │ │ │ │ └── Eq.java │ │ │ ├── query │ │ │ │ ├── Grouping.java │ │ │ │ ├── LastOrFirst.java │ │ │ │ ├── By.java │ │ │ │ ├── TimeConstraint.java │ │ │ │ ├── BySingle.java │ │ │ │ ├── ByTuple.java │ │ │ │ └── QueryFactory.java │ │ │ └── factory │ │ │ │ ├── RequestFactory.java │ │ │ │ ├── AgentReq.java │ │ │ │ ├── IndexFiltering.java │ │ │ │ ├── BatchAgentReq.java │ │ │ │ ├── IndexQuery.java │ │ │ │ ├── BatchReqFactory.java │ │ │ │ └── SingleReqFactory.java │ │ │ ├── JobCallback.java │ │ │ ├── ClientNotAvailableException.java │ │ │ ├── model │ │ │ ├── AgentResponse.java │ │ │ ├── ResponseStatus.java │ │ │ └── SmtpLogEntry.java │ │ │ ├── javax │ │ │ └── ClientIdMimeMessage.java │ │ │ ├── AgentUrl.java │ │ │ ├── SmtpAgentClient.java │ │ │ ├── ConnectionConfig.java │ │ │ └── JsonHttpClient.java │ └── test │ │ ├── java │ │ └── com │ │ │ └── fg │ │ │ └── mail │ │ │ └── smtp │ │ │ └── client │ │ │ └── SmtpAgentClientTest.java │ │ └── resources │ │ └── log4j.properties └── pom.xml ├── diagram.png ├── LICENSE.md ├── pom.xml └── README.md /deploy/.gitignore: -------------------------------------------------------------------------------- 1 | pda 2 | *.tar.gz -------------------------------------------------------------------------------- /deploy/db/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/db/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.ipr 3 | *.iws 4 | .idea 5 | target -------------------------------------------------------------------------------- /client/.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.ipr 3 | *.iws 4 | .idea 5 | target -------------------------------------------------------------------------------- /server/.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.ipr 3 | *.iws 4 | .idea 5 | target -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/arbiter/mail.log: -------------------------------------------------------------------------------- 1 | 10bytes--- -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/arbiter/mail.log.1: -------------------------------------------------------------------------------- 1 | 10bytes--- 2 | 11bytes--- -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/parser/tailed.log: -------------------------------------------------------------------------------- 1 | s 2 | s 3 | s 4 | s 5 | -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/arbiter/mail.log.2: -------------------------------------------------------------------------------- 1 | 10bytes--- 2 | 11bytes--- 3 | 11bytes--- -------------------------------------------------------------------------------- /diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FgForrest/Postfix-Deliverability-Analytics/HEAD/diagram.png -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/arbiter/mail.log.3: -------------------------------------------------------------------------------- 1 | 10bytes--- 2 | 11bytes--- 3 | 11bytes--- 4 | 11bytes--- -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/arbiter/mail.log.4: -------------------------------------------------------------------------------- 1 | 10bytes--- 2 | 11bytes--- 3 | 11bytes--- 4 | 11bytes--- 5 | 11bytes--- -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/real/mail.log.2.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FgForrest/Postfix-Deliverability-Analytics/HEAD/server/src/test/resources/META-INF/logs/real/mail.log.2.gz -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/parser/backup-multiple-clients.log.2.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FgForrest/Postfix-Deliverability-Analytics/HEAD/server/src/test/resources/META-INF/logs/parser/backup-multiple-clients.log.2.gz -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/Printable.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request; 2 | 3 | /** 4 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 5 | * @version $Id: 10/16/13 9:37 AM u_jli Exp $ 6 | */ 7 | public interface Printable { 8 | 9 | String print(); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/filter/AgentUrlPath.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.filter; 2 | 3 | import com.fg.mail.smtp.client.request.Printable; 4 | 5 | /** 6 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 7 | * @version $Id: 10/6/13 6:03 PM u_jli Exp $ 8 | */ 9 | public interface AgentUrlPath extends Printable { 10 | 11 | } 12 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/query/Grouping.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.query; 2 | 3 | /** 4 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 5 | * @version $Id: 10/5/13 5:20 PM u_jli Exp $ 6 | */ 7 | public interface Grouping { 8 | 9 | public static final String P_NAME = "groupBy"; 10 | 11 | By getProperty(); 12 | } 13 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/JobCallback.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client; 2 | 3 | import javax.annotation.Nullable; 4 | 5 | /** 6 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 7 | * @version $Id: 10/3/13 8:51 PM u_jli Exp $ 8 | */ 9 | public interface JobCallback { 10 | 11 | void execute(T job, @Nullable Long from, @Nullable Long to); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/query/LastOrFirst.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.query; 2 | 3 | /** 4 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 5 | * @version $Id: 10/5/13 5:18 PM u_jli Exp $ 6 | */ 7 | public interface LastOrFirst { 8 | 9 | public static final String P_NAME = "lastOrFirst"; 10 | 11 | Boolean getLastOrFirst(); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /server/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | [%-4level] %d{yyyy/MM/dd HH:mm:ss.SSS} [%.3thread] %logger{20} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/query/By.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.query; 2 | 3 | import com.fg.mail.smtp.client.request.Printable; 4 | 5 | /** 6 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 7 | * @version $Id: 10/16/13 9:10 AM u_jli Exp $ 8 | */ 9 | public interface By extends Printable { 10 | 11 | public enum Property { 12 | rcptEmail, queueId, msgId 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/index/NoPersistence.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.index 2 | 3 | import org.mapdb.DBMaker 4 | 5 | /** 6 | * 7 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 8 | * @version $Id: 10/28/13 8:17 PM u_jli Exp $ 9 | */ 10 | trait NoPersistence extends DbManager { 11 | 12 | override lazy val indexDb = DBMaker.newMemoryDB().make() 13 | override lazy val queueDb = DBMaker.newMemoryDB().make() 14 | 15 | } 16 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/ClientNotAvailableException.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client; 2 | 3 | /** 4 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 5 | * @version $Id: 7/3/13 10:59 AM u_jli Exp $ 6 | */ 7 | public class ClientNotAvailableException extends Exception { 8 | 9 | public ClientNotAvailableException(String message) { 10 | super(message); 11 | } 12 | 13 | public ClientNotAvailableException(String message, Exception e) { 14 | super(message, e); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/factory/RequestFactory.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.factory; 2 | 3 | import com.fg.mail.smtp.client.request.filter.Eq; 4 | import com.fg.mail.smtp.client.request.query.QueryFactory; 5 | 6 | /** 7 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 8 | * @version $Id: 10/9/13 8:01 PM u_jli Exp $ 9 | */ 10 | public interface RequestFactory { 11 | 12 | QueryFactory forClientId(String clientId); 13 | 14 | QueryFactory forClientIdAnd(String clientId, Eq equals); 15 | } 16 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/query/TimeConstraint.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.query; 2 | 3 | import javax.annotation.Nullable; 4 | 5 | /** 6 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 7 | * @version $Id: 10/5/13 8:58 AM u_jli Exp $ 8 | */ 9 | public interface TimeConstraint { 10 | 11 | public static final String P_NAME_FROM = "from"; 12 | public static final String P_NAME_TO = "to"; 13 | 14 | @Nullable 15 | Long getFrom(); 16 | 17 | @Nullable 18 | Long getTo(); 19 | 20 | } 21 | -------------------------------------------------------------------------------- /server/src/main/resources/logback-production.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ../logs/app-${bySecond}.log 5 | true 6 | 7 | [%-4level] %d{yyyy/MM/dd HH:mm:ss.SSS} [%.13thread] %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/query/BySingle.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.query; 2 | 3 | /** 4 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 5 | * @version $Id: 10/16/13 9:31 AM u_jli Exp $ 6 | */ 7 | public class BySingle implements By { 8 | 9 | private Property property; 10 | 11 | public BySingle(Property property) { 12 | assert property != null; 13 | this.property = property; 14 | } 15 | 16 | public String print() { 17 | return property.toString(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/filter/UrlPathPart.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.filter; 2 | 3 | /** 4 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 5 | * @version $Id: 10/6/13 4:35 PM u_jli Exp $ 6 | */ 7 | public enum UrlPathPart { 8 | 9 | clientId("agent-read"), rcptEmail("rcptEmail"), queueId("queue"), msgId("message"); 10 | 11 | private String name; 12 | 13 | UrlPathPart(String name) { 14 | this.name = name; 15 | } 16 | 17 | public String getName() { 18 | return name; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/Agent.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp 2 | 3 | import akka.actor.{Props, ActorSystem} 4 | import org.slf4j.LoggerFactory 5 | import com.fg.mail.smtp.index.DbManager 6 | 7 | /** 8 | * 9 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 10 | * @version $Id: 6/24/13 1:43 PM u_jli Exp $ 11 | */ 12 | object Agent extends App { 13 | val logger = LoggerFactory.getLogger("Agent") 14 | 15 | run() 16 | 17 | def run() = { 18 | val system = ActorSystem("agent") 19 | val o = Settings.options() 20 | system.actorOf(Props(new Supervisor(o, new DbManager(o))), "supervisor") 21 | system.awaitTermination() 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/util/CollectionImplicits.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.util 2 | 3 | /** 4 | * 5 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 6 | * @version $Id: 12/5/13 9:16 PM u_jli Exp $ 7 | */ 8 | trait CollectionImplicits { 9 | implicit class ListExtensions[K](val list: List[K]) { 10 | def copyWithout(item: K) = { 11 | val (left, right) = list span (_ != item) 12 | left ::: right.drop(1) 13 | } 14 | } 15 | 16 | implicit class MapExtensions[K, V](val map: Map[K, V]) { 17 | def updatedWith(key: K, default: V)(f: V => V) = { 18 | map.updated(key, f(map.getOrElse(key, default))) 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/query/ByTuple.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.query; 2 | 3 | /** 4 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 5 | * @version $Id: 10/16/13 9:14 AM u_jli Exp $ 6 | */ 7 | public class ByTuple implements By { 8 | 9 | private Property outer; 10 | private Property inner; 11 | 12 | public ByTuple(Property outer, Property inner) { 13 | assert inner != null; 14 | assert outer != null; 15 | this.outer = outer; 16 | this.inner = inner; 17 | } 18 | 19 | public String print() { 20 | return outer.toString() + "-and-" + inner.toString(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /server/common.properties: -------------------------------------------------------------------------------- 1 | # base directory where you cloned repository into 2 | base="/www" 3 | 4 | # scripts controls server over http which might be secured 5 | http_auth="disabledBy:default" 6 | 7 | # scripts secure-copies resources on remote server - it won't work over ssh password ! 8 | remote_user="lisak" 9 | remote_host="localhost" 10 | 11 | distribution_name="pda" 12 | distribution_path="./target/${distribution_name}.tar.gz" 13 | bounce_regex_list_path="./src/main/resources/bounce-regex-list.xml" 14 | deploy_path="${base}/Postfix-Deliverability-Analytics/deploy" 15 | deploy_conf_path="${deploy_path}/${distribution_name}/conf/" 16 | deploy_log_path="${deploy_path}/${distribution_name}/logs/" 17 | kill_timeout=10 -------------------------------------------------------------------------------- /client/src/test/java/com/fg/mail/smtp/client/SmtpAgentClientTest.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | 6 | /** 7 | * NOTE: All junit and integration client tests resides in server test suite ! These are just for playing around and bug fixing 8 | * 9 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 10 | * @version $Id: 10/2/13 11:50 AM u_jli Exp $ 11 | */ 12 | public class SmtpAgentClientTest { 13 | private static final Log log = LogFactory.getLog("SmtpAgentClientTest"); 14 | 15 | private ConnectionConfig cfg = new ConnectionConfig("host", 1523, "http-auth", 2*1000, 4*1000); 16 | private SmtpAgentClient client = new SmtpAgentClient(cfg); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/real/mail.log: -------------------------------------------------------------------------------- 1 | 2013 Jun 12 06:35:24.123 gds39d postfix/smtpd[25425]: connect from localhost[127.0.0.1] 2 | 2013 Jun 12 06:35:24.123 gds39d postfix/smtpd[25425]: disconnect from localhost[127.0.0.1] 3 | 2013 Jun 12 06:36:11.123 gds39d postfix/smtpd[25425]: connect from localhost[127.0.0.1] 4 | 2013 Jun 12 06:36:11.123 gds39d postfix/smtpd[25425]: disconnect from localhost[127.0.0.1] 5 | 2013 Jun 12 06:37:11.123 gds39d postfix/smtpd[25425]: connect from localhost[127.0.0.1] 6 | 2013 Jun 12 06:37:11.123 gds39d postfix/smtpd[25425]: disconnect from localhost[127.0.0.1] 7 | 2013 Jun 12 06:37:58.123 gds39d postfix/smtpd[25425]: connect from nagios.fg.cz[193.86.74.11] 8 | 2013 Jun 12 06:37:58.123 gds39d postfix/smtpd[25425]: disconnect from nagios.fg.cz[193.86.74.11] -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/real/mail.log.1: -------------------------------------------------------------------------------- 1 | 2013 Jun 11 06:35:24.123 gds39d postfix/smtpd[25425]: connect from localhost[127.0.0.1] 2 | 2013 Jun 11 06:35:24.123 gds39d postfix/smtpd[25425]: disconnect from localhost[127.0.0.1] 3 | 2013 Jun 11 06:36:11.123 gds39d postfix/smtpd[25425]: connect from localhost[127.0.0.1] 4 | 2013 Jun 11 06:36:11.123 gds39d postfix/smtpd[25425]: disconnect from localhost[127.0.0.1] 5 | 2013 Jun 11 06:37:11.123 gds39d postfix/smtpd[25425]: connect from localhost[127.0.0.1] 6 | 2013 Jun 11 06:37:11.123 gds39d postfix/smtpd[25425]: disconnect from localhost[127.0.0.1] 7 | 2013 Jun 11 06:37:58.123 gds39d postfix/smtpd[25425]: connect from nagios.fg.cz[193.86.74.11] 8 | 2013 Jun 11 06:37:58.123 gds39d postfix/smtpd[25425]: disconnect from nagios.fg.cz[193.86.74.11] -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/filter/AppendablePath.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.filter; 2 | 3 | /** 4 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 5 | * @version $Id: 10/5/13 6:04 PM u_jli Exp $ 6 | */ 7 | public class AppendablePath implements AgentUrlPath { 8 | 9 | protected StringBuilder path = new StringBuilder(); 10 | 11 | public AppendablePath(String path) { 12 | assert path != null; 13 | assert !path.startsWith("/"); 14 | this.path.append("/").append(path); 15 | } 16 | 17 | public AppendablePath appendSegment(String segment) { 18 | path.append("/").append(segment); 19 | return this; 20 | } 21 | 22 | public String print() { 23 | return path.toString(); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/model/AgentResponse.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.model; 2 | 3 | 4 | import com.fasterxml.jackson.annotation.JsonCreator; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | 7 | /** 8 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 9 | * @version $Id: 7/1/13 12:12 AM u_jli Exp $ 10 | */ 11 | public class AgentResponse { 12 | private T result; 13 | private ResponseStatus status; 14 | 15 | @JsonCreator 16 | public AgentResponse(@JsonProperty("result") T result, @JsonProperty("status") ResponseStatus status) { 17 | this.result = result; 18 | this.status = status; 19 | } 20 | public T getResult() { 21 | return result; 22 | } 23 | public ResponseStatus getStatus() { 24 | return status; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /client/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # use this file if you don't want to initialize Log4J during yours tests 2 | 3 | log4j.rootLogger=INFO, stdout 4 | log4j.category.com.fg=INFO, stdout 5 | log4j.category.org.springframework.jdbc.core=INFO, stdout 6 | log4j.category.com.fg.metadata=INFO, stdout 7 | log4j.category.com.fg.webapp.cps=INFO, stdout 8 | log4j.additivity.com.fg=false 9 | 10 | # stdout 11 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 12 | log4j.appender.stdout.target=System.out 13 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 14 | log4j.appender.stdout.layout.ConversionPattern=%-5p [%t][%d{ISO8601}][%c]: %m%n 15 | 16 | # stderr 17 | log4j.appender.stderr=org.apache.log4j.ConsoleAppender 18 | log4j.appender.stderr.target=System.err 19 | log4j.appender.stderr.layout=org.apache.log4j.PatternLayout 20 | log4j.appender.stderr.layout.ConversionPattern=%-5p [%t][%d{ISO8601}][%c]: %m%n 21 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Postfix Deliverability Analytics License 2 | ---- 3 | 4 | This library, Postfix Deliverability Analytics, is free software ("Licensed Software"); you can redistribute it and/or modify it under the terms of the [GNU Lesser General Public License] as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. 5 | 6 | This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; including but not limited to, the implied warranty of MERCHANTABILITY, NONINFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. 7 | 8 | You should have received a copy of the [GNU Lesser General Public License] along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 9 | 10 | [GNU Lesser General Public License]:http://www.gnu.org/licenses/lgpl-2.1.html -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/index/AutoCleanUpPersistence.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.index 2 | 3 | import org.mapdb.DBMaker 4 | import java.io.File 5 | 6 | /** 7 | * 8 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 9 | * @version $Id: 10/28/13 8:15 PM u_jli Exp $ 10 | */ 11 | trait AutoCleanUpPersistence extends DbManager { 12 | 13 | override def buildIndexDb = { 14 | val dbMaker = DBMaker 15 | .newFileDB(new File(o.dbDir + "/" + o.dbName)) 16 | .asyncWriteEnable() 17 | .mmapFileEnablePartial() 18 | .commitFileSyncDisable() 19 | .deleteFilesAfterClose() 20 | if (!o.dbAuth.isEmpty) 21 | dbMaker.encryptionEnable(o.dbAuth) 22 | dbMaker.make() 23 | } 24 | 25 | override lazy val queueDb = DBMaker.newFileDB(new File(o.dbDir + "/queue")) 26 | .commitFileSyncDisable() 27 | .deleteFilesAfterClose() 28 | .make() 29 | 30 | } 31 | -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/rest/ControllerSuite.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.rest 2 | 3 | import org.scalatest.{Matchers, FunSpec} 4 | import com.fg.mail.smtp.{ShutSystemDown, HomePage, Settings} 5 | import com.fg.mail.smtp.util.ServerInfoService 6 | 7 | 8 | /** 9 | * 10 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 11 | * @version $Id: 7/26/13 4:54 PM u_jli Exp $ 12 | */ 13 | class ControllerSuite extends FunSpec with Matchers { 14 | 15 | val options = Settings.options() 16 | 17 | val controller = new Controller(new ServerInfoService(options), options) 18 | 19 | describe("Controller should") { 20 | 21 | it("handle HomePage request") { 22 | controller.dispatch(HomePage(null), null) should not be 'empty 23 | } 24 | 25 | it("test") { 26 | ShutSystemDown("wtf") match { 27 | case ShutSystemDown(msg, ex) => 28 | ex match { 29 | case Some(e) => println("some ex") 30 | case None => println("none") 31 | } 32 | } 33 | } 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/index/IndexServiceSuite.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.index 2 | 3 | import com.fg.mail.smtp.{IndexQuery, IndexFilter, Client, TestSupport} 4 | import scala.collection.IterableView 5 | import scala.concurrent.Await 6 | import akka.pattern.ask 7 | import com.fg.mail.smtp.rest.Dispatcher 8 | 9 | /** 10 | * 11 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 12 | * @version $Id: 12/5/13 1:55 PM u_jli Exp $ 13 | */ 14 | class IndexServiceSuite extends TestSupport { 15 | 16 | val opt = loadOptions("application-test.conf").copy(httpServerStart = false) 17 | 18 | describe("test") { 19 | 20 | it("properly sorted") { 21 | val client = Client(IndexFilter("test-mail-module", None, None, None), Some(IndexQuery(None, None, None, Some(Dispatcher.groupBy_msgId)))) 22 | val result = Await.result(indexer ? client, timeout.duration).asInstanceOf[Option[Map[String, IterableView[IndexRecord, Iterable[IndexRecord]]]]].get 23 | 24 | result.keys.size should be (3) 25 | val logEntries: Iterable[IndexRecord] = result.last._2 26 | assert(logEntries.size === 7) 27 | } 28 | 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/filter/Eq.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.filter; 2 | 3 | public class Eq { 4 | 5 | private UrlPathPart property; 6 | private String value; 7 | 8 | public Eq(UrlPathPart property, String value) { 9 | this.property = property; 10 | this.value = value; 11 | } 12 | 13 | public String getPropertyName() { 14 | return property.getName(); 15 | } 16 | 17 | public String getValue() { 18 | return value; 19 | } 20 | 21 | @Override 22 | public boolean equals(Object o) { 23 | if (this == o) return true; 24 | if (o == null || getClass() != o.getClass()) return false; 25 | 26 | Eq equals = (Eq) o; 27 | 28 | if (property != equals.property) return false; 29 | if (value != null ? !value.equals(equals.value) : equals.value != null) return false; 30 | 31 | return true; 32 | } 33 | 34 | @Override 35 | public int hashCode() { 36 | int result = property != null ? property.getName().hashCode() : 0; 37 | result = 31 * result + (value != null ? value.hashCode() : 0); 38 | return result; 39 | } 40 | } -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/index/Queue.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.index 2 | 3 | import org.mapdb.DB 4 | import scala.collection.JavaConverters._ 5 | 6 | 7 | /** 8 | * It holds a lookup map[queueId, clientId] that is necessary due to the fact that a log entry doesn't contain information about clientId. 9 | * Postfix keeps a queue of messages. Message is removed from this queue if it is successfully sent, bounced or expired 10 | * 11 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 12 | * @version $Id: 10/28/13 8:24 PM u_jli Exp $ 13 | */ 14 | class Queue(val records: java.util.Map[String, QueueRecord]) { 15 | 16 | def insert(qid: String, msgIdClientId: QueueRecord): Option[QueueRecord] = Option(records.put(qid, msgIdClientId)) 17 | 18 | def lookup(qid: String): Option[QueueRecord] = Option(records.get(qid)) 19 | 20 | def invalidate(qid: String): Option[QueueRecord] = Option(records.remove(qid)) 21 | 22 | def getQueue = records.asScala.toMap 23 | } 24 | 25 | case class QueueRecord(msgId: String, cid: String, rcpt: String, hasBeenDeferred: Boolean) extends Serializable 26 | 27 | object Queue { 28 | 29 | def apply(db: DB): Queue = { 30 | new Queue(db.createHashMap("queue").makeOrGet()) 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/index/IndexRecord.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.index 2 | 3 | /** 4 | * LogEntry represents a relevant smtp log entry. All properties should be non-null except for sender and info message 5 | * 6 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 7 | * @version $Id: 6/24/13 5:46 PM u_jli Exp $ 8 | * 9 | * @param state - enum 0 = soft bounce, 1 = hard bounce, 2 = unknown bounce, 3 = OK 10 | */ 11 | case class IndexRecord( 12 | date: Long, 13 | queueId: String, 14 | msgId: String, 15 | rcptEmail: String, 16 | senderEmail: String, 17 | status: String, 18 | info: String, 19 | state: Int, 20 | stateInfo: String) extends Serializable with Comparable[IndexRecord] { 21 | 22 | def compareTo(that: IndexRecord): Int = { 23 | if (this.date > that.date) 24 | 1 25 | else if (this.date < that.date) 26 | -1 27 | else if (this == that) 28 | 0 29 | else 30 | this.toString.compareTo(that.toString) 31 | } 32 | 33 | } 34 | 35 | case class ClientIndexRecord(clientId: String, ir: IndexRecord, fromTailing: Boolean) -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/query/QueryFactory.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.query; 2 | 3 | import com.fg.mail.smtp.client.request.factory.AgentReq; 4 | import com.fg.mail.smtp.client.model.SmtpLogEntry; 5 | 6 | import javax.annotation.Nullable; 7 | import java.util.Map; 8 | import java.util.TreeSet; 9 | 10 | /** 11 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 12 | * @version $Id: 10/9/13 8:08 PM u_jli Exp $ 13 | */ 14 | public interface QueryFactory { 15 | 16 | AgentReq> forTimeConstraining(@Nullable Long from, @Nullable Long to); 17 | 18 | AgentReq forLastOrFirstConstraining(@Nullable Long from, @Nullable Long to, Boolean isLastOrFirst); 19 | 20 | AgentReq>> forGrouping(@Nullable Long from, @Nullable Long to, BySingle group); 21 | 22 | AgentReq>>> forMultipleGrouping(@Nullable Long from, @Nullable Long to, ByTuple group); 23 | 24 | AgentReq>> forConstraintMultipleGrouping(@Nullable Long from, @Nullable Long to, Boolean isLastOrFirst, ByTuple group); 25 | 26 | AgentReq> forConstrainedGrouping(@Nullable Long from, @Nullable Long to, Boolean isLastOrFirst, BySingle group); 27 | } 28 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/factory/AgentReq.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.factory; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fg.mail.smtp.client.model.AgentResponse; 5 | import com.fg.mail.smtp.client.request.filter.AgentUrlPath; 6 | import com.fg.mail.smtp.client.request.filter.AppendablePath; 7 | 8 | import javax.annotation.Nullable; 9 | 10 | /** 11 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 12 | * @version $Id: 10/5/13 8:15 PM u_jli Exp $ 13 | */ 14 | public class AgentReq extends TypeReference> { 15 | 16 | protected AgentUrlPath path; 17 | protected IndexQuery query; 18 | 19 | protected TypeReference> typeRef; 20 | 21 | protected AgentReq(AgentUrlPath path, @Nullable IndexQuery query, TypeReference> typeRef) { 22 | this.path = path; 23 | this.query = query; 24 | this.typeRef = typeRef; 25 | } 26 | 27 | public AgentReq(AppendablePath path, TypeReference> typeRef) { 28 | this(path, null, typeRef); 29 | } 30 | 31 | public AgentUrlPath getPath() { 32 | return path; 33 | } 34 | 35 | public IndexQuery getQuery() { 36 | return query; 37 | } 38 | 39 | public TypeReference> getTypeRef() { 40 | return typeRef; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/Supervisor.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp 2 | 3 | import akka.actor._ 4 | 5 | import com.fg.mail.smtp.notification.MailClient 6 | import com.fg.mail.smtp.index.{DbManager, Indexer} 7 | import com.fg.mail.smtp.stats.ProfilingCounter 8 | import akka.event.LoggingReceive 9 | 10 | 11 | /** 12 | * An akka actor that is supervising subordinate actors Indexer and Tailer. 13 | * It also starts jetty server that is to be subsequently 'joined' in Agent which puts its thread to sleep until server stops. 14 | * 15 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 16 | * @version $Id: 6/24/13 11:31 AM u_jli Exp $ 17 | */ 18 | class Supervisor(o: Options, dbManager: DbManager) extends Actor with ActorLogging { 19 | 20 | var indexer: ActorRef = _ 21 | 22 | override def preStart() { 23 | log.info(" is starting") 24 | val counter = context.actorOf(Props(new ProfilingCounter), "counter") 25 | indexer = context.actorOf(Props(new Indexer(counter, new DbManager(o), o)).withMailbox("bounded-deque-based"), "indexer") 26 | } 27 | 28 | override def postStop() { 29 | log.info(" is stopping") 30 | dbManager.close() 31 | } 32 | 33 | override def receive = LoggingReceive { 34 | 35 | case ShutSystemDown(why, ex) => 36 | MailClient.fail(why, ex, o) 37 | context.system.shutdown() 38 | 39 | case GetIndexer => 40 | sender ! indexer 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/stats/ProfilingCounter.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.stats 2 | 3 | import akka.actor.{ActorLogging, Actor} 4 | import com.fg.mail.smtp.{Request, ReqCtx, Client} 5 | 6 | /** 7 | * 8 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 9 | * @version $Id: 12/7/13 7:29 PM u_jli Exp $ 10 | */ 11 | class ProfilingCounter extends Actor with ActorLogging { 12 | 13 | /* index operations are recorded to status for having some overview */ 14 | var status: LastIndexingStatus = _ 15 | 16 | override def preStart() { 17 | log.info(" is starting") 18 | status = new LastIndexingStatus 19 | } 20 | 21 | override def postStop() { 22 | log.info(" is stopping") 23 | } 24 | 25 | override def receive = { 26 | 27 | case CountIndexedDeliveryAttemptLine(count) => 28 | status.logEntry(count) 29 | 30 | case CountIndexedCidLine(count) => 31 | status.cidLineIndexed(count) 32 | 33 | case CountIndexedMidLine(count) => 34 | status.midLineIndexed(count) 35 | 36 | case CountClientRequest(c) => 37 | status.clientRequest(c) 38 | 39 | case GetCountStatus(rc) => 40 | sender ! Some(status) 41 | } 42 | 43 | } 44 | 45 | case class CountIndexedCidLine(count: Long) 46 | case class CountRemovedLine(count: Long) 47 | case class CountClientRequest(c: Client) 48 | case class CountIndexedMidLine(count: Long) 49 | case class CountIndexedDeliveryAttemptLine(count: Long) 50 | case class GetCountStatus(ctx: ReqCtx) extends Request -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/model/ResponseStatus.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.model; 2 | 3 | 4 | import com.fasterxml.jackson.annotation.JsonCreator; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | 7 | /** 8 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 9 | * @version $Id: 7/1/13 12:12 AM u_jli Exp $ 10 | */ 11 | public class ResponseStatus { 12 | private Boolean succeeded; 13 | private String message; 14 | private Long timeStamp; 15 | private String version; 16 | private Long responseTime; 17 | 18 | @JsonCreator 19 | public ResponseStatus(@JsonProperty("succeeded") Boolean succeeded, 20 | @JsonProperty("message") String message, 21 | @JsonProperty("timeStamp") Long timeStamp, 22 | @JsonProperty("version") String version, 23 | @JsonProperty("responseTime") Long responseTime 24 | ) { 25 | this.succeeded = succeeded; 26 | this.message = message; 27 | this.timeStamp = timeStamp; 28 | } 29 | 30 | public Boolean getSucceeded() { 31 | return succeeded; 32 | } 33 | 34 | public String getMessage() { 35 | return message; 36 | } 37 | 38 | public Long getTimeStamp() { 39 | return timeStamp; 40 | } 41 | 42 | public String getVersion() { 43 | return version; 44 | } 45 | 46 | public Long getResponseTime() { 47 | return responseTime; 48 | } 49 | } -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/util/Profilable.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.util 2 | 3 | import org.slf4j.LoggerFactory 4 | import com.fg.mail.smtp.Options 5 | 6 | /** 7 | * 8 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 9 | * @version $Id: 11/11/13 9:43 PM u_jli Exp $ 10 | */ 11 | trait Profilable extends CollectionImplicits { 12 | 13 | val logger = LoggerFactory.getLogger(getClass.getName) 14 | 15 | val o: Options 16 | 17 | var overall = Map[String, (Long, Long)]() 18 | 19 | def profile[R](limit: Long, msg: String, argument: String = "profiling subject not provided")(block: => R): R = { 20 | if (!o.profilingEnabled) 21 | return block 22 | 23 | val t0 = System.nanoTime() 24 | val result = block 25 | val t1 = System.nanoTime() 26 | val tookInMicroSeconds = (t1 - t0) / 1000 27 | overall = overall.updatedWith(msg, (0, 0))(t => (t._1 + 1, t._2 + tookInMicroSeconds)) 28 | if (tookInMicroSeconds / 1000 > limit) { 29 | logger.warn(s"[PROFILE : '$msg' took $tookInMicroSeconds micro seconds] > $argument") 30 | logger.warn(getResultString) 31 | } 32 | result 33 | } 34 | 35 | def getResultString: String = { 36 | overall.foldLeft("\n") { (acc, t) => 37 | val callsCount = t._2._1 38 | val totalTime = t._2._2 39 | acc + " --- overall --- " + t._1 + " : " + (totalTime / 1000 / 1000) + " seconds in " + callsCount + " calls\n" 40 | } 41 | } 42 | 43 | def logResultString = { 44 | if (o.profilingEnabled) 45 | logger.info(getResultString) 46 | 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/db/MapDbSuite.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.db 2 | 3 | import com.fg.mail.smtp.TestSupport 4 | import scala.concurrent.duration.DurationDouble 5 | import java.io.File 6 | import java.text.SimpleDateFormat 7 | import java.util.Locale 8 | import akka.util.Timeout 9 | import org.scalatest.Ignore 10 | 11 | /** 12 | * 13 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 14 | * @version $Id: 11/9/13 5:53 PM u_jli Exp $ 15 | */ 16 | @Ignore 17 | class MapDbSuite extends TestSupport { 18 | 19 | val df = new SimpleDateFormat("mm:ss.SSS", Locale.US) 20 | def dbDir = new File(opt.dbDir) 21 | val opt = loadOptions("application.conf").copy( 22 | httpServerStart = false 23 | ) 24 | 25 | /* 26 | it("whatever whatever") { 27 | 28 | def s(index: Index) = { 29 | (0 to 10000000).foreach{ x => 30 | val i: Long = x + 100000000000L 31 | index.addRecord("cezdirectmail", new IndexRecord(i, "3dFjk95XgCz2plD", "<1995936274.3901383825361777.JavaMail.p_prj@gds39k>", "zp.stava@cez.cz", "", "sent", "(250 2.0.0 Mail 899273466 queued for delivery in session 7ea30000021e.),3,OK)", 1, "Lsoft"), true) 32 | if (i%100000==0) { 33 | println(df.format(new Date(System.currentTimeMillis()))) 34 | index.commit() 35 | } 36 | } 37 | } 38 | 39 | val index = provideIndex 40 | val now = System.currentTimeMillis() 41 | 42 | s(index) 43 | 44 | val then = System.currentTimeMillis() 45 | val thenDiff = then - now 46 | println(thenDiff) 47 | 48 | } 49 | */ 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/factory/IndexFiltering.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.factory; 2 | 3 | import com.fg.mail.smtp.client.request.filter.AgentUrlPath; 4 | import com.fg.mail.smtp.client.request.filter.Eq; 5 | import com.fg.mail.smtp.client.request.filter.UrlPathPart; 6 | 7 | import java.util.Collection; 8 | import java.util.Iterator; 9 | import java.util.LinkedHashSet; 10 | 11 | /** 12 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 13 | * @version $Id: 10/6/13 1:37 PM u_jli Exp $ 14 | */ 15 | public class IndexFiltering implements AgentUrlPath { 16 | 17 | private Collection filtering = new LinkedHashSet(); 18 | 19 | protected IndexFiltering(String clientId) { 20 | this.filtering.add(new Eq(UrlPathPart.clientId, clientId)); 21 | } 22 | 23 | protected IndexFiltering(String clientId, Eq filtering) { 24 | this(clientId); 25 | this.filtering.add(filtering); 26 | } 27 | 28 | public String print() { 29 | return buildPath(filtering.iterator(), new StringBuilder("/")); 30 | } 31 | 32 | private String buildPath(Iterator itr, StringBuilder result) { 33 | if (itr.hasNext()) { 34 | Eq eq = itr.next(); 35 | result.append(eq.getPropertyName()).append("/").append(eq.getValue()); 36 | if (itr.hasNext()) { 37 | result.append("/"); 38 | } 39 | return buildPath(itr, result); 40 | } else { 41 | return result.toString(); 42 | } 43 | } 44 | 45 | public String toString() { 46 | return print(); 47 | } 48 | } -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/index/DbManager.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.index 2 | 3 | import org.mapdb._ 4 | import java.io.File 5 | import com.fg.mail.smtp.Options 6 | import org.slf4j.LoggerFactory 7 | import com.fg.mail.smtp.util.Profilable 8 | 9 | /** 10 | * 11 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 12 | * @version $Id: 10/28/13 7:35 PM u_jli Exp $ 13 | */ 14 | class DbManager(val o: Options) extends Profilable { 15 | val log = LoggerFactory.getLogger(getClass.getName) 16 | 17 | lazy val indexDb: DB = buildIndexDb 18 | 19 | lazy val queueDb = DBMaker.newFileDB(new File(o.dbDir + "/queue")) 20 | .commitFileSyncDisable() 21 | .closeOnJvmShutdown() 22 | .make() 23 | 24 | def buildIndexDb = { 25 | val dbMaker = DBMaker 26 | .newFileDB(new File(o.dbDir + "/" + o.dbName)) 27 | .mmapFileEnablePartial() 28 | .commitFileSyncDisable() 29 | .asyncWriteFlushDelay(5000) 30 | .closeOnJvmShutdown() 31 | .freeSpaceReclaimQ(3) 32 | if (!o.dbAuth.isEmpty) 33 | dbMaker.encryptionEnable(o.dbAuth) 34 | dbMaker.make() 35 | } 36 | 37 | def commit() = profile(700, "Committing transaction") { 38 | indexDb.commit() 39 | queueDb.commit() 40 | } 41 | 42 | def close() { 43 | log.info("Closing database") 44 | indexDb.close() 45 | queueDb.close() 46 | log.info("Database closed") 47 | } 48 | 49 | def getPhysicalSize: Long = profile(900, "Measuring db size") { 50 | 0 //TODO 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/javax/ClientIdMimeMessage.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.javax; 2 | 3 | import javax.mail.MessagingException; 4 | import javax.mail.Session; 5 | import javax.mail.internet.MimeMessage; 6 | import java.security.SecureRandom; 7 | import java.util.Random; 8 | 9 | /** 10 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2014 11 | * @version $Id: 3/23/14 11:16 PM u_jli Exp $ 12 | */ 13 | public class ClientIdMimeMessage extends MimeMessage { 14 | private static final Random RND = new SecureRandom(); 15 | private static int ID; 16 | private final Integer id; 17 | private final String clientId; 18 | 19 | public static final String clientIdHashCode = String.valueOf("cid".hashCode()); // to recognize message-id with client-id information 20 | 21 | public ClientIdMimeMessage(Session session, String clientId, Integer id) { 22 | super(session); 23 | this.clientId = clientId; 24 | this.id = id; 25 | } 26 | 27 | @Override 28 | protected void updateMessageID() throws MessagingException { 29 | String id = this.id == null ? String.valueOf(ID++) : String.valueOf(this.id); 30 | setHeader("Message-ID", getUniqueMessageID(id, clientId)); 31 | } 32 | 33 | /** 34 | * a newsletter client complained about hostname and user presence in message-id, now there will be only 'clientId' in message-id 35 | * clientIdHashCode.id.random@clientId 36 | * clientIdHashCode - hashCode of 'cid' string - to recognize a new message-id format 37 | * id - mail message id 38 | * random - enforcing 100% uniqueness 39 | */ 40 | public static String getUniqueMessageID(String id, String clientId) { 41 | return "<" + clientIdHashCode + '.' + id + '.' + RND.nextInt(1000) + '@' + clientId + ">"; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/index/Digestor.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.index 2 | 3 | import java.util.NavigableSet 4 | import org.mapdb.{DB, BTreeKeySerializer} 5 | import scala.collection.JavaConverters._ 6 | import com.fg.mail.smtp.Options 7 | import java.io.File 8 | import com.fg.mail.smtp.util.{Profilable, Commons} 9 | import org.slf4j.LoggerFactory 10 | 11 | /** 12 | * 13 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 14 | * @version $Id: 10/29/13 5:53 PM u_jli Exp $ 15 | */ 16 | class Digestor(digests: NavigableSet[String], val o: Options) extends Profilable { 17 | 18 | val log = LoggerFactory.getLogger(getClass) 19 | 20 | def getDigests: collection.mutable.Set[String] = digests.asScala 21 | 22 | def getDigestedFiles: Array[File] = { 23 | profile(200, "Getting digested files") { 24 | new File(o.logDir).listFiles() 25 | .filterNot(_.isDirectory) 26 | .filterNot(_.length() < 10) 27 | .filter( (x: File) => o.rotatedPatternFn(x.getName) ) 28 | .filter( f => digests.contains(Commons.digestFirstLine(f))) 29 | .sorted( Ordering.by((_: File).getName) ) 30 | } 31 | } 32 | 33 | def store(file: File) { 34 | if (file.length() > 10) { 35 | store(Commons.digestFirstLine(file), file.getName) 36 | } else { 37 | log.warn(s"Digest of file ${file.getName} won't be stored because file is empty !") 38 | } 39 | } 40 | 41 | def store(md5: String, fileName: String) { 42 | if (digests.add(md5)) 43 | log.info(s"md5 digest for file $fileName successfully persisted") 44 | else 45 | log.warn(s"md5 digest for file $fileName already existed !") 46 | } 47 | } 48 | 49 | object Digestor { 50 | 51 | def apply(db: DB, name: String, o: Options): Digestor = { 52 | new Digestor(db.createTreeSet(name).serializer(BTreeKeySerializer.STRING).makeOrGet(), o) 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/AgentUrl.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client; 2 | 3 | import com.fg.mail.smtp.client.request.factory.AgentReq; 4 | import com.fg.mail.smtp.client.request.factory.IndexQuery; 5 | import com.fg.mail.smtp.client.request.filter.AgentUrlPath; 6 | 7 | import java.net.MalformedURLException; 8 | import java.net.URL; 9 | 10 | /** 11 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 12 | * @version $Id: 10/9/13 10:00 AM u_jli Exp $ 13 | */ 14 | public class AgentUrl { 15 | 16 | private String host; 17 | private int port; 18 | private AgentReq request; 19 | 20 | public AgentUrl(String host, int port, AgentReq request) { 21 | assert host != null; 22 | assert !host.endsWith("/"); 23 | this.host = host; 24 | this.port = port; 25 | this.request = request; 26 | } 27 | 28 | public String getHost() { 29 | return host; 30 | } 31 | 32 | public int getPort() { 33 | return port; 34 | } 35 | 36 | public AgentReq getRequest() { 37 | return request; 38 | } 39 | 40 | public URL getURL() { 41 | String url = print(); 42 | try { 43 | return new URL(url); 44 | } catch (MalformedURLException e) { 45 | throw new IllegalArgumentException("Url : " + url + " is not valid, please check your configuration !", e); 46 | } 47 | } 48 | 49 | public String print() { 50 | StringBuilder base = new StringBuilder("http://").append(host).append(":").append(port); 51 | AgentUrlPath path = request.getPath(); 52 | if (path != null) { 53 | base.append(path.print()); 54 | IndexQuery query = request.getQuery(); 55 | if (query != null) { 56 | base.append(query.print()); 57 | } 58 | } 59 | return base.toString(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /server/src/main/resources/assembly.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | bin 7 | 8 | tar.gz 9 | 10 | 11 | 12 | target/appassembler/repo 13 | repo 14 | 15 | 16 | target/appassembler/bin 17 | bin 18 | 0755 19 | 20 | 21 | start.sh.bat 22 | 23 | 24 | 25 | 26 | src/main/java 27 | logs 28 | 29 | */** 30 | 31 | 32 | 33 | 34 | 35 | src/main/resources/application.conf 36 | conf 37 | 38 | 39 | src/main/resources/bounce-regex-list.xml 40 | conf 41 | 42 | 43 | src/main/resources/control.sh 44 | 0755 45 | bin 46 | 47 | 48 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/tail/Rotation.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.tail 2 | 3 | import com.fg.mail.smtp.Options 4 | import java.io.{InputStream, File} 5 | import org.slf4j.LoggerFactory 6 | 7 | /** 8 | * Creates infinite enumeration of input streams from the same underlying path to a file that may be rotated 9 | * 10 | * @param file File handle to the log file 11 | * @param firstEOF possibility to execute this method when hitting EndOfFile 12 | * @param fileRotation possibility to execute this method when file is rotated 13 | * 14 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 15 | * @version $Id: 10/20/13 10:33 AM u_jli Exp $ 16 | */ 17 | class Rotation(o: Options, file: File)(firstEOF: () => Unit, fileRotation: () => Unit) extends java.util.Enumeration[InputStream] { 18 | val log = LoggerFactory.getLogger(getClass.getName) 19 | 20 | var firstIterationOver = false 21 | 22 | /** 23 | * reOpenTries - how many times to try to re-open the file - reasonable default might be 3 24 | * reOpenSleep - to sleep between re-open retries - reasonable default might be 1 second 25 | */ 26 | def hasMoreElements = { 27 | if (testExists()) { 28 | true 29 | } else { 30 | log.warn(s"Tailing is about to stop, file was not found in ${o.reOpenTries} attempts") 31 | false 32 | } 33 | } 34 | 35 | /** 36 | * eofWaitNewInput - to be called when the stream walked to the end of the file and need to wait for some more input - reasonable default might be 1 second 37 | */ 38 | def nextElement = { 39 | val result = new TailingInputStream(file, o.eofNewInputSleep, firstEOF, firstIterationOver) 40 | if (firstIterationOver) { 41 | log.info("Creating TailingInputStream for new file, current file is rotated") 42 | fileRotation() 43 | } else { 44 | log.info("Creating TailingInputStream for current file") 45 | firstIterationOver = true 46 | } 47 | result 48 | } 49 | 50 | /** 51 | * Test file existence N times, wait between retries 52 | * 53 | * @return true on success 54 | */ 55 | def testExists(): Boolean = { 56 | def tryExists(n: Int): Boolean = 57 | if (file.exists) 58 | true 59 | else if (n > o.reOpenTries) { 60 | false 61 | } else { 62 | o.reOpenSleep() 63 | tryExists(n+1) 64 | } 65 | 66 | tryExists(1) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/rest/Server.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.rest 2 | 3 | import com.fg.mail.smtp.{StartHttpServer, Options} 4 | import akka.actor.{ActorRef, ActorLogging, Actor} 5 | import scala.util.control.Exception._ 6 | import java.net.{InetSocketAddress, BindException} 7 | import com.sun.net.httpserver.{BasicAuthenticator, HttpServer} 8 | import com.fg.mail.smtp.notification.MailClient 9 | import java.util.concurrent.Executors 10 | 11 | /** 12 | * 13 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 14 | * @version $Id: 10/13/13 5:47 PM u_jli Exp $ 15 | */ 16 | class Server(indexer: ActorRef, counter: ActorRef, o: Options) extends Actor with ActorLogging { 17 | 18 | val executorService = Executors.newFixedThreadPool(1) 19 | var server: HttpServer = _ 20 | 21 | override def preStart() { 22 | catching(classOf[BindException]) 23 | .either(HttpServer.create(new InetSocketAddress(o.httpServerPort), 0)) match { 24 | case Left(e) => 25 | MailClient.fail("Smtp agent server probably started more than once, please stop currently running instance first. Shutting down !", Option(e), o) 26 | context.system.shutdown() 27 | case Right(s) => 28 | s.setExecutor(executorService) 29 | val ctx = s.createContext("/", new Handler(indexer, counter, o)) 30 | if (!o.httpServerAuth.isEmpty){ 31 | ctx.setAuthenticator(new SmtpAgentBasicHttpAuthenticator(o)) 32 | } 33 | server = s 34 | } 35 | } 36 | 37 | override def postStop() { 38 | try { 39 | log.info("stopping server") 40 | server.stop(1) 41 | executorService.shutdown() 42 | server = null 43 | } catch { 44 | case e: Throwable => log.warning("Failed to stop http server", e) 45 | } 46 | } 47 | 48 | override def receive = { 49 | case StartHttpServer => 50 | catching(classOf[IllegalStateException]) 51 | .either(server.start()) match { 52 | case Left(e) => log.info(s"Http server runs on port : ${o.httpServerPort}") 53 | case _ => log.info(s"Starting http server on port : ${o.httpServerPort}") 54 | } 55 | } 56 | 57 | class SmtpAgentBasicHttpAuthenticator(o: Options) extends BasicAuthenticator("smtp-agent") { 58 | def checkCredentials(user: String, pwd: String): Boolean = if (o.httpServerAuth.isEmpty) true else o.httpServerAuth.exists(_ == user + ":" + pwd) 59 | } 60 | } 61 | 62 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/tail/TailingInputStream.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.tail 2 | 3 | import java.io.{FileInputStream, InputStream, File} 4 | import org.slf4j.LoggerFactory 5 | import scala.annotation.tailrec 6 | 7 | /** 8 | * InputStream that handles log rotation. It will not return -1 on EOF. Instead it waits and continues reading. 9 | * When file is being rotated it behaves just if it found EOF and it returns -1. It is used together with SequenceInputStream 10 | * that is supplied new TailingInputStream (with the new file of the same name) when the old returns -1 (file rotated) 11 | * 12 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 13 | * @version $Id: 10/20/13 10:34 AM u_jli Exp $ 14 | */ 15 | class TailingInputStream(val file: File, val eofWaitForNewInput: () => Unit, val firstEOF: () => Unit, var firstEofReached: Boolean = false) extends InputStream { 16 | val log = LoggerFactory.getLogger(getClass.getName) 17 | 18 | require(file != null) 19 | assume(file.exists, "Attempt to tail a file that doesn't exists, somebody probably moved/removed postfix log file") 20 | 21 | private val inputStream = new FileInputStream(file) 22 | 23 | def read: Int = handle(inputStream.read) 24 | 25 | override def read(b: Array[Byte]): Int = read(b, 0, b.length) 26 | 27 | override def read(b: Array[Byte], off: Int, len: Int): Int = handle(inputStream.read(b, off, len)) 28 | 29 | override def close() { 30 | log.warn("Closing input stream of tailed file") 31 | inputStream.close() 32 | } 33 | 34 | /* NOTE !!! rotating a file with just one line and immediate writing to the new one makes this method return false !!! */ 35 | protected def rotatedOrClosed_? = inputStream.getChannel.position > file.length || !inputStream.getChannel.isOpen 36 | 37 | @tailrec 38 | private def handle(read: => Int): Int = read match { 39 | case -1 if rotatedOrClosed_? => 40 | log.info(s"File ${file.getName} was rotated or closed") 41 | -1 42 | case i if i > -1 => i 43 | case -1 => // when reading a big file I'm getting -1 return value several times, which leads to premature firstEOF ! 44 | try { 45 | eofWaitForNewInput() 46 | } catch { 47 | case ie: InterruptedException => 48 | log.warn("Tailing interrupted while sleeping and waiting for new input !") 49 | close() 50 | } 51 | if (!firstEofReached) { 52 | firstEOF() 53 | firstEofReached = true 54 | } 55 | handle(read) 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/notification/MailClient.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.notification 2 | 3 | import javax.mail._ 4 | import java.util.Properties 5 | import javax.mail.internet.InternetAddress 6 | import com.sun.mail.smtp.SMTPMessage 7 | import javax.mail.Message.RecipientType.TO 8 | import org.slf4j.LoggerFactory 9 | import scala.util.control.Exception._ 10 | import java.io.{PrintWriter, StringWriter} 11 | import com.fg.mail.smtp.Options 12 | 13 | /** 14 | * Serves only for notification emails when something goes wrong 15 | * 16 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 17 | * @version $Id: 6/24/13 2:36 PM u_jli Exp $ 18 | */ 19 | object MailClient { 20 | val log = LoggerFactory.getLogger("MailClient") 21 | 22 | val smtpHost = "localhost" 23 | val smtpPort = "25" 24 | 25 | val props = new Properties() 26 | props.put("mail.smtp.host", smtpHost) 27 | props.put("mail.smtp.port", smtpPort) 28 | 29 | def send(subject: String, text: String, recipients: List[String]) { 30 | log.warn(s"Mailing notification to $recipients\n subject: $subject\n text: $text") 31 | 32 | this.synchronized { 33 | recipients.foreach( rcpt => { 34 | val message = new SMTPMessage(Session.getInstance(props)) 35 | message.setFrom(new InternetAddress("liska@fg.cz")) 36 | message.setRecipients(TO, rcpt) 37 | message.setSubject(subject) 38 | message.setText(text) 39 | catching(classOf[Throwable]).either(Transport.send(message)) match { 40 | case Left(e) => log.warn(s"Unable to connect to mail server : $smtpHost:$smtpPort, notification not sent", e) 41 | case _ => log.info("Notification sent") 42 | } 43 | } 44 | ) 45 | } 46 | } 47 | 48 | def fail(problem: String, e: Option[Throwable], o: Options, status: String = "") { 49 | val stackTrace = { 50 | e match { 51 | case Some(ex) => 52 | val sw = new StringWriter() 53 | ex.printStackTrace(new PrintWriter(sw)) 54 | sw.toString 55 | case _ => "" 56 | } 57 | } 58 | val msg = s"host: ${o.hostName} - $problem ${e.fold(" : ")(_.getMessage)}'" 59 | log.warn(s"$msg \n $stackTrace") 60 | if (System.getProperty("agentProductionMode") ne null) 61 | send(msg, s"$status \n $stackTrace", o.notifRcpts.toList) 62 | } 63 | 64 | def info(problem: String, o: Options, status: String = "") { 65 | log.warn(problem) 66 | if (System.getProperty("agentProductionMode") ne null) 67 | send(s"host: ${o.hostName} - $problem", status, o.notifRcpts.toList) 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | parent 4 | com.fg.pda 5 | pom 6 | 1.1.0-SNAPSHOT 7 | Postfix Deliverability Analytics Suite 8 | 9 | client 10 | server 11 | 12 | 13 | 14 | 15 | liska.jakub@gmail.com 16 | Jakub Liška 17 | FG Forrest, a.s. 18 | http://www.fg.cz 19 | 20 | 21 | 22 | 23 | scm:git:git@github.com:FgForrest/Postfix-Deliverability-Analytics.git 24 | scm:git:git@github.com:FgForrest/Postfix-Deliverability-Analytics.git 25 | scm:git:git@github.com:FgForrest/Postfix-Deliverability-Analytics.git 26 | HEAD 27 | 28 | 29 | 30 | 31 | fg-central 32 | FG Central repository 33 | https://nexus.fg.cz/content/repositories/internal 34 | 35 | 36 | fg-central 37 | FG Central repository 38 | https://nexus.fg.cz/content/repositories/snapshots 39 | 40 | 41 | 42 | 43 | 44 | typesafe 45 | typesafe 46 | http://repo.typesafe.com/typesafe/releases 47 | 48 | 49 | oss-sonatype 50 | oss-sonatype 51 | https://oss.sonatype.org/content/repositories/snapshots/ 52 | 53 | true 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | org.apache.maven.plugins 62 | maven-release-plugin 63 | 2.5 64 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/factory/BatchAgentReq.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.factory; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fg.mail.smtp.client.ClientNotAvailableException; 5 | import com.fg.mail.smtp.client.model.AgentResponse; 6 | import com.fg.mail.smtp.client.request.filter.AgentUrlPath; 7 | 8 | import javax.annotation.Nullable; 9 | import java.util.*; 10 | import java.util.concurrent.TimeUnit; 11 | 12 | /** 13 | * Distribution of a batch into 2 or more jobs. 14 | * It is defined by a time interval that splits batch into two or more jobs by creating an interval from timeframe. 15 | * Interval can be right-open if upper limit is not provided, last element is null in that case. 16 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 17 | * @version $Id: 10/9/13 9:01 PM u_jli Exp $ 18 | */ 19 | public class BatchAgentReq extends AgentReq { 20 | 21 | private final Collection times; 22 | 23 | public BatchAgentReq(@Nullable AgentUrlPath path, IndexQuery query, long batchTimeframe, TimeUnit batchTimeframeUnit, TypeReference> typeRef) { 24 | super(path, query, typeRef); 25 | assert query != null; 26 | assert query.getFrom() != null; 27 | assert batchTimeframeUnit != null; 28 | this.times = computeInterval( 29 | query.getFrom(), 30 | query.getTo() == null ? System.currentTimeMillis() : query.getTo(), 31 | batchTimeframeUnit.toMillis(batchTimeframe) 32 | ); 33 | assert times.size() > 0; 34 | } 35 | 36 | /** 37 | * interval of the least possible size of 2, if the last element is null, the interval is right-open 38 | */ 39 | private Collection computeInterval(long current, long to, long batchTimeframe) { 40 | List interval = new LinkedList(); 41 | while (current < to) { 42 | interval.add(current); 43 | current += batchTimeframe; 44 | } 45 | interval.add(to); 46 | return interval; 47 | } 48 | 49 | public Collection> getRequests() throws ClientNotAvailableException { 50 | ArrayList> result = new ArrayList>(); 51 | Iterator iterator = times.iterator(); 52 | 53 | if (!iterator.hasNext()) { 54 | throw new IllegalStateException("Please fix computeInterval method, it should never return empty collection !"); 55 | } 56 | Long from = iterator.next(); 57 | while (iterator.hasNext()) { 58 | Long to = iterator.next(); 59 | result.add(new AgentReq(path, query.copy(from, to), typeRef)); 60 | from = to; 61 | } 62 | return result; 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/factory/IndexQuery.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.factory; 2 | 3 | import com.fg.mail.smtp.client.request.Printable; 4 | import com.fg.mail.smtp.client.request.query.By; 5 | import com.fg.mail.smtp.client.request.query.Grouping; 6 | import com.fg.mail.smtp.client.request.query.LastOrFirst; 7 | import com.fg.mail.smtp.client.request.query.TimeConstraint; 8 | import com.google.common.collect.Lists; 9 | 10 | import javax.annotation.Nullable; 11 | import java.util.ArrayList; 12 | import java.util.Iterator; 13 | import java.util.LinkedHashMap; 14 | import java.util.Map; 15 | 16 | /** 17 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 18 | * @version $Id: 10/5/13 8:38 AM u_jli Exp $ 19 | */ 20 | public class IndexQuery implements TimeConstraint, Grouping, LastOrFirst, Printable { 21 | 22 | protected Long from; 23 | protected Long to; 24 | protected Boolean lastOrFirst; 25 | protected By property; 26 | 27 | protected IndexQuery(@Nullable Long from, @Nullable Long to, @Nullable Boolean lastOrFirst, @Nullable By property) { 28 | this.from = from; 29 | this.to = to; 30 | this.lastOrFirst = lastOrFirst; 31 | this.property = property; 32 | } 33 | 34 | public Long getFrom() { 35 | return from; 36 | } 37 | 38 | public Long getTo() { 39 | return to; 40 | } 41 | 42 | public Boolean getLastOrFirst() { 43 | return lastOrFirst; 44 | } 45 | 46 | public By getProperty() { 47 | return property; 48 | } 49 | 50 | public IndexQuery copy(@Nullable Long from, @Nullable Long to) { 51 | return new IndexQuery(from, to, lastOrFirst, property); 52 | } 53 | 54 | public String print() { 55 | Map m = new LinkedHashMap(); 56 | m.put(TimeConstraint.P_NAME_FROM, from == null ? null : String.valueOf(from)); 57 | m.put(TimeConstraint.P_NAME_TO, to == null ? null : String.valueOf(to)); 58 | m.put(Grouping.P_NAME, property == null ? null : property.print()); 59 | m.put(LastOrFirst.P_NAME, lastOrFirst == null ? null : lastOrFirst.toString()); 60 | return buildQueryString(Lists.reverse(new ArrayList>(m.entrySet())).iterator(), ""); 61 | } 62 | 63 | private String buildQueryString(Iterator> itr, String result) { 64 | if (itr.hasNext()) { 65 | Map.Entry entry = itr.next(); 66 | String value = entry.getValue(); 67 | if (value != null) { 68 | if (result.length() > 0 && !result.startsWith("&")) { 69 | result = "&" + result; 70 | } 71 | result = entry.getKey() + "=" + value + result; 72 | } 73 | return buildQueryString(itr, result); 74 | } else { 75 | return result.equals("") ? "" : "?" + result; 76 | } 77 | } 78 | 79 | public String toString() { 80 | return print(); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/TestSupport.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp 2 | 3 | import java.text.SimpleDateFormat 4 | import java.util.{Date, Locale} 5 | import com.typesafe.config.ConfigFactory 6 | import java.io.File 7 | import akka.actor.{ActorRef, Props, ActorSystem} 8 | import akka.testkit.{ImplicitSender, TestKit} 9 | import org.scalatest.{Matchers, BeforeAndAfterAll, FunSpecLike} 10 | import scala.concurrent.Await 11 | import akka.pattern.ask 12 | import com.fg.mail.smtp.index.{DbManager, IndexRecord, AutoCleanUpPersistence} 13 | import com.fg.mail.smtp.client.model.SmtpLogEntry 14 | 15 | 16 | /** 17 | * 18 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 19 | * @version $Id: 6/24/13 5:13 PM u_jli Exp $ 20 | */ 21 | abstract class TestSupport(_system: ActorSystem) extends TestKit(_system) with ImplicitSender with FunSpecLike with Matchers with BeforeAndAfterAll { 22 | 23 | def this() = this(ActorSystem("agent")) 24 | 25 | val opt: Options 26 | var supervisor: ActorRef = _ 27 | var indexer: ActorRef = _ 28 | var counter: ActorRef = _ 29 | var tailer: ActorRef = _ 30 | 31 | lazy implicit val timeout = opt.askTimeout 32 | lazy implicit val rc = new ReqCtx(Map[String, String]("client-version" -> "123", "User-Agent" -> "test")) 33 | 34 | def providePersistence: DbManager = new DbManager(opt) with AutoCleanUpPersistence 35 | 36 | override def beforeAll() { 37 | supervisor = system.actorOf(Props(new Supervisor(opt, providePersistence)), "supervisor") 38 | indexer = Await.result(supervisor ? GetIndexer, timeout.duration).asInstanceOf[ActorRef] 39 | counter = Await.result(indexer ? GetCouter, timeout.duration).asInstanceOf[ActorRef] 40 | tailer = Await.result(indexer ? GetTailer, timeout.duration).asInstanceOf[ActorRef] 41 | val index = Await.result(indexer ? GetDisposableRecordsByClientId(rc), timeout.duration) 42 | assert(index != null) 43 | } 44 | 45 | override def afterAll() { 46 | TestKit.shutdownActorSystem(system) 47 | } 48 | 49 | def entriesShouldNotContainSentOnes(col: Iterable[SmtpLogEntry]) = col filter (_.getState == 3) should be('empty) 50 | def recordsShouldNotContainSentOnes(col: Iterable[IndexRecord]) = col filter (_.state == 3) should be('empty) 51 | 52 | val cl = getClass.getClassLoader 53 | cl.loadClass("org.slf4j.LoggerFactory").getMethod("getLogger",cl.loadClass("java.lang.String")).invoke(null,"ROOT") 54 | 55 | val testLogDir = "src/test/resources/META-INF/logs/" 56 | 57 | def sleepFor(t: Long) { Thread.sleep(t) } 58 | def sleep(ms: Long) = () => Thread.sleep(ms) 59 | 60 | def toDate(d: String): Date = { new SimpleDateFormat("yyyy MMM dd HH:mm:ss.SSS", Locale.US).parse(d) } 61 | def fromDate(d: Date): String = { new SimpleDateFormat("yyyy MMM dd HH:mm:ss.SSS", Locale.US).format(d) } 62 | 63 | def loadOptions(fileName: String): Options = 64 | Settings.buildOptions( 65 | ConfigFactory.parseFile( 66 | Option(getClass.getClassLoader.getResource(fileName)).fold(new File("../conf/" + fileName))(url => new File(url.toURI)) 67 | ).resolve() 68 | ) 69 | } 70 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/SmtpAgentClient.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client; 2 | 3 | import com.fg.mail.smtp.client.model.SmtpLogEntry; 4 | import com.fg.mail.smtp.client.request.factory.AgentReq; 5 | import com.fg.mail.smtp.client.request.factory.BatchAgentReq; 6 | import com.fg.mail.smtp.client.request.factory.SingleReqFactory; 7 | 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | /** 12 | * Http client for accessing restful interface of Smtp Agent 13 | * 14 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 15 | * @version $Id: 6/30/13 11:15 AM u_jli Exp $ 16 | */ 17 | public class SmtpAgentClient { 18 | 19 | private JsonHttpClient jsonHttpClient; 20 | private SingleReqFactory reqFactory = new SingleReqFactory(); 21 | 22 | public SmtpAgentClient(ConnectionConfig conCfg) { 23 | this.jsonHttpClient = new JsonHttpClient(conCfg); 24 | } 25 | 26 | public JsonHttpClient getJsonHttpClient() { 27 | return jsonHttpClient; 28 | } 29 | 30 | /** 31 | * @return map of recipient email address counts by client id 32 | */ 33 | public String shutAgentDown() throws ClientNotAvailableException { 34 | return jsonHttpClient.resolveResult(reqFactory.forAgentShutdown()); 35 | } 36 | 37 | /** 38 | * @return map of recipient email address counts by client id 39 | */ 40 | public Map pullRcptAddressCounts() throws ClientNotAvailableException { 41 | return jsonHttpClient.resolveResult(reqFactory.forRcptAddressCounts()); 42 | } 43 | 44 | /** 45 | * @return map of recipient email addresses by client id 46 | */ 47 | public Map> pullRcptAddresses() throws ClientNotAvailableException { 48 | return jsonHttpClient.resolveResult(reqFactory.forRcptAddresses()); 49 | } 50 | 51 | /** 52 | * @return map of unknown bounces by client id 53 | */ 54 | public Map> pullUnknownBounces() throws ClientNotAvailableException { 55 | return jsonHttpClient.resolveResult(reqFactory.forUnknownBounces()); 56 | } 57 | 58 | /** 59 | * @param request with filter (narrow down the result set by mandatory clientId and optional rcptEmail, queueId or msgId) and query (result can be grouped and constrained by time and last entry) 60 | * @return all log entries for client id sorted by date and possibly grouped by a property or two and constrained by time or chronologically last log entry 61 | */ 62 | public T pull(AgentReq request) throws ClientNotAvailableException { 63 | return jsonHttpClient.resolveResult(request); 64 | } 65 | 66 | /** 67 | * @param request with filter (narrow down the result set by mandatory clientId and optional rcptEmail, queueId or msgId) and query (result can be grouped and constrained by time and last entry) 68 | * @param callback to be supplied with 2 - x jobs from a batch based on provided timeframe 69 | */ 70 | public void pull(BatchAgentReq request, JobCallback callback) throws ClientNotAvailableException { 71 | jsonHttpClient.processBatch(request, callback); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/ConnectionConfig.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client; 2 | 3 | import org.apache.commons.codec.binary.Base64; 4 | 5 | import java.net.URLConnection; 6 | import java.util.LinkedHashMap; 7 | import java.util.Map; 8 | 9 | /** 10 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 11 | * @version $Id: 10/17/13 9:04 AM u_jli Exp $ 12 | */ 13 | public class ConnectionConfig { 14 | 15 | public static final String httpUserAgent = "smtp-agent-http-client"; 16 | 17 | private String clientVersion = SmtpAgentClient.class.getPackage().getImplementationVersion(); 18 | private final String httpAuth; 19 | private final String host; 20 | private final int port; 21 | private final int connectionTimeout; 22 | private final int readTimeout; 23 | 24 | private Map headers = new LinkedHashMap(); 25 | 26 | public ConnectionConfig(String host, int port, String httpAuth, int connectionTimeout, int readTimeout) { 27 | this.host = host; 28 | this.port = port; 29 | assert httpAuth != null && httpAuth.contains(":"); 30 | this.httpAuth = "Basic " + Base64.encodeBase64String(httpAuth.getBytes()); 31 | this.clientVersion = clientVersion != null ? clientVersion : "unknown"; 32 | this.connectionTimeout = connectionTimeout; 33 | this.readTimeout = readTimeout; 34 | } 35 | 36 | public String addHeader(String name, String value) { 37 | return headers.put(name, value); 38 | } 39 | 40 | public void setHeaders(Map headers) { 41 | headers.putAll(headers); 42 | } 43 | 44 | public Map getHeaders() { 45 | return headers; 46 | } 47 | 48 | public String getHost() { 49 | return host; 50 | } 51 | 52 | public int getPort() { 53 | return port; 54 | } 55 | 56 | public String getHttpAuth() { 57 | return httpAuth; 58 | } 59 | 60 | public String getClientVersion() { 61 | return clientVersion; 62 | } 63 | 64 | public int getConnectionTimeout() { 65 | return connectionTimeout; 66 | } 67 | 68 | public int getReadTimeout() { 69 | return readTimeout; 70 | } 71 | 72 | public URLConnection configure(URLConnection conn) { 73 | conn.setRequestProperty("User-Agent", httpUserAgent); 74 | conn.setRequestProperty("client-version", clientVersion); 75 | conn.setRequestProperty("Authorization", httpAuth); 76 | for (Map.Entry entry : headers.entrySet()) { 77 | conn.setRequestProperty(entry.getKey(), entry.getValue()); 78 | } 79 | conn.setConnectTimeout(connectionTimeout); 80 | conn.setReadTimeout(readTimeout); 81 | return conn; 82 | } 83 | 84 | @Override 85 | public String toString() { 86 | return "\nConnectionConfig {" + 87 | "host='" + host + '\'' + 88 | ", port=" + port + 89 | ", clientVersion='" + clientVersion + '\'' + 90 | ", connectionTimeout=" + connectionTimeout + 91 | ", readTimeout=" + readTimeout + 92 | '}'; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/parser/BounceMapSuite.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.parser 2 | 3 | import org.scalatest.{Matchers, FunSpec} 4 | import com.fg.mail.smtp.Settings 5 | 6 | import java.net.URL 7 | import com.sun.xml.internal.messaging.saaj.util.Base64 8 | import scala.util.matching.Regex 9 | import com.fg.mail.smtp.util.ParsingUtils 10 | 11 | /** 12 | * 13 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 14 | * @version $Id: 7/18/13 4:53 PM u_jli Exp $ 15 | */ 16 | class BounceMapSuite extends FunSpec with Matchers { 17 | 18 | val opt = Settings.options() 19 | 20 | val knownHardBounce = "this email was blacklisted" 21 | val knownSoftBounce = "network connection timed out" 22 | val unknownBounce = "nejakej vykonstruovanej duvod v cizim jazyce" 23 | 24 | val prioritizedBounceList = new BounceListParser().parse(getClass.getClassLoader.getResourceAsStream("bounce-regex-list.xml")).fold( 25 | left => throw new IllegalStateException(left), 26 | right => right 27 | ) 28 | 29 | describe("parser should return regex map") { 30 | 31 | it("valid") { 32 | prioritizedBounceList should not be 'empty 33 | prioritizedBounceList.find(t => t._1 == "hard") should not be None 34 | prioritizedBounceList.find(t => t._1 == "soft") should not be None 35 | prioritizedBounceList.exists(_ eq null) should be (false) 36 | prioritizedBounceList.iterator.next()._1 should equal("soft") 37 | 38 | prioritizedBounceList.foldLeft(21L) { 39 | case(acc, t@(bounceType, prioritizedOrder, defaultOrder, bounceCategory, r)) => 40 | assert(defaultOrder === acc) 41 | acc - 1 42 | } 43 | } 44 | 45 | //TODO nasmerovat na github az to pujde 46 | it("from remote host") { 47 | val urlConnection = new URL(opt.bounceListUrlAndAuth._1).openConnection() 48 | urlConnection.setRequestProperty("Authorization", "Basic " + new String(Base64.encode(opt.bounceListUrlAndAuth._2.getBytes))) 49 | 50 | new BounceListParser().parse(urlConnection.getInputStream).fold( 51 | left => throw new IllegalStateException(left), 52 | right => right 53 | ) 54 | } 55 | 56 | it("that will resolve states correctly") { 57 | ParsingUtils.resolveState(knownSoftBounce, "bounced", false, prioritizedBounceList) should be ((0, "bad domain: connection timeout")) 58 | ParsingUtils.resolveState(knownHardBounce, "deferred", true, prioritizedBounceList) should be ((1, "spam detection and blacklisting")) 59 | ParsingUtils.resolveState("not important", "sent", false, prioritizedBounceList) should be ((3, "OK")) 60 | ParsingUtils.resolveState("not important", "sent", true, prioritizedBounceList) should be ((4, "finally OK")) 61 | } 62 | } 63 | 64 | describe("regex should not take too much time") { 65 | 66 | it("matching common error message") { 67 | prioritizedBounceList.foreach { tuple: (String, Long, Long, String, Regex) => 68 | val now = System.currentTimeMillis() 69 | tuple._5.pattern.matcher("Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again").matches() 70 | val after = System.currentTimeMillis() - now 71 | if (after > 4 ) fail(s"$after ms takes {$tuple._4}") 72 | } 73 | } 74 | 75 | } 76 | 77 | } 78 | 79 | -------------------------------------------------------------------------------- /client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | client 5 | Postfix Deliverability Analytics Client 6 | Client application for requesting data from server 7 | 8 | 9 | 10 | liska.jakub@gmail.com 11 | Jakub Liška 12 | FG Forrest, a.s. 13 | http://www.fg.cz 14 | 15 | 16 | 17 | 18 | com.fg.pda 19 | parent 20 | 1.1.0-SNAPSHOT 21 | ../pom.xml 22 | 23 | 24 | 25 | 26 | com.fasterxml.jackson.core 27 | jackson-databind 28 | 2.9.10.7 29 | 30 | 31 | com.fasterxml.jackson.core 32 | jackson-annotations 33 | 2.1.5 34 | 35 | 36 | commons-codec 37 | commons-codec 38 | 1.9 39 | 40 | 41 | com.google.guava 42 | guava 43 | 16.0.1 44 | 45 | 46 | com.google.code.findbugs 47 | jsr305 48 | 2.0.3 49 | 50 | 51 | javax.mail 52 | mail 53 | 1.4 54 | 55 | 56 | 57 | 58 | 59 | 60 | org.slf4j 61 | jcl-over-slf4j 62 | 1.6.4 63 | 64 | 65 | commons-logging 66 | commons-logging 67 | provided 68 | 1.1.1 69 | 70 | 71 | org.slf4j 72 | slf4j-api 73 | 1.6.4 74 | 75 | 76 | org.slf4j 77 | slf4j-log4j12 78 | 1.6.4 79 | 80 | 81 | 82 | 83 | 84 | junit 85 | junit 86 | 4.10 87 | test 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/tail/TailSuite.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.tail 2 | 3 | import java.io._ 4 | import org.scalatest._ 5 | import scala.concurrent.Await 6 | import com.fg.mail.smtp._ 7 | import akka.pattern.ask 8 | 9 | import com.fg.mail.smtp.stats.{GetCountStatus, LastIndexingStatus} 10 | 11 | /** 12 | * 13 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 14 | * @version $Id: 6/24/13 2:34 PM u_jli Exp $ 15 | */ 16 | class TailSuite extends TestSupport with BeforeAndAfter { 17 | 18 | val opt = loadOptions("application-test.conf").copy( 19 | httpServerStart = false, 20 | eofNewInputSleep = sleep(25) 21 | ) 22 | 23 | val existingFile = new File(testLogDir + "parser/existingFile") 24 | val tailedFile = new File(testLogDir + "parser/tailed.log") 25 | val absentFile = new File(testLogDir + "parser/absentFile") 26 | val rotatedFile = new File(testLogDir + "parser/rotatedFile") 27 | 28 | val msgIdLine = "2013 Jun 11 06:38:52.123 gds39d postfix/cleanup[7565]: B0236C2C111XXXX: message-id=msgId-B0236C2C111XXXX\n" 29 | 30 | before { 31 | existingFile.createNewFile 32 | absentFile.delete 33 | } 34 | after { 35 | absentFile.delete 36 | rotatedFile.delete 37 | existingFile.delete 38 | } 39 | 40 | override def beforeAll() { 41 | super.beforeAll() 42 | } 43 | 44 | override def afterAll() { 45 | val out = new FileWriter(tailedFile, false) 46 | out.append("s\ns\ns\ns\n") 47 | out.flush() 48 | out.close() 49 | super.afterAll() 50 | } 51 | 52 | describe("tail should") { 53 | 54 | it("find existing file") { 55 | assert(existingFile.exists()) 56 | assert(new Rotation(opt.copy(reOpenTries = 1), existingFile)(() => Unit, () => Unit).testExists()) 57 | assert(new Rotation(opt.copy(reOpenTries = 0), existingFile)(() => Unit, () => Unit).testExists()) 58 | } 59 | 60 | it("fail on absent file") { 61 | assert(!absentFile.exists()) 62 | assert(!new Rotation(opt.copy(reOpenTries = 1), absentFile)(() => Unit, () => Unit).testExists()) 63 | assert(!new Rotation(opt.copy(reOpenTries = 0), absentFile)(() => Unit, () => Unit).testExists()) 64 | } 65 | 66 | it("find existing file after x attempts") { 67 | def testCreate(f: File, n: Int) = try { 68 | var count = 0 69 | val result = new Rotation(opt.copy(reOpenTries = n, reOpenSleep = () => { count += 1; if (count >= n) f.createNewFile }), f)(() => Unit, () => Unit).testExists() 70 | (result, count) 71 | } finally { 72 | f.delete 73 | } 74 | 75 | assert(existingFile.exists()) 76 | assert(!absentFile.exists()) 77 | assert(testCreate(absentFile, 0) === (false, 0)) 78 | assert(testCreate(absentFile, 1) === (true, 1)) 79 | assert(testCreate(existingFile, 10) === (true, 0)) 80 | } 81 | } 82 | 83 | describe("tailing") { 84 | 85 | it("inputStream should read lines as they are being added and survive file rotation") { 86 | 87 | def testLine(s: String, expectedCount: Long) { 88 | system.log.info("writing a line") 89 | val out = new FileWriter(tailedFile, true) 90 | out.append(s) 91 | out.flush() 92 | out.close() 93 | } 94 | 95 | testLine(msgIdLine, 1) 96 | testLine(msgIdLine, 2) 97 | testLine(msgIdLine + msgIdLine + msgIdLine, 5) 98 | 99 | tailedFile.renameTo(rotatedFile) should be (true) 100 | 101 | testLine(msgIdLine, 6) 102 | } 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/Message.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp 2 | 3 | import java.io.{File, BufferedReader} 4 | import com.fg.mail.smtp.index.ClientIndexRecord 5 | 6 | /** 7 | * A bunch of Akka messages 8 | * 9 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 10 | * @version $Id: 6/24/13 8:55 AM u_jli Exp $ 11 | */ 12 | trait Request { def ctx: ReqCtx } 13 | case class ReqCtx(underlying: Map[String, String]) 14 | 15 | case class ShutdownAgent(ctx: ReqCtx) extends Request 16 | case class ReindexAgent(ctx: ReqCtx) extends Request 17 | case class RestartAgent(ctx: ReqCtx) extends Request 18 | case class RefreshBounceList(ctx: ReqCtx) extends Request 19 | case class MemoryUsage(ctx: ReqCtx) extends Request 20 | case class IndexMemoryFootprint(ctx: ReqCtx) extends Request 21 | case class IndexedLogFiles(ctx: ReqCtx) extends Request 22 | case class RcptAddressCounts(ctx: ReqCtx) extends Request 23 | case class RcptAddresses(ctx: ReqCtx) extends Request 24 | case class UnknownBounces(ctx: ReqCtx) extends Request 25 | case class IndexAge(ctx: ReqCtx) extends Request 26 | case class GetDisposableRecordsByClientId(ctx: ReqCtx) extends Request 27 | case class GetQueue(ctx: ReqCtx) extends Request 28 | 29 | case class Html(body: String) 30 | object ContentType { 31 | val json = "application/json;charset=utf-8" 32 | val html = "text/html;charset=utf-8" 33 | } 34 | 35 | case class Client(filter: IndexFilter, query: Option[IndexQuery])(implicit reqCtx: ReqCtx) extends Request { 36 | def ctx: ReqCtx = reqCtx 37 | } 38 | 39 | case class IndexFilter(clientId: String, email: Option[String], queue: Option[String], message: Option[String]) 40 | case class IndexQuery(from: Option[Long], to: Option[Long], lastOrFirst: Option[Boolean], groupBy: Option[String]) 41 | 42 | sealed trait View 43 | case class HomePage(ctx: ReqCtx) extends Request with View 44 | case class QueryPage(ctx: ReqCtx) extends Request with View 45 | case class ServerInfo(ctx: ReqCtx) extends Request with View 46 | 47 | sealed trait Message 48 | case class ShutSystemDown(why: String, ex: Option[Throwable] = None) extends Message 49 | case class RestartSystem(why: String, ex: Option[Throwable] = None) extends Message 50 | case object StartHttpServer extends Message 51 | case object GetIndexer extends Message 52 | 53 | sealed trait Indexing 54 | case object GetTailer extends Message with Indexing 55 | case object GetCouter extends Message with Indexing 56 | case class RestartIndexer(why: String, ex: Option[Throwable] = None) extends Message with Indexing 57 | case object ParsingBackupFinished extends Message with Indexing 58 | case class IndexTailedRecords(records: Iterable[ClientIndexRecord]) extends Message with Indexing 59 | case class IndexBackupRecords(records: Iterable[ClientIndexRecord], file: File, digest: Option[String], last: Boolean) extends Message with Indexing 60 | case object LogFileRotated extends Message with Indexing 61 | case object IndexingTailFinished extends Message with Indexing 62 | 63 | sealed trait Tailing 64 | case class ReadLines(reader: BufferedReader, batchSize: Int) extends Message with Tailing 65 | case class ReadBackup(digests: collection.mutable.Set[String]) extends Message with Tailing 66 | case object StartTailing extends Message with Tailing 67 | 68 | sealed trait Line 69 | case class MessageLine(queueId: String, msgId: String) extends Line 70 | case class ClientLine(queueId: String, clientId: String) extends Line 71 | case class DeliveryAttemptLine(date: Long, queueId: String, recipient: String, status: String, info: String) extends Line 72 | case class RemovedQueueLine(queueId: String) extends Line 73 | case class ExpiredLine(date: Long, queueId: String, sender: String, status: String, info: String) extends Line 74 | 75 | class RestartException extends Exception -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/parser/LogParserSuite.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.parser 2 | 3 | import scala.collection.immutable.TreeSet 4 | import java.io.File 5 | import com.fg.mail.smtp._ 6 | 7 | import akka.pattern.ask 8 | 9 | import scala.io.Source 10 | import com.fg.mail.smtp.util.{ParsingUtils, Commons} 11 | import scala.concurrent.Await 12 | import com.fg.mail.smtp.index.IndexRecord 13 | import scala.collection.IterableView 14 | 15 | /** 16 | * 17 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 18 | * @version $Id: 6/24/13 11:03 PM u_jli Exp $ 19 | */ 20 | class LogParserSuite extends TestSupport { 21 | 22 | def arbiterLogDir = new File(getClass.getClassLoader.getResource("META-INF/logs/arbiter").toURI) 23 | 24 | val opt = loadOptions("application-test.conf").copy(httpServerStart = false) 25 | 26 | describe("parser should") { 27 | 28 | it("group messages over multiple log files from directory") { 29 | 30 | Await.result(indexer ? GetDisposableRecordsByClientId(rc), timeout.duration).asInstanceOf[Option[Map[String, IterableView[IndexRecord, Iterable[IndexRecord]]]]] match { 31 | case Some(i) => 32 | assert(i.keys.size === 4) 33 | val test= i("test-mail-module").toList 34 | val first = i("first-client-id").toList 35 | val second = i("second-client-id").toList 36 | val third = i("third-client-id").toList 37 | assert(test.size === 16) 38 | assert(first.size === 9) 39 | assert(second.size === 9) 40 | assert(third.size === 9) 41 | val records = test ++ first ++ second ++ third 42 | assert(records.size === 9 * 3 + 16) 43 | val rcptLessRecords = records.foldLeft(0) { case (acc: Int, r) => if (Option(r.rcptEmail).isEmpty) acc+1 else acc } 44 | val rcptFullRecords = records.foldLeft(0) { case (acc: Int, r) => if (Option(r.rcptEmail).isDefined) acc+1 else acc } 45 | assert(rcptLessRecords === 0) 46 | assert(rcptFullRecords === 9 * 3 + 16) 47 | 48 | noMsgHasEmptyValue(records) 49 | case _ => 50 | fail("GetIndex should always return stuff") 51 | } 52 | } 53 | 54 | it("split backup files into groups for indexing and storing") { 55 | //65 bytes => index 2 and backup 2 files 56 | val arbiter = ParsingUtils.splitFiles(arbiterLogDir, Settings.options().copy(maxFileSizeToIndex = 0.000065D)) 57 | assert(arbiter.remainingSize < 0) 58 | assert(arbiter.toIndex.size === 2) 59 | assert(arbiter.toIgnore.size === 2) 60 | 61 | val indexed = arbiter.toIndex 62 | val backup = arbiter.toIgnore 63 | 64 | val expectedIndexed = TreeSet[String]("mail.log.2", "mail.log.1") 65 | val expectedBackup = TreeSet[String]("mail.log.4", "mail.log.3") 66 | 67 | assert(indexed.map(f => f.getName).subsetOf(expectedIndexed)) 68 | assert(backup.map(f => f.getName).subsetOf(expectedBackup)) 69 | 70 | assert(indexed.head.getName === "mail.log.2") 71 | assert(backup.head.getName === "mail.log.4") 72 | } 73 | 74 | it("digest files always with the same m5d") { 75 | 76 | def digest(f: File) = Commons.digestFirstLine(f) 77 | 78 | val indigestibles = new File(getClass.getClassLoader.getResource("META-INF/logs/real").toURI).listFiles().map(f => (digest(f), f)).filterNot(tuple => tuple._1 == digest(tuple._2)).map(_._2) 79 | 80 | assert(indigestibles.size === 0) 81 | 82 | } 83 | } 84 | 85 | def noMsgHasEmptyValue(index: Iterable[IndexRecord]) { 86 | val emptyRecords = index.filterNot { 87 | case IndexRecord(date: Long, queueId: String, msgId: String, rcptEmail: String, senderEmail: String, status: String, info: String, state: Int, stateInfo: String) => true 88 | case _ => false 89 | } 90 | assert(emptyRecords.isEmpty) 91 | } 92 | } -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/rest/Controller.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.rest 2 | 3 | import com.fg.mail.smtp._ 4 | import scala.xml.Xhtml 5 | import scala.collection.immutable.ListMap 6 | import com.fg.mail.smtp.HomePage 7 | import com.fg.mail.smtp.QueryPage 8 | import com.fg.mail.smtp.Options 9 | import com.fg.mail.smtp.index.Index 10 | import com.fg.mail.smtp.util.ServerInfoService 11 | 12 | /** 13 | * 14 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 15 | * @version $Id: 7/26/13 3:35 PM u_jli Exp $ 16 | */ 17 | class Controller(serverInfoService: ServerInfoService, o: Options) { 18 | 19 | lazy val url = s"http://${o.hostName}:${o.httpServerPort}/" 20 | 21 | lazy val actionHrefs = ListMap( 22 | "Shutdown" -> "agent-shutdown", 23 | "Restart (restart http server, reload configuration, refresh bounce list, reindex logs)" -> "agent-restart", 24 | "Reindex (refresh bounce list, reindex logs)" -> "agent-reindex", 25 | "Refresh bounce list" -> "agent-refresh-bouncelist" 26 | ).mapValues(url + _) 27 | 28 | lazy val readHrefs = ListMap( 29 | "Index status (since last start only)" -> "agent-status", 30 | "Total count of recipient email addresses" -> "agent-status/rcpt-address-counts", 31 | "Recipient email addresses" -> "agent-status/rcpt-addresses", 32 | "Unclassified bounce messages" -> "agent-status/unknown-bounces", 33 | "Memory info (RAM usage)" -> "agent-status/memory-usage", 34 | "Index and queue size" -> "agent-status/index-memory-footprint", 35 | "Indexed log files" -> "agent-status/indexed-log-files", 36 | "Environment info (memory, threads, GC, variables)" -> "agent-status/server-info", 37 | "Querying" -> "agent-read" 38 | ).mapValues(url + _) 39 | 40 | def dispatch(req: View, index: Index): String = req match { 41 | case HomePage(_) => 42 | Xhtml.toXhtml( 43 | 44 | 45 |
    {for (href <- actionHrefs) yield
  • {href._1}
    • }
46 | 47 | 48 | 49 | ) 50 | case QueryPage(_) => 51 | Xhtml.toXhtml( 52 | 53 | 54 | {for (clientId <- index.getClientIds) yield 55 |
56 |
order by :
57 | recipient email address 58 | message id 59 | queue id 60 |
61 | } 62 | 63 | 64 | ) 65 | 66 | case ServerInfo(_) => 67 | Xhtml.toXhtml( 68 | 69 | 70 | 
{serverInfoService.getEnvironment.toString}
71 |
72 | 73 | 74 | ) 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/util/Commons.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.util 2 | 3 | import java.security.MessageDigest 4 | import java.io._ 5 | import java.util.zip.GZIPInputStream 6 | import scala.io.Source 7 | import java.net.URL 8 | import scala.util.Try 9 | import com.sun.xml.internal.messaging.saaj.util.Base64 10 | 11 | /** 12 | * 13 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 14 | * @version $Id: 10/30/13 5:02 PM u_jli Exp $ 15 | */ 16 | object Commons { 17 | 18 | def digest(input: String): String = MessageDigest.getInstance("MD5").digest(input.getBytes).map("%02x".format(_)).mkString 19 | 20 | def digestFirstLine(file: File): String = { 21 | val source = getSource(file) 22 | try { 23 | val it = source.getLines().take(1) 24 | require(it.hasNext, s"md5 digesting of the first line of file $file expect it not to be empty !") 25 | digest(it.next()) 26 | } finally { 27 | source.close() 28 | } 29 | } 30 | 31 | def getInputStream(resourceLocation: String, auth: Option[String]): InputStream = { 32 | val u: URL = if (resourceLocation.startsWith("./") || resourceLocation.startsWith("../") || resourceLocation.startsWith("/")) 33 | new File(resourceLocation).toURI.toURL 34 | else if (!resourceLocation.startsWith("classpath:")) 35 | new URL(resourceLocation) 36 | else 37 | Try(Thread.currentThread.getContextClassLoader) 38 | .getOrElse(getClass.getClassLoader) 39 | .getResource(resourceLocation.substring("classpath:".length)) 40 | 41 | val connection = u.openConnection() 42 | auth.foreach( a => 43 | connection.setRequestProperty("Authorization", "Basic " + new String(Base64.encode(a.getBytes))) 44 | ) 45 | connection.getInputStream 46 | } 47 | 48 | 49 | def getSource(file: File): Source = 50 | if (file.getName.endsWith(".gz")) { 51 | val is = gunzipFileSafely(file) 52 | Source.createBufferedSource(is, 32768, reset = () => Source.fromInputStream(is), close = () => is.close()) 53 | } else 54 | Source.fromFile(file) 55 | 56 | def gunzipFileSafely(file: File): InputStream = { 57 | 58 | def throwAndClose(msg: String, e: Throwable, in: InputStream) { 59 | if (in != null) in.close() 60 | throw new IllegalStateException(msg, e) 61 | } 62 | 63 | var in: InputStream = null 64 | try { 65 | in = new GZIPInputStream(new FileInputStream(file), 32768) 66 | } catch { 67 | case fnf: FileNotFoundException => 68 | throwAndClose(s"Unable to find file ${file.getAbsolutePath}", fnf, in) 69 | case ioe: IOException => 70 | throwAndClose(s"Unexpected IO error during gunziping file ${file.getAbsolutePath}", ioe, in) 71 | case e: Throwable => 72 | throwAndClose(s"Unexpected error during reading file ${file.getAbsolutePath}", e, in) 73 | } 74 | in 75 | } 76 | 77 | def buildTable(header: List[Any], rows: List[List[Any]]) = { 78 | 79 | def formatRows(rowSeparator: String, rows: Seq[String]): String = ( 80 | rowSeparator :: 81 | rows.head :: 82 | rowSeparator :: 83 | rows.tail.toList ::: 84 | rowSeparator :: 85 | List()).mkString("\n") 86 | 87 | def formatRow(row: Seq[Any], colSizes: Seq[Int]) = { 88 | val cells = for ((item, size) <- row.zip(colSizes)) yield if (size == 0) "" else ("%" + size + "s").format(item) 89 | cells.mkString("|", "|", "|") 90 | } 91 | 92 | def rowSeparator(colSizes: Seq[Int]) = colSizes map { "-" * _ } mkString("+", "+", "+") 93 | 94 | val table = List(header) ::: rows 95 | table match { 96 | case Seq() => "" 97 | case _ => 98 | val sizes = for (row <- table) yield for (cell <- row) yield if (cell == null) 0 else cell.toString.length 99 | val colSizes = for (col <- sizes.transpose) yield col.max 100 | val rows = for (row <- table) yield formatRow(row, colSizes) 101 | formatRows(rowSeparator(colSizes), rows) 102 | } 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/stats/LastIndexingStatus.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.stats 2 | 3 | import com.fasterxml.jackson.annotation.{JsonProperty, JsonCreator} 4 | import scala.annotation.meta.field 5 | import java.lang.String 6 | import java.text.SimpleDateFormat 7 | import java.util.{Date, Locale} 8 | import com.fg.mail.smtp.Client 9 | import scala.collection.mutable 10 | import com.fasterxml.jackson.databind.{SerializerProvider, JsonSerializer} 11 | import com.fasterxml.jackson.core.JsonGenerator 12 | import com.fasterxml.jackson.databind.annotation.JsonSerialize 13 | 14 | /** 15 | * Subject for json serialization to provide brief status via http 16 | * 17 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 18 | * @version $Id: 6/26/13 11:55 PM u_jli Exp $ 19 | */ 20 | case class LastIndexingStatus( 21 | @JsonSerialize(using = classOf[DateSerializer]) 22 | @(JsonProperty@field)("Agent restarted at") 23 | restartedAt: Long = System.currentTimeMillis(), 24 | 25 | @JsonSerialize(using = classOf[DateSerializer]) 26 | @(JsonProperty@field)("Agent started at") 27 | agentStartTime: Long = LastIndexingStatus.agentStarted, 28 | 29 | @JsonSerialize(using = classOf[DateSerializer]) 30 | @(JsonProperty@field)("Last line received at") 31 | var lastLineReceivedAt: Long = 0, 32 | 33 | @(JsonProperty@field)("Count of indexed client-id log lines from backup") 34 | var clientIdMessagesIndexedFromBackup: Long = 0, 35 | 36 | @(JsonProperty@field)("Count of indexed message-id log lines from backup") 37 | var messageIdMessagesIndexedFromBackup: Long = 0, 38 | 39 | @(JsonProperty@field)("Count of indexed log entries (delivery attempts regardless of the status) from backup") 40 | var indexedLogEntriesFromBackup: Long = 0, 41 | 42 | @(JsonProperty@field)("Client statistics") 43 | clientStatistics: mutable.Map[String, Statistics] = mutable.Map[String, Statistics]() 44 | ) { 45 | 46 | 47 | def cidLineIndexed(count: Long) { 48 | clientIdMessagesIndexedFromBackup = count 49 | lastLineReceivedAt = System.currentTimeMillis() 50 | } 51 | 52 | def midLineIndexed(count: Long) { 53 | messageIdMessagesIndexedFromBackup = count 54 | lastLineReceivedAt = System.currentTimeMillis() 55 | } 56 | 57 | def logEntry(count: Long) { 58 | indexedLogEntriesFromBackup = count 59 | lastLineReceivedAt = System.currentTimeMillis() 60 | } 61 | 62 | def clientRequest(r: Client) { 63 | val clientVersion = r.ctx.underlying("client-version") 64 | val clientId = r.filter.clientId 65 | clientStatistics.get(clientId) match { 66 | case Some(stats) => clientStatistics(clientId) = stats.modify(clientVersion) 67 | case None => clientStatistics(clientId) = new Statistics(clientId, clientVersion, 1, System.currentTimeMillis()) 68 | } 69 | } 70 | 71 | } 72 | 73 | object LastIndexingStatus { 74 | val agentStarted = System.currentTimeMillis() 75 | } 76 | 77 | case class Statistics @JsonCreator() ( 78 | @(JsonProperty@field)("clientId") clientId: String, 79 | @(JsonProperty@field)("clientVersion") clientVersion: String, 80 | @(JsonProperty@field)("requestsCount") requestsCount: Int, 81 | @(JsonProperty@field)("lastRequestAt") @JsonSerialize(using = classOf[DateSerializer]) lastRequestAt: Long 82 | ) { 83 | 84 | def modify(clientVersion: String): Statistics = { 85 | new Statistics(clientId, clientVersion, requestsCount + 1, System.currentTimeMillis()) 86 | } 87 | 88 | } 89 | 90 | class DateSerializer extends JsonSerializer[Long] { 91 | 92 | private def format(l: Long): String = new SimpleDateFormat("yyyy MMM dd HH:mm:ss.SSS", Locale.US).format(new Date(l)) 93 | 94 | def serialize(value: Long, jgen: JsonGenerator, provider: SerializerProvider) = jgen.writeString(if (value == 0) "not yet" else format(value)) 95 | } 96 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/parser/BounceListParser.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.parser 2 | 3 | import scala.xml.Node 4 | import scala.util.matching.Regex 5 | import java.io.InputStream 6 | import scala.xml.XML 7 | import scala.util.control.Exception._ 8 | import java.util.regex.{Pattern, PatternSyntaxException} 9 | import scala.collection.mutable.TreeSet 10 | import org.slf4j.LoggerFactory 11 | import com.fg.mail.smtp.util.Commons 12 | 13 | /** 14 | * Parser of regular expressions xml list. It returns either suitable data structure or a xml validation error. 15 | * There are to types of categories: 16 | * Soft bounce means that message was deferred and it is to be tried again later on, it is kept in queue 17 | * Hard bounce means that the reason of not delivering a message was too serious to try to deliver message again, it is removed from queue right away 18 | * 19 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 20 | * @version $Id: 7/18/13 4:54 PM u_jli Exp $ 21 | */ 22 | class BounceListParser { 23 | val log = LoggerFactory.getLogger(getClass) 24 | 25 | def parse(uriAndCredentials: (String, String)): Either[Throwable, TreeSet[(String, Long, Long, String, Regex)]] = { 26 | log.info("Resolving regex bounce list from " + uriAndCredentials._1) 27 | catching(classOf[Throwable]) 28 | .either(Commons.getInputStream(uriAndCredentials._1, Option(uriAndCredentials._2).filter(_.trim.nonEmpty))) 29 | match { 30 | case Left(e) => Left(e) 31 | case Right(is: InputStream) => parse(is) 32 | } 33 | } 34 | 35 | def parse(is: InputStream): Either[Exception, TreeSet[(String, Long, Long, String, Regex)]] = { 36 | log.info("parsing regex bounce list") 37 | /** 38 | * @param n xml bounces node 39 | * @return n if xml is valid or reason why it is not valid 40 | */ 41 | def getValidNodeOnly(n: Node): Either[Exception, Node] = { 42 | if (!(Set[String]() ++ (n \ "_").map(_.label)).equals(Set[String]("regex"))) 43 | Left(new IllegalArgumentException("bounces element must have just 'regex' children")) 44 | else { 45 | (n \ "_") 46 | .foldLeft[Either[Exception,Node]](Right(n))((acc, group) => 47 | (group \ "regex").partition(regex => (regex \ "@category").length == 1 && (regex \ "or").length > 0) match { 48 | case (valids, invalids) if !invalids.isEmpty => 49 | Left(new IllegalArgumentException("regex elements must have a 'category' attribute and at least one 'or' child element:\n" + invalids.mkString("\n"))) 50 | case (valids, _) => 51 | (for { 52 | r <- valids 53 | or <- r \ "or" 54 | if catching(classOf[PatternSyntaxException]).either(Pattern.compile(or.text)).isLeft || or.text.isEmpty 55 | } yield or).toList match { 56 | case Nil => acc 57 | case invalidRegexes => Left(new IllegalArgumentException("regex elements must have valid regular expressions:\n" + invalidRegexes.mkString("\n"))) 58 | } 59 | } 60 | ) 61 | } 62 | } 63 | 64 | implicit val regexOrdering = Ordering.by[Regex, String](_.toString()) 65 | val ordering = Ordering[(String, Long, Long, String, Regex)].on((t: (String, Long, Long, String, Regex)) => (t._1, t._2, t._3, t._4, t._5)).reverse 66 | getValidNodeOnly(XML.load(is)) match { 67 | case Right(n) => 68 | val regexElements = n \ "regex" 69 | Right( 70 | regexElements.foldLeft(regexElements.size, TreeSet[(String, Long, Long, String, Regex)]()(ordering))( 71 | (accumulator, regexElm) => { 72 | accumulator._2.add( 73 | ( 74 | (regexElm \ "@type").text, 75 | 0L, 76 | accumulator._1, 77 | (regexElm \ "@category").text, 78 | new Regex("(" + (regexElm \ "or").map("("+_.text+")").reduceLeft[String]((ac, g) => ac+"|"+g)+")") 79 | ) 80 | ) 81 | accumulator.copy(_1 = accumulator._1 - 1) 82 | } 83 | )._2 84 | ) 85 | case Left(v) => Left(v) 86 | } 87 | } 88 | 89 | } -------------------------------------------------------------------------------- /server/src/test/resources/application-test.conf: -------------------------------------------------------------------------------- 1 | app { 2 | 3 | # several methods are being profiled for measuring execution count and time, should be disabled for production 4 | profiling { 5 | 6 | enabled = true 7 | 8 | } 9 | 10 | # email notification is sent to recipients if exception or some problem occurs 11 | notification { 12 | 13 | # email addresses that are to be sent notifications about warnings, errors, unknown bounces etc. 14 | recipients = [""] 15 | 16 | } 17 | 18 | # jdk's HttpServer settings 19 | http-server { 20 | 21 | # hostname http server is running on 22 | host = localhost 23 | 24 | # port number of http server 25 | port = 6666 26 | 27 | # basic authentication credentials of smtp agent's http server 28 | # empty string means that auth will be disabled 29 | auth = [] 30 | 31 | # it is possible to not start http server 32 | start = true 33 | 34 | } 35 | 36 | # bounce regex list is an xml file containing regular expressions used for bounce message categorization 37 | # it's the only way how to help application with log heuristics 38 | # it needs to be updated once in a while to categorize unresolved bounces that can be retrieved from rest method agent-status/unknown-bounces 39 | bounce-regex-list { 40 | 41 | # remote or local location of bounce regex list (use 'http://', 'classpath:', 'file://', absolute or relative path) 42 | url = "./src/main/resources/bounce-regex-list.xml" 43 | 44 | # basic base64 authentication credentials of remote server that serves bounce regex list file 45 | # empty string means it won't attempt to authenticate 46 | auth = "" 47 | 48 | } 49 | 50 | # information about postfix logs being analyzed 51 | # Logrotate utility is periodically rotating log files so that the log file will be renamed and compressed as follows : 52 | # mail.log -> mail.log.1 -> mail.log.1.gz -> mail.log.1.gz 53 | logs { 54 | 55 | # path of directory that contains postfix log files 56 | dir = "src/test/resources/META-INF/logs/parser/" 57 | 58 | # name of the log file that is being written to by postfix and tailed by agent 59 | tailed-log-file = "tailed.log" 60 | 61 | # name of the file that was actually rotated 62 | rotated-file = "tailed.log.1" 63 | 64 | # regex for matching backup log files that has been rotated. 65 | # NOTE that pattern must have a first capturing group pointing at number that says how many times log was rotated (important for log files indexing order) 66 | rotated-file-pattern = "backup-.*.(\\d{1,3}).*" 67 | 68 | # maximum size of log files to index in Mega Bytes this property must be explicitly specified 69 | # because files must be processed by reverse alphabetical order (from the oldest to the newest) 70 | # and a log file might contain 10% or 90% of relevant log entries because postfix does not have to be dedicated necessarily. 71 | # Please note that this property doesn't see whether a file is zipped or not 72 | max-file-size-to-index : 900 73 | 74 | # how many relevant lines (client-id, message-id, sentOrDeferred, expired) is in a batch to be indexed 75 | # this influences MapDB commit frequency which is now once in per index-batch-size records 76 | index-batch-size : 1000 77 | } 78 | 79 | # MapDB setup 80 | db { 81 | 82 | # path of directory that contains database files 83 | dir = "src/test/resources/META-INF/db/" 84 | 85 | # name of the database 86 | name = "test" 87 | 88 | # e.n.c.r.y.p.t.i.o.n key, empty string means that DB won't be encrypted 89 | auth = "" 90 | 91 | } 92 | 93 | timing { 94 | 95 | # how many seconds an actor thread is given for answering a request (eq. requesting indexer) before it fails 96 | request-timeout = 20 97 | 98 | # number of attempts to re-open a log file after it is moved during log rotation 99 | re-open-tries = 5 100 | 101 | # how many miliseconds to wait between re-open tries 102 | re-open-sleep = 10 103 | 104 | # how many miliseconds to wait after reaching log file EOF before reading next line 105 | eof-wait-for-new-input-sleep = 20 106 | 107 | } 108 | 109 | } -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/StressSuite.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp 2 | 3 | import akka.pattern.ask 4 | import scala.concurrent.Await 5 | import scala.concurrent.duration.Duration 6 | import com.fg.mail.smtp.index.{DbManager, IndexRecord, QueueRecord} 7 | import java.io.File 8 | import akka.actor.{ActorRef, Props, ActorSystem} 9 | import scala.collection.IterableView 10 | import com.fg.mail.smtp.stats.{GetCountStatus, LastIndexingStatus} 11 | 12 | /** 13 | * 14 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 15 | * @version $Id: 10/17/13 5:21 PM u_jli Exp $ 16 | */ 17 | class StressSuite extends TestSupport { 18 | 19 | val opt = loadOptions("application-test.conf") 20 | def dbDir = new File(opt.dbDir) 21 | 22 | override def providePersistence: DbManager = new DbManager(opt) 23 | 24 | override def afterAll() { 25 | dbDir.listFiles().filter(_.getName.startsWith(opt.dbName)).foreach(_.delete()) 26 | super.afterAll() 27 | } 28 | 29 | describe("Actor system should") { 30 | 31 | describe("survive restart of supervisor") { 32 | 33 | it("without modifying index, queue or status") { 34 | val beforeIndex = Await.result(indexer ? GetDisposableRecordsByClientId(rc), timeout.duration).asInstanceOf[Option[Map[String, IterableView[IndexRecord, Iterable[IndexRecord]]]]].get.map { case (k, v) => (k, v.force) } 35 | val beforeQueue = Await.result(tailer ? GetQueue(rc), timeout.duration).asInstanceOf[Option[Map[String, QueueRecord]]].get 36 | val beforeStatus = Await.result(counter ? GetCountStatus(rc), timeout.duration).asInstanceOf[Option[LastIndexingStatus]].get 37 | 38 | indexer ! RestartIndexer("OK") 39 | 40 | indexer = Await.result(supervisor ? GetIndexer, timeout.duration).asInstanceOf[ActorRef] 41 | counter = Await.result(indexer ? GetCouter, timeout.duration).asInstanceOf[ActorRef] 42 | tailer = Await.result(indexer ? GetTailer, timeout.duration).asInstanceOf[ActorRef] 43 | val afterIndex = Await.result(indexer ? GetDisposableRecordsByClientId(rc), timeout.duration).asInstanceOf[Option[Map[String, IterableView[IndexRecord, Iterable[IndexRecord]]]]].get.map { case (k, v) => (k, v.force) } 44 | val afterStatus = Await.result(counter ? GetCountStatus(rc), timeout.duration).asInstanceOf[Option[LastIndexingStatus]].get 45 | val afterQueue = Await.result(tailer ? GetQueue(rc), timeout.duration).asInstanceOf[Option[Map[String, QueueRecord]]].get 46 | 47 | assert(beforeIndex.equals(afterIndex)) 48 | assert(beforeQueue.equals(afterQueue)) 49 | assert(beforeStatus.indexedLogEntriesFromBackup === afterStatus.indexedLogEntriesFromBackup) 50 | } 51 | 52 | } 53 | 54 | describe("survive shutdown of supervisor") { 55 | 56 | it("and load into the exact same state") { 57 | val beforeIndex = Await.result(indexer ? GetDisposableRecordsByClientId(rc), timeout.duration).asInstanceOf[Option[Map[String, IterableView[IndexRecord, Iterable[IndexRecord]]]]].get.map { case (k, v) => (k, v.force) } 58 | val beforeQueue = Await.result(tailer ? GetQueue(rc), timeout.duration).asInstanceOf[Option[Map[String, QueueRecord]]].get 59 | 60 | indexer ! ShutdownAgent(rc) 61 | system.awaitTermination(Duration.create(2, "s")) 62 | val newSystem = ActorSystem("newName") 63 | val supervisor = newSystem.actorOf(Props(new Supervisor(opt, new DbManager(opt))), "supervisor") 64 | val newIndexer = Await.result(supervisor ? GetIndexer, timeout.duration).asInstanceOf[ActorRef] 65 | val newTailer = Await.result(newIndexer ? GetTailer, timeout.duration).asInstanceOf[ActorRef] 66 | val newCounter = Await.result(newIndexer ? GetCouter, timeout.duration).asInstanceOf[ActorRef] 67 | 68 | val afterIndex = Await.result(newIndexer ? GetDisposableRecordsByClientId(rc), timeout.duration).asInstanceOf[Option[Map[String, IterableView[IndexRecord, Iterable[IndexRecord]]]]].get.map { case (k, v) => (k, v.force) } 69 | val afterQueue = Await.result(newTailer ? GetQueue(rc), timeout.duration).asInstanceOf[Option[Map[String, QueueRecord]]].get 70 | val afterStatus = Await.result(newCounter ? GetCountStatus(rc), timeout.duration).asInstanceOf[Option[LastIndexingStatus]].get 71 | 72 | assert(beforeIndex.equals(afterIndex)) 73 | assert(beforeQueue.equals(afterQueue)) 74 | assert(afterStatus.lastLineReceivedAt.longValue() === 0) 75 | 76 | newIndexer ! ShutdownAgent(rc) 77 | newSystem.awaitTermination(Duration.create(2, "s")) 78 | } 79 | 80 | } 81 | 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /server/src/main/resources/bounce-regex-list.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 11 | try.*later 12 | connection.*lost 13 | lost.*connection 14 | cannot deliver to this user at this time 15 | (:?temporarily|temporary).*(:?deferred|error|unavailable) 16 | (:?problem|zkuste pozdeji|temporary|temporarily|insufficient system storage) 17 | 450 |451 |400 18 | 19 | 20 | connection timed out|session timeout exceeded|timed out while|timeout waiting for client input 21 | 22 | 23 | 552 24 | (:?mailbox|schranka|full|over|exceeded).*(:?preplnena|exceeded|mailbox|quota|full|storage) 25 | 26 | 27 | (:?denied|ip address|overeni|refused|access forbidden).*(:?|ip |blocked) 28 | refused to talk to me|not our customer 29 | 30 | 31 | connection.*refused 32 | syntax error in parameters or arguments 33 | command not allowed 34 | 35 | 36 | (:?host|domain).*(:?not found|disabled) 37 | 38 | 39 | (:?relay|access|relaying|relay access).*(:?denied|not permitted|not allowed) 40 | (:?we do not|unable to|isn't allowed).*(:?relay|relayed) 41 | 42 | 43 | network.*unreachable 44 | no route to host 45 | (:?unrouteable|unroutable).*(:?address|host) 46 | 47 | 48 | sender policy framework|spf|intrusion prevention 49 | 50 | 51 | (:?too many|too much).*(:?connections|mail|send|recipients) 52 | connection limit exceeded 53 | one recipient domain per session 54 | 55 | 56 | service.*(:?unavailable|not available) 57 | internal error|try that server first|local error in processing|insufficient system resources|system failure 58 | 59 | 60 | (:?rejected|denied|policy).*(:?policy|violation) 61 | 62 | 63 | (:?greylisting|graylisted|greylisted|graylisting) 64 | 65 | 66 | (:?dns|name service).*(:?timeout|error) 67 | 68 | 69 | (:?content|message|rules).*(:?rejection|rejected|refused|size exceeds) 70 | 71 | 72 | loops back to myself|bad address syntax 73 | 74 | 75 | alias expansion.*error 76 | unable to figure out my ip addresses 77 | unable to accept message 78 | 79 | 80 | (:?mailbox|address|user|account|recipient|@).*(:?rejected|unknown|disabled|unavailable|invalid|inactive|not exist) 81 | (:?rejected|unknown|unavailable|no|illegal|invalid|no such).*(:?mailbox|address|user|account|recipient|alias) 82 | (:?address|user|recipient) does(n't| not) have .*(:?mailbox|account) 83 | 550 |553 |501 84 | 85 | 86 | (:?spam|unsolicited|blacklisting|blacklisted|blacklist|554 ) 87 | 88 | 89 | returned to sender 90 | 91 | 92 | (:?auth).*(:?required) 93 | 94 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/factory/BatchReqFactory.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.factory; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fg.mail.smtp.client.model.AgentResponse; 5 | import com.fg.mail.smtp.client.model.SmtpLogEntry; 6 | import com.fg.mail.smtp.client.request.filter.Eq; 7 | import com.fg.mail.smtp.client.request.query.BySingle; 8 | import com.fg.mail.smtp.client.request.query.ByTuple; 9 | import com.fg.mail.smtp.client.request.query.QueryFactory; 10 | 11 | import javax.annotation.Nullable; 12 | import java.util.Map; 13 | import java.util.TreeSet; 14 | import java.util.concurrent.TimeUnit; 15 | 16 | /** 17 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 18 | * @version $Id: 10/9/13 7:59 PM u_jli Exp $ 19 | */ 20 | public class BatchReqFactory implements RequestFactory { 21 | 22 | private long jobTimeframe; 23 | private TimeUnit jobTimeframeUnit; 24 | 25 | public BatchReqFactory(long jobTimeframe, TimeUnit jobTimeframeUnit) { 26 | this.jobTimeframe = jobTimeframe; 27 | this.jobTimeframeUnit = jobTimeframeUnit; 28 | } 29 | 30 | public BatchQueryFactory forClientId(String clientId) { 31 | return new BatchQueryFactory(new IndexFiltering(clientId), jobTimeframe, jobTimeframeUnit); 32 | } 33 | 34 | public BatchQueryFactory forClientIdAnd(String clientId, Eq equals) { 35 | return new BatchQueryFactory(new IndexFiltering(clientId, equals), jobTimeframe, jobTimeframeUnit); 36 | } 37 | 38 | public static class BatchQueryFactory implements QueryFactory { 39 | 40 | private IndexFiltering filtering; 41 | private long jobTimeframe; 42 | private TimeUnit jobTimeframeUnit; 43 | 44 | public BatchQueryFactory(IndexFiltering filtering, long jobTimeframe, TimeUnit jobTimeframeUnit) { 45 | this.filtering = filtering; 46 | this.jobTimeframe = jobTimeframe; 47 | this.jobTimeframeUnit = jobTimeframeUnit; 48 | } 49 | 50 | public BatchAgentReq> forTimeConstraining(Long from, @Nullable Long to) { 51 | assert from != null; 52 | IndexQuery query = new IndexQuery(from, to, null, null); 53 | return new BatchAgentReq>(filtering, query, jobTimeframe, jobTimeframeUnit, new TypeReference>>() {}); 54 | } 55 | 56 | public BatchAgentReq forLastOrFirstConstraining(Long from, @Nullable Long to, Boolean isLastOrFirst) { 57 | assert from != null; 58 | assert isLastOrFirst != null; 59 | IndexQuery query = new IndexQuery(from, to, isLastOrFirst, null); 60 | return new BatchAgentReq(filtering, query, jobTimeframe, jobTimeframeUnit, new TypeReference>() {}); 61 | } 62 | 63 | public BatchAgentReq>> forGrouping(Long from, @Nullable Long to, BySingle group) { 64 | assert from != null; 65 | assert group != null; 66 | IndexQuery query = new IndexQuery(from, to, null, group); 67 | return new BatchAgentReq>>(filtering, query, jobTimeframe, jobTimeframeUnit, new TypeReference>>>() {}); 68 | } 69 | 70 | public BatchAgentReq>>> forMultipleGrouping(Long from, @Nullable Long to, ByTuple group) { 71 | assert from != null; 72 | assert group != null; 73 | IndexQuery query = new IndexQuery(from, to, null, group); 74 | return new BatchAgentReq>>>(filtering, query, jobTimeframe, jobTimeframeUnit, new TypeReference>>>>() {}); 75 | } 76 | 77 | public BatchAgentReq>> forConstraintMultipleGrouping(Long from, @Nullable Long to, Boolean isLastOrFirst, ByTuple group) { 78 | assert from != null; 79 | assert group != null; 80 | assert isLastOrFirst != null; 81 | IndexQuery query = new IndexQuery(from, to, isLastOrFirst, group); 82 | return new BatchAgentReq>>(filtering, query, jobTimeframe, jobTimeframeUnit, new TypeReference>>>() {}); 83 | } 84 | 85 | public BatchAgentReq> forConstrainedGrouping(Long from, @Nullable Long to, Boolean isLastOrFirst, BySingle group) { 86 | assert from != null; 87 | assert group != null; 88 | assert isLastOrFirst != null; 89 | IndexQuery query = new IndexQuery(from, to, isLastOrFirst, group); 90 | return new BatchAgentReq>(filtering, query, jobTimeframe, jobTimeframeUnit, new TypeReference>>() {}); 91 | } 92 | 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/rest/Dispatcher.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.rest 2 | 3 | import com.fg.mail.smtp.rest.RestDSL._ 4 | import com.fg.mail.smtp._ 5 | import scala.Some 6 | import com.fg.mail.smtp.Client 7 | import com.fg.mail.smtp.IndexQuery 8 | import com.fg.mail.smtp.stats.GetCountStatus 9 | 10 | /** 11 | * A lightweight DSL based router of URIs with query strings 12 | * 13 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 14 | * @version $Id: 6/26/13 9:28 PM u_jli Exp $ 15 | */ 16 | object Dispatcher { 17 | 18 | val groupBy_rcptEmail = "rcptEmail" 19 | val groupBy_queueId = "queueId" 20 | val groupBy_msgId = "msgId" 21 | 22 | val from = QueryMatcher(LONG("from")) 23 | val to = QueryMatcher(LONG("to")) 24 | val fromTo = QueryMatcher(LONG("from"), LONG("to")) 25 | 26 | val lastOrFirst = QueryMatcher(BOOLEAN("lastOrFirst")) 27 | val toLastOrFirst = QueryMatcher(LONG("to"), BOOLEAN("lastOrFirst")) 28 | val fromLastOrFirst = QueryMatcher(LONG("from"), BOOLEAN("lastOrFirst")) 29 | val groupBy = QueryMatcher(*("groupBy")) 30 | val fromGroupBy = QueryMatcher(LONG("from"), *("groupBy")) 31 | val toGroupBy = QueryMatcher(LONG("to"), *("groupBy")) 32 | val fromToGroupBy = QueryMatcher(LONG("from"), LONG("to"), *("groupBy")) 33 | val lastOrFirstGroupBy = QueryMatcher(BOOLEAN("lastOrFirst"), *("groupBy")) 34 | 35 | def queryDispatch(rc: ReqCtx) = / { 36 | case "agent-shutdown" => ShutdownAgent(rc) 37 | case "agent-restart" => RestartAgent(rc) 38 | case "agent-reindex" => ReindexAgent(rc) 39 | case "agent-refresh-bouncelist" => RefreshBounceList(rc) 40 | case "agent-status" => / { 41 | case "rcpt-address-counts" => RcptAddressCounts(rc) 42 | case "rcpt-addresses" => RcptAddresses(rc) 43 | case "unknown-bounces" => UnknownBounces(rc) 44 | case "index-age" => IndexAge(rc) 45 | case "memory-usage" => MemoryUsage(rc) 46 | case "index-memory-footprint" => IndexMemoryFootprint(rc) 47 | case "indexed-log-files" => IndexedLogFiles(rc) 48 | case "server-info" => ServerInfo(rc) 49 | case $() => GetCountStatus(rc) 50 | } 51 | case "agent-read" => / { 52 | case *(clientId) => / { 53 | case $() => ? { 54 | getMatcher(clientId, None, None, None)(rc) 55 | } 56 | case "rcptEmail" => / { 57 | case `@`(email) => / { 58 | case $() => ? { 59 | getMatcher(clientId, Some(email), None, None)(rc) 60 | } 61 | } 62 | } 63 | case "queue" => / { 64 | case *(queueId) => / { 65 | case $() => ? { 66 | getMatcher(clientId, None, Some(queueId), None)(rc) 67 | } 68 | } 69 | } 70 | case "message" => / { 71 | case *(msgId) => / { 72 | case $() => ? { 73 | getMatcher(clientId, None, None, Some(msgId))(rc) 74 | } 75 | } 76 | } 77 | } 78 | case $() => QueryPage(rc) 79 | } 80 | case $() => HomePage(rc) 81 | } 82 | 83 | private def getMatcher(clientId: String, email: Option[String], queue: Option[String], message: Option[String])(rc: ReqCtx): PartialFunction[URIQuery, Request] = { 84 | case fromToGroupBy(Some(f), Some(t), Some(gb)) => 85 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(Some(f), Some(t), None, Some(gb))))(rc) 86 | case fromTo(Some(f), Some(t)) => 87 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(Some(f), Some(t), None, None)))(rc) 88 | case fromLastOrFirst(Some(f), Some(l)) => 89 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(Some(f), None, Some(l), None)))(rc) 90 | case toLastOrFirst(Some(t), Some(l)) => 91 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(None, Some(t), Some(l), None)))(rc) 92 | case fromGroupBy(Some(f), Some(gb)) => 93 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(Some(f), None, None, Some(gb))))(rc) 94 | case toGroupBy(Some(t), Some(gb)) => 95 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(None, Some(t), None, Some(gb))))(rc) 96 | case lastOrFirstGroupBy(Some(l), Some(gb)) => 97 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(None, None, Some(l), Some(gb))))(rc) 98 | case groupBy(Some(gb)) => 99 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(None, None, None, Some(gb))))(rc) 100 | case lastOrFirst(Some(l)) => 101 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(None, None, Some(l), None)))(rc) 102 | case from(Some(f)) => 103 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(Some(f), None, None, None)))(rc) 104 | case to(Some(t)) => 105 | Client(IndexFilter(clientId, email, queue, message), Some(IndexQuery(None, Some(t), None, None)))(rc) 106 | case _ => 107 | Client(IndexFilter(clientId, email, queue, message), None)(rc) 108 | } 109 | 110 | def dispatch(url: String, query: String)(implicit rc: ReqCtx): Option[Request] = { 111 | queryDispatch(rc)(url) match { 112 | case Some(f) => f.apply(query) 113 | case _ => None 114 | } 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/model/SmtpLogEntry.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.model; 2 | 3 | import java.util.Comparator; 4 | import java.util.Date; 5 | 6 | /** 7 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 8 | * @version $Id: 6/30/13 7:02 PM u_jli Exp $ 9 | */ 10 | public class SmtpLogEntry implements Comparable { 11 | 12 | /* retrieved from logs */ 13 | private Date date; 14 | private String queueId; 15 | private String msgId; 16 | private String rcptEmail; 17 | private String senderEmail; 18 | private String status; 19 | private String info; 20 | 21 | /* computed */ 22 | /** 23 | * state enum 0 = soft bounce, 1 = hard bounce, 2 = unknown bounce 24 | */ 25 | private Integer state; 26 | private String stateInfo; 27 | 28 | public Date getDate() { 29 | return date; 30 | } 31 | 32 | public void setDate(Date date) { 33 | this.date = date; 34 | } 35 | 36 | public String getQueueId() { 37 | return queueId; 38 | } 39 | 40 | public void setQueueId(String queueId) { 41 | this.queueId = queueId; 42 | } 43 | 44 | public String getMsgId() { 45 | return msgId; 46 | } 47 | 48 | public void setMsgId(String msgId) { 49 | this.msgId = msgId; 50 | } 51 | 52 | public String getRcptEmail() { 53 | return rcptEmail; 54 | } 55 | 56 | public void setRcptEmail(String rcptEmail) { 57 | this.rcptEmail = rcptEmail; 58 | } 59 | 60 | public String getSenderEmail() { 61 | return senderEmail; 62 | } 63 | 64 | public void setSenderEmail(String senderEmail) { 65 | this.senderEmail = senderEmail; 66 | } 67 | 68 | public String getStatus() { 69 | return status; 70 | } 71 | 72 | public void setStatus(String status) { 73 | this.status = status; 74 | } 75 | 76 | public String getInfo() { 77 | return info; 78 | } 79 | 80 | public void setInfo(String info) { 81 | this.info = info; 82 | } 83 | 84 | public Integer getState() { 85 | return state; 86 | } 87 | 88 | public void setState(Integer state) { 89 | this.state = state; 90 | } 91 | 92 | public String getStateInfo() { 93 | return stateInfo; 94 | } 95 | 96 | public void setStateInfo(String stateInfo) { 97 | this.stateInfo = stateInfo; 98 | } 99 | 100 | @Override 101 | public boolean equals(Object o) { 102 | if (this == o) return true; 103 | if (o == null || getClass() != o.getClass()) return false; 104 | 105 | SmtpLogEntry logEntry = (SmtpLogEntry) o; 106 | 107 | if (!date.equals(logEntry.date)) return false; 108 | if (info != null ? !info.equals(logEntry.info) : logEntry.info != null) return false; 109 | if (stateInfo != null ? !stateInfo.equals(logEntry.stateInfo) : logEntry.stateInfo != null) return false; 110 | if (!queueId.equals(logEntry.queueId)) return false; 111 | if (!msgId.equals(logEntry.msgId)) return false; 112 | if (rcptEmail != null ? !rcptEmail.equals(logEntry.rcptEmail) : logEntry.rcptEmail != null) return false; 113 | if (!status.equals(logEntry.status)) return false; 114 | if (!state.equals(logEntry.state)) return false; 115 | 116 | return true; 117 | } 118 | 119 | @Override 120 | public int hashCode() { 121 | int result = date.hashCode(); 122 | result = 31 * result + queueId.hashCode(); 123 | result = 31 * result + msgId.hashCode(); 124 | result = 31 * result + rcptEmail.hashCode(); 125 | result = 31 * result + status.hashCode(); 126 | result = 31 * result + state.hashCode(); 127 | result = 31 * result + (info != null ? info.hashCode() : 0); 128 | result = 31 * result + (stateInfo != null ? stateInfo.hashCode() : 0); 129 | return result; 130 | } 131 | 132 | @Override 133 | public String toString() { 134 | return "LogEntry{" + 135 | "date=" + date + 136 | ", queueId='" + queueId + '\'' + 137 | ", msgId='" + msgId + '\'' + 138 | ", rcptEmail='" + rcptEmail + '\'' + 139 | ", senderEmail='" + senderEmail + '\'' + 140 | ", status='" + status + '\'' + 141 | ", info='" + info + '\'' + 142 | ", state='" + state + '\'' + 143 | ", stateInfo='" + stateInfo + '\'' + 144 | '}'; 145 | } 146 | 147 | /* TreeSet would be sorted from newest to oldest */ 148 | public int compareTo(SmtpLogEntry o) { 149 | if (this.date.after(o.date)) 150 | return -1; 151 | else if (this.equals(o)) 152 | return 0; 153 | else 154 | return 1; 155 | } 156 | 157 | public static class ReverseComparator implements Comparator { 158 | 159 | /* TreeSet would be sorted from oldest to newest */ 160 | public int compare(SmtpLogEntry o1, SmtpLogEntry o2) { 161 | if (o1.date.after(o2.date)) 162 | return 1; 163 | else if (o1.equals(o2)) 164 | return 0; 165 | else 166 | return -1; 167 | } 168 | } 169 | } 170 | 171 | -------------------------------------------------------------------------------- /client/src/main/java/com/fg/mail/smtp/client/request/factory/SingleReqFactory.java: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.client.request.factory; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fg.mail.smtp.client.model.AgentResponse; 5 | import com.fg.mail.smtp.client.model.SmtpLogEntry; 6 | import com.fg.mail.smtp.client.request.filter.AppendablePath; 7 | import com.fg.mail.smtp.client.request.filter.Eq; 8 | import com.fg.mail.smtp.client.request.query.BySingle; 9 | import com.fg.mail.smtp.client.request.query.ByTuple; 10 | import com.fg.mail.smtp.client.request.query.QueryFactory; 11 | 12 | import javax.annotation.Nullable; 13 | import java.util.Map; 14 | import java.util.Set; 15 | import java.util.TreeSet; 16 | 17 | /** 18 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 19 | * @version $Id: 10/7/13 9:21 PM u_jli Exp $ 20 | */ 21 | public class SingleReqFactory implements RequestFactory { 22 | 23 | public SingleQueryFactory forClientId(String clientId) { 24 | return new SingleQueryFactory(new IndexFiltering(clientId)); 25 | } 26 | 27 | public SingleQueryFactory forClientIdAnd(String clientId, Eq equals) { 28 | return new SingleQueryFactory(new IndexFiltering(clientId, equals)); 29 | } 30 | 31 | public AgentReq> forRcptAddressCounts() { 32 | return new AgentReq>(new AppendablePath("agent-status").appendSegment("rcpt-address-counts"), new TypeReference>>() {}); 33 | } 34 | 35 | public AgentReq forAgentShutdown() { 36 | return new AgentReq(new AppendablePath("agent-shutdown"), new TypeReference>() {}); 37 | } 38 | 39 | public AgentReq>> forRcptAddresses() { 40 | return new AgentReq>>(new AppendablePath("agent-status").appendSegment("rcpt-addresses"), new TypeReference>>>() {}); 41 | } 42 | 43 | public AgentReq>> forUnknownBounces() { 44 | return new AgentReq>>(new AppendablePath("agent-status").appendSegment("unknown-bounces"), new TypeReference>>>() {}); 45 | } 46 | 47 | public static class SingleQueryFactory implements QueryFactory { 48 | 49 | private IndexFiltering filtering; 50 | 51 | public SingleQueryFactory(IndexFiltering filtering) { 52 | this.filtering = filtering; 53 | } 54 | 55 | public AgentReq> forTimeConstraining(@Nullable Long from, @Nullable Long to) { 56 | IndexQuery query = new IndexQuery(from, to, null, null); 57 | return new AgentReq>(filtering, query, new TypeReference>>() {}); 58 | } 59 | 60 | public AgentReq forLastOrFirstConstraining(@Nullable Long from, @Nullable Long to, Boolean isLastOrFirst) { 61 | assert isLastOrFirst != null; 62 | IndexQuery query = new IndexQuery(from, to, isLastOrFirst, null); 63 | return new AgentReq(filtering, query, new TypeReference>() {}); 64 | } 65 | 66 | public AgentReq>> forGrouping(@Nullable Long from, @Nullable Long to, BySingle group) { 67 | assert group != null; 68 | IndexQuery query = new IndexQuery(from, to, null, group); 69 | return new AgentReq>>(filtering, query, new TypeReference>>>() {}); 70 | } 71 | 72 | public AgentReq>>> forMultipleGrouping(@Nullable Long from, @Nullable Long to, ByTuple group) { 73 | assert group != null; 74 | IndexQuery query = new IndexQuery(from, to, null, group); 75 | return new AgentReq>>>(filtering, query, new TypeReference>>>>() {}); 76 | } 77 | 78 | public AgentReq>> forConstraintMultipleGrouping(@Nullable Long from, @Nullable Long to, Boolean isLastOrFirst, ByTuple group) { 79 | assert group != null; 80 | assert isLastOrFirst != null; 81 | IndexQuery query = new IndexQuery(from, to, isLastOrFirst, group); 82 | return new AgentReq>>(filtering, query, new TypeReference>>>() {}); 83 | } 84 | 85 | public AgentReq> forConstrainedGrouping(@Nullable Long from, @Nullable Long to, Boolean isLastOrFirst, BySingle group) { 86 | assert group != null; 87 | assert isLastOrFirst != null; 88 | IndexQuery query = new IndexQuery(from, to, isLastOrFirst, group); 89 | return new AgentReq>(filtering, query, new TypeReference>>() {}); 90 | } 91 | 92 | public AgentReq> queryLess() { 93 | return new AgentReq>(filtering, null, new TypeReference>>() {}); 94 | } 95 | } 96 | 97 | } 98 | 99 | -------------------------------------------------------------------------------- /server/src/test/scala/com/fg/mail/smtp/regexp/RegexpSuite.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.regexp 2 | 3 | import org.scalatest.{Matchers, FunSuite} 4 | import scala.collection.immutable.HashSet 5 | import scala.io.Source 6 | 7 | import com.fg.mail.smtp.Settings 8 | import com.fg.mail.smtp.util.ParsingUtils 9 | 10 | /** 11 | * 12 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 13 | * @version $Id: 6/24/13 9:36 PM u_jli Exp $ 14 | */ 15 | class RegexpSuite extends FunSuite with Matchers { 16 | 17 | val midLine = "2013 Jun 16 16:05:07.123 gds39d postfix/cleanup[26547]: 3bZLDk1V50z37Dv: message-id=whatever" 18 | val cidLine = "2013 Jun 16 16:05:07.123 gds39d postfix/cleanup[26547]: 3bZLDk1V50z37Dv: info: header client-id: test-mail-module from localhost[127.0.0.1]; from= to= proto=ESMTP helo=" 19 | val expiredLine = "2013 Jun 15 16:28:52.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: from=, status=expired, returned to sender" 20 | val recipientLine = "2013 Jun 16 16:05:07.123 gds39d postfix/smtp[20945]: 3bZLDk1V50z37Dv: to=, relay=hermes.fg.cz[193.86.74.5]:25, delay=0.16, delays=0.01/0/0.05/0.1, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 5058FBC3A)" 21 | 22 | val o = Settings.options().copy(tailedLogFileName = "tailed.log", rotatedPatternFn = _.matches("backup-.*"), indexBatchSize = 1001) 23 | 24 | def testFile = Source.fromInputStream(getClass.getClassLoader.getResourceAsStream("META-INF/logs/parser/backup-single-client.log.1")) 25 | def expectedSizeOf[E](i: Int)(c: Iterable[E]) { new HashSet[E] ++ c should have size i } 26 | 27 | test("message id regex should capture message id") { 28 | midLine match { 29 | case ParsingUtils.midRegex(queueId, msgId) => 30 | queueId should be ("3bZLDk1V50z37Dv") 31 | msgId should be ("whatever") 32 | case _ => 33 | throw new IllegalStateException(cidLine.toString + " is not valid!") 34 | } 35 | } 36 | 37 | test("client id regex should capture message id and client id") { 38 | cidLine match { 39 | case ParsingUtils.cidRegex(queueId, clientId) => 40 | queueId should be ("3bZLDk1V50z37Dv") 41 | clientId should be ("test-mail-module") 42 | case _ => 43 | throw new IllegalStateException(cidLine.toString + " is not valid!") 44 | } 45 | } 46 | 47 | test("there should be only 1 client id lines in test file, the previous 3 have client-id information in message-id header") { 48 | val clientIdsByMessageId = testFile.getLines().toIterable 49 | .collect { 50 | case ParsingUtils.cidRegex(queueId, clientId) => (queueId, clientId) 51 | }.toMap 52 | 53 | clientIdsByMessageId should not be null 54 | clientIdsByMessageId should have size 1 55 | expectedSizeOf(1)(clientIdsByMessageId.values) 56 | } 57 | 58 | test("recipient regex should capture all groups") { 59 | recipientLine match { 60 | case ParsingUtils.deliveryAttempt(date, queueId, recipient, status, info) => 61 | date should be ("2013 Jun 16 16:05:07.123") 62 | queueId should be ("3bZLDk1V50z37Dv") 63 | recipient should be ("liska@fg.cz") 64 | status should be ("sent") 65 | info should be ("250 2.0.0 Ok: queued as 5058FBC3A") 66 | case _ => 67 | throw new IllegalStateException(recipientLine.toString + " is not valid!") 68 | } 69 | } 70 | 71 | test("there should be 7 (sent || deferred || bounced) and 1 expired lines in test file") { 72 | val sentDeferredOrBounced = testFile.getLines().toIterable 73 | .collect { 74 | case ParsingUtils.deliveryAttempt(date, queueId, recipient, status, info) 75 | => (date, queueId, recipient, status, info) 76 | } 77 | 78 | val expired = testFile.getLines().toIterable 79 | .collect { 80 | case ParsingUtils.expiredRegex(date, queueId, recipient, status, info) 81 | => (date, queueId, recipient, status, info) 82 | } 83 | 84 | sentDeferredOrBounced should not be null 85 | sentDeferredOrBounced should have size 15 86 | sentDeferredOrBounced foreach ( 87 | tuple => tuple.productIterator.foreach( 88 | Option(_) should not be None 89 | ) 90 | ) 91 | 92 | expired should have size 1 93 | } 94 | 95 | test("expired regex should capture all groups") { 96 | expiredLine match { 97 | case ParsingUtils.expiredRegex(date, queueId, sender, status, info) => 98 | date should be ("2013 Jun 15 16:28:52.123") 99 | queueId should be ("3bYscs22wBz37Dv") 100 | sender should be ("no-reply@directmail.fg.cz") 101 | status should be ("expired") 102 | info should be ("returned to sender") 103 | case _ => 104 | throw new IllegalStateException(expiredLine.toString + " is not valid!") 105 | } 106 | } 107 | 108 | test("there should be only 1 expired line in test file") { 109 | val messages = testFile.getLines().toIterable 110 | .collect { 111 | case ParsingUtils.expiredRegex(date, queueId, sender, status, info) 112 | => (date, queueId, sender, status, info) 113 | } 114 | 115 | messages should not be null 116 | messages should have size 1 117 | messages foreach ( 118 | tuple => tuple.productIterator.foreach( 119 | Option(_) should not be None 120 | ) 121 | ) 122 | } 123 | 124 | } 125 | -------------------------------------------------------------------------------- /server/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | 3 | loggers = ["akka.event.slf4j.Slf4jLogger"] 4 | loglevel = INFO 5 | daemonic = on 6 | 7 | actor { 8 | mailbox { 9 | requirements { 10 | "akka.dispatch.BoundedDequeBasedMessageQueueSemantics" = akka.actor.mailbox.bounded-deque-based 11 | } 12 | } 13 | debug { 14 | receive = on 15 | lifecycle = on 16 | } 17 | } 18 | } 19 | 20 | bounded-deque-based { 21 | mailbox-type = "akka.dispatch.BoundedDequeBasedMailbox" 22 | mailbox-capacity = 5000 23 | mailbox-push-timeout-time = 10s 24 | } 25 | 26 | atmos { 27 | trace { 28 | enabled = false 29 | 30 | node = agent 31 | 32 | traceable { 33 | 34 | "/user/supervisor" = on 35 | "/user/supervisor/indexer" = on 36 | "/user/supervisor/indexer/tailer" = on 37 | 38 | "*" = off 39 | } 40 | 41 | sampling { 42 | "/user/supervisor" = 1 # sample every trace for supervisor 43 | "/user/supervisor/indexer" = 10 # sample every 10th trace for indexer 44 | "/user/supervisor/indexer/tailer" = 100 # sample every 100th trace for tailer 45 | } 46 | 47 | } 48 | } 49 | 50 | app { 51 | 52 | base = "/www/Postfix-Deliverability-Analytics" 53 | 54 | # several methods are being profiled for measuring execution count and time, should be disabled for production 55 | profiling { 56 | 57 | enabled = false 58 | 59 | } 60 | 61 | # email notification is sent to recipients if exception or some problem occurs 62 | notification { 63 | 64 | # email addresses that are to be sent notifications about warnings, errors, unknown bounces etc. 65 | recipients = ["liska.jakub@gmail.com"] 66 | 67 | } 68 | 69 | # jdk's HttpServer settings 70 | http-server { 71 | 72 | # hostname http server is running on 73 | host = localhost 74 | 75 | # port number of http server 76 | port = 1523 77 | 78 | # basic authentication credentials of smtp agent's http server 79 | # empty string means that auth will be disabled 80 | auth = [] 81 | 82 | # it is possible to not start http server 83 | start = true 84 | 85 | } 86 | 87 | # bounce regex list is an xml file containing regular expressions used for bounce message categorization 88 | # it's the only way how to help application with log heuristics 89 | # it needs to be updated once in a while to categorize unresolved bounces that can be retrieved from rest method agent-status/unknown-bounces 90 | bounce-regex-list { 91 | 92 | # remote or local location of bounce regex list (use 'http://', 'classpath:' or 'file://') 93 | url = "file://"${app.base}"/server/src/main/resources/bounce-regex-list.xml" 94 | 95 | # basic base64 authentication credentials of remote server that serves bounce regex list file 96 | # empty string means it won't attempt to authenticate 97 | auth = "" 98 | 99 | } 100 | 101 | # information about postfix logs being analyzed 102 | # Logrotate utility is periodically rotating log files so that the log file will be renamed and compressed as follows : 103 | # mail.log -> mail.log.1 -> mail.log.1.gz -> mail.log.1.gz 104 | logs { 105 | 106 | # absolute path of directory that contains postfix log files 107 | dir = "/var/log/" 108 | 109 | # name of the log file that is being written to by postfix and tailed by agent 110 | tailed-log-file = "mail.log" 111 | 112 | # name of the file that was actually rotated 113 | rotated-file = "mail.log.1" 114 | 115 | # regex for matching backup log files that has been rotated. 116 | # NOTE that pattern must have a first capturing group pointing at number that says how many times log was rotated (important for log files indexing order) 117 | rotated-file-pattern = "mail\\.log\\.(\\d{1,3}).*" 118 | 119 | # maximum size of log files to index in Mega Bytes this property must be explicitly specified 120 | # because files must be processed by reverse alphabetical order (from the oldest to the newest) 121 | # and a log file might contain 10% or 90% of relevant log entries because postfix does not have to be dedicated necessarily. 122 | # Please note that this property doesn't see whether a file is zipped or not 123 | max-file-size-to-index : 1000 124 | 125 | # how many relevant lines (client-id, message-id, sentOrDeferred, expired) is in a batch to be indexed 126 | # this influences MapDB commit frequency which is now once in per index-batch-size records 127 | index-batch-size : 1000 128 | } 129 | 130 | # MapDB setup 131 | db { 132 | 133 | # absolute path of directory that contains database files 134 | dir = ${app.base}"/deploy/db/" 135 | 136 | # name of the database 137 | name = "smtpLogDb" 138 | 139 | # e.n.c.r.y.p.t.i.o.n key, empty string means that DB won't be encrypted 140 | auth = "" 141 | 142 | } 143 | 144 | timing { 145 | 146 | # how many seconds an actor thread is given for answering a request (eq. requesting indexer) before it fails 147 | request-timeout = 100 148 | 149 | # number of attempts to re-open a log file after it is moved during log rotation 150 | re-open-tries = 15 151 | 152 | # how many miliseconds to wait between re-open tries 153 | re-open-sleep = 1000 154 | 155 | # how many miliseconds to wait after reaching log file EOF before reading next line 156 | eof-wait-for-new-input-sleep = 1000 157 | 158 | } 159 | 160 | } -------------------------------------------------------------------------------- /server/src/main/resources/control.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURL_BASE="curl http://127.0.0.1:1523" 4 | CURL_BASE_HEADERS="curl -D - http://127.0.0.1:1523" 5 | W_TIMEOUT=10; 6 | PID_REGEX="^[0-9]+$" 7 | START_SCRIPT="./start.sh" 8 | 9 | if [ -z "$JAVA_HOME" ] ; then 10 | export JAVA_HOME="/www/server/java/jdk" 11 | fi 12 | 13 | if [[ -n "$1" ]] && [[ "$1" != "start" ]] && [[ -z "$2" ]] && [[ `curl -sI http://127.0.0.1:1523 | tr -d '\r' | sed -En 's/^HTTP\/1\.1 (.*)/\1/p'` == "401 Unauthorized" ]] 14 | then 15 | echo " !!! Http server requires authentication, provide 'username:password' as last script argument !!! " 16 | exit 17 | fi 18 | 19 | if [ -n "$2" ] 20 | then 21 | CURL_BASE="curl -u $2 http://127.0.0.1:1523" 22 | fi 23 | 24 | 25 | function startDaemon () { 26 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 27 | if [ -z "$DPID" ]; then 28 | /bin/bash $START_SCRIPT /dev/null 2>&1 & 29 | echo "Application started."; 30 | else 31 | echo "Application is already running, you might want to restart it"; 32 | fi 33 | } 34 | 35 | function getStatus () { 36 | STATUS=0; 37 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 38 | AGSTATUS=`${CURL_BASE}/agent-status 2>/dev/null` 39 | if [ $? == 0 ]; then 40 | echo "Application running with PID=$DPID and status:" 41 | echo "$AGSTATUS"; 42 | elif [ "$DPID" != "" ]; then 43 | echo "Application running with PID=$DPID but not responding to status command." 44 | else 45 | echo "Application is not running." 46 | fi 47 | 48 | return $STATUS; 49 | } 50 | 51 | function getUnknownBounces () { 52 | STATUS=0; 53 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 54 | AGSTATUS=`${CURL_BASE}/agent-status/unknown-bounces 2>/dev/null` 55 | if [ $? == 0 ]; then 56 | echo "Application running with PID=$DPID and status:" 57 | echo "$AGSTATUS"; 58 | elif [ "$DPID" != "" ]; then 59 | echo "Application running with PID=$DPID but not responding to getUnknownBounces command." 60 | else 61 | echo "Application is not running." 62 | fi 63 | 64 | return $STATUS; 65 | } 66 | 67 | function restart () { 68 | STATUS=0; 69 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 70 | if [ $? == 0 ] && [[ "$DPID" =~ $PID_REGEX ]]; then 71 | AGSTATUS=`${CURL_BASE}/agent-restart 2>/dev/null` 72 | echo "Application is being restarted with PID=$DPID and status:" 73 | echo "$AGSTATUS"; 74 | elif [ -z "$DPID" ]; then 75 | echo "Application is not running. Starting now..." 76 | startDaemon; 77 | else 78 | echo "Please fix restart function of control.sh script, it does not work as expected" 79 | fi 80 | 81 | return $STATUS; 82 | } 83 | 84 | function reindex () { 85 | STATUS=0; 86 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 87 | AGSTATUS=`${CURL_BASE}/agent-reindex 2>/dev/null` 88 | if [ $? == 0 ]; then 89 | echo "Application running with PID=$DPID and status:" 90 | echo "$AGSTATUS"; 91 | elif [ "$DPID" != "" ]; then 92 | echo "Application running with PID=$DPID but not responding to reindex command." 93 | else 94 | echo "Application is not running." 95 | fi 96 | 97 | return $STATUS; 98 | } 99 | 100 | function refreshBouncelist () { 101 | STATUS=0; 102 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 103 | AGSTATUS=`${CURL_BASE}/agent-refresh-bouncelist 2>/dev/null` 104 | if [ $? == 0 ]; then 105 | echo "Application running with PID=$DPID and status:" 106 | echo "$AGSTATUS"; 107 | elif [ "$DPID" != "" ]; then 108 | echo "Application running with PID=$DPID but not responding to refreshBouncelist command." 109 | else 110 | echo "Application is not running." 111 | fi 112 | 113 | return $STATUS; 114 | } 115 | 116 | function stopDaemon () { 117 | AGSTOP=`${CURL_BASE}/agent-shutdown 2>/dev/null` 118 | if [ $? == 0 ]; then 119 | echo "Trying to stop Application with agent-shutdown command." 120 | echo "Waiting ${W_TIMEOUT}s before kill." 121 | sleep $W_TIMEOUT; 122 | fi 123 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 124 | if [ "$DPID" != "" ]; then 125 | echo "Trying to kill app." 126 | kill -9 $DPID 127 | fi 128 | echo "Application stopped." 129 | } 130 | 131 | case "$1" in 132 | start) 133 | startDaemon; 134 | ;; 135 | stop) 136 | stopDaemon; 137 | ;; 138 | restart) 139 | restart; 140 | ;; 141 | reindex) 142 | reindex; 143 | ;; 144 | refreshBouncelist) 145 | refreshBouncelist; 146 | ;; 147 | unknownBounces) 148 | getUnknownBounces; 149 | ;; 150 | status) 151 | getStatus; STATUS_RC=$?; 152 | exit $STATUS_RC; 153 | ;; 154 | *) 155 | cat < Thread.sleep(ms) 39 | Try( 40 | new Options( 41 | c.getBoolean("app.profiling.enabled"), 42 | c.getString("app.http-server.host"), 43 | c.getStringList("app.notification.recipients").asScala, 44 | Timeout(c.getInt("app.timing.request-timeout").second), 45 | c.getDouble("app.logs.max-file-size-to-index"), 46 | c.getInt("app.http-server.port"), 47 | (c.getString("app.bounce-regex-list.url"), c.getString("app.bounce-regex-list.auth")), 48 | c.getStringList("app.http-server.auth").asScala, 49 | c.getBoolean("app.http-server.start"), 50 | if (c.getString("app.logs.dir").endsWith("/")) c.getString("app.logs.dir") else c.getString("app.logs.dir") + "/", 51 | c.getString("app.logs.tailed-log-file"), 52 | c.getString("app.logs.rotated-file"), 53 | c.getString("app.logs.rotated-file-pattern"), 54 | _.matches(c.getString("app.logs.rotated-file-pattern")), 55 | c.getInt("app.logs.index-batch-size"), 56 | c.getString("app.db.dir"), 57 | c.getString("app.db.name"), 58 | c.getString("app.db.auth"), 59 | c.getInt("app.timing.re-open-tries"), 60 | sleep(c.getInt("app.timing.re-open-sleep")), 61 | sleep(c.getInt("app.timing.eof-wait-for-new-input-sleep")) 62 | ) 63 | ) match { 64 | case Success(o) => 65 | logger.info("Settings successfully parsed...") 66 | o 67 | case Failure(ex) => 68 | logger.info(s"There is an error in settings", ex) 69 | throw ex 70 | } 71 | } 72 | } 73 | 74 | /** 75 | * @param profilingEnabled profiling has some overhead so you can turn it off 76 | * @param hostName http server is running on 77 | * @param notifRcpts email addresses that are to be sent notifications about warnings, errors, unknown bunces etc. 78 | * @param askTimeout how much time an actor thread is given for answering a request (eq. requesting indexer) before it fails 79 | * @param maxFileSizeToIndex maximum size of log files to index in Mega Bytes this property must be explicitly specified 80 | * because files must be processed by reverse alphabetical order (from the oldest to the newest) 81 | * and a log file might contain 10% or 90% of relevant log entries because postfix does not have to be dedicated necessarily. 82 | * Please note that this property doesn't see whether a file is zipped or not 83 | * @param httpServerPort port number of http server 84 | * @param bounceListUrlAndAuth remote or local location of bounce regex list (use file:// protocol if local, http:// if remote) 85 | * @param httpServerAuth basic base64 authentication credentials of remote server that serves bounce regex list file 86 | * @param httpServerStart it is possible to not start http server 87 | * @param logDir absolute path of directory that contains postfix log files 88 | * @param indexBatchSize how many relevant lines (client-id, message-id, sentOrDeferred, expired) is in a batch to be indexed 89 | * @param dbDir absolute path of directory that contains database files 90 | * @param dbName database name 91 | * @param dbAuth encryption key 92 | * @param reOpenTries number of attempts to re-open a log file after it is moved during log rotation 93 | * @param reOpenSleep how many miliseconds to wait between re-open tries 94 | * @param eofNewInputSleep how many miliseconds to wait after reaching log file EOF before reading next line 95 | * @param rotatedPatternFn backup file name matching constraint so that only backup files in a directory are read (regex for matching backup log files that has been rotated) 96 | * @param tailedLogFileName name of the log file that is being written to by postfix and tailed by agent 97 | */ 98 | case class Options( 99 | profilingEnabled: Boolean, 100 | hostName: String, 101 | notifRcpts: mutable.Buffer[String], 102 | askTimeout: Timeout, 103 | maxFileSizeToIndex: Double, 104 | httpServerPort: Int, 105 | bounceListUrlAndAuth: (String, String), 106 | httpServerAuth: mutable.Buffer[String], 107 | httpServerStart: Boolean, 108 | logDir: String, 109 | tailedLogFileName: String, 110 | rotatedFileName: String, 111 | rotatedPattern: String, 112 | rotatedPatternFn: String => Boolean, 113 | indexBatchSize: Int, 114 | dbDir: String, 115 | dbName: String, 116 | dbAuth: String, 117 | reOpenTries: Int, 118 | reOpenSleep: () => Unit, 119 | eofNewInputSleep: () => Unit 120 | ) -------------------------------------------------------------------------------- /server/common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . ./common.properties 3 | 4 | PID_REGEX="^[0-9]+$" 5 | 6 | function removeDeploy () { 7 | echo "*** Removing deploy ***"; 8 | rm -r "$deploy_path/$distribution_name" > /dev/null 9 | } 10 | 11 | function stop () { 12 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 13 | if [[ "$DPID" =~ $PID_REGEX ]]; then 14 | echo "Application is running with PID=$DPID" 15 | AGSTATUS=`curl -u ${http_auth} http://127.0.0.1:1523/agent-shutdown 2>/dev/null` 16 | echo "Application shut down with status:" 17 | echo "$AGSTATUS"; 18 | echo "Waiting ${kill_timeout}s before kill." 19 | sleep $kill_timeout; 20 | DPID=`ps aux 2>/dev/null | grep "[c]om.fg.mail.smtp.Agent" 2>/dev/null | awk '{ print $2 }' 2>/dev/null` 21 | if [[ "$DPID" =~ $PID_REGEX ]]; then 22 | echo "Trying to kill Mail Agent." 23 | kill -9 $DPID 24 | fi 25 | else 26 | echo "Application is not running" 27 | fi 28 | } 29 | 30 | function start () { 31 | stop 32 | 33 | if [ ! -d "${deploy_path}/${distribution_name}/bin" ]; then 34 | echo 35 | echo " **** Cannot start application because ${deploy_path}/${distribution_name}/bin does not exist " 36 | echo 37 | exit 38 | fi 39 | 40 | cd "${deploy_path}/${distribution_name}/bin" 41 | echo "*** Starting application ***"; 42 | /bin/bash ./control.sh start ${http_auth} 43 | sleep 1 44 | cd ${deploy_log_path} 45 | tail -f $(ls -1t | head -1) 46 | 47 | } 48 | 49 | function copyToLocal () { 50 | if [ -d "$deploy_path/$distribution_name" ]; then 51 | stop 52 | removeDeploy 53 | fi 54 | 55 | if [ $? != 0 ]; then 56 | exit 57 | fi 58 | 59 | echo "*** Gunzipping distribution to destination $deploy_path ***"; 60 | tar -C "$deploy_path" -zxf "$distribution_path" > /dev/null 61 | } 62 | 63 | function copyToLocalAndRun () { 64 | copyToLocal 65 | if [ $? == 0 ]; then 66 | start 67 | fi 68 | } 69 | 70 | function backupDB () { 71 | if [ "$(ls -A ${deploy_path}/db/* 2>/dev/null)" ]; then 72 | echo "*** backing up existing DB files ***"; 73 | zip -r "/tmp/db_backup.zip" "${deploy_path}/db"; 74 | else 75 | echo "${deploy_path}/db is empty. No backup." 76 | fi 77 | } 78 | 79 | function deleteDB () { 80 | echo "*** removing existing DB files from ${deploy_path}/db/ ***"; 81 | rm -f ${deploy_path}/db/* 82 | } 83 | 84 | function copyToLocalDeleteDBAndRun () { 85 | copyToLocal 86 | deleteDB 87 | if [ $? == 0 ]; then 88 | start 89 | fi 90 | } 91 | 92 | function copyToLocalBackupDeleteDBAndRun () { 93 | copyToLocal 94 | backupDB 95 | deleteDB 96 | if [ $? == 0 ]; then 97 | start 98 | fi 99 | } 100 | 101 | function copyToRemote () { 102 | if [ ! -f "$distribution_path" ] || [ -z "$remote_user" ] || [ -z "$remote_host" ]; then 103 | echo 104 | echo " **** Please provide remote user and host and verify $distribution_path exists " 105 | echo 106 | echo " common.sh copyToRemote admin charon.example.com " 107 | echo 108 | exit 109 | fi 110 | 111 | echo "*** Secure copying distribution to $remote_user@$remote_host:${deploy_path} ***" 112 | scp "$distribution_path" "$remote_user@$remote_host:${deploy_path}" 113 | } 114 | 115 | function copyBounceToRemote () { 116 | if [ ! -f "$distribution_path" ] || [ -z "$remote_user" ] || [ -z "$remote_host" ]; then 117 | echo 118 | echo " **** Please provide remote user and host and verify $distribution_path exists " 119 | echo 120 | echo " common.sh copyBounceToRemote admin charon.example.com " 121 | echo 122 | exit 123 | fi 124 | 125 | echo "*** Secure copying bounce regex list to $remote_user@$remote_host:${deploy_conf_path} ***" 126 | scp "$bounce_regex_list_path" "$remote_user@$remote_host:${deploy_conf_path}" 127 | } 128 | 129 | case "$1" in 130 | copyToLocal) 131 | copyToLocal; 132 | ;; 133 | copyToLocalAndRun) 134 | copyToLocalAndRun; 135 | ;; 136 | copyToLocalDeleteDBAndRun) 137 | copyToLocalDeleteDBAndRun; 138 | ;; 139 | copyToLocalBackupDeleteDBAndRun) 140 | copyToLocalBackupDeleteDBAndRun; 141 | ;; 142 | copyToRemote) 143 | copyToRemote; 144 | ;; 145 | copyBounceToRemote) 146 | copyBounceToRemote; 147 | ;; 148 | removeDeploy) 149 | removeDeploy; 150 | ;; 151 | backupDB) 152 | backupDB; 153 | ;; 154 | deleteDB) 155 | deleteDB; 156 | ;; 157 | stop) 158 | stop; 159 | ;; 160 | start) 161 | start; 162 | ;; 163 | *) 164 | cat < void processBatch(BatchAgentReq request, JobCallback callback) throws ClientNotAvailableException { 35 | for (AgentReq req : request.getRequests()) { 36 | IndexQuery query = req.getQuery(); 37 | callback.execute(resolveResult(req), query.getFrom(), query.getTo()); 38 | } 39 | } 40 | 41 | protected T resolveResult(AgentReq request) throws ClientNotAvailableException { 42 | AgentResponse response = resolve(request); 43 | ResponseStatus status = response.getStatus(); 44 | if (status.getSucceeded()) { 45 | log.info("Request was successfully served. Result is ready for further processing"); 46 | return response.getResult(); 47 | } else { 48 | throw new ClientNotAvailableException(status.getMessage()); 49 | } 50 | } 51 | 52 | public AgentResponse resolve(AgentReq request) throws ClientNotAvailableException { 53 | AgentUrl url = new AgentUrl(conCfg.getHost(), conCfg.getPort(), request); 54 | long start = System.currentTimeMillis(); 55 | URLConnection urlConn = openConnection(url.getURL()); 56 | AgentResponse result = deserialize(getInputStream(urlConn), request); 57 | log.info("Response retrieved and deserialized in " + (System.currentTimeMillis() - start) + " ms"); 58 | return result; 59 | } 60 | 61 | protected AgentResponse deserialize(InputStream inputStream, AgentReq request) throws ClientNotAvailableException { 62 | try { 63 | return mapper.readValue(new InputStreamReader(inputStream), request.getTypeRef()); 64 | } catch (JsonMappingException e) { 65 | throw new IllegalStateException("Unexpected JSON mapping error occurred during deserialization", e); 66 | } catch (JsonParseException e) { 67 | throw new IllegalStateException("Unexpected JSON parsing error occurred during deserialization", e); 68 | } catch (IOException e) { 69 | throw new ClientNotAvailableException("Connection to remote server failed, unable to read the stream", e); 70 | } 71 | } 72 | 73 | private URLConnection openConnection(URL url) throws ClientNotAvailableException { 74 | log.info("Connecting to : " + url.toString() + " with connection timeout : " + conCfg.getConnectionTimeout()/1000 + " and read timeout : " + conCfg.getReadTimeout()/1000 + " seconds"); 75 | try { 76 | return conCfg.configure(url.openConnection()); 77 | } catch (IOException e) { 78 | throw new ClientNotAvailableException("Connection to remote server failed - unable to open connection - probably a networking problem !", e); 79 | } 80 | } 81 | 82 | private InputStream getInputStream(URLConnection urlConn) throws ClientNotAvailableException { 83 | try { 84 | return urlConn.getInputStream(); 85 | } catch (IOException e) { 86 | if (urlConn instanceof HttpURLConnection) { 87 | try { 88 | int responseCode = ((HttpURLConnection) urlConn).getResponseCode(); 89 | InputStream errorStream = ((HttpURLConnection) urlConn).getErrorStream(); 90 | ByteArrayOutputStream os = new ByteArrayOutputStream(); 91 | byte[] buf = new byte[4096]; 92 | try { 93 | // read the response body 94 | int ret; 95 | while ((ret = errorStream.read(buf)) > 0) os.write(buf, 0, ret); 96 | } finally { 97 | // close the error stream 98 | errorStream.close(); 99 | } 100 | throw new ClientNotAvailableException("Error http response " + responseCode + ": " + new String(os.toByteArray()), e); 101 | } catch (IOException omg) { 102 | throw new ClientNotAvailableException("Connection to remote server failed, unable to retrieve error code, connection was probably refused", e); 103 | } 104 | } else { 105 | throw new ClientNotAvailableException("Connection to remote server failed, unable to read the stream", e); 106 | } 107 | } 108 | } 109 | 110 | /** 111 | * helper method for testing purposes 112 | * @param request 113 | */ 114 | public String resolveWithoutDeserialization(AgentReq request) { 115 | try { 116 | URLConnection urlConn = conCfg.configure(new AgentUrl(conCfg.getHost(), conCfg.getPort(), request).getURL().openConnection()); 117 | BufferedReader reader = new BufferedReader(new InputStreamReader(urlConn.getInputStream())); 118 | String line = ""; 119 | StringBuilder sb = new StringBuilder(); 120 | do { 121 | sb.append(line); 122 | line = reader.readLine(); 123 | } while (line != null); 124 | return sb.toString(); 125 | } catch (IOException e) { 126 | throw new RuntimeException("Connection to remote server failed !", e); 127 | } 128 | } 129 | 130 | } 131 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/util/ParsingUtils.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.util 2 | 3 | import scala.util.matching.Regex 4 | import java.io.File 5 | import java.math.BigDecimal 6 | import scala.collection.immutable.TreeSet 7 | import org.slf4j.LoggerFactory 8 | import scala.Some 9 | import com.fg.mail.smtp.Options 10 | 11 | /** 12 | * 13 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 14 | * @version $Id: 12/7/13 1:01 PM u_jli Exp $ 15 | */ 16 | object ParsingUtils { 17 | 18 | val log = LoggerFactory.getLogger(getClass) 19 | 20 | /* when email expired or was successfully sent, queue is removed by postfix which leads to following log entry */ 21 | val removedQueueRegex = """.*?: ([a-zA-Z0-9]{15}): removed$""".r 22 | /* message-id log entry has a value generated by mail client, it is available in LogEntry for future use */ 23 | val midRegex = """.*?: ([a-zA-Z0-9]{15}): message-id=(.*?)$""".r 24 | /* client-id log entry is generated by postfix because it is set up to do so if agent client's (mail module) request header contains client-id 25 | * It allows for partitioning Index by clients so that we can run multiple newsletter campaigns at the same time */ 26 | val cidRegex = """.*?: ([a-zA-Z0-9]{15}): info: header client-id: (.*?) .*$""".r 27 | /* Log entry of this form follows mail that was either deferred or sent successfully and it is to be removed from queue */ 28 | val deliveryAttempt = """^(\d{4} [a-zA-Z]+ \d{1,2} \d{2}:\d{2}:\d{2}.\d{3}).*?([a-zA-Z0-9]{15}): to=<(.*?@.*?)>.*?status=(sent|deferred|bounced) \((.*)\)$""".r 29 | /* Log entry of this form follows mail that is considered expired and it is to be removed from queue */ 30 | val expiredRegex = """^(\d{4} [a-zA-Z]+ \d{1,2} \d{2}:\d{2}:\d{2}.\d{3}).*?([a-zA-Z0-9]{15}): from=<(.*?@.*?)>.*?status=(expired), (.*)$""".r 31 | 32 | /** 33 | * Decide on type of bounce, soft bounce means that message was deferred and it is to be tried again later on. Hard bounce 34 | * means that the reason of not delivering a message was too serious to try to deliver message again, it is removed from queue right away 35 | * 36 | * @return tuple (hard-1/soft-0, bounce reason message) 37 | */ 38 | def resolveState(info: String, status: String, hasBeenDeferred: Boolean, prioritizedBounceList: scala.collection.mutable.TreeSet[(String, Long, Long, String, Regex)]): (Int, String) = { 39 | 40 | def stripPrefix(target: String): String = { 41 | if (target.startsWith("host ")) { 42 | val relevantIndex = target.indexOf(" said: ") 43 | if (relevantIndex > 0) 44 | target.substring(relevantIndex + " said: ".length) 45 | } else if (target.startsWith("connect to ")) { 46 | val relevantPart = target.substring("(connect to ".length) 47 | val relevantIndex = relevantPart.indexOf(' ') 48 | if (relevantIndex > 0) 49 | relevantPart.substring(relevantIndex + 1) 50 | } 51 | target 52 | } 53 | 54 | status match { 55 | case "sent" if hasBeenDeferred => (4, "finally OK") 56 | case "sent" => (3, "OK") 57 | case "expired" => (0, "expired, returned to sender") 58 | case _ => 59 | prioritizedBounceList.find { 60 | case (bounceType, prioritizedOrder, defaultOrder, bounceCategory, regex) => 61 | regex.pattern.matcher(stripPrefix(info.toLowerCase)).find() 62 | } match { 63 | case Some(t) => 64 | prioritizedBounceList.remove(t) 65 | prioritizedBounceList.add(t.copy(_2 = t._2 + 1L)) 66 | if (t._1 == "soft") 67 | (0, t._4) 68 | else 69 | (1, t._4) 70 | case None => 71 | (2, "unable to decide on type of bounce") 72 | } 73 | } 74 | } 75 | 76 | case class Arbiter(remainingSize: Double, toIndex: TreeSet[File], toIgnore: TreeSet[File]) { 77 | 78 | def toIndexInit: TreeSet[File] = if (toIndex.isEmpty) TreeSet[File]() else toIndex.init 79 | 80 | } 81 | 82 | def splitFiles(d: File, o: Options): Arbiter = { 83 | val rotatedPatternRegex = o.rotatedPattern.r 84 | 85 | def toMB(value: Double, places: Int): Double = { 86 | new BigDecimal(value / 1024D / 1024D).setScale(places, BigDecimal.ROUND_HALF_UP).doubleValue 87 | } 88 | 89 | def fileOrderName(f: File): Option[Int] = { 90 | rotatedPatternRegex 91 | .findFirstMatchIn(f.getName) 92 | .flatMap(m => if (m.groupCount > 0) Some(m.group(1)).map(_.toInt) else None) 93 | } 94 | 95 | def fileLength(f: File): Long = { 96 | if (f.getName.endsWith("gz") || f.getName.endsWith("tar")) f.length() * 10 else f.length() 97 | } 98 | 99 | if (new File(o.logDir + o.tailedLogFileName).createNewFile()) { 100 | log.warn(s"File to be tailed ${o.tailedLogFileName} doesn't exist, it was created...") 101 | } 102 | 103 | val allFiles = d.listFiles().filter(!_.isDirectory) 104 | val tailedFileSize = allFiles.find(_.getName == o.tailedLogFileName).get.length() 105 | val fileSizeToIndex = o.maxFileSizeToIndex * 1024 * 1024 - tailedFileSize 106 | 107 | log.info(f"Postfix is currently logging to file '${o.tailedLogFileName}' having size : ${toMB(tailedFileSize, 7)}%f MB") 108 | log.info(f"There is ${toMB(fileSizeToIndex, 7)}%f MB available for file indexing") 109 | log.info(s"Processing directory ${d.getCanonicalPath} searching for backup files :") 110 | 111 | val ordering = Ordering.by(fileOrderName) 112 | 113 | allFiles. 114 | filter( // filter out files that don't match pattern of rotated log files 115 | f => o.rotatedPatternFn(f.getName) 116 | ). 117 | sorted( // order matters, this deals with the fact that mail.log.12.gz file would incorrectly precede file mail.log.9.gz even though it's older 118 | ordering 119 | ). 120 | foldLeft( Arbiter(fileSizeToIndex, TreeSet[File]()(ordering.reverse), TreeSet[File]()(ordering.reverse)) ) { 121 | (arbiter, f) => { 122 | if (f.length() < 10) { 123 | log.warn(s"Skipping file ${f.getName} because it's empty !") 124 | Arbiter(arbiter.remainingSize, arbiter.toIndex, arbiter.toIgnore + f) 125 | } else if (arbiter.remainingSize < 0) { 126 | Arbiter(arbiter.remainingSize, arbiter.toIndex, arbiter.toIgnore + f) 127 | } else { 128 | val fileSize = fileLength(f) 129 | val remaining = arbiter.remainingSize - fileSize 130 | if (remaining > 0) { 131 | log.info(f"\t file ${f.getName} of size ${toMB(fileSize, 7)}%f MB is to be indexed, ${toMB(remaining, 7)}%f MB remains" ) 132 | Arbiter(remaining, arbiter.toIndex + f, arbiter.toIgnore) 133 | } else { 134 | log.warn(f"\t file ${f.getName} of size ${toMB(fileSize, 7)}%f MB is to be persisted, limit ${o.maxFileSizeToIndex}%f MB was reached, ${toMB(remaining * -1, 7)}%f MB is missing for it to be indexed") 135 | Arbiter(remaining, arbiter.toIndex, arbiter.toIgnore + f) 136 | } 137 | } 138 | } 139 | } 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /server/src/test/resources/META-INF/logs/parser/backup-single-client.log.1: -------------------------------------------------------------------------------- 1 | 2013 Jun 16 16:05:06.123 gds39d postfix/smtpd[7052]: 3bZLDk1V50z37Dv: client=gds39k.active24.cz[81.95.110.19] 2 | 2013 Jun 16 16:05:07.123 gds39d postfix/cleanup[26547]: 3bZLDk1V50z37Dv: message-id=<98494.123.456@test-mail-module> 3 | 2013 Jun 16 16:05:07.123 gds39d postfix/qmgr[6273]: 3bZLDk1V50z37Dv: from=, size=793, nrcpt=1 (queue active) 4 | 2013 Jun 16 16:05:07.123 gds39d postfix/smtp[20945]: 3bZLDk1V50z37Dv: to=, relay=hermes.fg.cz[193.86.74.5]:25, delay=0.16, delays=0.01/0/0.05/0.1, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 5058FBC3A) 5 | 2013 Jun 16 16:05:07.123 gds39d postfix/qmgr[6273]: 3bZLDk1V50z37Dv: removed 6 | 2013 Jun 16 14:49:42.123 gds39d postfix/smtpd[10275]: 3bYk6B34Hkz37Dv: client=gds39k.active24.cz[81.95.110.19] 7 | 2013 Jun 16 14:49:42.123 gds39d postfix/cleanup[1345]: 3bYk6B34Hkz37Dv: message-id=<98494.456.789@test-mail-module> 8 | 2013 Jun 16 14:49:42.123 gds39d postfix/qmgr[6273]: 3bYk6B34Hkz37Dv: from=, size=849, nrcpt=1 (queue active) 9 | 2013 Jun 16 14:49:42.123 gds39d postfix/smtp[31323]: 3bYk6B34Hkz37Dv: to=, relay=none, delay=0.02, delays=0.01/0/0.01/0, dsn=5.4.4, status=bounced (Host or domain name not found. Name service error for name=nonexistingserver.cz type=A: Host not found) 10 | 2013 Jun 16 14:49:42.123 gds39d postfix/bounce[1346]: 3bYk6B34Hkz37Dv: sender non-delivery notification: C1ABDC2C11B 11 | 2013 Jun 16 14:49:42.123 gds39d postfix/qmgr[6273]: 3bYk6B34Hkz37Dv: removed 12 | 2013 Jun 10 15:34:07.123 gds39d postfix/smtpd[14765]: 3bYscs22wBz37Dv: client=gds39k.active24.cz[81.95.110.19] 13 | 2013 Jun 10 15:34:07.123 gds39d postfix/cleanup[14685]: 3bYscs22wBz37Dv: message-id=<98494.321.654@test-mail-module> 14 | 2013 Jun 10 15:34:07.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: from=, size=9236, nrcpt=1 (queue active) 15 | 2013 Jun 10 15:34:42.123 gds39d postfix/smtp[12216]: 3bYscs22wBz37Dv: to=, relay=none, delay=35, delays=0/0/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 16 | 2013 Jun 10 15:43:17.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: from=, size=9236, nrcpt=1 (queue active) 17 | 2013 Jun 10 15:43:52.123 gds39d postfix/smtp[1505]: 3bYscs22wBz37Dv: to=, relay=none, delay=585, delays=549/0.06/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 18 | 2013 Jun 10 15:58:17.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: from=, size=9236, nrcpt=1 (queue active) 19 | 2013 Jun 10 15:58:52.123 gds39d postfix/smtp[30111]: 3bYscs22wBz37Dv: to=, relay=none, delay=1485, delays=1450/0/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 20 | 2013 Jun 10 16:28:17.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: from=, size=9236, nrcpt=1 (queue active) 21 | 2013 Jun 10 16:28:52.123 gds39d postfix/smtp[30083]: 3bYscs22wBz37Dv: to=, relay=none, delay=3285, delays=3250/0.01/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 22 | 2013 Jun 10 17:28:17.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: from=, size=9236, nrcpt=1 (queue active) 23 | 2013 Jun 10 17:28:52.123 gds39d postfix/smtp[25891]: 3bYscs22wBz37Dv: to=, relay=none, delay=6885, delays=6850/0.03/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 24 | 2013 Jun 10 18:38:17.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: from=, size=9236, nrcpt=1 (queue active) 25 | 2013 Jun 10 18:38:52.123 gds39d postfix/smtp[6704]: 3bYscs22wBz37Dv: to=, relay=none, delay=11085, delays=11050/0.03/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 26 | 2013 Jun 15 16:28:52.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: from=, status=expired, returned to sender 27 | 2013 Jun 15 16:28:52.123 gds39d postfix/bounce[28279]: 3bYscs22wBz37Dv: sender non-delivery notification: 6B67B3FC013 28 | 2013 Jun 15 16:28:52.123 gds39d postfix/qmgr[6273]: 3bYscs22wBz37Dv: removed 29 | 30 | 2013 Jun 18 17:34:07.123 gds39d postfix/cleanup[14685]: 3bYsfs23wBzs7Dv: message-id=msgId-3bYsfs23wBzs7Dv 31 | 2013 Jun 18 17:34:07.123 gds39d postfix/cleanup[14685]: 3bYsfs23wBzs7Dv: info: header client-id: test-mail-module from localhost[127.0.0.1]; from= to= proto=ESMTP helo= 32 | 2013 Jun 18 17:34:07.123 gds39d postfix/qmgr[6273]: 3bYsfs23wBzs7Dv: from=, size=9236, nrcpt=1 (queue active) 33 | 2013 Jun 18 17:34:42.123 gds39d postfix/smtp[12216]: 3bYsfs23wBzs7Dv: to=, relay=none, delay=35, delays=0/0/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 34 | 2013 Jun 18 17:43:17.123 gds39d postfix/qmgr[6273]: 3bYsfs23wBzs7Dv: from=, size=9236, nrcpt=1 (queue active) 35 | 2013 Jun 18 17:43:52.123 gds39d postfix/smtp[1505]: 3bYsfs23wBzs7Dv: to=, relay=none, delay=585, delays=549/0.06/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 36 | 2013 Jun 18 17:58:17.123 gds39d postfix/qmgr[6273]: 3bYsfs23wBzs7Dv: from=, size=9236, nrcpt=1 (queue active) 37 | 2013 Jun 18 17:58:52.123 gds39d postfix/smtp[30111]: 3bYsfs23wBzs7Dv: to=, relay=none, delay=1485, delays=1450/0/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 38 | 2013 Jun 18 18:28:17.123 gds39d postfix/qmgr[6273]: 3bYsfs23wBzs7Dv: from=, size=9236, nrcpt=1 (queue active) 39 | 2013 Jun 18 18:28:52.123 gds39d postfix/smtp[30083]: 3bYsfs23wBzs7Dv: to=, relay=none, delay=3285, delays=3250/0.01/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 40 | 2013 Jun 18 18:28:17.123 gds39d postfix/qmgr[6273]: 3bYsfs23wBzs7Dv: from=, size=9236, nrcpt=1 (queue active) 41 | 2013 Jun 18 18:48:52.123 gds39d postfix/smtp[25891]: 3bYsfs23wBzs7Dv: to=, relay=none, delay=6885, delays=6850/0.03/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 42 | 2013 Jun 18 19:38:17.123 gds39d postfix/qmgr[6273]: 3bYsfs23wBzs7Dv: from=, size=9236, nrcpt=1 (queue active) 43 | 2013 Jun 18 19:38:52.123 gds39d postfix/smtp[6704]: 3bYsfs23wBzs7Dv: to=, relay=none, delay=11085, delays=11050/0.03/35/0, dsn=4.4.3, status=deferred (Host or domain name not found. Name service error for name=cedefop.eu.int type=MX: Host not found, try again) 44 | 2013 Jun 18 20:05:07.123 gds39d postfix/smtp[20945]: 3bYsfs23wBzs7Dv: to=, relay=hermes.fg.cz[193.86.74.5]:25, delay=0.16, delays=0.01/0/0.05/0.1, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 5058FBC3A) 45 | -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/rest/RestDSL.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.rest 2 | 3 | import com.fg.mail.smtp.Request 4 | 5 | /** 6 | * DSL backend for {@link com.fg.mail.smtp.rest.Dispatcher} 7 | * 8 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 9 | * @version $Id: 6/24/13 1:56 PM u_jli Exp $ 10 | */ 11 | object RestDSL { 12 | 13 | type KeyValueMatcher[T] = (String, PartialFunction[Seq[String], T]) 14 | 15 | object $ { 16 | def unapply(pathSegment: String): Boolean = { 17 | pathSegment.eq(null) || pathSegment.length == 0 18 | } 19 | } 20 | 21 | object * { 22 | def apply(key: String): KeyValueMatcher[String] = ( key, { case Seq(head, _*) => head } ) 23 | def unapply(pathSegment: String): Option[String] = { 24 | if (!(pathSegment eq null) && pathSegment.length > 0) 25 | Some(pathSegment) 26 | else 27 | None 28 | } 29 | } 30 | 31 | object `@` { 32 | def apply(key: String): KeyValueMatcher[String] = ( key, { case Seq(head, _*) => head } ) 33 | def unapply(emailSegment: String): Option[String] = { 34 | emailSegment match { 35 | case (es: String) if es.contains("@") => Some(es) 36 | case _ => None 37 | } 38 | } 39 | } 40 | 41 | object BOOLEAN { 42 | def apply(key: String): KeyValueMatcher[Boolean] = ( 43 | key, 44 | new PartialFunction[Seq[String], Boolean] { 45 | def isDefinedAt(values: Seq[String]): Boolean = { 46 | values.mkString match { 47 | case path: String if path.equalsIgnoreCase("true") || path.equalsIgnoreCase("false") => true 48 | case _ => false 49 | } 50 | } 51 | 52 | def apply(values: Seq[String]): Boolean = { 53 | values.mkString match { 54 | case BOOLEAN(b) => b 55 | } 56 | } 57 | } 58 | ) 59 | def unapply(pathSegment: String): Option[Boolean] = { 60 | pathSegment match { 61 | case path: String if path.equalsIgnoreCase("true") => Some(true) 62 | case path: String if path.equalsIgnoreCase("false") => Some(false) 63 | case _ => None 64 | } 65 | } 66 | } 67 | 68 | object LONG { 69 | def apply(key: String): KeyValueMatcher[Long] = ( 70 | key, 71 | new PartialFunction[Seq[String], Long] { 72 | def isDefinedAt(values: Seq[String]): Boolean = { 73 | values.exists(_ match { case LONG(_) => true; case _ => false }) 74 | } 75 | 76 | def apply(values: Seq[String]): Long = { 77 | values.collectFirst({ case LONG(l) => l }).get 78 | } 79 | } 80 | ) 81 | def unapply(string: String): Option[Long] = { 82 | if (string eq null) 83 | None 84 | else 85 | try { 86 | Some(java.lang.Long.parseLong(string, 10)) 87 | } 88 | catch { 89 | case e: NumberFormatException => None 90 | case e: Throwable => throw e 91 | } 92 | } 93 | } 94 | 95 | type Dispatch = (String) => Option[(String) => Option[Request]] 96 | 97 | implicit def RequestToDispatch(h: Request): Dispatch = { 98 | (pathSegment: String) => 99 | { 100 | if (pathSegment eq null) 101 | Some((query: String) => Some(h)) 102 | else 103 | None 104 | } 105 | } 106 | 107 | case class /(matcher: PartialFunction[String, Dispatch]) extends Dispatch { 108 | 109 | /** 110 | * @param trailingPath A valid URI path (or the yet unmatched part of the path of an URI). 111 | * The trailingPath is either null or is a string that starts with a "/". 112 | * The semantics of null is that the complete path was matched; i.e., there 113 | * is no remaining part. 114 | */ 115 | def apply(trailingPath: String): Option[String => Option[Request]] = { 116 | if (trailingPath == null) 117 | return { 118 | if (matcher.isDefinedAt(null)) 119 | matcher(null)(null) 120 | else 121 | // the provided path was completely matched, however it is too short 122 | // w.r.t. a RESTful application; hence, this points to a design problem 123 | None 124 | } 125 | 126 | if (trailingPath.charAt(0) != '/') 127 | throw new IllegalArgumentException("The provided path: \""+trailingPath+"\" is invalid; it must start with a /.") 128 | 129 | val path = trailingPath.substring(1) // we truncate the trailing "/" 130 | val separatorIndex = path.indexOf('/') 131 | val head = if (separatorIndex == -1) path else path.substring(0, separatorIndex) 132 | val tail = if (separatorIndex == -1 || separatorIndex == 0 && path.length == 1) null else path.substring(separatorIndex) 133 | if (matcher.isDefinedAt(head)) 134 | matcher(head)(tail) 135 | else 136 | None 137 | } 138 | } 139 | 140 | type URIQuery = Map[String, Seq[String]] 141 | 142 | case class ?(matcher: PartialFunction[URIQuery, Request]) extends Dispatch { 143 | def apply(path: String): Option[String => Option[Request]] = { 144 | if (path ne null) 145 | None 146 | else { 147 | Some((query: String) => { 148 | val splitUpQuery = decodeRawURLQueryString(query) 149 | if (matcher.isDefinedAt(splitUpQuery)) 150 | Some(matcher(splitUpQuery)) 151 | else 152 | None 153 | }) 154 | } 155 | } 156 | } 157 | 158 | trait QueryMatcher { 159 | protected def apply[T](kvMatcher: KeyValueMatcher[T], uriQuery: URIQuery): Option[T] = { 160 | val (key, valueMatcher) = kvMatcher 161 | uriQuery.get(key) match { 162 | case Some(values) => { 163 | if (valueMatcher.isDefinedAt(values)) { 164 | Some(valueMatcher(values)) 165 | } else { 166 | None 167 | } 168 | } 169 | case None => None 170 | } 171 | } 172 | } 173 | 174 | object QueryMatcher { 175 | def apply[T1](kvm1: KeyValueMatcher[T1]) = 176 | new QueryMatcher1(kvm1) 177 | 178 | def apply[T1, T2](kvm1: KeyValueMatcher[T1], kvm2: KeyValueMatcher[T2]) = 179 | new QueryMatcher2(kvm1, kvm2) 180 | 181 | def apply[T1, T2, T3](kvm1: KeyValueMatcher[T1], kvm2: KeyValueMatcher[T2], kvm3: KeyValueMatcher[T3]) = 182 | new QueryMatcher3(kvm1, kvm2, kvm3) 183 | 184 | } 185 | 186 | class QueryMatcher1[T1](val kvMatcher1: KeyValueMatcher[T1]) extends QueryMatcher { 187 | def unapply(uriQuery: URIQuery): Some[Option[T1]] = { 188 | Some(apply(kvMatcher1, uriQuery)) 189 | } 190 | } 191 | 192 | class QueryMatcher2[T1, T2](val kvm1: KeyValueMatcher[T1], val kvm2: KeyValueMatcher[T2]) extends QueryMatcher { 193 | def unapply(uriQuery: URIQuery): Some[(Option[T1], Option[T2])] = { 194 | Some((apply(kvm1, uriQuery), apply(kvm2, uriQuery))) 195 | } 196 | } 197 | 198 | class QueryMatcher3[T1, T2, T3](val kvm1: KeyValueMatcher[T1], val kvm2: KeyValueMatcher[T2], val kvm3: KeyValueMatcher[T3]) extends QueryMatcher { 199 | def unapply(uriQuery: URIQuery): Some[(Option[T1], Option[T2], Option[T3])] = { 200 | Some((apply(kvm1, uriQuery), apply(kvm2, uriQuery), apply(kvm3, uriQuery))) 201 | } 202 | } 203 | 204 | def decodeRawURLQueryString(query: String, encoding: String = "UTF-8"): Map[String, List[String]] = { 205 | import java.net.URLDecoder.decode 206 | 207 | var param_values = Map[String, List[String]]().withDefaultValue(List[String]()) 208 | if ((query eq null) || query.length == 0) 209 | param_values 210 | else { 211 | for (param_value <- query.split('&')) { 212 | val index = param_value.indexOf('=') 213 | if (index == -1) { 214 | val param = decode(param_value, encoding) 215 | param_values = param_values.updated(param, param_values(param)) 216 | } else { 217 | val param = decode(param_value.substring(0, index), encoding) 218 | val value = decode(param_value.substring(index + 1), encoding) 219 | param_values = param_values.updated(param, param_values(param) :+ value) 220 | } 221 | } 222 | param_values 223 | } 224 | } 225 | 226 | 227 | } 228 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Postfix Deliverability Analytics 2 | ========= 3 | 4 | Due to the smtp protocol imperfection there are very limited possibilities for a mail client to learn much about what happens with a message after being sent. However we can resolve this information from logs of Postfix smtp server that is logging the entire course of message delivery process. PDA runs on the same host and it is parsing postfix log files and indexing data so that mail clients can query PDA over http and restful interface for whatever information it is interested in. 5 | 6 | Build instructions 7 | ----- 8 | 9 | PDA is not meant to be an embeddable application but rather standalone agent monitoring postfix logs and serving client requests. So far it uses maven and you don't need to use anything else then basic maven goals like clean, test and install. Install makes a package with generated shell scripts that are self documented. 10 | 11 | Use [common.sh](https://github.com/FgForrest/Postfix-Deliverability-Analytics/blob/master/server/common.sh) for package distribution. 12 | 13 | Use [control.sh](https://github.com/FgForrest/Postfix-Deliverability-Analytics/blob/master/server/src/main/resources/control.sh) for server lifecycle management 14 | 15 | This is a directory structure you're end up with : 16 | 17 | ``` 18 | ├── db 19 | │   └── main 20 | │   ├── queue 21 | │   ├── queue.p 22 | │   ├── queue.t 23 | │   ├── smtpLogDb 24 | │   └── smtpLogDb.p 25 | └── pda 26 | ├── bin 27 | │   ├── control.sh 28 | │   └── start.sh 29 | ├── conf 30 | │   ├── application.conf 31 | │   └── bounce-regex-list.xml 32 | ├── logs 33 | │   └── app-2014-03-22_205457.log 34 | └── repo (jars) 35 | 36 | ``` 37 | 38 | PDA general info 39 | ---- 40 | 41 | - Standalone application written in Scala, using Akka and MapDB as it's core dependencies 42 | - Apart from HttpServer single-threaded pool there are 4 more threads/actors : 43 | - Superviser 44 | - Indexer - receives relevant log records and indexes / stores them to MapDB. 45 | - Tailer - controls another thread that tails log files. It is blocked all the time 46 | - Memory footprint is from 20 - 400 MG RAM depending on databaze size that affects memory size required for handling restful queries 47 | - Database is encrypted and http server supports only basic authentication 48 | - Prefer using dedicated postfix server being shared by a set of PDA client applications. Do not expect PDA to be able of handling tens of GigaBytes of log files. On a dedicated postfix server PDA is able to index and serve around 5GB of uncompressed log files. It is not optimized and tested beyond that. 49 | - PDA is using bounce-regex-list.xml file to gather regular expressions for categorizing bounces by messages. Once in a while it needs to be updated with regular expression for newly encountered bounce messages that was not categorized by current rules. Beware that amount of regular expressions and their complexity influence indexing speed because it might be executed in order of magnitude of 7 50 | - PDA's user interface is served by HttpServer running on localhost:1523 by default 51 | - PDA's initialization has a several steps 52 | 1. regex-bounce-list.xml file is processed 53 | 2. log files that were backed up by logrotate are indexed 54 | 3. tailing of the actual log file that is being written to starts, indexing all its current and future content 55 | 4. HttpServer starts listening to http requests 56 | 57 | ![Diagram](https://github.com/FgForrest/Postfix-Deliverability-Analytics/blob/master/diagram.png) 58 | 59 | Postfix settings 60 | ---- 61 | 62 | - All necessary configuration regarding to postfix can be set in /etc/postfix/main.cf 63 | - deferred queue tuning - when postfix is unable to deliver a message due to a soft bounce, it attempts to do so again using interval where next value is a double of the previous. By default it is trying to do so too frequently which might produce GigaBytes of data. Desired time serie would rather look like this : 4m, 8m, 16m, 32m, 1h, 2h, 4h, 8h, 16h, 32h, 64h 64 | - minimal_backoff_time = 240s 65 | - maximal_backoff_time = 40h 66 | - maximal_queue_lifetime = 6d 67 | - enable_long_queue_ids = yes 68 | - this is an essential setting that makes queue IDs unique which eliminates records duplication 69 | - everything else can be set in PDA's agent.conf file, especially : 70 | - logs.dir = "/var/log/" 71 | - tailed-log-file = "mail.log" 72 | - rotated-file = "mail.log.1" 73 | - rotated-file-pattern = "mail\\.log\\.(\\d{1,3}).*" 74 | - max-file-size-to-index : 1000 75 | 76 | Mail client settings 77 | ---- 78 | 79 | Client applications must identify themselves so that PDA is able to associate message queues with corresponding smtp clients. There are 2 ways how to do that: 80 | - **simple** : override Message-ID header with value of this format : 81 | ``` 82 | "<" + "cid".hashCode() + '.' + id + '.' + RND.nextInt(1000) + '@' + clientId + ">" 83 | ``` 84 | 85 | - **complicated** : consists in adding a new header "client-id" to a message and setting up postfix to log these headers so that PDA can see that. Header_checks and logging requires creating a configuration file /etc/postfix/maps/header_checks which makes postfix log messages that have a "client-id" MIME header 86 | ``` 87 | "/^client-id:.* / INFO" 88 | ``` 89 | 90 | Syslog settings 91 | ------ 92 | 93 | Syslog is logging timestamps without miliseconds which is not quite useful. In /etc/rsyslog.conf : 94 | - define MailLogFormat template 95 | ``` 96 | $template MailLogFormat,"%timegenerated:1:4:date-rfc3339% %timegenerated:1:6:date-rfc3164-buggyday% %timegenerated:12:23:date-rfc3339% %source% %syslogtag%%msg%\n" 97 | ``` 98 | 99 | - tell syslog to use template MailLogFormat for logging into mail.* log files 100 | ``` 101 | mail.* -/var/log/mail.log;MailLogFormat 102 | ``` 103 | 104 | Logrotate settings 105 | ------ 106 | 107 | Logrotate is periodically rotating log files so that the log file will be renamed and compressed as follows : 108 | - mail.log -> mail.log.1 -> mail.log.1.gz -> mail.log.1.gz 109 | - these settings makes it easier for PDA to process everything. Property 'size 100MB' doesn't allow log files grow more then 100MB 110 | - /etc/logrotate.d/rsyslog 111 | ``` 112 | /var/log/mail.log { 113 | rotate 100 114 | size 100M 115 | missingok 116 | notifempty 117 | compress 118 | delaycompress 119 | sharedscripts 120 | postrotate 121 | invoke-rc.d rsyslog reload > /dev/null 122 | endscript 123 | } 124 | ``` 125 | 126 | PDA setting 127 | ----- 128 | 129 | All PDA settings are available in [application.conf](https://github.com/FgForrest/Postfix-Deliverability-Analytics/blob/master/server/src/main/resources/application.conf) 130 | 131 | 132 | Bounce classification by regular expressions 133 | ------ 134 | 135 | Considering the fact that individual email services make up their own error messages and reasons why email can not be delivered, there is no easy way to classify them. [bounce-regex-list.xml](https://github.com/FgForrest/Postfix-Deliverability-Analytics/blob/master/server/src/main/resources/bounce-regex-list.xml) groups regular expressions into a several categories to match every error message encountered at log file in order to classify it as a soft or hard bounce and to find general reason for it's nondelivery. 136 | After some time you can check unclassified bounces in UI [http://localhost:1523/](http://localhost:1523/) under 'Unclassified bounce messages' or [http://localhost:1523/agent-status/unknown-bounces](http://localhost:1523/agent-status/unknown-bounces). There is a list of clients, each might contain a list of bounced messages that were not classified, value of 'info' property is the actual error message that was not possible to classify using bounce-regex-list. You need to edit file bounce-regex-list.xml (it's location depends on application.conf settings), add or change some regular expression so that next time this kind of error will be classified. 'Refresh bounce list' command reloads bounce-regex-list.xml that you edited. Significant amount of bounces might mean that there is something wrong with your postfix settings or mx records -------------------------------------------------------------------------------- /server/src/main/scala/com/fg/mail/smtp/index/Index.scala: -------------------------------------------------------------------------------- 1 | package com.fg.mail.smtp.index 2 | 3 | import scala.collection.IterableView 4 | import java.util 5 | import org.mapdb._ 6 | import java.util.{Comparator, NavigableSet} 7 | import java.io.{DataInput, DataOutput} 8 | import scala.collection.JavaConverters._ 9 | 10 | /** 11 | * Index is a set of [clientId, date, indexRecord] 12 | * Client id log entry is preceding following log entries with the same queue id 13 | * 14 | * @author Jakub Liška (liska@fg.cz), FG Forrest a.s. (c) 2013 15 | * @version $Id: 6/24/13 7:18 PM u_jli Exp $ 16 | */ 17 | class Index(val records: NavigableSet[Fun.Tuple3[String, Long, IndexRecord]]) { 18 | 19 | def isEmpty = records.isEmpty 20 | 21 | /* 22 | def sizeFor(clientId: String): Int = 23 | profile(500, "Counting db records for clientId", "clientId") { 24 | records.subSet(Fun.t3(clientId, 0L, null), Fun.t3(clientId, Fun.HI[Long], Fun.HI[IndexRecord])).size() 25 | } 26 | */ 27 | 28 | def addRecord(cir: ClientIndexRecord) = { 29 | val clientId = cir.clientId 30 | val r = cir.ir 31 | records.add(Fun.t3(clientId, r.date, r)) 32 | } 33 | 34 | /** 35 | * @return lazy view of records by clientId 36 | * @note that you can iterate Map values only once because iteration over millions takes more than a few seconds on a single-core machine with NFS 37 | */ 38 | def getAsMapWithDisposableValues(interval: Interval): Map[String, IterableView[IndexRecord, Iterable[IndexRecord]]] = { 39 | new Iterable[(String, IterableView[IndexRecord, Iterable[IndexRecord]])] { 40 | def iterator: Iterator[(String, IterableView[IndexRecord, Iterable[IndexRecord]])] = 41 | new Iterator[(String, IterableView[IndexRecord, Iterable[IndexRecord]])] { 42 | val clientIt = getClientIds.iterator 43 | def hasNext: Boolean = clientIt.hasNext 44 | def next(): (String, IterableView[IndexRecord, Iterable[IndexRecord]]) = { 45 | val next = clientIt.next() 46 | val subIter = records.subSet( 47 | Fun.t3(next, interval.from.getOrElse(0L), null), 48 | Fun.t3(next, if (interval.to.isEmpty) Fun.HI[Long] else interval.to.get, Fun.HI[IndexRecord]) 49 | ).iterator() 50 | ( 51 | next, 52 | new Iterable[IndexRecord] { 53 | def iterator: Iterator[IndexRecord] = { 54 | new Iterator[IndexRecord] { 55 | def hasNext: Boolean = subIter.hasNext 56 | def next(): IndexRecord = subIter.next().c 57 | } 58 | } 59 | }.view 60 | ) 61 | } 62 | } 63 | }.toMap 64 | } 65 | 66 | /** 67 | * @return lazy view of all tuple[clientId, record] - in case it would be really necessary to iterate through possibly tens of millions of records, it better be lazy (such an iteration takes more than a few seconds on a single-core machine with NFS) 68 | * @note that you can iterate it only once because iteration over millions takes more than a few seconds on a single-core machine with NFS 69 | */ 70 | def getClientIdRecordTuples: IterableView[(String, IndexRecord), Iterable[(String, IndexRecord)]] = { 71 | new Iterable[(String, IndexRecord)] { 72 | def iterator: Iterator[(String, IndexRecord)] = { 73 | new Iterator[(String, IndexRecord)] { 74 | val iter = records.iterator 75 | def hasNext: Boolean = iter.hasNext 76 | def next(): (String, IndexRecord) = { 77 | val next = iter.next() 78 | (next.a, next.c) 79 | } 80 | } 81 | } 82 | }.view 83 | } 84 | 85 | /** 86 | * @return lazy view of records - in case it would be really necessary to iterate through possibly tens of millions of records, it better be lazy (such an iteration takes more than a few seconds on a single-core machine with NFS) 87 | * @note that you can iterate it only once because iteration over millions takes more than a few seconds on a single-core machine with NFS 88 | */ 89 | def getRecords: IterableView[IndexRecord, Iterable[IndexRecord]] = { 90 | new Iterable[IndexRecord] { 91 | def iterator: Iterator[IndexRecord] = { 92 | new Iterator[IndexRecord] { 93 | val iter = records.iterator 94 | def hasNext: Boolean = iter.hasNext 95 | def next(): IndexRecord = iter.next().c 96 | } 97 | } 98 | }.view 99 | } 100 | 101 | /** 102 | * @param interval time constraint - if it contains None values, it means the interval is open 103 | * @param reverse true means descending order, false ascending 104 | * @return lazy view of all records for particular clientId constrained by specified interval 105 | * @note that you can iterate it only once because iteration over millions takes more than a few seconds on a single-core machine with NFS 106 | */ 107 | def getRecordsFor(clientId: String, interval: Interval, reverse: Boolean = false): IterableView[IndexRecord, Iterable[IndexRecord]] = { 108 | new Iterable[IndexRecord] { 109 | def iterator: Iterator[IndexRecord] = { 110 | new Iterator[IndexRecord] { 111 | val iter = records.subSet( 112 | Fun.t3(clientId, interval.from.getOrElse(0L), null), 113 | Fun.t3(clientId, if (interval.to.isEmpty) Fun.HI[Long] else interval.to.get, Fun.HI[IndexRecord]) 114 | ) match { 115 | case ss: util.NavigableSet[Fun.Tuple3[String, Long, IndexRecord]] if reverse => ss.descendingIterator() 116 | case ss: util.NavigableSet[Fun.Tuple3[String, Long, IndexRecord]] if !reverse => ss.iterator() 117 | case _ => throw new IllegalStateException("BTreeMap KeySet's subset ain't NavigableSet") 118 | } 119 | def hasNext: Boolean = iter.hasNext 120 | def next(): IndexRecord = iter.next.c 121 | } 122 | } 123 | }.view 124 | } 125 | 126 | /** Resolving client ids from MapDb would have O(n) complexity which is deadly for millions of records (such an iteration takes more than a few seconds on a single-core machine with NFS) */ 127 | def getClientIds: Iterable[String] = { 128 | val s = new java.util.TreeSet[Fun.Tuple3[String, Long, IndexRecord]]( 129 | new Comparator[Fun.Tuple3[String, Long, IndexRecord]] { 130 | def compare(o1: Fun.Tuple3[String, Long, IndexRecord], o2: Fun.Tuple3[String, Long, IndexRecord]): Int = o1.a.compare(o2.a) 131 | } 132 | ) 133 | s.addAll(records) 134 | s.asScala.map(_.a) 135 | } 136 | } 137 | 138 | 139 | /** 140 | * It tells MapDb how to serialize IndexRecord for it to perform better 141 | */ 142 | class IndexRecordSerializer extends Serializer[IndexRecord] with Serializable { 143 | 144 | def serialize(out: DataOutput, r: IndexRecord) { 145 | out.writeLong(r.date) 146 | out.writeUTF(r.queueId) 147 | out.writeUTF(r.msgId) 148 | out.writeUTF(r.rcptEmail) 149 | out.writeUTF(r.senderEmail) 150 | out.writeUTF(r.status) 151 | out.writeUTF(r.info) 152 | out.writeInt(r.state) 153 | out.writeUTF(r.stateInfo) 154 | } 155 | 156 | def deserialize(in: DataInput, available: Int): IndexRecord = { 157 | IndexRecord(in.readLong(), in.readUTF(), in.readUTF(), in.readUTF(), in.readUTF(), in.readUTF(), in.readUTF(), in.readInt(), in.readUTF()) 158 | } 159 | 160 | def fixedSize(): Int = -1 161 | } 162 | 163 | /** Time constraint for getting records that occurred at a period of time 'from - to'. None means the interval is open. */ 164 | case class Interval(from: Option[Long], to: Option[Long]) 165 | 166 | object Index { 167 | 168 | val serializer = new BTreeKeySerializer.Tuple3KeySerializer[String, java.lang.Long, IndexRecord](null, null, Serializer.STRING, Serializer.LONG, new IndexRecordSerializer) 169 | 170 | def apply(db: DB, name: String) = { 171 | /** Node size 6 proved to be the most optimal value for IndexRecord persistence */ 172 | new Index(db.createTreeSet(name).counterEnable().nodeSize(6).serializer(serializer).makeOrGet()) 173 | } 174 | 175 | } --------------------------------------------------------------------------------