├── .gitignore ├── .scalafix.conf ├── .scalafmt.conf ├── CNAME ├── LICENSE ├── README.md ├── Resources.md ├── analytics ├── dataset │ ├── .gitignore │ ├── enron-small.parquet │ ├── roles.csv │ ├── rolesDataPrep.scs │ └── src │ │ ├── it │ │ ├── resources │ │ │ └── log4j.xml │ │ └── scala │ │ │ └── com │ │ │ └── uebercomputing │ │ │ └── spark │ │ │ └── dataset │ │ │ └── TopNEmailMessageSendersIntegrationTest.scala │ │ └── main │ │ └── scala │ │ └── com │ │ └── uebercomputing │ │ └── spark │ │ ├── dataset │ │ └── TopNEmailMessageSenders.scala │ │ └── sql │ │ ├── DataFrameFromRddDynamicSchema.scala │ │ ├── DataFrameFromRddWithKnownSchema.scala │ │ ├── DataFrameOps.scala │ │ ├── DataFrameToRdd.scala │ │ ├── EmailsPerUserDataFrame.scala │ │ ├── EmailsPerUserRdd.scala │ │ ├── MySqlJdbcDataFrame.scala │ │ ├── ParquetPartitions.scala │ │ └── UniqueFromsCounter.scala └── rdd │ └── src │ ├── main │ └── scala │ │ └── com │ │ └── uebercomputing │ │ └── background │ │ └── FunctionalCombinators.scala │ └── test │ └── resources │ ├── enron │ └── maildir │ │ ├── kean-s │ │ └── deleted_items │ │ │ └── 338.txt │ │ ├── lay-k │ │ ├── enron │ │ │ ├── 2.txt │ │ │ └── 3.txt │ │ └── inbox │ │ │ ├── 568.txt │ │ │ └── 898.txt │ │ ├── mims-thurston-p │ │ └── inbox │ │ │ ├── 28.txt │ │ │ └── 98.txt │ │ └── neal-s │ │ ├── all_documents │ │ ├── 10.txt │ │ ├── 20.txt │ │ ├── 94.txt │ │ └── 99.txt │ │ └── regulatory │ │ ├── 1.txt │ │ └── 2.txt │ └── log4j.properties ├── enron-small.parquet ├── mailrecord-utils ├── .gitignore └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── uebercomputing │ │ │ └── mailrecord │ │ │ ├── Attachment.java │ │ │ ├── MailRecord.java │ │ │ └── MailRecordProtocol.java │ ├── resources │ │ ├── log4j.xml │ │ └── log4j2.xml │ └── scala │ │ └── com │ │ └── uebercomputing │ │ ├── hadoop │ │ └── HadoopUtils.scala │ │ ├── io │ │ ├── IoConstants.scala │ │ ├── PathUtils.scala │ │ └── Utf8Codec.scala │ │ ├── mailparser │ │ └── enronfiles │ │ │ ├── AvroMain.scala │ │ │ ├── AvroMessageProcessor.scala │ │ │ ├── FileSystemMetadata.scala │ │ │ ├── MailDirectoryProcessor.scala │ │ │ ├── MessageParser.scala │ │ │ ├── MessageProcessor.scala │ │ │ ├── MessageUtils.scala │ │ │ ├── ParquetMain.scala │ │ │ ├── ParquetMessageProcessor.scala │ │ │ ├── ParseException.scala │ │ │ └── ParsedMessageToMailRecordConverter.scala │ │ ├── mailrecord │ │ ├── Implicits.scala │ │ ├── MailRecordAvroReader.scala │ │ ├── MailRecordAvroWriter.scala │ │ ├── MailRecordOps.scala │ │ ├── MailRecordParquetReader.scala │ │ └── MailRecordParquetWriter.scala │ │ └── utils │ │ └── DatePartitioner.scala │ └── test │ ├── resources │ ├── enron │ │ └── maildir │ │ │ ├── kean-s │ │ │ └── deleted_items │ │ │ │ └── 338.txt │ │ │ ├── lay-k │ │ │ ├── enron │ │ │ │ ├── 2.txt │ │ │ │ └── 3.txt │ │ │ └── inbox │ │ │ │ ├── 568.txt │ │ │ │ └── 898.txt │ │ │ ├── mims-thurston-p │ │ │ └── inbox │ │ │ │ ├── 28.txt │ │ │ │ └── 98.txt │ │ │ └── neal-s │ │ │ ├── all_documents │ │ │ ├── 10.txt │ │ │ ├── 20.txt │ │ │ ├── 94.txt │ │ │ └── 99.txt │ │ │ └── regulatory │ │ │ ├── 1.txt │ │ │ └── 2.txt │ ├── log4j.xml │ └── log4j2-test.xml │ └── scala │ └── com │ └── uebercomputing │ ├── mailparser │ └── enronfiles │ │ ├── AvroMessageProcessorTest.scala │ │ ├── MailDirectoryProcessorTest.scala │ │ └── MessageParserTest.scala │ ├── mailrecord │ ├── MailRecordOpsTest.scala │ ├── MailRecordParquetWriterTest.scala │ └── MailRecordProvider.scala │ ├── test │ ├── AvroFileFixtureTest.scala │ ├── AvroMailRecordsFileProvider.scala │ └── UnitTest.scala │ └── utils │ └── DatePartitionerTest.scala ├── notebooks ├── ApacheSparkThroughEmail1.ipynb ├── ApacheSparkThroughEmail2.ipynb ├── ApacheSparkThroughEmail3.ipynb ├── exec │ └── .gitignore ├── html │ ├── ApacheSparkThroughEmail1.html │ ├── ApacheSparkThroughEmail2.html │ └── ApacheSparkThroughEmail3.html └── pdf │ ├── ApacheSparkThroughEmail1.pdf │ ├── ApacheSparkThroughEmail2.pdf │ └── ApacheSparkThroughEmail3.pdf ├── presentation ├── .gitignore ├── ApacheSparkThroughEmail.md ├── ApacheSparkThroughEmail.pdf ├── SpeakerNotes.md ├── SpeakerNotes.pdf ├── graphics │ ├── AsymmetrikPingPong.png │ ├── Catalyst.png │ ├── CatalystUnifiedInterface.png │ ├── Chemo.png │ ├── Column.png │ ├── DataFrameReader.png │ ├── DatasetApi.png │ ├── Farley.png │ ├── FunctionApi.png │ ├── Goal.png │ ├── HadoodEcosystem.xcf │ ├── Hadoop.png │ ├── HadoopEcosystem.png │ ├── HorizontalScaling.png │ ├── Laptop.png │ ├── Notebook1Job2Dag.png │ ├── QueryPlanningPhases.png │ ├── SparkApplication.png │ ├── SparkComponents.png │ ├── SparkJobsNotebook1.png │ ├── SparkLogo.png │ ├── SparkRdd.png │ ├── StandaloneServer1.png │ └── VerticalScaling.png ├── makeNotes └── makePresentation ├── project ├── Dependencies.scala ├── build.properties └── plugins.sbt ├── roles.csv └── version.sbt /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/.gitignore -------------------------------------------------------------------------------- /.scalafix.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/.scalafix.conf -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/.scalafmt.conf -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | uebercomputing.com 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/README.md -------------------------------------------------------------------------------- /Resources.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/Resources.md -------------------------------------------------------------------------------- /analytics/dataset/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /analytics/dataset/enron-small.parquet: -------------------------------------------------------------------------------- 1 | /datasets/enron/enron-small.parquet -------------------------------------------------------------------------------- /analytics/dataset/roles.csv: -------------------------------------------------------------------------------- 1 | /datasets/enron/roles.csv -------------------------------------------------------------------------------- /analytics/dataset/rolesDataPrep.scs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/rolesDataPrep.scs -------------------------------------------------------------------------------- /analytics/dataset/src/it/resources/log4j.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/it/resources/log4j.xml -------------------------------------------------------------------------------- /analytics/dataset/src/it/scala/com/uebercomputing/spark/dataset/TopNEmailMessageSendersIntegrationTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/it/scala/com/uebercomputing/spark/dataset/TopNEmailMessageSendersIntegrationTest.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/dataset/TopNEmailMessageSenders.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/dataset/TopNEmailMessageSenders.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/DataFrameFromRddDynamicSchema.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/DataFrameFromRddDynamicSchema.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/DataFrameFromRddWithKnownSchema.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/DataFrameFromRddWithKnownSchema.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/DataFrameOps.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/DataFrameOps.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/DataFrameToRdd.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/DataFrameToRdd.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/EmailsPerUserDataFrame.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/EmailsPerUserDataFrame.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/EmailsPerUserRdd.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/EmailsPerUserRdd.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/MySqlJdbcDataFrame.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/MySqlJdbcDataFrame.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/ParquetPartitions.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/ParquetPartitions.scala -------------------------------------------------------------------------------- /analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/UniqueFromsCounter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/dataset/src/main/scala/com/uebercomputing/spark/sql/UniqueFromsCounter.scala -------------------------------------------------------------------------------- /analytics/rdd/src/main/scala/com/uebercomputing/background/FunctionalCombinators.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/main/scala/com/uebercomputing/background/FunctionalCombinators.scala -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/kean-s/deleted_items/338.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/kean-s/deleted_items/338.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/lay-k/enron/2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/lay-k/enron/2.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/lay-k/enron/3.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/lay-k/enron/3.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/lay-k/inbox/568.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/lay-k/inbox/568.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/lay-k/inbox/898.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/lay-k/inbox/898.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/mims-thurston-p/inbox/28.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/mims-thurston-p/inbox/28.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/mims-thurston-p/inbox/98.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/mims-thurston-p/inbox/98.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/neal-s/all_documents/10.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/neal-s/all_documents/10.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/neal-s/all_documents/20.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/neal-s/all_documents/20.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/neal-s/all_documents/94.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/neal-s/all_documents/94.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/neal-s/all_documents/99.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/neal-s/all_documents/99.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/neal-s/regulatory/1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/neal-s/regulatory/1.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/enron/maildir/neal-s/regulatory/2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/enron/maildir/neal-s/regulatory/2.txt -------------------------------------------------------------------------------- /analytics/rdd/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/analytics/rdd/src/test/resources/log4j.properties -------------------------------------------------------------------------------- /enron-small.parquet: -------------------------------------------------------------------------------- 1 | /datasets/enron/enron-small.parquet -------------------------------------------------------------------------------- /mailrecord-utils/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/.gitignore -------------------------------------------------------------------------------- /mailrecord-utils/src/main/java/com/uebercomputing/mailrecord/Attachment.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/java/com/uebercomputing/mailrecord/Attachment.java -------------------------------------------------------------------------------- /mailrecord-utils/src/main/java/com/uebercomputing/mailrecord/MailRecord.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/java/com/uebercomputing/mailrecord/MailRecord.java -------------------------------------------------------------------------------- /mailrecord-utils/src/main/java/com/uebercomputing/mailrecord/MailRecordProtocol.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/java/com/uebercomputing/mailrecord/MailRecordProtocol.java -------------------------------------------------------------------------------- /mailrecord-utils/src/main/resources/log4j.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/resources/log4j.xml -------------------------------------------------------------------------------- /mailrecord-utils/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/resources/log4j2.xml -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/hadoop/HadoopUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/hadoop/HadoopUtils.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/io/IoConstants.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/io/IoConstants.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/io/PathUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/io/PathUtils.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/io/Utf8Codec.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/io/Utf8Codec.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/AvroMain.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/AvroMain.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/AvroMessageProcessor.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/AvroMessageProcessor.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/FileSystemMetadata.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/FileSystemMetadata.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/MailDirectoryProcessor.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/MailDirectoryProcessor.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/MessageParser.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/MessageParser.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/MessageProcessor.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/MessageProcessor.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/MessageUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/MessageUtils.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/ParquetMain.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/ParquetMain.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/ParquetMessageProcessor.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/ParquetMessageProcessor.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/ParseException.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/ParseException.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/ParsedMessageToMailRecordConverter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailparser/enronfiles/ParsedMessageToMailRecordConverter.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/Implicits.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/Implicits.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordAvroReader.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordAvroReader.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordAvroWriter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordAvroWriter.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordOps.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordOps.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordParquetReader.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordParquetReader.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordParquetWriter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/mailrecord/MailRecordParquetWriter.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/main/scala/com/uebercomputing/utils/DatePartitioner.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/main/scala/com/uebercomputing/utils/DatePartitioner.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/kean-s/deleted_items/338.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/kean-s/deleted_items/338.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/lay-k/enron/2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/lay-k/enron/2.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/lay-k/enron/3.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/lay-k/enron/3.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/lay-k/inbox/568.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/lay-k/inbox/568.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/lay-k/inbox/898.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/lay-k/inbox/898.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/mims-thurston-p/inbox/28.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/mims-thurston-p/inbox/28.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/mims-thurston-p/inbox/98.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/mims-thurston-p/inbox/98.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/neal-s/all_documents/10.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/neal-s/all_documents/10.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/neal-s/all_documents/20.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/neal-s/all_documents/20.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/neal-s/all_documents/94.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/neal-s/all_documents/94.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/neal-s/all_documents/99.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/neal-s/all_documents/99.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/neal-s/regulatory/1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/neal-s/regulatory/1.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/enron/maildir/neal-s/regulatory/2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/enron/maildir/neal-s/regulatory/2.txt -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/log4j.xml -------------------------------------------------------------------------------- /mailrecord-utils/src/test/resources/log4j2-test.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/resources/log4j2-test.xml -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/mailparser/enronfiles/AvroMessageProcessorTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/mailparser/enronfiles/AvroMessageProcessorTest.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/mailparser/enronfiles/MailDirectoryProcessorTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/mailparser/enronfiles/MailDirectoryProcessorTest.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/mailparser/enronfiles/MessageParserTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/mailparser/enronfiles/MessageParserTest.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/mailrecord/MailRecordOpsTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/mailrecord/MailRecordOpsTest.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/mailrecord/MailRecordParquetWriterTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/mailrecord/MailRecordParquetWriterTest.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/mailrecord/MailRecordProvider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/mailrecord/MailRecordProvider.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/test/AvroFileFixtureTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/test/AvroFileFixtureTest.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/test/AvroMailRecordsFileProvider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/test/AvroMailRecordsFileProvider.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/test/UnitTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/test/UnitTest.scala -------------------------------------------------------------------------------- /mailrecord-utils/src/test/scala/com/uebercomputing/utils/DatePartitionerTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/mailrecord-utils/src/test/scala/com/uebercomputing/utils/DatePartitionerTest.scala -------------------------------------------------------------------------------- /notebooks/ApacheSparkThroughEmail1.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/ApacheSparkThroughEmail1.ipynb -------------------------------------------------------------------------------- /notebooks/ApacheSparkThroughEmail2.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/ApacheSparkThroughEmail2.ipynb -------------------------------------------------------------------------------- /notebooks/ApacheSparkThroughEmail3.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/ApacheSparkThroughEmail3.ipynb -------------------------------------------------------------------------------- /notebooks/exec/.gitignore: -------------------------------------------------------------------------------- 1 | ApacheSparkThroughEmail*.ipynb 2 | -------------------------------------------------------------------------------- /notebooks/html/ApacheSparkThroughEmail1.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/html/ApacheSparkThroughEmail1.html -------------------------------------------------------------------------------- /notebooks/html/ApacheSparkThroughEmail2.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/html/ApacheSparkThroughEmail2.html -------------------------------------------------------------------------------- /notebooks/html/ApacheSparkThroughEmail3.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/html/ApacheSparkThroughEmail3.html -------------------------------------------------------------------------------- /notebooks/pdf/ApacheSparkThroughEmail1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/pdf/ApacheSparkThroughEmail1.pdf -------------------------------------------------------------------------------- /notebooks/pdf/ApacheSparkThroughEmail2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/pdf/ApacheSparkThroughEmail2.pdf -------------------------------------------------------------------------------- /notebooks/pdf/ApacheSparkThroughEmail3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/notebooks/pdf/ApacheSparkThroughEmail3.pdf -------------------------------------------------------------------------------- /presentation/.gitignore: -------------------------------------------------------------------------------- 1 | index.html 2 | reveal.js 3 | -------------------------------------------------------------------------------- /presentation/ApacheSparkThroughEmail.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/ApacheSparkThroughEmail.md -------------------------------------------------------------------------------- /presentation/ApacheSparkThroughEmail.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/ApacheSparkThroughEmail.pdf -------------------------------------------------------------------------------- /presentation/SpeakerNotes.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/SpeakerNotes.md -------------------------------------------------------------------------------- /presentation/SpeakerNotes.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/SpeakerNotes.pdf -------------------------------------------------------------------------------- /presentation/graphics/AsymmetrikPingPong.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/AsymmetrikPingPong.png -------------------------------------------------------------------------------- /presentation/graphics/Catalyst.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/Catalyst.png -------------------------------------------------------------------------------- /presentation/graphics/CatalystUnifiedInterface.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/CatalystUnifiedInterface.png -------------------------------------------------------------------------------- /presentation/graphics/Chemo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/Chemo.png -------------------------------------------------------------------------------- /presentation/graphics/Column.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/Column.png -------------------------------------------------------------------------------- /presentation/graphics/DataFrameReader.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/DataFrameReader.png -------------------------------------------------------------------------------- /presentation/graphics/DatasetApi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/DatasetApi.png -------------------------------------------------------------------------------- /presentation/graphics/Farley.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/Farley.png -------------------------------------------------------------------------------- /presentation/graphics/FunctionApi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/FunctionApi.png -------------------------------------------------------------------------------- /presentation/graphics/Goal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/Goal.png -------------------------------------------------------------------------------- /presentation/graphics/HadoodEcosystem.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/HadoodEcosystem.xcf -------------------------------------------------------------------------------- /presentation/graphics/Hadoop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/Hadoop.png -------------------------------------------------------------------------------- /presentation/graphics/HadoopEcosystem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/HadoopEcosystem.png -------------------------------------------------------------------------------- /presentation/graphics/HorizontalScaling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/HorizontalScaling.png -------------------------------------------------------------------------------- /presentation/graphics/Laptop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/Laptop.png -------------------------------------------------------------------------------- /presentation/graphics/Notebook1Job2Dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/Notebook1Job2Dag.png -------------------------------------------------------------------------------- /presentation/graphics/QueryPlanningPhases.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/QueryPlanningPhases.png -------------------------------------------------------------------------------- /presentation/graphics/SparkApplication.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/SparkApplication.png -------------------------------------------------------------------------------- /presentation/graphics/SparkComponents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/SparkComponents.png -------------------------------------------------------------------------------- /presentation/graphics/SparkJobsNotebook1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/SparkJobsNotebook1.png -------------------------------------------------------------------------------- /presentation/graphics/SparkLogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/SparkLogo.png -------------------------------------------------------------------------------- /presentation/graphics/SparkRdd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/SparkRdd.png -------------------------------------------------------------------------------- /presentation/graphics/StandaloneServer1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/StandaloneServer1.png -------------------------------------------------------------------------------- /presentation/graphics/VerticalScaling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/graphics/VerticalScaling.png -------------------------------------------------------------------------------- /presentation/makeNotes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/makeNotes -------------------------------------------------------------------------------- /presentation/makePresentation: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/presentation/makePresentation -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/project/Dependencies.scala -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.10.0 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/medale/spark-mail/HEAD/project/plugins.sbt -------------------------------------------------------------------------------- /roles.csv: -------------------------------------------------------------------------------- 1 | /datasets/enron/roles.csv -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | ThisBuild / version := "3.0.0-SNAPSHOT" 2 | --------------------------------------------------------------------------------