├── project ├── build.properties ├── build.sbt └── plugins.sbt ├── version.sbt ├── .env ├── .env.test ├── .scala-steward.conf ├── lihua └── src │ └── main │ └── scala │ └── lihua │ ├── Entity.scala │ ├── package.scala │ ├── playJson │ └── Formats.scala │ └── EntityDAO.scala ├── stream └── src │ └── main │ ├── scala │ └── com │ │ └── iheart │ │ └── thomas │ │ └── stream │ │ ├── package.scala │ │ ├── MessageSubscriber.scala │ │ ├── ArmKPIEvents.scala │ │ ├── PipeSyntax.scala │ │ ├── AdminEvent.scala │ │ ├── JobEvent.scala │ │ ├── ArmExtractor.scala │ │ ├── Job.scala │ │ ├── BanditProcessAlg.scala │ │ ├── BusAlg.scala │ │ └── JobSpec.scala │ └── resources │ └── reference.conf ├── .scalafmt.conf ├── docs ├── src │ └── main │ │ └── resources │ │ └── microsite │ │ └── img │ │ ├── newGroup.png │ │ ├── Thomas_Bayes.gif │ │ ├── resizeGroup.png │ │ └── thomas-the-tank-engine.png └── docs │ ├── permission.md │ └── core.md ├── .jvmopts ├── analysis └── src │ └── main │ └── scala │ └── com │ └── iheart │ └── thomas │ └── analysis │ ├── implicits.scala │ ├── bayesian │ ├── Evaluation.scala │ ├── models │ │ ├── LogNormalModel.scala │ │ ├── BetaModel.scala │ │ └── NormalModel.scala │ ├── fit │ │ ├── Measurable.scala │ │ └── DistributionSpec.scala │ ├── KPIIndicator.scala │ ├── BenchmarkResult.scala │ ├── Variable.scala │ └── Posterior.scala │ ├── KPIEventQuery.scala │ ├── monitor │ └── ExperimentKPIHistory.scala │ ├── package.scala │ ├── PerUserSamples.scala │ ├── AccumulativeKPIQueryRepo.scala │ ├── syntax │ └── AllSyntax.scala │ ├── KPI.scala │ ├── KPIStats.scala │ └── KPIRepo.scala ├── bandit └── src │ └── main │ └── scala │ └── com │ └── iheart │ └── thomas │ └── bandit │ ├── bayesian │ ├── package.scala │ ├── BayesianMAB.scala │ └── BanditSpec.scala │ ├── package.scala │ ├── Error.scala │ ├── ArmSpec.scala │ ├── BanditStatus.scala │ └── tracking │ └── BanditEvent.scala ├── .gitignore ├── mongo └── src │ └── main │ └── scala │ ├── lihua │ ├── mongo │ │ ├── package.scala │ │ ├── Crypt.scala │ │ ├── ShutdownHook.scala │ │ ├── DBError.scala │ │ ├── Query.scala │ │ └── JsonFormats.scala │ └── crypt │ │ └── CryptTsec.scala │ └── com │ └── iheart │ └── thomas │ └── mongo │ ├── FeatureDAO.scala │ └── AbtestDAO.scala ├── kafka └── src │ └── main │ ├── resources │ └── reference.conf │ └── scala │ └── com │ └── iheart │ └── thomas │ └── kafka │ ├── KafkaConfig.scala │ ├── MessageProcessor.scala │ └── JsonMessageSubscriber.scala ├── core └── src │ └── main │ └── scala │ └── com │ └── iheart │ └── thomas │ ├── abtest │ ├── protocol │ │ └── protocols.scala │ ├── json │ │ └── play │ │ │ └── Instances.scala │ └── QueryDSL.scala │ ├── package.scala │ ├── ThrowableExtension.scala │ ├── admin │ ├── AuthRecord.scala │ └── User.scala │ ├── tracking │ └── EventLogger.scala │ └── utils │ └── time │ └── Period.scala ├── http4s └── src │ └── main │ ├── twirl │ └── com │ │ └── iheart │ │ └── thomas │ │ ├── abtest │ │ └── admin │ │ │ ├── featureTestsLink.scala.html │ │ │ ├── newRevision.scala.html │ │ │ ├── editTest.scala.html │ │ │ ├── newTest.scala.html │ │ │ ├── formParts │ │ │ └── mutualExclusivity.scala.html │ │ │ └── assignments.scala.html │ │ ├── analysis │ │ ├── perUserSamplesSummary.scala.html │ │ ├── conversionsStats.scala.html │ │ ├── newConversionKPI.scala.html │ │ ├── newAccumulativeKPI.scala.html │ │ ├── editAccumulativeKPI.scala.html │ │ ├── editConversionKPI.scala.html │ │ ├── kpiBasics.scala.html │ │ ├── updatePrior.scala.html │ │ ├── index.scala.html │ │ └── kpiState.scala.html │ │ ├── errorMsg.scala.html │ │ ├── stream │ │ ├── jobStatusBadge.scala.html │ │ └── background.scala.html │ │ ├── redirect.scala.html │ │ ├── bandit │ │ ├── newBandit.scala.html │ │ ├── index.scala.html │ │ └── banditView.scala.html │ │ ├── auth │ │ ├── resetPassLink.scala.html │ │ ├── login.scala.html │ │ ├── registration.scala.html │ │ ├── resetPass.scala.html │ │ └── users.scala.html │ │ └── topNav.scala.html │ ├── scala │ └── com │ │ └── iheart │ │ └── thomas │ │ └── http4s │ │ ├── package.scala │ │ ├── UIConfig.scala │ │ ├── ConfigResource.scala │ │ ├── auth │ │ ├── package.scala │ │ └── AuthedEndpointsUtils.scala │ │ ├── bandit │ │ ├── Bandit.scala │ │ └── ManagerAlg.scala │ │ ├── ReverseRoutes.scala │ │ ├── MongoResources.scala │ │ ├── stream │ │ └── UI.scala │ │ ├── StreamControlTest.scala │ │ ├── Formatters.scala │ │ └── CommonFormDecoders.scala │ └── resources │ └── reference.conf ├── docker └── dynamodb │ └── Dockerfile ├── tests └── src │ ├── it │ ├── resources │ │ ├── logback.xml │ │ └── application.conf │ └── scala │ │ └── com │ │ └── iheart │ │ └── thomas │ │ ├── analysis │ │ └── ConversionKPIRepoSuite.scala │ │ └── abtest │ │ ├── EndpointSuite.scala │ │ └── TestUtils.scala │ ├── test │ ├── scala │ │ └── com │ │ │ └── iheart │ │ │ └── thomas │ │ │ ├── http4s │ │ │ ├── AdminUIConfigSuite.scala │ │ │ └── analysis │ │ │ │ └── DecoderSuite.scala │ │ │ ├── stream │ │ │ ├── PubSub.scala │ │ │ └── JValueParserSuite.scala │ │ │ ├── analysis │ │ │ ├── bayesian │ │ │ │ ├── fit │ │ │ │ │ └── NormalSuite.scala │ │ │ │ ├── ModelEvaluatorSuite.scala │ │ │ │ └── models │ │ │ │ │ └── BetaKPISuite.scala │ │ │ └── BenchmarkResultSuite.scala │ │ │ ├── abtest │ │ │ ├── model │ │ │ │ └── ModelSuite.scala │ │ │ └── TestsDataSuite.scala │ │ │ └── bandit │ │ │ └── bayesian │ │ │ └── BayesianMABAlgSuite.scala │ └── resources │ │ └── logback.xml │ └── main │ ├── resources │ └── logback.xml │ └── scala │ └── com │ └── iheart │ └── thomas │ └── example │ └── ExampleApps.scala ├── testkit └── src │ └── main │ ├── resources │ ├── application.conf │ └── logback.xml │ └── scala │ └── com │ └── iheart │ └── thomas │ └── testkit │ ├── ExampleParsers.scala │ ├── LocalDynamo.scala │ ├── TestMessageKafkaProducer.scala │ ├── Resources.scala │ └── MockQueryAccumulativeKPIAlg.scala ├── CODE_OF_CONDUCT.md ├── COPY.md ├── .mergify.yml ├── monitor └── src │ └── main │ └── scala │ └── com │ └── iheart │ └── thomas │ └── monitor │ ├── MonitorEvent.scala │ ├── Reporter.scala │ └── DatadogClient.scala ├── README.md ├── stress └── src │ └── test │ ├── resources │ └── logback.xml │ └── scala │ └── com │ └── iheart │ └── thomas │ └── stress │ └── Simulations.scala ├── spark └── src │ ├── test │ └── scala │ │ └── com │ │ └── iheart │ │ └── thomas │ │ └── spark │ │ └── AutoRefreshAssignerSuite.scala │ └── main │ └── scala │ └── com │ └── iheart │ └── thomas │ └── spark │ ├── AutoRefreshAssigner.scala │ └── Assigner.scala ├── client └── src │ ├── it │ └── scala │ │ └── com │ │ └── iheart │ │ └── thomas │ │ └── client │ │ └── JavaAPISuite.scala │ └── main │ └── scala │ └── com │ └── iheart │ └── thomas │ └── client │ ├── PlayJsonHttp4Client.scala │ └── JavaAbtestAssignments.scala ├── dynamo └── src │ └── main │ └── scala │ └── com │ └── iheart │ └── thomas │ └── dynamo │ ├── BanditsDAOs.scala │ ├── package.scala │ └── AdminDAOs.scala ├── docker-compose.yml └── .github └── workflows └── ci.yml /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.9.7 2 | -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | ThisBuild / version := "1.4.9-SNAPSHOT" 2 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | THOMAS_ENV=dev 2 | 3 | THOMAS_MONGO_PORT=27017 4 | THOMAS_DYNAMO_PORT=8042 -------------------------------------------------------------------------------- /.env.test: -------------------------------------------------------------------------------- 1 | THOMAS_ENV=test 2 | THOMAS_MONGO_PORT=27027 3 | THOMAS_DYNAMO_PORT=8043 -------------------------------------------------------------------------------- /.scala-steward.conf: -------------------------------------------------------------------------------- 1 | updates.pin = [ { groupId = "com.typesafe.play", version = "2.7." } ] 2 | -------------------------------------------------------------------------------- /lihua/src/main/scala/lihua/Entity.scala: -------------------------------------------------------------------------------- 1 | package lihua 2 | 3 | case class Entity[T](_id: EntityId, data: T) 4 | -------------------------------------------------------------------------------- /project/build.sbt: -------------------------------------------------------------------------------- 1 | ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always 2 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/package.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | 3 | package object stream {} 4 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 3.8.5 2 | 3 | maxColumn = 85 4 | 5 | verticalMultiline.atDefnSite = true 6 | newlines.beforeTypeBounds = keep -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/newGroup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iheartradio/thomas/HEAD/docs/src/main/resources/microsite/img/newGroup.png -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/Thomas_Bayes.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iheartradio/thomas/HEAD/docs/src/main/resources/microsite/img/Thomas_Bayes.gif -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/resizeGroup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iheartradio/thomas/HEAD/docs/src/main/resources/microsite/img/resizeGroup.png -------------------------------------------------------------------------------- /.jvmopts: -------------------------------------------------------------------------------- 1 | -Dfile.encoding=UTF8 2 | -Xms1G 3 | -Xmx6G 4 | -XX:MaxMetaspaceSize=2G 5 | -XX:ReservedCodeCacheSize=250M 6 | -XX:+TieredCompilation 7 | -XX:-UseGCOverheadLimit 8 | -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/thomas-the-tank-engine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iheartradio/thomas/HEAD/docs/src/main/resources/microsite/img/thomas-the-tank-engine.png -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/implicits.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package analysis 3 | 4 | import syntax.AllSyntax 5 | 6 | object implicits extends AllSyntax 7 | -------------------------------------------------------------------------------- /bandit/src/main/scala/com/iheart/thomas/bandit/bayesian/package.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package bandit 3 | 4 | package object bayesian { 5 | type Likelihood = Double 6 | 7 | } 8 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/MessageSubscriber.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.stream 2 | import fs2.Stream 3 | 4 | trait MessageSubscriber[F[_], Message] { 5 | def subscribe: Stream[F, Message] 6 | } 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bsp 2 | dynamodb-local 3 | crit.json 4 | derby.log 5 | metastore_db 6 | logs 7 | release 8 | plots 9 | target 10 | /.idea 11 | /.idea_modules 12 | /.classpath 13 | /.project 14 | /.settings 15 | /RUNNING_PID 16 | -------------------------------------------------------------------------------- /mongo/src/main/scala/lihua/mongo/package.scala: -------------------------------------------------------------------------------- 1 | package lihua 2 | 3 | import reactivemongo.api.bson.BSONObjectID 4 | 5 | package object mongo { 6 | def generateId: EntityId = EntityId(BSONObjectID.generate().stringify) 7 | } 8 | -------------------------------------------------------------------------------- /kafka/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | thomas.stream.kafka { 2 | kafka-servers: "127.0.0.1:9092" 3 | kafka-servers: ${?KAFKA_SERVER_STRING} 4 | topic: "thomas-events" 5 | group-id: "thomas-stream" 6 | parse-parallelization: 4 7 | } -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/abtest/protocol/protocols.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.abtest.protocol 2 | 3 | import com.iheart.thomas.abtest.model.UserMetaCriteria 4 | 5 | case class UpdateUserMetaCriteriaRequest( 6 | criteria: UserMetaCriteria, 7 | auto: Boolean) 8 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/Evaluation.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package analysis 3 | package bayesian 4 | 5 | case class Evaluation( 6 | name: ArmName, 7 | probabilityBeingOptimal: Probability, 8 | resultAgainstBenchmark: Option[BenchmarkResult]) 9 | -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/package.scala: -------------------------------------------------------------------------------- 1 | package com.iheart 2 | 3 | package object thomas { 4 | type FeatureName = String 5 | type GroupName = String 6 | type Username = String 7 | type UserId = String 8 | type ArmName = GroupName 9 | type Period = utils.time.Period 10 | } 11 | -------------------------------------------------------------------------------- /docs/docs/permission.md: -------------------------------------------------------------------------------- 1 | ### Roles in Thomas 2 | 3 | 1. Admin - omnipotent 4 | 2. Developer - create/manage tests 5 | 3. Tester - add white list users 6 | 4. Analyst - manage real time analysis 7 | 5. Scientist - Manage bandits 8 | 6. User - read only 9 | 7. Guest - default role with no permission for anything 10 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/abtest/admin/featureTestsLink.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.FeatureName 2 | @import com.iheart.thomas.http4s.UIEnv 3 | 4 | @(feature: FeatureName, text: Option[String] = None)(implicit env: UIEnv) 5 | 6 | @text.getOrElse(feature) -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/perUserSamplesSummary.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.analysis._ 2 | 3 | @(summary: PerUserSamplesLnSummary) 4 | 5 |
Count: @summary.count
6 |
Mean Exp: @Math.exp(summary.mean)
-------------------------------------------------------------------------------- /docker/dynamodb/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | FROM amazon/dynamodb-local 3 | 4 | WORKDIR /home/dynamodblocal 5 | 6 | # make database directory and change owner to dynamodb user 7 | RUN mkdir ./db && chown -R 1000 ./db 8 | 9 | CMD ["-jar", "DynamoDBLocal.jar", "-dbPath", "/home/dynamodblocal/db", "-sharedDb"] 10 | VOLUME ["/home/dynamodblocal/db"] -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/errorMsg.scala.html: -------------------------------------------------------------------------------- 1 | 2 | @(msg: String) 3 | 4 | 5 | 6 | 7 | @main("Error!") { 8 | 11 | 12 | 13 | } 14 | 15 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/ArmKPIEvents.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.stream 2 | 3 | import cats.data.NonEmptyChain 4 | import com.iheart.thomas.ArmName 5 | 6 | import java.time.Instant 7 | 8 | case class ArmKPIEvents[Event]( 9 | armName: ArmName, 10 | es: NonEmptyChain[Event], 11 | timeStamp: Instant) 12 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/PipeSyntax.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.stream 2 | 3 | import fs2.Pipe 4 | import cats.implicits._ 5 | object PipeSyntax { 6 | implicit class pipeSyntax[F[_], A, B](private val self: Pipe[F, A, B]) 7 | extends AnyVal { 8 | 9 | def void: Pipe[F, A, Unit] = self.andThen(_.void) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /stream/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | thomas.stream { 2 | job { 3 | job-check-frequency: 10s 4 | job-obsolete-count: 10 5 | max-chunk-size: 20 6 | job-process-frequency: 3s 7 | assigner { 8 | refresh-period: 5m 9 | stale-timeout: 1h 10 | tests-range: 5h 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /mongo/src/main/scala/lihua/mongo/Crypt.scala: -------------------------------------------------------------------------------- 1 | package lihua 2 | package mongo 3 | 4 | import cats.tagless.FunctorK 5 | 6 | trait Crypt[F[_]] { 7 | def encrypt(value: String): F[String] 8 | def decrypt(value: String): F[String] 9 | } 10 | 11 | object Crypt { 12 | implicit val functorKInstanceForCrypt: FunctorK[Crypt] = 13 | cats.tagless.Derive.functorK[Crypt] 14 | } 15 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("io.gatling" %% "gatling-sbt" % "4.3.3") 2 | 3 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") 4 | 5 | addSbtPlugin( 6 | "com.kailuowang" %% "sbt-catalysts" % "1.3.3" 7 | ) 8 | 9 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") 10 | 11 | addSbtPlugin("com.typesafe.play" % "sbt-twirl" % "1.5.2") 12 | 13 | addSbtPlugin("io.spray" % "sbt-revolver" % "0.10.0") 14 | -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/ThrowableExtension.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | 3 | import java.io.{PrintWriter, StringWriter} 4 | 5 | object ThrowableExtension { 6 | implicit class throwableExtensions(private val t: Throwable) extends AnyVal { 7 | 8 | def fullStackTrace: String = { 9 | val sw = new StringWriter 10 | t.printStackTrace(new PrintWriter(sw)) 11 | sw.toString 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/package.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | 3 | import org.http4s.Uri 4 | import org.http4s.headers.Location 5 | import tsec.mac.jca.HMACSHA256 6 | 7 | package object http4s { 8 | 9 | type AuthImp = HMACSHA256 10 | 11 | implicit class StringOps(private val self: String) extends AnyVal { 12 | def uri = Uri.unsafeFromString(self) 13 | def location = Location(uri) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/stream/jobStatusBadge.scala.html: -------------------------------------------------------------------------------- 1 | @import java.time.Instant 2 | @import com.iheart.thomas.http4s.Formatters._ 3 | 4 | @(startedO: Option[Instant]) 5 | 6 | @if(startedO.isEmpty) { 7 | 8 | Scheduled 9 | 10 | } 11 | 12 | @for(started <- startedO) { 13 | 14 | 15 | Started @dateTimeMid(started) 16 | 17 | } 18 | 19 | -------------------------------------------------------------------------------- /bandit/src/main/scala/com/iheart/thomas/bandit/package.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package bandit 3 | 4 | import com.iheart.thomas.analysis.KPIStats 5 | 6 | object `package` { 7 | 8 | type Reward = Double 9 | type ExpectedReward = Reward 10 | type Weight = Double 11 | 12 | type ArmState[KS <: KPIStats] = analysis.monitor.ExperimentKPIState.ArmState[KS] 13 | val ArmState = analysis.monitor.ExperimentKPIState.ArmState 14 | } 15 | -------------------------------------------------------------------------------- /tests/src/it/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/conversionsStats.scala.html: -------------------------------------------------------------------------------- 1 | 2 | @import com.iheart.thomas.analysis._ 3 | @import com.iheart.thomas.http4s.Formatters, Formatters._ 4 | 5 | @(conversions: Conversions) 6 | 7 |
Total: @conversions.total
8 |
Converted: @conversions.converted
9 |
Rate: @formatPercentage(conversions.rate)
-------------------------------------------------------------------------------- /bandit/src/main/scala/com/iheart/thomas/bandit/Error.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.bandit 2 | 3 | import com.iheart.thomas.FeatureName 4 | import com.iheart.thomas.analysis.KPIName 5 | 6 | import scala.util.control.NoStackTrace 7 | 8 | sealed trait Error extends RuntimeException with NoStackTrace 9 | 10 | case class KPINotFound(name: KPIName) extends Error 11 | case class KPIIncorrectType(name: KPIName) extends Error 12 | 13 | case class AbtestNotFound(feature: FeatureName) extends Error 14 | -------------------------------------------------------------------------------- /tests/src/it/resources/application.conf: -------------------------------------------------------------------------------- 1 | mongoDB { 2 | dbs: { 3 | abtest: { 4 | name: ABTest 5 | collections: { 6 | tests: { 7 | name: ABTests 8 | } 9 | feature: { 10 | name: feature 11 | } 12 | } 13 | } 14 | } 15 | hosts = ["localhost:27027"] 16 | read-preference = "primary" 17 | } 18 | mongo-async-driver { 19 | akka { 20 | log-dead-letters = off 21 | log-dead-letters-during-shutdown = off 22 | } 23 | } -------------------------------------------------------------------------------- /testkit/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | mongoDB { 2 | dbs: { 3 | abtest: { 4 | name: ABTest 5 | collections: { 6 | tests: { 7 | name: ABTests 8 | } 9 | feature: { 10 | name: feature 11 | } 12 | } 13 | } 14 | } 15 | hosts = ["localhost:27017"] 16 | read-preference = "primary" 17 | } 18 | mongo-async-driver { 19 | akka { 20 | log-dead-letters = off 21 | log-dead-letters-during-shutdown = off 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/AdminEvent.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package stream 3 | 4 | import com.iheart.thomas.abtest.model.TestId 5 | 6 | import java.time.Instant 7 | 8 | 9 | sealed trait AdminEvent extends Serializable 10 | 11 | object AdminEvent { 12 | case class FeatureChanged(name: FeatureName, updated: Instant) extends AdminEvent 13 | case class TestCreated(name: FeatureName, testId: TestId) extends AdminEvent 14 | case class TestUpdated(testId: TestId) extends AdminEvent 15 | } 16 | -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/admin/AuthRecord.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.admin 2 | 3 | import java.time.Instant 4 | 5 | case class AuthRecord( 6 | id: String, 7 | jwtEncoded: String, 8 | identity: String, 9 | expiry: Instant, 10 | lastTouched: Option[Instant]) 11 | 12 | trait AuthRecordDAO[F[_]] { 13 | def insert(record: AuthRecord): F[AuthRecord] 14 | 15 | def update(record: AuthRecord): F[AuthRecord] 16 | 17 | def remove(id: String): F[Unit] 18 | 19 | def find(id: String): F[Option[AuthRecord]] 20 | } 21 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/UIConfig.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s 2 | 3 | import com.iheart.thomas.admin.User 4 | import com.iheart.thomas.http4s.AdminUI.AdminUIConfig 5 | 6 | case class UIEnv( 7 | routes: ReverseRoutes, 8 | currentUser: User, 9 | siteName: String, 10 | docUrl: String) 11 | 12 | object UIEnv { 13 | def apply( 14 | currentUser: User 15 | )(implicit cfg: AdminUIConfig 16 | ): UIEnv = 17 | UIEnv(ReverseRoutes(cfg.rootPath), currentUser, cfg.siteName, cfg.docUrl) 18 | } 19 | -------------------------------------------------------------------------------- /lihua/src/main/scala/lihua/package.scala: -------------------------------------------------------------------------------- 1 | package lihua 2 | 3 | import io.estatico.newtype.macros.newtype 4 | 5 | object `package` { 6 | @newtype 7 | case class EntityId(value: String) 8 | 9 | implicit def toDataOps[A](a: A): DataOps[A] = new DataOps(a) 10 | 11 | val idFieldName = "_id" // determined by the field name of Entity 12 | } 13 | 14 | private[lihua] class DataOps[A](private val a: A) extends AnyVal { 15 | def toEntity(id: String): Entity[A] = toEntity(EntityId(id)) 16 | def toEntity(entityId: EntityId): Entity[A] = Entity(entityId, a) 17 | } 18 | -------------------------------------------------------------------------------- /bandit/src/main/scala/com/iheart/thomas/bandit/ArmSpec.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package bandit 3 | 4 | import com.iheart.thomas.abtest.model.{Group, GroupMeta, GroupSize} 5 | 6 | case class ArmSpec( 7 | name: ArmName, 8 | initialSize: Option[GroupSize] = None, 9 | meta: Option[GroupMeta] = None, 10 | reserved: Boolean = false, 11 | description: Option[String] = None) 12 | 13 | object ArmSpec { 14 | def fromGroup(group: Group): ArmSpec = 15 | ArmSpec(group.name, Some(group.size), group.meta, false, description = group.description) 16 | } 17 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | We are committed to providing a friendly, safe and welcoming environment for all, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, nationality, or other such characteristics. 4 | 5 | Everyone is expected to follow the [Scala Code of Conduct](https://www.scala-lang.org/conduct/) when discussing the project on the available communication channels. If you are being harassed, please contact us immediately so that we can support you. 6 | -------------------------------------------------------------------------------- /bandit/src/main/scala/com/iheart/thomas/bandit/bayesian/BayesianMAB.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package bandit 3 | package bayesian 4 | 5 | import com.iheart.thomas.abtest.model.Abtest 6 | import com.iheart.thomas.analysis.{KPIName, KPIStats} 7 | import com.iheart.thomas.analysis.monitor.ExperimentKPIState 8 | import lihua.Entity 9 | 10 | case class BayesianMAB( 11 | abtest: Entity[Abtest], 12 | spec: BanditSpec, 13 | state: Option[ExperimentKPIState[KPIStats]]) { 14 | def feature: FeatureName = abtest.data.feature 15 | def kpiName: KPIName = spec.kpiName 16 | } 17 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/redirect.scala.html: -------------------------------------------------------------------------------- 1 | 2 | @(directToUrl: String, message: String, seconds: Int = 5) 3 | 4 | @main("Redirecting...") { 5 | 6 | 9 | 10 |
11 | You will be redirected to @directToUrl 12 | in 5 seconds. 13 |
14 | 23 | } -------------------------------------------------------------------------------- /COPY.md: -------------------------------------------------------------------------------- 1 | Copyright 2019 iHeartMedia + Entertainment, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /bandit/src/main/scala/com/iheart/thomas/bandit/BanditStatus.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.bandit 2 | 3 | import cats.kernel.Eq 4 | 5 | sealed trait BanditStatus extends Serializable with Product 6 | 7 | object BanditStatus { 8 | case object Running extends BanditStatus 9 | case object Paused 10 | extends BanditStatus // the treatments are still provided but policy no longer in action. 11 | case object Stopped 12 | extends BanditStatus // The treatments are no longer provided. i.e. backing A/B test is no longer running. 13 | 14 | implicit val eqInst: Eq[BanditStatus] = Eq.fromUniversalEquals 15 | } 16 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/KPIEventQuery.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | 3 | import com.iheart.thomas.{ArmName, FeatureName} 4 | 5 | import java.time.Instant 6 | import scala.annotation.implicitAmbiguous 7 | 8 | @implicitAmbiguous( 9 | "Query $Event for $K. If you don't need this. Use KPIEventQuery.alwaysFail" 10 | ) 11 | trait KPIEventQuery[F[_], K <: KPI, Event] { 12 | def apply( 13 | k: K, 14 | at: Instant 15 | ): F[List[Event]] 16 | 17 | def apply( 18 | k: K, 19 | feature: FeatureName, 20 | at: Instant 21 | ): F[List[(ArmName, Event)]] 22 | } 23 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/monitor/ExperimentKPIHistory.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis.monitor 2 | 3 | import com.iheart.thomas.analysis.KPIStats 4 | import com.iheart.thomas.analysis.monitor.ExperimentKPIState.Key 5 | 6 | case class ExperimentKPIHistory[+KS <: KPIStats]( 7 | key: Key, 8 | history: List[ExperimentKPIState[KS]]) 9 | 10 | trait ExperimentKPIHistoryRepo[F[_]] { 11 | def get(key: Key): F[ExperimentKPIHistory[KPIStats]] 12 | def append(state: ExperimentKPIState[KPIStats]): F[ExperimentKPIHistory[KPIStats]] 13 | def delete(k: Key): F[Option[ExperimentKPIHistory[KPIStats]]] 14 | } 15 | -------------------------------------------------------------------------------- /kafka/src/main/scala/com/iheart/thomas/kafka/KafkaConfig.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.kafka 2 | 3 | import cats.effect.Sync 4 | import com.typesafe.config.Config 5 | import pureconfig.ConfigSource 6 | 7 | case class KafkaConfig( 8 | kafkaServers: String, 9 | topic: String, 10 | groupId: String, 11 | parseParallelization: Int) 12 | 13 | object KafkaConfig { 14 | def fromConfig[F[_]: Sync](cfg: Config): F[KafkaConfig] = { 15 | import pureconfig.generic.auto._ 16 | import pureconfig.module.catseffect.syntax._ 17 | ConfigSource.fromConfig(cfg).at("thomas.stream.kafka").loadF[F, KafkaConfig]() 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | pull_request_rules: 2 | - name: label scala-steward's PRs 3 | conditions: 4 | - author=scala-steward 5 | actions: 6 | label: 7 | add: [dependencies] 8 | - name: automatically merge scala-steward's patch PRs 9 | conditions: 10 | - author=scala-steward 11 | - status-success=Build and Test (ubuntu-latest, adopt@1.11) 12 | actions: 13 | merge: 14 | method: merge 15 | - name: automatically merge kailuowang PRs 16 | conditions: 17 | - author=kailuowang 18 | - status-success=Build and Test (ubuntu-latest, adopt@1.11) 19 | actions: 20 | merge: 21 | method: merge 22 | -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/abtest/json/play/Instances.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.abtest.json.play 2 | 3 | import cats.Applicative 4 | import _root_.play.api.libs.json.JsResult 5 | 6 | trait Instances { 7 | implicit val instancesJSResult: Applicative[JsResult] = new Applicative[JsResult] { 8 | override def pure[A](x: A): JsResult[A] = JsResult.applicativeJsResult.pure(x) 9 | 10 | override def ap[A, B](ff: JsResult[A => B])(fa: JsResult[A]): JsResult[B] = 11 | JsResult.applicativeJsResult.apply(ff, fa) 12 | 13 | override def map[A, B](m: JsResult[A])(f: A => B): JsResult[B] = m.map(f) 14 | } 15 | 16 | } 17 | 18 | object Instances extends Instances 19 | -------------------------------------------------------------------------------- /monitor/src/main/scala/com/iheart/thomas/monitor/MonitorEvent.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package monitor 3 | 4 | import play.api.libs.json._ 5 | 6 | case class MonitorEvent( 7 | title: String, 8 | text: String, 9 | status: Status.Status, 10 | tags: List[String]) 11 | 12 | object MonitorEvent { 13 | type Status = Status.Status 14 | val Status = com.iheart.thomas.monitor.Status 15 | implicit val format: Writes[MonitorEvent] = Json.writes[MonitorEvent] 16 | } 17 | 18 | private[monitor] object Status extends Enumeration { 19 | type Status = Value 20 | 21 | val success, error = Value 22 | 23 | implicit val format: Format[Status] = Json.formatEnum(this) 24 | } 25 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/http4s/AdminUIConfigSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s 2 | 3 | import cats.effect.IO 4 | import cats.effect.testing.scalatest.AsyncIOSpec 5 | import com.iheart.thomas.admin.Role 6 | import org.scalatest.freespec.AsyncFreeSpec 7 | import org.scalatest.matchers.should.Matchers 8 | 9 | class AdminUIConfigSuite extends AsyncFreeSpec with AsyncIOSpec with Matchers { 10 | 11 | "AdminUIConfig" - { 12 | "can read reference conf" in { 13 | ConfigResource 14 | .cfg[IO]() 15 | .map(AdminUI.loadConfig[IO](_)) 16 | .use(identity) 17 | .asserting(cfg => Role.values.contains(cfg.initialRole) shouldBe true) 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/models/LogNormalModel.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis.bayesian.models 2 | 3 | import cats.data.ValidatedNel 4 | import cats.implicits._ 5 | import com.stripe.rainier.compute.Real 6 | 7 | case class LogNormalModel(inner: NormalModel) { 8 | lazy val mean: Real = (inner.mean + (inner.variance / 2d)).exp 9 | } 10 | object LogNormalModel { 11 | def apply( 12 | miu0: Double, 13 | n0: Double, 14 | alpha: Double, 15 | beta: Double 16 | ): LogNormalModel = LogNormalModel(NormalModel(miu0, n0, alpha, beta)) 17 | 18 | def validate(model: LogNormalModel): ValidatedNel[String, LogNormalModel] = 19 | NormalModel.validate(model.inner).as(model) 20 | } 21 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/newConversionKPI.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | 3 | @import com.iheart.thomas.http4s.UIEnv 4 | 5 | @(successMsg: Option[String] = None)(implicit env: UIEnv) 6 | 7 | @topNav("New Conversion KPI", "Analysis") { 8 | 9 |
10 | 11 | @conversionKPIForm(None, successMsg) 12 | 13 |
14 |
15 | 18 | Never Mind 19 |
20 |
21 | 22 | 23 |
24 | } -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/ConfigResource.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s 2 | 3 | import cats.effect.{Resource, Sync} 4 | import com.typesafe.config.Config 5 | import pureconfig.ConfigSource 6 | import pureconfig.module.catseffect.syntax._ 7 | 8 | trait ConfigResource { 9 | def cfg[F[_]]( 10 | cfgResourceName: Option[String] = None 11 | )(implicit F: Sync[F] 12 | ): Resource[F, Config] = 13 | Resource.eval( 14 | cfgResourceName 15 | .fold(ConfigSource.default)(name => 16 | ConfigSource 17 | .resources(name) 18 | .withFallback(ConfigSource.default) 19 | ) 20 | .loadF[F, Config]() 21 | ) 22 | } 23 | 24 | object ConfigResource extends ConfigResource 25 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/auth/package.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package http4s 3 | 4 | import com.iheart.thomas.admin.{Role, User} 5 | import tsec.authentication.{AugmentedJWT, SecuredRequestHandler} 6 | import tsec.authorization.{AuthGroup, SimpleAuthEnum} 7 | import cats.implicits._ 8 | 9 | package object auth { 10 | 11 | type Token[A] = AugmentedJWT[A, Username] 12 | 13 | type AuthedRequestHandler[F[_], Auth] = SecuredRequestHandler[ 14 | F, 15 | Username, 16 | User, 17 | Token[Auth] 18 | ] 19 | 20 | implicit object Roles extends SimpleAuthEnum[Role, String] { 21 | val values: AuthGroup[Role] = 22 | AuthGroup.fromSeq(Role.values) 23 | 24 | def getRepr(t: Role): String = t.name 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /bandit/src/main/scala/com/iheart/thomas/bandit/tracking/BanditEvent.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package bandit.tracking 3 | 4 | import com.iheart.thomas.abtest.model.Abtest 5 | import com.iheart.thomas.analysis.KPIStats 6 | import com.iheart.thomas.analysis.monitor.ExperimentKPIState 7 | import com.iheart.thomas.tracking.Event 8 | 9 | sealed abstract class BanditEvent extends Event 10 | 11 | object BanditEvent { 12 | 13 | object BanditPolicyUpdate { 14 | case object Initiated extends BanditEvent 15 | 16 | case class Calculated(newState: ExperimentKPIState[KPIStats]) extends BanditEvent 17 | 18 | case class Reallocated(test: Abtest) extends BanditEvent 19 | 20 | case object UpdatePolicyAllRunningTriggered extends BanditEvent 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/bandit/newBandit.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | @import com.iheart.thomas.http4s.UIEnv 3 | @import com.iheart.thomas.analysis.KPIName 4 | 5 | @(kpis: Seq[KPIName])(implicit env: UIEnv) 6 | 7 | 8 | @topNav("New Bandit", "Bandit") { 9 |
10 | Creating a new multi-arm bandit 11 |
12 |
13 | @banditSpecForm(None, kpis) 14 |
15 |
16 | 17 | Cancel 18 |
19 |
20 |
21 | 22 | 23 | } -------------------------------------------------------------------------------- /mongo/src/main/scala/lihua/mongo/ShutdownHook.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] [iHeartMedia Inc] 3 | * All rights reserved 4 | */ 5 | package lihua 6 | package mongo 7 | 8 | trait ShutdownHook { 9 | def onShutdown[T](code: => T): Unit 10 | } 11 | 12 | object ShutdownHook { 13 | 14 | /** a shutdownhook 15 | */ 16 | object ignore extends ShutdownHook { 17 | override def onShutdown[T](code: => T): Unit = () 18 | } 19 | 20 | class Manual extends ShutdownHook { 21 | @volatile 22 | private[mongo] var callbacks: List[() => _] = Nil 23 | override def onShutdown[T](code: => T): Unit = { 24 | callbacks = (() => code) :: callbacks 25 | } 26 | def shutdown(): Unit = callbacks.foreach(_()) 27 | } 28 | 29 | def manual = new Manual 30 | } 31 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/auth/resetPassLink.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | @import com.iheart.thomas.http4s.UIEnv 3 | 4 | @(username: String, link: String 5 | )(implicit env: UIEnv) 6 | 7 | @topNav("Reset Password Link", "Users") { 8 | 16 | 17 | 18 | 19 | 20 | } -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/fit/Measurable.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis.bayesian.fit 2 | 3 | import cats.Contravariant 4 | import com.iheart.thomas.GroupName 5 | import com.iheart.thomas.abtest.model.Abtest 6 | 7 | import java.time.Instant 8 | 9 | trait Measurable[F[_], M, K] { 10 | def measureAbtest( 11 | k: K, 12 | abtest: Abtest, 13 | start: Option[Instant], 14 | end: Option[Instant] 15 | ): F[Map[GroupName, M]] 16 | 17 | def measureHistory( 18 | k: K, 19 | start: Instant, 20 | end: Instant 21 | ): F[M] 22 | } 23 | 24 | object Measurable { 25 | implicit def contravariantInst[F[_], M]: Contravariant[Measurable[F, M, *]] = 26 | cats.tagless.Derive.contravariant[Measurable[F, M, *]] 27 | } 28 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/stream/PubSub.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.stream 2 | 3 | import cats.effect.Concurrent 4 | import cats.implicits._ 5 | import fs2.Stream 6 | import fs2.concurrent.Topic 7 | import org.typelevel.jawn.ast.JValue 8 | 9 | class PubSub[F[_]: Concurrent] private (topic: Topic[F, JValue]) 10 | extends MessageSubscriber[F, JValue] { 11 | 12 | def publish(js: JValue*): F[Unit] = js.toList.traverse(topic.publish1).void 13 | def publishS(js: JValue*): Stream[F, Unit] = Stream.eval(publish(js: _*)) 14 | 15 | def subscribe: Stream[F, JValue] = 16 | topic.subscribe(10) // topic returns always returns the last message 17 | } 18 | 19 | object PubSub { 20 | def create[F[_]: Concurrent]: F[PubSub[F]] = 21 | Topic[F, JValue].map(new PubSub[F](_)) 22 | } 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Thomas - a new A/B test library 3 | 4 | 5 | [![Build](https://github.com/iheartradio/thomas/workflows/Continuous%20Integration/badge.svg)](https://github.com/iheartradio/thomas/actions?query=workflow%3A%22Continuous+Integration%22) 6 | [![Gitter](https://badges.gitter.im/iheartradio/thomas.svg)](https://gitter.im/iheartradio/thomas?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) 7 | 8 | 9 | 10 | 11 | ### Go to [the website](https://iheartradio.github.io/thomas/) for documentation. 12 | 13 | 14 | ## Copyright and License 15 | 16 | All code is available to you under the Apache 2 license, available in the COPY file. 17 | 18 | Copyright iHeartMedia + Entertainment, Inc., 2018-2020. 19 | -------------------------------------------------------------------------------- /http4s/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | mongoDB { 2 | dbs: { 3 | abtest: { 4 | name: ABTest 5 | collections: { 6 | tests: { 7 | name: ABTests 8 | } 9 | feature: { 10 | name: feature 11 | } 12 | } 13 | } 14 | } 15 | hosts = ["localhost:27017"] 16 | } 17 | 18 | thomas.abtest.get-groups { 19 | ttl = 1m 20 | } 21 | 22 | thomas.admin-ui { 23 | key =${?THOMAS_ADMIN_KEY} 24 | root-path = /admin 25 | admin-tables-read-capacity = 10 26 | admin-tables-write-capacity = 10 27 | initial-admin-username = admin 28 | initial-role: User 29 | site-name: A/B test Admin UI 30 | dynamo { 31 | access-key: SET_ME 32 | secret-key: SET_ME 33 | region: SET_ME 34 | } 35 | doc-url: "https://iheartradio.github.io" 36 | } -------------------------------------------------------------------------------- /stress/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx 7 | false 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/analysis/bayesian/fit/NormalSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis.bayesian.fit 2 | 3 | import com.iheart.thomas.analysis.bayesian.fit.DistributionSpec.Normal 4 | import com.iheart.thomas.analysis.syntax.AllSyntax 5 | import com.stripe.rainier.core.Model 6 | import com.stripe.rainier.sampler.RNG 7 | import org.scalatest.funsuite.AnyFunSuiteLike 8 | import org.scalatest.matchers.should.Matchers 9 | 10 | class NormalSuite extends AnyFunSuiteLike with Matchers with AllSyntax { 11 | implicit val rng = RNG.default 12 | 13 | test("Normal fit consistent with spec") { 14 | val data = 15 | Model.sample(Normal(13, 4).distribution.latent) 16 | val fitSpec = Normal.fit(data) 17 | fitSpec.location shouldBe (13d +- 0.65) 18 | fitSpec.scale shouldBe (4d +- 0.3) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/newAccumulativeKPI.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | @import com.iheart.thomas.analysis._ 3 | 4 | @import com.iheart.thomas.http4s.UIEnv 5 | 6 | @(queryNames: List[QueryName], 7 | successMsg: Option[String] = None)(implicit env: UIEnv) 8 | 9 | @topNav("New Accumulative KPI", "Analysis") { 10 | 11 |
12 | 13 | @accumulativeKPIForm(None, successMsg, true, queryNames) 14 | 15 |
16 |
17 | 20 | Never Mind 21 |
22 |
23 | 24 | 25 |
26 | } -------------------------------------------------------------------------------- /spark/src/test/scala/com/iheart/thomas/spark/AutoRefreshAssignerSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.spark 2 | import cats.kernel.laws.discipline.SerializableTests 3 | import cats.tests.CatsSuite 4 | import concurrent.duration._ 5 | 6 | class AutoRefreshAssignerSuite extends CatsSuite { 7 | // / disable this test to see if this is still required since it seems that the whole UDF is no longer serializable. 8 | // checkAll( 9 | // "AutoRefreshAssigner.udf Serializable", 10 | // SerializableTests.serializable( 11 | // AutoRefreshAssigner("fakeUrl", 10.seconds).assignUdf("fakeFeature") 12 | // ) 13 | // ) 14 | 15 | checkAll( 16 | "AutoRefreshAssigner.udf Serializable", 17 | SerializableTests.serializable( 18 | AutoRefreshAssigner("fakeUrl", 10.seconds).assignFunction("fakeFeature") 19 | ) 20 | ) 21 | } 22 | -------------------------------------------------------------------------------- /tests/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | [%level] %logger{15} - %message%n%xException{10} 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/bandit/Bandit.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s.bandit 2 | 3 | import com.iheart.thomas.FeatureName 4 | import com.iheart.thomas.abtest.model.Abtest 5 | import com.iheart.thomas.analysis.{KPIName, KPIStats} 6 | import com.iheart.thomas.analysis.monitor.ExperimentKPIState 7 | import com.iheart.thomas.bandit.BanditStatus 8 | import com.iheart.thomas.bandit.bayesian.{BanditSpec, BayesianMAB} 9 | import lihua.Entity 10 | 11 | case class Bandit( 12 | abtest: Entity[Abtest], 13 | spec: BanditSpec, 14 | state: Option[ExperimentKPIState[KPIStats]], 15 | status: BanditStatus) { 16 | def feature: FeatureName = abtest.data.feature 17 | def kpiName: KPIName = spec.kpiName 18 | } 19 | 20 | object Bandit { 21 | def apply(b: BayesianMAB, status: BanditStatus): Bandit = 22 | Bandit(b.abtest, b.spec, b.state, status) 23 | } 24 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/http4s/analysis/DecoderSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s.analysis 2 | 3 | import cats.data.Chain 4 | import cats.data.Validated.Valid 5 | import com.iheart.thomas.analysis.bayesian.models.BetaModel 6 | import com.iheart.thomas.analysis.{ConversionKPI, KPIName} 7 | import org.scalatest.funsuite.AnyFunSuiteLike 8 | import org.scalatest.matchers.should.Matchers 9 | 10 | class DecoderSuite extends AnyFunSuiteLike with Matchers { 11 | 12 | test("can read conversion kpi form") { 13 | UI.Decoders.conversionKPIDecoder.apply( 14 | Map( 15 | "name" -> Chain("foo"), 16 | "author" -> Chain("bar"), 17 | "model.alpha" -> Chain("1"), 18 | "model.beta" -> Chain("2") 19 | ) 20 | ) should be( 21 | Valid(ConversionKPI(KPIName("foo"), "bar", None, BetaModel(1, 2), None)) 22 | ) 23 | 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /stress/src/test/scala/com/iheart/thomas/stress/Simulations.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2018] [iHeartMedia Inc] 3 | * All rights reserved 4 | */ 5 | 6 | package com.iheart.thomas.stress 7 | 8 | import io.gatling.core.Predef._ 9 | import io.gatling.http.Predef._ 10 | 11 | import scala.concurrent.duration._ 12 | 13 | class GetGroupsSimulation extends Simulation { 14 | 15 | val userId = 4331241 16 | val testName = "getGroups" 17 | 18 | val host = "http://localhost:9000" 19 | 20 | setUp( 21 | scenario(testName) 22 | .during(500.seconds) { 23 | exec( 24 | http(testName) 25 | .get(s"$host/internal/users/$userId/tests/groups") 26 | .check(status.is(200)) 27 | ) 28 | } 29 | .inject(rampUsers(10) over (60.seconds)) 30 | ).protocols(http.disableCaching) 31 | .assertions( 32 | global.requestsPerSec.gte(1000) 33 | ) 34 | 35 | } 36 | -------------------------------------------------------------------------------- /tests/src/it/scala/com/iheart/thomas/analysis/ConversionKPIRepoSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | 3 | import cats.effect.IO 4 | import cats.effect.testing.scalatest.AsyncIOSpec 5 | import com.iheart.thomas.analysis.bayesian.models.BetaModel 6 | import com.iheart.thomas.dynamo.AnalysisDAOs 7 | import com.iheart.thomas.testkit.Resources.localDynamoR 8 | import org.scalatest.freespec.AsyncFreeSpec 9 | import org.scalatest.matchers.should.Matchers 10 | 11 | class ConversionKPIRepoSuite extends AsyncFreeSpec with AsyncIOSpec with Matchers { 12 | val daoR = localDynamoR.map(implicit ld => AnalysisDAOs.conversionKPIRepo[IO]) 13 | 14 | "Can insert a new KPI" in { 15 | val toInsert = 16 | ConversionKPI(KPIName("a"), "kai", None, BetaModel(0.1, 0.1), None) 17 | 18 | daoR 19 | .use( 20 | _.create(toInsert) 21 | ) 22 | .asserting(_ shouldBe toInsert) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/JobEvent.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.stream 2 | 3 | import com.iheart.thomas.{ArmName, FeatureName} 4 | import com.iheart.thomas.analysis.QueryAccumulativeKPI 5 | import com.iheart.thomas.tracking.Event 6 | 7 | sealed trait JobEvent extends Event 8 | 9 | object JobEvent { 10 | case class MessagesReceived[M](sample: M, size: Int) extends JobEvent 11 | case class MessagesParseError[M](err: Throwable, rawMsg: M) extends JobEvent 12 | case class RunningJobsUpdated(jobs: Seq[Job]) extends JobEvent 13 | case class EventsQueriedForFeature( 14 | k: QueryAccumulativeKPI, 15 | feature: FeatureName, 16 | armsCount: List[(ArmName, Int)]) 17 | extends JobEvent 18 | 19 | case class EventsQueried( 20 | k: QueryAccumulativeKPI, 21 | count: Int) 22 | extends JobEvent 23 | 24 | case class EventQueryInitiated( 25 | k: QueryAccumulativeKPI) 26 | extends JobEvent 27 | } 28 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/package.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package analysis 3 | 4 | import com.iheart.thomas.analysis.bayesian.Variable 5 | import com.stripe.rainier.compute.Real 6 | import io.estatico.newtype.macros.newsubtype 7 | 8 | object `package` { 9 | @newsubtype case class Probability(p: Double) 10 | @newsubtype case class KPIDouble(d: Double) 11 | @newsubtype case class KPIName(n: String) 12 | @newsubtype case class QueryName(n: String) 13 | 14 | type Diff = Double 15 | type Samples[A] = List[A] 16 | 17 | object KPIName { 18 | def fromString(n: String): Either[String, KPIName] = 19 | if (n.matches("[-_.A-Za-z0-9]+")) 20 | Right(KPIName(n)) 21 | else Left("KPI Name can only consists alphanumeric characters, '_' or '-'.") 22 | 23 | } 24 | 25 | type Indicator = Variable[Real] 26 | 27 | type ConversionEvent = Boolean 28 | val Converted = true 29 | val Initiated = false 30 | 31 | } 32 | -------------------------------------------------------------------------------- /lihua/src/main/scala/lihua/playJson/Formats.scala: -------------------------------------------------------------------------------- 1 | package lihua.playJson 2 | 3 | import lihua.{Entity, EntityId} 4 | import play.api.libs.json.Json.toJson 5 | import play.api.libs.json.{Format, JsObject, JsResult, JsValue, Json, OFormat} 6 | 7 | trait Formats { 8 | implicit object EntityIdFormat extends Format[EntityId] { 9 | 10 | override def reads(json: JsValue): JsResult[EntityId] = 11 | (json \ "$oid").validate[String].map(EntityId(_)) 12 | 13 | override def writes(o: EntityId): JsValue = Json.obj("$oid" -> o.value) 14 | } 15 | 16 | implicit def entityFormat[T: Format]: OFormat[Entity[T]] = new OFormat[Entity[T]] { 17 | def writes(e: Entity[T]): JsObject = 18 | toJson(e.data).as[JsObject] + ("_id" -> toJson(e._id)) 19 | 20 | def reads(json: JsValue): JsResult[Entity[T]] = for { 21 | id <- (json \ "_id").validate[EntityId] 22 | t <- json.validate[T] 23 | } yield Entity(id, t) 24 | } 25 | 26 | } 27 | 28 | object Formats extends Formats 29 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/editAccumulativeKPI.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.analysis._ 2 | @import com.iheart.thomas.stream._, JobSpec._ 3 | @import com.iheart.thomas.html._ 4 | 5 | @import com.iheart.thomas.http4s.UIEnv 6 | 7 | 8 | @( 9 | kpi: QueryAccumulativeKPI, 10 | runningJobO: Option[JobInfo[UpdateKPIPrior]], 11 | successMsg: Option[String] = None 12 | )(implicit env: UIEnv) 13 | 14 | @topNav(s"Editing Accumulative KPI ${kpi.name}", "Analysis") { 15 | 16 |
17 | @accumulativeKPIForm(Some(kpi), successMsg, false, Nil) 18 |
19 |
20 | 23 | Never Mind 24 |
25 |
26 |
27 | 28 | 29 | @updatePrior(kpi.name, runningJobO) 30 | 31 | } -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/editConversionKPI.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.analysis._ 2 | @import com.iheart.thomas.stream._, JobSpec._ 3 | @import com.iheart.thomas.html._ 4 | 5 | @import com.iheart.thomas.http4s.UIEnv 6 | 7 | 8 | @( 9 | kpi: ConversionKPI, 10 | runningJobO: Option[JobInfo[UpdateKPIPrior]], 11 | successMsg: Option[String] = None 12 | )(implicit env: UIEnv) 13 | 14 | @topNav(s"Editing Conversion KPI ${kpi.name}", "Analysis") { 15 | 16 |
17 | 18 | @conversionKPIForm(Some(kpi), successMsg, false) 19 | 20 |
21 |
22 | 25 | Never Mind 26 |
27 |
28 | 29 | 30 |
31 | 32 | 33 | @updatePrior(kpi.name, runningJobO) 34 | 35 | } -------------------------------------------------------------------------------- /monitor/src/main/scala/com/iheart/thomas/monitor/Reporter.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.monitor 2 | 3 | import cats.effect.{Concurrent, Fiber} 4 | import org.typelevel.log4cats.Logger 5 | import cats.implicits._ 6 | 7 | /** Fire and forget reporter 8 | * 9 | * @tparam F 10 | */ 11 | trait Reporter[F[_]] { 12 | def report(event: MonitorEvent): F[Fiber[F, Throwable, Unit]] 13 | 14 | } 15 | 16 | object Reporter { 17 | def datadog[F[_]]( 18 | client: DatadogClient[F], 19 | logger: Logger[F] 20 | )(implicit F: Concurrent[F] 21 | ): Reporter[F] = 22 | (event: MonitorEvent) => 23 | F.start { 24 | (event.status match { 25 | case MonitorEvent.Status.error => logger.error(event.toString) 26 | case _ => F.unit 27 | }) *> 28 | client.send(event)(e => 29 | logger.error( 30 | s"Failed to report $event due to " + 31 | e.toString 32 | ) 33 | ) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/PerUserSamples.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | 3 | import breeze.stats.meanAndVariance 4 | import PerUserSamples.LnSummary 5 | import cats.kernel.CommutativeMonoid 6 | import henkan.convert.Syntax._ 7 | 8 | case class PerUserSamples( 9 | values: Array[Double]) { 10 | def map(f: Double => Double): PerUserSamples = PerUserSamples(values.map(f)) 11 | 12 | def ln: PerUserSamples = map(Math.log) 13 | 14 | lazy val lnSummary: LnSummary = meanAndVariance(ln.values).to[LnSummary]() 15 | 16 | } 17 | 18 | object PerUserSamples { 19 | type LnSummary = PerUserSamplesLnSummary 20 | 21 | implicit val instances: CommutativeMonoid[PerUserSamples] = 22 | new CommutativeMonoid[PerUserSamples] { 23 | def empty: PerUserSamples = PerUserSamples(Array.empty) 24 | 25 | override def combine( 26 | x: PerUserSamples, 27 | y: PerUserSamples 28 | ): PerUserSamples = PerUserSamples(Array.concat(x.values, y.values)) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/KPIIndicator.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | package bayesian 3 | 4 | import com.iheart.thomas.analysis.bayesian.models.{ 5 | BetaModel, 6 | LogNormalModel, 7 | NormalModel 8 | } 9 | import com.stripe.rainier.sampler.{RNG, SamplerConfig} 10 | 11 | trait KPIIndicator[Model] { 12 | def apply( 13 | model: Model 14 | ): Indicator 15 | } 16 | 17 | object KPIIndicator { 18 | def sample[Model]( 19 | model: Model 20 | )(implicit indicator: KPIIndicator[Model], 21 | sampler: SamplerConfig, 22 | rng: RNG 23 | ): Seq[Double] = indicator(model).predict() 24 | 25 | implicit val betaInstance: KPIIndicator[BetaModel] = 26 | (model: BetaModel) => Variable(model.prediction) 27 | 28 | implicit val normalInstance: KPIIndicator[NormalModel] = 29 | (model: NormalModel) => Variable(model.mean) 30 | 31 | implicit val logNormalInstance: KPIIndicator[LogNormalModel] = 32 | (model: LogNormalModel) => Variable(model.mean) 33 | } -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/BenchmarkResult.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis.bayesian 2 | 3 | import com.iheart.thomas.ArmName 4 | import com.iheart.thomas.analysis.`package`.{Diff, Samples} 5 | import com.iheart.thomas.analysis.{KPIDouble, Probability} 6 | import io.estatico.newtype.ops._ 7 | 8 | case class BenchmarkResult( 9 | rawSample: Samples[Diff], 10 | benchmarkArm: ArmName) { 11 | 12 | lazy val sorted = rawSample.sorted 13 | def findMinimum(threshold: Diff): KPIDouble = 14 | KPIDouble(sorted.take((sorted.size.toDouble * (1.0 - threshold)).toInt).last) 15 | 16 | lazy val indicatorSample = rawSample.coerce[List[KPIDouble]] 17 | lazy val probabilityOfImprovement = Probability( 18 | rawSample.count(_ > 0).toDouble / rawSample.length 19 | ) 20 | lazy val riskOfUsing = findMinimum(0.95) 21 | lazy val expectedEffect = KPIDouble(rawSample.sum / rawSample.size) 22 | lazy val medianEffect = findMinimum(0.5) 23 | lazy val riskOfNotUsing = KPIDouble(-findMinimum(0.05).d) 24 | 25 | } 26 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/models/BetaModel.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | package bayesian.models 3 | import syntax.all._ 4 | import cats.data.ValidatedNel 5 | import com.stripe.rainier.core.Beta 6 | import cats.implicits._ 7 | case class BetaModel( 8 | alpha: Double, 9 | beta: Double) { 10 | lazy val prediction = Beta(alpha, beta).latent 11 | 12 | override def toString: String = { 13 | val converted = alpha - 1d 14 | val total = beta - 1 + converted 15 | s"Beta Model based on converted: $converted total: $total" 16 | } 17 | } 18 | 19 | object BetaModel { 20 | def apply(conversions: Conversions): BetaModel = 21 | BetaModel( 22 | alpha = conversions.converted + 1d, 23 | beta = conversions.total - conversions.converted + 1d 24 | ) 25 | 26 | def validate(model: BetaModel): ValidatedNel[String, BetaModel] = 27 | (model.beta > 0).toValidatedNel(model, "Beta must be larger than zero") <* 28 | (model.alpha > 0).toValidatedNel(model, "Alpha must be larger than zero") 29 | 30 | } 31 | -------------------------------------------------------------------------------- /mongo/src/main/scala/com/iheart/thomas/mongo/FeatureDAO.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2018] [iHeartMedia Inc] 3 | * All rights reserved 4 | */ 5 | 6 | package com.iheart 7 | package thomas 8 | package mongo 9 | 10 | import cats.effect.Async 11 | import cats.implicits._ 12 | import com.iheart.thomas.abtest.model._ 13 | import com.iheart.thomas.abtest.json.play.Formats._ 14 | import lihua.mongo.EitherTDAOFactory 15 | import reactivemongo.api.bson.collection.BSONCollection 16 | import reactivemongo.api.indexes.IndexType 17 | 18 | import scala.concurrent.ExecutionContext 19 | 20 | class FeatureDAOFactory[F[_]](implicit ec: ExecutionContext, F: Async[F]) 21 | extends EitherTDAOFactory[Feature, F]("abtest", "feature") { 22 | def ensure(collection: BSONCollection): F[Unit] = { 23 | 24 | F.fromFuture( 25 | F.delay( 26 | collection.indexesManager 27 | .ensure( 28 | index( 29 | Seq( 30 | ("name", IndexType.Ascending) 31 | ), 32 | unique = true 33 | ) 34 | ) 35 | .void 36 | ) 37 | ) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/analysis/BenchmarkResultSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | 3 | import com.iheart.thomas.analysis.bayesian.BenchmarkResult 4 | import com.iheart.thomas.analysis.syntax.AllSyntax 5 | import org.scalatest.funsuite.AnyFunSuiteLike 6 | import org.scalatest.matchers.should.Matchers 7 | 8 | class BenchmarkResultSuite extends AnyFunSuiteLike with Matchers with AllSyntax { 9 | test("probability of Improvement") { 10 | val subject = bayesian.BenchmarkResult(List(-1, 1, -3, 2, -3), "B") 11 | subject.probabilityOfImprovement shouldBe Probability(0.4d) 12 | } 13 | 14 | test("riskOfUsing") { 15 | val subject = BenchmarkResult((-6 to 93).toList.map(_.toDouble), "B") 16 | subject.riskOfUsing shouldBe -2d 17 | } 18 | 19 | test("riskOf Not Using") { 20 | val subject = bayesian.BenchmarkResult((-50 to 50).toList.map(_.toDouble), "B") 21 | subject.riskOfNotUsing shouldBe -44d 22 | } 23 | 24 | test("expected effect") { 25 | val subject = bayesian.BenchmarkResult((-50 to 50).toList.map(_.toDouble), "B") 26 | subject.expectedEffect shouldBe 0d 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /testkit/src/main/scala/com/iheart/thomas/testkit/ExampleParsers.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.testkit 2 | 3 | import cats.{Applicative, MonadThrow} 4 | import cats.implicits._ 5 | import com.iheart.thomas.{ArmName, FeatureName, UserId} 6 | import com.iheart.thomas.stream.{ArmParser, TimeStampParser, UserParser} 7 | import org.typelevel.jawn.ast.JValue 8 | 9 | object ExampleParsers { 10 | import com.iheart.thomas.stream.JValueSyntax._ 11 | implicit def armParser[F[_]: Applicative]: ArmParser[F, JValue] = 12 | new ArmParser[F, JValue] { 13 | def apply( 14 | m: JValue, 15 | feature: FeatureName 16 | ): F[Option[ArmName]] = 17 | m.getPath(s"treatment-groups.$feature").getString.pure[F] 18 | } 19 | 20 | implicit def userParser[F[_]: Applicative]: UserParser[F, JValue] = 21 | new UserParser[F, JValue] { 22 | def apply( 23 | m: JValue 24 | ): F[Option[UserId]] = 25 | m.getPath(s"userId").getString.pure[F] 26 | } 27 | 28 | implicit def timeStampParser[F[_]: MonadThrow]: TimeStampParser[F, JValue] = 29 | TimeStampParser.fromField[F]("timeStamp") 30 | } 31 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/models/NormalModel.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | package bayesian.models 3 | import syntax.all._ 4 | import cats.implicits._ 5 | import cats.data.ValidatedNel 6 | import com.stripe.rainier.core.{Gamma, Normal} 7 | 8 | /** xi | µ, τ ∼ N (µ, τ ) i.i.d. µ | τ ∼ N (µ0, n0τ ) τ ∼ Ga(α, β) 9 | * 10 | * @param miu0 11 | * µ 12 | * @param n0 13 | * n0 14 | * @param alpha 15 | * α 16 | * @param beta 17 | * β 18 | */ 19 | case class NormalModel( 20 | miu0: Double, 21 | n0: Double, 22 | alpha: Double, 23 | beta: Double) { 24 | 25 | lazy val τ = Gamma(shape = alpha, scale = 1d / beta).latent 26 | lazy val variance = 1d / τ 27 | lazy val mean = Normal(miu0, (1d / (τ * n0)).pow(0.5)).latent 28 | } 29 | 30 | object NormalModel { 31 | def validate(model: NormalModel): ValidatedNel[String, NormalModel] = 32 | (model.beta > 0).toValidatedNel(model, "Beta must be larger than zero") <* 33 | (model.n0 > 0).toValidatedNel(model, "n0 must be larger than zero") <* 34 | (model.alpha > 0).toValidatedNel(model, "Alpha must be larger than zero") 35 | } 36 | -------------------------------------------------------------------------------- /tests/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${application.home:-.}/logs/application.log 7 | 8 | %date [%level] from %logger in %thread - %message%n%xException 9 | 10 | 11 | 12 | 13 | 14 | [%level] %logger{15} - %message%n%xException{10} 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/ReverseRoutes.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s 2 | 3 | import com.iheart.thomas.FeatureName 4 | import com.iheart.thomas.analysis.KPIName 5 | import com.iheart.thomas.http4s.AdminUI.AdminUIConfig 6 | 7 | case class ReverseRoutes(rootPath: String) { 8 | val tests = s"$rootPath/tests" 9 | val assignments = s"$rootPath/assignments" 10 | val home = tests 11 | val features = s"$rootPath/features" 12 | def login(redirectTo: String): String = s"$login?redirectTo=$redirectTo" 13 | val login: String = s"$rootPath/login" 14 | val users = s"$rootPath/users" 15 | val register = s"$rootPath/register" 16 | val logout = s"$rootPath/logout" 17 | val analysis = s"$rootPath/analysis/" 18 | val bandits = s"$rootPath/bandits/" 19 | def bandit(feature: FeatureName): String = bandits + feature + "/" 20 | def kpi(kpi: KPIName) = s"$rootPath/analysis/kpis/$kpi" 21 | val background = s"$rootPath/stream/background" 22 | def analysisOf(feature: FeatureName) = s"$rootPath/analysis/abtests/$feature/" 23 | } 24 | 25 | object ReverseRoutes { 26 | implicit def apply(implicit cfg: AdminUIConfig): ReverseRoutes = 27 | ReverseRoutes(cfg.rootPath) 28 | } 29 | -------------------------------------------------------------------------------- /testkit/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${application.home:-.}/logs/application.log 7 | 8 | %date [%level] from %logger in %thread - %message%n%xException 9 | 10 | 11 | 12 | 13 | 14 | [%level] %logger{15} - %message%n%xException{10} 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /client/src/it/scala/com/iheart/thomas/client/JavaAPISuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2018] [iHeartMedia Inc] 3 | * All rights reserved 4 | */ 5 | 6 | package com.iheart.thomas 7 | package client 8 | 9 | import java.time.{Duration, LocalDateTime} 10 | 11 | import org.scalatest.funsuite.AnyFunSuite 12 | import org.scalatest.matchers.should.Matchers 13 | import collection.JavaConverters._ 14 | 15 | class JavaAPISuite extends AnyFunSuite with Matchers { 16 | 17 | test("integration") { 18 | val host = 19 | sys.env 20 | .get("ABTEST_HOST_ROOT_PATH") 21 | .getOrElse("http://localhost:9000/internal") 22 | val api = 23 | JavaAbtestAssignments.create(s"${host}/testsWithFeatures") 24 | val begin = LocalDateTime.now 25 | println(begin + " --- Begin") 26 | val n = 1000000 27 | (1 to n).foreach { uid => 28 | api.assignments( 29 | uid.toString, 30 | new java.util.ArrayList[String](), 31 | new java.util.HashMap[String, String](), 32 | List("Radio_Model").asJava 33 | ) 34 | } 35 | 36 | val span = Duration.between(begin, LocalDateTime.now) 37 | 38 | println("spent" + span) 39 | println(n / span.toMillis + " assignments / ms") 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dynamo/src/main/scala/com/iheart/thomas/dynamo/BanditsDAOs.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package dynamo 3 | 4 | import cats.effect.Async 5 | import com.iheart.thomas.bandit.bayesian.{BanditSpec, BanditSpecDAO} 6 | import com.iheart.thomas.dynamo.DynamoFormats._ 7 | import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient 8 | 9 | object BanditsDAOs extends ScanamoManagement { 10 | val banditStateTableName = "ds-bandit-state" 11 | val banditSpecTableName = "ds-bandit-setting" 12 | val banditKeyName = "feature" 13 | val banditKey = ScanamoDAOHelperStringKey.keyOf(banditKeyName) 14 | 15 | val tables = 16 | List((banditStateTableName, banditKey), (banditSpecTableName, banditKey)) 17 | 18 | def ensureBanditTables[F[_]: Async]( 19 | readCapacity: Long, 20 | writeCapacity: Long 21 | )(implicit dc: DynamoDbAsyncClient 22 | ): F[Unit] = 23 | ensureTables(tables, readCapacity, writeCapacity) 24 | 25 | implicit def banditSpec[F[_]: Async]( 26 | implicit dynamoClient: DynamoDbAsyncClient 27 | ): BanditSpecDAO[F] = 28 | new ScanamoDAOHelperStringKey[F, BanditSpec]( 29 | banditSpecTableName, 30 | banditKeyName, 31 | dynamoClient 32 | ) with BanditSpecDAO[F] 33 | 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/tracking/EventLogger.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.tracking 2 | 3 | import cats.Applicative 4 | import cats.effect.Sync 5 | 6 | import scala.annotation.implicitNotFound 7 | import org.typelevel.log4cats.Logger 8 | @implicitNotFound( 9 | "Logger for tracking. Check `com.iheart.thomas.tracking.EventLogger` for options" 10 | ) 11 | trait EventLogger[F[_]] { 12 | def apply(e: Event): F[Unit] 13 | 14 | def debug(m: => String): F[Unit] 15 | } 16 | 17 | object EventLogger { 18 | def noop[F[_]: Applicative]: EventLogger[F] = 19 | new EventLogger[F] { 20 | def apply(e: Event): F[Unit] = Applicative[F].unit 21 | def debug(s: => String): F[Unit] = Applicative[F].unit 22 | } 23 | 24 | def stdout[F[_]: Sync]: EventLogger[F] = 25 | new EventLogger[F] { 26 | def apply(e: Event): F[Unit] = Sync[F].delay(println(e)) 27 | def debug(s: => String): F[Unit] = Sync[F].delay(println(s)) 28 | } 29 | 30 | def catsLogger[F[_]](logger: Logger[F]): EventLogger[F] = 31 | new EventLogger[F] { 32 | def apply(e: Event): F[Unit] = logger.info(e.toString) 33 | def debug(s: => String): F[Unit] = logger.debug(s) 34 | } 35 | } 36 | 37 | trait Event extends Serializable with Product 38 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/fit/DistributionSpec.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis.bayesian.fit 2 | 3 | import com.stripe.rainier 4 | import com.stripe.rainier.core.Distribution 5 | 6 | sealed trait DistributionSpec[T] extends Serializable with Product { 7 | def distribution: Distribution[T] 8 | } 9 | 10 | object DistributionSpec { 11 | case class Normal( 12 | location: Double, 13 | scale: Double) 14 | extends DistributionSpec[Double] { 15 | val distribution = rainier.core.Normal(location, scale) 16 | } 17 | 18 | object Normal { 19 | def fit(data: List[Double]): Normal = { 20 | import breeze.stats.meanAndVariance 21 | import meanAndVariance.MeanAndVariance 22 | val MeanAndVariance(m, v, _) = meanAndVariance(data) 23 | Normal(m, Math.sqrt(v)) 24 | } 25 | } 26 | 27 | case class LogNormal( 28 | location: Double, 29 | scale: Double) 30 | extends DistributionSpec[Double] { 31 | val distribution = rainier.core.LogNormal(location, scale) 32 | 33 | } 34 | 35 | case class Uniform( 36 | from: Double, 37 | to: Double) 38 | extends DistributionSpec[Double] { 39 | val distribution = rainier.core.Uniform(from, to) 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /spark/src/main/scala/com/iheart/thomas/spark/AutoRefreshAssigner.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package spark 3 | 4 | import cats.effect.IO 5 | import mau.RefreshRef 6 | import cats.effect.unsafe.implicits.global 7 | import org.apache.spark.sql.expressions.UserDefinedFunction 8 | 9 | import concurrent.duration._ 10 | import org.apache.spark.sql.functions.udf 11 | 12 | /** Provides a `udf` that assigns based on auto refreshed test data. 13 | * 14 | * @param url 15 | * @param refreshPeriod 16 | * how often test data is refreshed 17 | */ 18 | case class AutoRefreshAssigner( 19 | url: String, 20 | refreshPeriod: FiniteDuration = 10.minutes) { 21 | private lazy val inner: RefreshRef[IO, Assigner] = { 22 | import scala.concurrent.ExecutionContext.Implicits.global 23 | RefreshRef 24 | .create[IO, Assigner](_ => IO.unit) 25 | .flatTap(ref => ref.getOrFetch(refreshPeriod)(Assigner.create[IO](url, None))) 26 | 27 | }.unsafeRunSync() 28 | 29 | def assignUdf(feature: FeatureName): UserDefinedFunction = udf( 30 | assignFunction(feature) 31 | ) 32 | 33 | def assignFunction(feature: FeatureName): String => Option[GroupName] = 34 | (userId: String) => { 35 | inner.get.unsafeRunSync().flatMap(_.assign(feature, userId)) 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/kpiBasics.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.analysis._ 2 | 3 | @( 4 | draftO: Option[KPI], 5 | showNameField: Boolean = true 6 | ) 7 | 8 | 9 |
10 |
11 | Basics 12 |
13 |
14 | @if( showNameField ) { 15 |
16 | 17 | 24 |
25 | } else { 26 | 29 | } 30 | 31 |
32 | 33 | 37 |
38 | 39 |
40 |
-------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/AccumulativeKPIQueryRepo.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | 3 | import cats.{Applicative, Functor} 4 | import com.iheart.thomas.analysis.MessageQuery.FieldName 5 | import com.iheart.thomas.analysis.Query.DefaultParamValue 6 | import cats.implicits._ 7 | 8 | import scala.concurrent.duration.FiniteDuration 9 | trait AccumulativeKPIQueryRepo[F[_]] { 10 | def queries: F[List[PerUserSamplesQuery[F]]] 11 | def implemented: Boolean = true 12 | def findQuery( 13 | name: QueryName 14 | )(implicit F: Functor[F] 15 | ): F[Option[PerUserSamplesQuery[F]]] = 16 | queries.map(_.find(_.name == name)) 17 | } 18 | 19 | object AccumulativeKPIQueryRepo { 20 | def unsupported[F[_]: Applicative]: AccumulativeKPIQueryRepo[F] = 21 | new AccumulativeKPIQueryRepo[F] { 22 | override def implemented: Boolean = false 23 | 24 | def queries: F[List[PerUserSamplesQuery[F]]] = Nil.pure[F].widen 25 | } 26 | } 27 | 28 | trait PerUserSamplesQuery[F[_]] 29 | extends KPIEventQuery[F, QueryAccumulativeKPI, PerUserSamples] { 30 | def name: QueryName 31 | def params: Map[FieldName, DefaultParamValue] = Map.empty 32 | def frequency: FiniteDuration 33 | } 34 | 35 | object Query { 36 | type DefaultParamValue = Option[String] 37 | } 38 | -------------------------------------------------------------------------------- /mongo/src/main/scala/com/iheart/thomas/mongo/AbtestDAO.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2018] [iHeartMedia Inc] 3 | * All rights reserved 4 | */ 5 | package com.iheart 6 | package thomas 7 | package mongo 8 | 9 | import cats.effect.Async 10 | import cats.implicits._ 11 | import com.iheart.thomas.abtest.model._ 12 | import lihua.mongo.EitherTDAOFactory 13 | import reactivemongo.api.indexes.IndexType 14 | import com.iheart.thomas.abtest.json.play.Formats._ 15 | import reactivemongo.api.bson.collection.BSONCollection 16 | 17 | import scala.concurrent.ExecutionContext 18 | 19 | class AbtestDAOFactory[F[_]](implicit ec: ExecutionContext, F: Async[F]) 20 | extends EitherTDAOFactory[Abtest, F]("abtest", "tests") { 21 | 22 | def ensure(collection: BSONCollection): F[Unit] = { 23 | F.fromFuture( 24 | F.delay( 25 | collection.indexesManager.ensure( 26 | index( 27 | Seq( 28 | ("start", IndexType.Descending), 29 | ("end", IndexType.Descending) 30 | ) 31 | ) 32 | ) *> collection.indexesManager 33 | .ensure( 34 | index( 35 | Seq( 36 | ("feature", IndexType.Ascending) 37 | ) 38 | ) 39 | ) 40 | .void 41 | ) 42 | ) 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/MongoResources.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package http4s 3 | 4 | import cats.effect.{Async, Resource} 5 | 6 | import com.iheart.thomas.abtest.AbtestAlg 7 | import com.iheart.thomas.mongo.DAOs 8 | import com.typesafe.config.Config 9 | 10 | import scala.compat.java8.DurationConverters._ 11 | import scala.concurrent.ExecutionContext 12 | 13 | object MongoResources extends ConfigResource { 14 | 15 | def abtestAlg[F[_]: Async]( 16 | cfg: Config, 17 | daos: mongo.DAOs[F] 18 | ): Resource[F, AbtestAlg[F]] = { 19 | 20 | implicit val (abtestDAO, featureDAO) = daos 21 | val refreshPeriod = 22 | cfg.getDuration("thomas.abtest.get-groups.ttl").toScala 23 | AbtestAlg.defaultResource[F](refreshPeriod) 24 | } 25 | 26 | def abtestAlg[F[_]: Async]( 27 | cfgResourceName: Option[String] = None 28 | )(implicit ex: ExecutionContext 29 | ): Resource[F, AbtestAlg[F]] = 30 | cfg[F](cfgResourceName).flatMap(abtestAlg(_)) 31 | 32 | def abtestAlg[F[_]: Async]( 33 | cfg: Config 34 | )(implicit ex: ExecutionContext 35 | ): Resource[F, AbtestAlg[F]] = 36 | dAOs[F](cfg).flatMap(abtestAlg(cfg, _)) 37 | 38 | def dAOs[F[_]: Async]( 39 | config: Config 40 | )(implicit ex: ExecutionContext 41 | ): Resource[F, DAOs[F]] = 42 | mongo.daosResource[F](config) 43 | 44 | } 45 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/syntax/AllSyntax.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | package syntax 3 | 4 | import cats.data.{Validated, ValidatedNel} 5 | import com.iheart.thomas.analysis.bayesian.fit.KPISyntax 6 | import com.iheart.thomas.analysis.syntax.RealSyntax.RealSyntaxOps 7 | import com.iheart.thomas.analysis.syntax.ValidationSyntax.BooleanOps 8 | import com.stripe.rainier.compute.Real 9 | import com.stripe.rainier.core.Model 10 | import com.stripe.rainier.sampler.SamplerConfig 11 | trait AllSyntax extends KPISyntax with RealSyntax with ValidationSyntax 12 | 13 | object all extends AllSyntax 14 | 15 | trait RealSyntax { 16 | implicit def toOps(real: Real): RealSyntaxOps = new RealSyntaxOps(real) 17 | } 18 | 19 | object RealSyntax { 20 | private[syntax] class RealSyntaxOps(private val real: Real) extends AnyVal { 21 | def sample(implicit sc: SamplerConfig) = Model.sample(real, sc) 22 | } 23 | } 24 | 25 | trait ValidationSyntax { 26 | implicit def toOps(boolean: Boolean): BooleanOps = new BooleanOps(boolean) 27 | } 28 | 29 | object ValidationSyntax { 30 | private[syntax] class BooleanOps(private val bool: Boolean) extends AnyVal { 31 | def toValidatedNel[L, R]( 32 | ifTrue: => R, 33 | ifFalse: => L 34 | ): ValidatedNel[L, R] = 35 | if (bool) Validated.validNel(ifTrue) else Validated.invalidNel(ifFalse) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/KPI.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | import MessageQuery._ 3 | import bayesian.models.{BetaModel, LogNormalModel} 4 | 5 | import scala.util.matching.Regex 6 | 7 | sealed trait KPI { 8 | def name: KPIName 9 | def author: String 10 | def description: Option[String] 11 | } 12 | 13 | sealed trait AccumulativeKPI extends KPI { 14 | def model: LogNormalModel 15 | } 16 | 17 | case class QueryAccumulativeKPI( 18 | name: KPIName, 19 | author: String, 20 | description: Option[String], 21 | model: LogNormalModel, 22 | queryName: QueryName, 23 | queryParams: Map[String, String]) 24 | extends AccumulativeKPI 25 | 26 | case class ConversionKPI( 27 | name: KPIName, 28 | author: String, 29 | description: Option[String], 30 | model: BetaModel, 31 | messageQuery: Option[ConversionMessageQuery]) 32 | extends KPI 33 | 34 | case class ConversionMessageQuery( 35 | initMessage: MessageQuery, 36 | convertedMessage: MessageQuery) 37 | 38 | case class MessageQuery( 39 | description: Option[String], 40 | criteria: List[Criteria]) 41 | 42 | case class Criteria( 43 | fieldName: FieldName, 44 | matchingRegex: FieldRegex) { 45 | lazy val regex: Regex = new Regex(matchingRegex) 46 | } 47 | 48 | object MessageQuery { 49 | type FieldName = String 50 | type FieldRegex = String 51 | } 52 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/ArmExtractor.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package stream 3 | 4 | import cats.Monad 5 | import com.iheart.thomas.abtest.PerformantAssigner 6 | import com.iheart.thomas.abtest.model.AssignmentTruthAt.{Message, Realtime} 7 | import com.iheart.thomas.abtest.model.{Feature, UserGroupQuery} 8 | import cats.implicits._ 9 | import utils.time._ 10 | 11 | trait ArmExtractor[F[_], Message] { 12 | def apply(feature: Feature, message: Message): F[Option[ArmName]] 13 | } 14 | 15 | object ArmExtractor { 16 | implicit def default[F[_]: Monad, Message]( 17 | implicit 18 | userParser: UserParser[F, Message], 19 | timeStampParser: TimeStampParser[F, Message], 20 | armParser: ArmParser[F, Message], 21 | assigner: PerformantAssigner[F] 22 | ): ArmExtractor[F, Message] = (feature, message) => { 23 | feature.assignmentTruthAt match { 24 | case Realtime => 25 | for { 26 | uid <- userParser(message) 27 | ts <- timeStampParser(message) 28 | assignments <- assigner.assign( 29 | UserGroupQuery( 30 | uid, 31 | at = Some(ts.toOffsetDateTimeUTC), 32 | features = List(feature.name) 33 | ) 34 | ) 35 | } yield assignments.get(feature.name).map(_.groupName) 36 | case Message => armParser(message, feature.name) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/Variable.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis.bayesian 2 | 3 | import cats.Apply 4 | import cats.implicits._ 5 | import com.stripe.rainier.core.{Model, ToGenerator} 6 | import com.stripe.rainier.sampler.{RNG, SamplerConfig} 7 | 8 | case class Variable[A]( 9 | v: A, 10 | model: Option[Model] = None) { 11 | 12 | def predict[U]( 13 | nChains: Int = 4 14 | )(implicit 15 | sampler: SamplerConfig, 16 | g: ToGenerator[A, U], 17 | rng: RNG 18 | ): List[U] = 19 | model.fold(Model.sample(v, sampler))( 20 | _.sample(sampler, nChains = nChains) 21 | .predict(v) 22 | ) 23 | 24 | def map2[B, C](that: Variable[B])(f: (A, B) => C): Variable[C] = 25 | Variable( 26 | f(v, that.v), 27 | (model, that.model).mapN(_ merge _) orElse model orElse that.model 28 | ) 29 | 30 | def map[B](f: A => B): Variable[B] = Variable(f(v), model) 31 | } 32 | 33 | object Variable { 34 | 35 | def apply[A]( 36 | a: A, 37 | model: Model 38 | ): Variable[A] = Variable(a, Some(model)) 39 | 40 | implicit def applyInstance: Apply[Variable] = 41 | new Apply[Variable] { 42 | override def ap[A, B](ff: Variable[A => B])(fa: Variable[A]): Variable[B] = 43 | ff.map2(fa)((f, b) => f(b)) 44 | 45 | override def map[A, B](fa: Variable[A])(f: A => B): Variable[B] = fa.map(f) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/bandit/index.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | @import com.iheart.thomas.http4s.UIEnv 3 | @import com.iheart.thomas.http4s.bandit.Bandit 4 | 5 | @( 6 | bandits: Seq[Bandit] 7 | )(implicit env: UIEnv) 8 | 9 | 10 | @topNav("Bandit", "Bandit") { 11 | 12 | New Bandit 13 | @for(bandit <- bandits) { 14 | 15 | 16 |
17 |
18 |
19 | 20 |
21 | Feature 22 | @bandit.feature 23 |
24 |
25 | @bandit.spec.title 26 |
27 | 28 |
29 |
30 | 31 | Author: 32 | @bandit.spec.author 33 | 34 |
35 |
36 | @bandit.status 37 |
38 |
39 |
40 | } 41 | } -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/abtest/admin/newRevision.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.abtest.model._ 2 | @import lihua.Entity 3 | @import com.iheart.thomas.http4s.UIEnv 4 | @import com.iheart.thomas.html._ 5 | 6 | 7 | @( 8 | fromTest: Entity[Abtest], 9 | draft: Option[AbtestSpec], 10 | errorMsg: Option[String] = None, 11 | operatingSettingsOnly: Boolean)(implicit env: UIEnv) 12 | 13 | @topNav("Create A New Test", "A/B Tests") { 14 |
15 |
16 | Creating an A/B Test to follow 17 | 18 | @fromTest.data.name 19 | for 20 | 21 | @featureTestsLink(fromTest.data.feature) 22 |
23 | @testForm(draft, errorMsg, operatingSettingsOnly = operatingSettingsOnly) 24 |
25 |
26 | 29 | 30 |
31 |
32 |
33 | } -------------------------------------------------------------------------------- /tests/src/main/scala/com/iheart/thomas/example/ExampleApps.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package example 3 | 4 | import cats.effect._ 5 | import com.iheart.thomas.http4s.AdminUI 6 | import com.iheart.thomas.http4s.abtest.AbtestService 7 | import com.iheart.thomas.tracking.EventLogger 8 | import org.http4s.blaze.server.BlazeServerBuilder 9 | import org.typelevel.log4cats.slf4j.Slf4jLogger 10 | import testkit.{LocalDynamo, MockQueryAccumulativeKPIAlg} 11 | 12 | import scala.concurrent.ExecutionContext.Implicits.global 13 | 14 | object ExampleAbtestServerApp extends IOApp { 15 | implicit val logger = EventLogger.catsLogger(Slf4jLogger.getLogger[IO]) 16 | 17 | def run(args: List[String]): IO[ExitCode] = 18 | AbtestService.fromMongo[IO]().use { s => 19 | BlazeServerBuilder[IO] 20 | .bindHttp(8080, "0.0.0.0") 21 | .withHttpApp(s.routes) 22 | .serve 23 | .compile 24 | .drain 25 | .as(ExitCode.Success) 26 | } 27 | } 28 | 29 | object ExampleAbtestAdminUIApp extends IOApp { 30 | import testkit.ExampleParsers._ 31 | 32 | implicit val queryAlg = MockQueryAccumulativeKPIAlg[IO]() 33 | 34 | implicit val logger = EventLogger.catsLogger(Slf4jLogger.getLogger[IO]) 35 | 36 | def run(args: List[String]): IO[ExitCode] = { 37 | LocalDynamo 38 | .client[IO]() 39 | .flatMap(implicit c => AdminUI.serverResourceAutoLoadConfig[IO]) 40 | .use(_ => IO.never) 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/abtest/model/ModelSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package abtest 3 | package model 4 | 5 | import org.scalatest.funsuite.AnyFunSuiteLike 6 | import org.scalatest.matchers.should.Matchers 7 | 8 | import java.time.Instant 9 | 10 | class ModelSuite extends AnyFunSuiteLike with Matchers { 11 | 12 | val existingFeatureConfig: Feature = Feature( 13 | name = "test feature", 14 | overrides = Map( 15 | "user1" -> "control", 16 | "user2" -> "treatment" 17 | ), 18 | lastUpdated = Some(Instant.EPOCH) 19 | ) 20 | 21 | test("a new feature config with new override should not trigger nonTestSettingsChangedFrom") { 22 | val newConfig: Feature = Feature( 23 | name = "test feature", 24 | overrides = Map( 25 | "user1" -> "treatment", 26 | "user2" -> "control" 27 | ), 28 | lastUpdated = None 29 | ) 30 | existingFeatureConfig nonTestSettingsChangedFrom newConfig shouldBe false 31 | } 32 | 33 | test("a new feature config with new description should trigger nonTestSettingsChangedFrom") { 34 | val newConfig: Feature = Feature( 35 | name = "test feature", 36 | description = Some("test description"), 37 | overrides = Map( 38 | "user1" -> "control", 39 | "user2" -> "treatment" 40 | ), 41 | lastUpdated = None 42 | ) 43 | existingFeatureConfig nonTestSettingsChangedFrom newConfig shouldBe true 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /testkit/src/main/scala/com/iheart/thomas/testkit/LocalDynamo.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package testkit 3 | 4 | import cats.Parallel 5 | import cats.effect.{Async, Resource, Sync} 6 | import com.iheart.thomas.dynamo.ScanamoManagement 7 | import org.scanamo.LocalDynamoDB 8 | import cats.implicits._ 9 | import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient 10 | import software.amazon.awssdk.services.dynamodb.model.{ 11 | ResourceNotFoundException, 12 | ScalarAttributeType 13 | } 14 | 15 | object LocalDynamo extends ScanamoManagement { 16 | def client[F[_]]( 17 | port: Int = 8042 18 | )(implicit F: Sync[F] 19 | ): Resource[F, DynamoDbAsyncClient] = 20 | Resource.make { 21 | F.delay(LocalDynamoDB.client(port)) 22 | } { client => 23 | F.delay(client.close()) 24 | } 25 | 26 | def clientWithTables[F[_]: Parallel]( 27 | tables: (String, Seq[(String, ScalarAttributeType)])* 28 | )(implicit F: Async[F] 29 | ): Resource[F, DynamoDbAsyncClient] = 30 | client[F](8043).flatTap { client => 31 | Resource.make { 32 | tables.toList.parTraverse { case (tableName, keyAttributes) => 33 | ensureTable[F](client, tableName, keyAttributes, 10L, 10L) 34 | } 35 | } { _ => 36 | tables.toList.parTraverse { t => 37 | F.delay(LocalDynamoDB.deleteTable(client)(t._1)).void.recover { 38 | case _: ResourceNotFoundException => () 39 | } 40 | }.void 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /mongo/src/main/scala/lihua/mongo/DBError.scala: -------------------------------------------------------------------------------- 1 | package lihua 2 | package mongo 3 | 4 | import cats.data.NonEmptyList 5 | 6 | sealed trait DBError extends RuntimeException with Product with Serializable 7 | 8 | object DBError { 9 | case object NotFound extends DBError 10 | 11 | case class DBException( 12 | throwable: Throwable, 13 | collection: String) 14 | extends DBError { 15 | override def getCause: Throwable = throwable 16 | 17 | override def getMessage: String = 18 | s"Error occurred (collection: $collection): ${throwable.getMessage} " 19 | } 20 | case class DBLastError(override val getMessage: String) extends DBError 21 | 22 | case class WriteError(details: NonEmptyList[WriteErrorDetail]) extends DBError { 23 | override def getMessage: String = details.toString() 24 | } 25 | 26 | sealed trait WriteErrorDetail extends Product with Serializable { 27 | def code: Int 28 | def msg: String 29 | override def toString: String = s"code: $code, message: $msg" 30 | } 31 | 32 | case class ItemWriteErrorDetail( 33 | code: Int, 34 | msg: String) 35 | extends WriteErrorDetail 36 | case class WriteConcernErrorDetail( 37 | code: Int, 38 | msg: String) 39 | extends WriteErrorDetail 40 | 41 | case class UpdatedCountErrorDetail( 42 | expectedCount: Int, 43 | actual: Int) 44 | extends DBError { 45 | override def getMessage = 46 | s"updated count is $actual, expected $expectedCount" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | zookeeper: 5 | image: 'docker.io/bitnami/zookeeper:3-debian-10' 6 | ports: 7 | - '2181:2181' 8 | volumes: 9 | - zookeeper_data_${THOMAS_ENV}:/bitnami 10 | environment: 11 | - ALLOW_ANONYMOUS_LOGIN=yes 12 | 13 | kafka: 14 | image: 'docker.io/bitnami/kafka:2-debian-10' 15 | ports: 16 | - '9092:9092' 17 | volumes: 18 | - kafka_data_${THOMAS_ENV}:/bitnami 19 | environment: 20 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 21 | - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true 22 | - ALLOW_PLAINTEXT_LISTENER=yes 23 | - KAFKA_LISTENERS=PLAINTEXT://:9092 24 | - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 25 | depends_on: 26 | - zookeeper 27 | 28 | mongo: 29 | container_name: thomas-mongodb-${THOMAS_ENV} 30 | image: 'mongo:6.0.13' 31 | ports: 32 | - ${THOMAS_MONGO_PORT}:27017 33 | volumes: 34 | - mongo_data_${THOMAS_ENV}:/data/db 35 | 36 | dynamo: 37 | container_name: thomas-dynamo-${THOMAS_ENV} 38 | build: 39 | context: . 40 | dockerfile: docker/dynamodb/Dockerfile 41 | ports: 42 | - ${THOMAS_DYNAMO_PORT}:8000 43 | volumes: 44 | - dynamo_data_${THOMAS_ENV}:/home/dynamodblocal/db 45 | 46 | 47 | volumes: 48 | zookeeper_data_test: 49 | kafka_data_test: 50 | zookeeper_data_dev: 51 | kafka_data_dev: 52 | mongo_data_test: 53 | dynamo_data_test: 54 | mongo_data_dev: 55 | dynamo_data_dev: 56 | -------------------------------------------------------------------------------- /client/src/main/scala/com/iheart/thomas/client/PlayJsonHttp4Client.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.client 2 | 3 | import cats.effect.Async 4 | import org.http4s.client.dsl.Http4sClientDsl 5 | import _root_.play.api.libs.json.{JsObject, Reads, Writes} 6 | import cats.implicits._ 7 | import org.http4s.{InvalidResponseException, Request, Uri} 8 | 9 | /** Utility class for writing http4 based client using play json 10 | * @tparam F 11 | */ 12 | abstract class PlayJsonHttp4sClient[F[_]: Async](c: org.http4s.client.Client[F]) 13 | extends Http4sClientDsl[F] 14 | with lihua.playJson.Formats { 15 | import org.http4s.play._ 16 | import org.http4s.{EntityDecoder, EntityEncoder} 17 | 18 | implicit def autoEntityEncoderFromJsonWrites[A: Writes]: EntityEncoder[F, A] = 19 | jsonEncoderOf[F, A] 20 | 21 | implicit def jsObjectEncoder: EntityEncoder[F, JsObject] = jsonEncoder[F].narrow 22 | implicit def jsonDeoder[A: Reads]: EntityDecoder[F, A] = jsonOf 23 | 24 | def expect[A](reqF: F[Request[F]])(implicit d: EntityDecoder[F, A]): F[A] = 25 | reqF.flatMap(expect(_)) 26 | 27 | def expect[A](req: Request[F])(implicit d: EntityDecoder[F, A]): F[A] = 28 | c.expectOr(req) { err => 29 | err.bodyText.compile.toList 30 | .map(body => 31 | InvalidResponseException( 32 | s"status: ${err.status.code} \n body: ${body.mkString}" 33 | ) 34 | ) 35 | } 36 | 37 | def encode(urlString: String): Uri = 38 | Uri.unsafeFromString(Uri.encode(urlString)) 39 | 40 | } 41 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/stream/UI.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s.stream 2 | 3 | import cats.effect.Async 4 | import com.iheart.thomas.admin 5 | import com.iheart.thomas.http4s.{AuthImp, ReverseRoutes, UIEnv} 6 | import com.iheart.thomas.http4s.auth.AuthedEndpointsUtils 7 | import com.iheart.thomas.stream.JobAlg 8 | import org.http4s.dsl.Http4sDsl 9 | import tsec.authentication._ 10 | import cats.implicits._ 11 | import com.iheart.thomas.html.redirect 12 | import com.iheart.thomas.http4s.AdminUI.AdminUIConfig 13 | import org.http4s.twirl._ 14 | import com.iheart.thomas.stream.html._ 15 | import org.http4s.Uri.Path.Segment 16 | 17 | class UI[F[_]: Async]( 18 | implicit 19 | jobAlg: JobAlg[F], 20 | adminUICfg: AdminUIConfig) 21 | extends AuthedEndpointsUtils[F, AuthImp] 22 | with Http4sDsl[F] { 23 | 24 | val rootPath = Root / Segment("stream") 25 | val reverseRoutes = implicitly[ReverseRoutes] 26 | 27 | val readonlyRoutes = roleBasedService(admin.Authorization.backgroundManagerRoles) { 28 | case GET -> `rootPath` / "background" asAuthed u => { 29 | jobAlg.allJobs.flatMap { jobs => 30 | Ok(background(jobs)(UIEnv(u))) 31 | } 32 | } 33 | 34 | case GET -> `rootPath` / "background" / jobKey / "stop" asAuthed _ => { 35 | jobAlg.stop(jobKey) *> Ok( 36 | redirect( 37 | reverseRoutes.background, 38 | s"Process $jobKey is stopped." 39 | ) 40 | ) 41 | } 42 | } 43 | 44 | val routes = readonlyRoutes 45 | 46 | } 47 | -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/utils/time/Period.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.utils.time 2 | 3 | import cats.{Foldable, Reducible, Semigroup} 4 | import cats.implicits._ 5 | 6 | import java.time.Instant 7 | 8 | case class Period(from: Instant, to: Instant) { 9 | assert( 10 | !from.isAfter(to), 11 | s"Invalid Period($from, $to). The from and to is reverse." 12 | ) 13 | def isBefore(instant: Instant): Boolean = to.isBefore(instant) 14 | def isAfter(instant: Instant): Boolean = from.isAfter(instant) 15 | def includes(instant: Instant): Boolean = !isBefore(instant) && !isAfter(instant) 16 | def extendsToInclude(instant: Instant): Period = { 17 | if (isBefore(instant)) Period(from, instant) 18 | else if (isAfter(instant)) Period(instant, to) 19 | else this 20 | } 21 | } 22 | 23 | object Period { 24 | implicit object instance extends Semigroup[Period] { 25 | def combine(x: Period, y: Period): Period = Period( 26 | first(x.from, y.from), 27 | last(x.to, y.to) 28 | ) 29 | } 30 | 31 | def of(a: Instant, b: Instant): Period = 32 | if (a.isBefore(b)) Period(a, b) else Period(b, a) 33 | 34 | def fromStamps[F[_]: Reducible](stamps: F[Instant]): Period = 35 | stamps.reduceLeftTo(i => Period(i, i))(_.extendsToInclude(_)) 36 | 37 | def of[F[_]: Foldable, A]( 38 | withStamps: F[A], 39 | stamp: A => Instant 40 | ): Option[Period] = 41 | withStamps.reduceLeftToOption { a => 42 | val s = stamp(a) 43 | Period(s, s) 44 | } { (p, a) => 45 | p.extendsToInclude(stamp(a)) 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/StreamControlTest.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s 2 | 3 | import java.time.LocalDateTime 4 | import cats.effect.{Concurrent, ExitCode, IO, IOApp} 5 | import fs2.concurrent.SignallingRef 6 | import fs2.Stream 7 | import cats.implicits._ 8 | import org.http4s.HttpRoutes 9 | import org.http4s.blaze.server.BlazeServerBuilder 10 | import org.http4s.dsl.Http4sDsl 11 | import org.http4s.syntax.all._ 12 | 13 | import concurrent.duration._ 14 | 15 | object StreamControlTest extends IOApp with Http4sDsl[IO] { 16 | def run(args: List[String]): IO[ExitCode] = 17 | banditUpdateController[IO]( 18 | Stream 19 | .repeatEval(IO.sleep(1.second) *> IO(println(LocalDateTime.now))) 20 | .interruptAfter(120.seconds) 21 | ).flatMap { case (runs, pause) => 22 | BlazeServerBuilder[IO] 23 | .bindHttp(9000, "localhost") 24 | .withHttpApp(routes(pause).orNotFound) 25 | .serve 26 | .concurrently(runs) 27 | .compile 28 | .drain 29 | .as(ExitCode.Success) 30 | } 31 | 32 | def banditUpdateController[F[_]: Concurrent](stream: Stream[F, Unit]) = { 33 | SignallingRef[F, Boolean](false).map { paused => 34 | val consumerStream = stream.pauseWhen(paused) 35 | (consumerStream, paused) 36 | } 37 | } 38 | 39 | def routes(signallingRef: SignallingRef[IO, Boolean]) = 40 | HttpRoutes.of[IO] { 41 | case GET -> Root / "start" => signallingRef.set(false) >> Ok("started") 42 | case GET -> Root / "stop" => signallingRef.set(true) >> Ok("stopped") 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/abtest/admin/editTest.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.abtest.model._ 2 | @import lihua._ 3 | @import com.iheart.thomas.http4s.UIEnv 4 | @import com.iheart.thomas.html._ 5 | 6 | 7 | @( 8 | test: Entity[Abtest], 9 | errMessage: Option[String] = None, 10 | spec: Option[AbtestSpec] = None, 11 | followUpO: Option[Entity[Abtest]] = None, 12 | operatingSettingsOnly: Boolean)(implicit env: UIEnv) 13 | 14 | @topNav("A/B Test for feature " + test.data.feature, "A/B Tests") { 15 | 16 |
17 | 18 |
19 | 20 | Editing A/B Test @test.data.name 21 | for 22 | 23 | @featureTestsLink(test.data.feature) 24 |
25 | @testForm(spec.orElse(Some(test.data.toSpec)), errMessage, false, followUpO.isEmpty, operatingSettingsOnly= operatingSettingsOnly) 26 | 27 |
28 |
29 | 32 | 33 |
34 | @for(followUp <- followUpO) { 35 |
36 | Note: this test is followed by @followUp.data.name 37 |
38 | } 39 |
40 | 41 |
42 | } -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/Job.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.stream 2 | 3 | import java.time.Instant 4 | 5 | /** Job to process Stream of messages 6 | * 7 | * @param spec 8 | * @param checkedOut 9 | * last time a worker reported that it's working on it 10 | * @param started 11 | * latest start time when a worker started working on it. 12 | */ 13 | case class Job( 14 | key: String, 15 | spec: JobSpec, 16 | checkedOut: Option[Instant], 17 | started: Option[Instant]) 18 | 19 | case class JobInfo[JS <: JobSpec]( 20 | started: Option[Instant], 21 | spec: JS) 22 | 23 | object Job { 24 | def apply(spec: JobSpec): Job = Job(spec.key, spec, None, None) 25 | } 26 | 27 | /** A DAO for job. Implemenation should pass thomas.stream.JobDAOSuite in the tests 28 | * module. 29 | * @tparam F 30 | */ 31 | trait JobDAO[F[_]] { 32 | 33 | /** @return 34 | * None if job with the same key already exist. 35 | */ 36 | def insertO(job: Job): F[Option[Job]] 37 | 38 | /** Update checkedOut but fails when the existing data is inconsistent with given 39 | * `job` 40 | * @return 41 | * None if either the job no longer exist or its signature is different, i.e. 42 | * checkedOut is inconsistent 43 | */ 44 | def updateCheckedOut( 45 | job: Job, 46 | at: Instant 47 | ): F[Option[Job]] 48 | 49 | def setStarted( 50 | job: Job, 51 | at: Instant 52 | ): F[Job] 53 | 54 | def remove(jobKey: String): F[Unit] 55 | 56 | def find(jobKey: String): F[Option[Job]] 57 | 58 | def all: F[Vector[Job]] 59 | 60 | } 61 | -------------------------------------------------------------------------------- /kafka/src/main/scala/com/iheart/thomas/kafka/MessageProcessor.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package kafka 3 | 4 | import com.iheart.thomas.FeatureName 5 | import com.iheart.thomas.analysis.KPIName 6 | import com.iheart.thomas.analysis.ConversionEvent 7 | import cats.effect.kernel.Resource 8 | import fs2.Pipe 9 | import fs2.kafka.Deserializer 10 | 11 | trait MessageProcessor[F[_]] { 12 | type RawMessage 13 | type PreprocessedMessage 14 | 15 | implicit def deserializer: Resource[F, Deserializer[F, RawMessage]] 16 | def preprocessor: Pipe[F, RawMessage, PreprocessedMessage] 17 | def toConversionEvent( 18 | featureName: FeatureName, 19 | KPIName: KPIName 20 | ): F[ 21 | Pipe[F, PreprocessedMessage, (ArmName, ConversionEvent)] 22 | ] 23 | } 24 | 25 | object MessageProcessor { 26 | def apply[F[_], Message]( 27 | toEvent: ( 28 | FeatureName, 29 | KPIName 30 | ) => F[Pipe[F, Message, (ArmName, ConversionEvent)]] 31 | )(implicit ev: Resource[F, Deserializer[F, Message]] 32 | ): MessageProcessor[F] { 33 | type RawMessage = Message; 34 | type PreprocessedMessage = Message 35 | } = 36 | new MessageProcessor[F] { 37 | 38 | type RawMessage = Message 39 | type PreprocessedMessage = Message 40 | implicit def deserializer: Resource[F, Deserializer[F, RawMessage]] = ev 41 | def preprocessor: Pipe[F, Message, Message] = identity 42 | 43 | def toConversionEvent( 44 | featureName: FeatureName, 45 | kpiName: KPIName 46 | ): F[Pipe[F, Message, (ArmName, ConversionEvent)]] = 47 | toEvent(featureName, kpiName) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/auth/login.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | @import com.iheart.thomas.http4s.ReverseRoutes 3 | @(errorMsg: Option[String] = None)(implicit reverseRoutes: ReverseRoutes) 4 | 5 | 6 | @main("Login"){ 7 | 8 |
9 |
10 |
11 |
12 | Login 13 |
14 |
15 | @for(msg <- errorMsg) { 16 | 19 | } 20 |
21 |
22 | 23 | 25 |
26 | 27 |
28 | 29 | 31 |
32 |
33 | 34 |
35 |
36 |
37 |
New User? Register here
38 | Forget Password? Please contact admin for a reset link. 39 |
40 |
41 |
42 |
43 | 44 |
45 | 46 | } -------------------------------------------------------------------------------- /dynamo/src/main/scala/com/iheart/thomas/dynamo/package.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.dynamo 2 | 3 | import cats.effect.{Resource, Sync} 4 | import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient 5 | import io.estatico.newtype.Coercible 6 | import pureconfig.ConfigSource 7 | import software.amazon.awssdk.auth.credentials.AwsBasicCredentials 8 | import software.amazon.awssdk.regions.Region 9 | 10 | import java.net.URI 11 | 12 | object `package` { 13 | 14 | implicit def coercible[A]: Coercible[A, A] = new Coercible[A, A] {} 15 | 16 | def client[F[_]]( 17 | config: ClientConfig 18 | )(implicit F: Sync[F] 19 | ): Resource[F, DynamoDbAsyncClient] = { 20 | import config._ 21 | Resource.make( 22 | F.delay { 23 | val builder = 24 | DynamoDbAsyncClient 25 | .builder() 26 | .region(Region.of(region)) 27 | .credentialsProvider(() => 28 | AwsBasicCredentials.create(accessKey, secretKey) 29 | ) 30 | config.overrideEndpoint.fold(builder.build())(ep => 31 | builder.endpointOverride(URI.create(ep)).build() 32 | ) 33 | } 34 | )(c => F.delay(c.close())) 35 | 36 | } 37 | 38 | def client[F[_]: Sync](cfg: ConfigSource): Resource[F, DynamoDbAsyncClient] = { 39 | import pureconfig.generic.auto._ 40 | import pureconfig.module.catseffect.syntax._ 41 | Resource 42 | .eval(cfg.loadF[F, ClientConfig]()) 43 | .flatMap(client[F](_)) 44 | } 45 | } 46 | 47 | case class ClientConfig( 48 | accessKey: String, 49 | secretKey: String, 50 | region: String, 51 | overrideEndpoint: Option[String] = None) 52 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | 3 | on: 4 | pull_request: 5 | branches: ['*'] 6 | push: 7 | branches: ['master'] 8 | 9 | env: 10 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 11 | THOMAS_ADMIN_KEY: "ecf9b0704e8bfc153a99d9e269c4fdfba54f9d9297881a99317d907a29c8cabccb9c23a4ab0f9dc70663cf74f11f257914e95e99e7b0e3a5f16f321269a239c5be7dd46c03e4f78f80f161e138723ceff4a00e893dc712f9b23881e3c7c00f7367a1d7b36fba5d92640979029bdf6fdbf48b740771613edaba9d73d42146b8ee2638e08fecb1c78ce3a216f4493024ff444a4f9fe8c2f5ca21edd5f4c775205fecacce1eaaaa0bdd84704944816b19d0d2061a4841e4e6939a7ab9b931ccf314332ac00d4bf249ccdbb12f4467aad01405d4b37e187507678334f596678f415e6395d8a4df9498c862b37342f03f52ff7d0425d6f40b649fa3f19fe13cb65183" 12 | 13 | jobs: 14 | build: 15 | name: Build and Test 16 | strategy: 17 | matrix: 18 | os: [ubuntu-latest] 19 | java: [adopt@1.11] 20 | 21 | runs-on: ${{ matrix.os }} 22 | steps: 23 | - name: Checkout current branch (full) 24 | uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | 28 | - name: Set up JDK 11 29 | uses: actions/setup-java@v4 30 | with: 31 | java-version: '11' 32 | distribution: 'adopt' 33 | 34 | - name: Set up sbt 35 | uses: sbt/setup-sbt@v1 36 | 37 | - name: Validation 38 | run: sbt validate 39 | 40 | - name: Cache sbt 41 | uses: actions/cache@v4 42 | with: 43 | path: | 44 | ~/.sbt 45 | ~/.cache/coursier/v1 46 | key: ${{ runner.os }}-sbt-cache-v4-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} 47 | -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/abtest/QueryDSL.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package abtest 3 | 4 | import java.time.{Instant, OffsetDateTime} 5 | 6 | import model._ 7 | import lihua.{Entity, EntityDAO} 8 | import _root_.play.api.libs.json.{JsObject, Json, Writes} 9 | import utils.time._ 10 | import scala.concurrent.duration.FiniteDuration 11 | 12 | object QueryDSL { 13 | 14 | object abtests { 15 | 16 | def byTime(time: OffsetDateTime): JsObject = byTime(time.toInstant, None) 17 | 18 | def byTime( 19 | time: Instant, 20 | duration: Option[FiniteDuration] 21 | ): JsObject = 22 | Json.obj( 23 | "start" -> Json.obj("$lte" -> duration.fold(time)(time.plusDuration)) 24 | ) ++ endTimeAfter(time) 25 | 26 | def endTimeAfter(time: Instant): JsObject = 27 | Json.obj( 28 | "$or" -> Json.arr( 29 | Json.obj("end" -> Json.obj("$gt" -> time)), 30 | Json.obj("end" -> Json.obj("$exists" -> false)) 31 | ) 32 | ) 33 | } 34 | 35 | implicit class ExtendedOps[F[_]](self: EntityDAO[F, Feature, JsObject]) { 36 | 37 | def byName(name: FeatureName): F[Entity[Feature]] = 38 | self.findOne(Symbol("name") -> name) 39 | def byNameOption(name: FeatureName): F[Option[Entity[Feature]]] = 40 | self.findOneOption(Symbol("name") -> name) 41 | } 42 | 43 | implicit def fromField1[A: Writes](tp: (Symbol, A)): JsObject = 44 | Json.obj(tp._1.name -> Json.toJson(tp._2)) 45 | 46 | implicit def fromFields2[A: Writes, B: Writes]( 47 | p: ((Symbol, A), (Symbol, B)) 48 | ): JsObject = 49 | p match { 50 | case ((s1, a), (s2, b)) => Json.obj(s1.name -> a, s2.name -> b) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /core/src/main/scala/com/iheart/thomas/admin/User.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.admin 2 | 3 | import cats.Eq 4 | import com.iheart.thomas.Username 5 | import play.api.libs.json.{Format, Json} 6 | 7 | import java.time.Instant 8 | 9 | final case class User( 10 | username: Username, 11 | hash: String, 12 | role: Role, 13 | resetToken: Option[PassResetToken] = None) {} 14 | 15 | final case class Role(name: String) 16 | 17 | object User { 18 | implicit val userFmt: Format[User] = Json.format[User] 19 | } 20 | 21 | object Role { 22 | val Admin: Role = Role("Admin") 23 | val Developer: Role = 24 | Role("Developer") // can start their own test and become feature manager 25 | val Tester: Role = Role("Tester") // can change overrides 26 | val User: Role = Role("User") // readonly but can be feature admin 27 | val Analyst: Role = Role("Analyst") 28 | val Scientist: Role = Role("Scientist") 29 | val Guest: Role = Role("Guest") // Cannot do anything 30 | 31 | val values = List(Admin, User, Developer, Tester, Analyst, Scientist, Guest) 32 | 33 | implicit val roleFmt: Format[Role] = Json.format[Role] 34 | implicit val eqRole: Eq[Role] = Eq.fromUniversalEquals[Role] 35 | } 36 | 37 | trait UserDAO[F[_]] { 38 | def update(user: User): F[User] 39 | 40 | def insert(user: User): F[User] 41 | 42 | def remove(username: String): F[Unit] 43 | 44 | def find(username: String): F[Option[User]] 45 | 46 | def all: F[Vector[User]] 47 | 48 | def get(username: String): F[User] 49 | } 50 | 51 | case class PassResetToken( 52 | value: String, 53 | expires: Instant) 54 | 55 | object PassResetToken { 56 | implicit val passResetTokenFmt: Format[PassResetToken] = 57 | Json.format[PassResetToken] 58 | } 59 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/stream/JValueParserSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package stream 3 | 4 | import com.iheart.thomas.analysis._ 5 | import org.scalatest.freespec.AnyFreeSpecLike 6 | import org.scalatest.matchers.should.Matchers 7 | import org.typelevel.jawn.ast._ 8 | 9 | class JValueParserSuite extends AnyFreeSpecLike with Matchers { 10 | import KpiEventParser.parseConversionEvent 11 | "JValue Parser" - { 12 | "parse single event from regex" in { 13 | val query = ConversionMessageQuery( 14 | initMessage = MessageQuery(None, List(Criteria("foo.bar", "abc"))), 15 | convertedMessage = MessageQuery(None, List(Criteria("bar", "^abc$"))) 16 | ) 17 | 18 | parseConversionEvent( 19 | JObject.fromSeq( 20 | Seq("foo" -> JObject.fromSeq(Seq(("bar" -> JString("xxxabcxxx"))))) 21 | ), 22 | query 23 | ) shouldBe List(Initiated) 24 | 25 | parseConversionEvent( 26 | JObject.fromSeq(Seq("bar" -> JString("abc"))), 27 | query 28 | ) shouldBe List(Converted) 29 | 30 | parseConversionEvent( 31 | JObject.fromSeq(Seq("bar" -> JString("abc2"))), 32 | query 33 | ) shouldBe Nil 34 | } 35 | 36 | "parse multiple event from regex" in { 37 | val query = ConversionMessageQuery( 38 | initMessage = MessageQuery(None, List(Criteria("display", "^search$"))), 39 | convertedMessage = MessageQuery(None, List(Criteria("action", "^click$"))) 40 | ) 41 | 42 | parseConversionEvent( 43 | JObject.fromSeq( 44 | Seq("display" -> JString("search"), "action" -> JString("click")) 45 | ), 46 | query 47 | ).toSet shouldBe Set(Converted, Initiated) 48 | 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/auth/registration.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | 3 | 4 | @(errorMsg: Option[String] = None, username: Option[String] = None) 5 | 6 | @main("Registration"){ 7 | 8 |
9 |
10 |
11 |
12 | 13 | 14 |
15 |
16 | User Registration 17 |
18 |
19 | @for(msg <- errorMsg) { 20 | 23 | } 24 |
25 |
26 | 27 | s"value=$u") 29 | /> 30 |
31 | 32 |
33 | 34 | 36 |
37 |
38 | 39 | 41 |
42 |
43 | 44 |
45 |
46 |
47 |
48 |
49 |
50 | 51 |
52 |
53 | } -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/auth/resetPass.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | 3 | 4 | @(username: String, errorMsg: Option[String] = None) 5 | 6 | 7 | 8 | @main("Reset Password"){ 9 | 10 | @for(msg <- errorMsg) { 11 | 14 | } 15 | 16 |
17 | 18 |
19 |
20 |
21 | Reset Password for @username 22 |
23 |
24 | @for(msg <- errorMsg) { 25 | 28 | } 29 |
30 | 31 |
32 | 33 | 35 |
36 |
37 | 38 | 40 |
41 |
42 | 43 |
44 |
45 |
46 |
47 |
48 | 49 |
50 | } -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/BanditProcessAlg.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.stream 2 | 3 | import cats.effect.Temporal 4 | import com.iheart.thomas.FeatureName 5 | import com.iheart.thomas.bandit.bayesian.BayesianMABAlg 6 | import com.iheart.thomas.stream.JobSpec.ProcessSettings 7 | import fs2.Pipe 8 | import cats.implicits._ 9 | import com.iheart.thomas.analysis.monitor.ExperimentKPIState.Specialization 10 | 11 | trait BanditProcessAlg[F[_], Message] { 12 | def process( 13 | feature: FeatureName 14 | ): F[(Pipe[F, Message, Unit], ProcessSettings)] 15 | } 16 | 17 | object BanditProcessAlg { 18 | implicit def default[F[_]: Temporal, Message]( 19 | implicit allKPIProcessAlg: AllKPIProcessAlg[F, Message], 20 | banditAlg: BayesianMABAlg[F] 21 | ): BanditProcessAlg[F, Message] = new BanditProcessAlg[F, Message] { 22 | 23 | def process( 24 | feature: FeatureName 25 | ): F[(Pipe[F, Message, Unit], ProcessSettings)] = { 26 | for { 27 | bandit <- banditAlg.get(feature) 28 | settings = ProcessSettings( 29 | bandit.spec.stateMonitorFrequency, 30 | bandit.spec.stateMonitorEventChunkSize, 31 | None 32 | ) 33 | monitorPipe <- allKPIProcessAlg.monitorExperiment( 34 | feature, 35 | bandit.kpiName, 36 | Specialization.BanditCurrent, 37 | settings 38 | ) 39 | } yield { 40 | ( 41 | monitorPipe.andThen { states => 42 | states 43 | .groupWithin( 44 | bandit.spec.updatePolicyStateChunkSize, 45 | bandit.spec.updatePolicyFrequency 46 | ) 47 | .evalMapFilter(_.last.traverse(banditAlg.updatePolicy)) 48 | .void 49 | }, 50 | settings 51 | ) 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/abtest/admin/newTest.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.abtest.model._ 2 | @import com.iheart.thomas.FeatureName 3 | @import com.iheart.thomas.http4s.UIEnv 4 | @import com.iheart.thomas.html._ 5 | @import com.iheart.thomas.http4s.Formatters._ 6 | @import lihua._ 7 | @( 8 | feature: FeatureName, 9 | draft: Option[AbtestSpec], 10 | errorMsg: Option[String] = None, 11 | followUpCandidateO: Option[Entity[Abtest]] = None 12 | )(implicit env: UIEnv) 13 | 14 | 15 | @topNav("Create New Test", "A/B Tests") { 16 | 17 |
18 | Creating a new A/B Test for 19 | 20 | @featureTestsLink(feature) 21 |
22 | @if(followUpCandidateO.nonEmpty && draft.isEmpty) { 23 | @for(followUp <- followUpCandidateO) { 24 | 35 | } 36 | } else { 37 | 38 |
39 | @testForm(draft, errorMsg) 40 |
41 |
42 | 43 | 44 |
45 |
46 |
47 | } 48 | } -------------------------------------------------------------------------------- /kafka/src/main/scala/com/iheart/thomas/kafka/JsonMessageSubscriber.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.kafka 2 | 3 | import cats.effect.Async 4 | import cats.implicits._ 5 | import com.iheart.thomas.stream.MessageSubscriber 6 | import com.typesafe.config.Config 7 | import fs2.Stream 8 | import fs2.kafka.{AutoOffsetReset, ConsumerSettings} 9 | import org.typelevel.log4cats.Logger 10 | import org.typelevel.jawn.ast 11 | import org.typelevel.jawn.ast.JValue 12 | 13 | object JsonMessageSubscriber { 14 | 15 | implicit def apply[F[_]: Async]( 16 | implicit config: Config, 17 | log: Logger[F] 18 | ): MessageSubscriber[F, JValue] = 19 | new MessageSubscriber[F, JValue] { 20 | override def subscribe: Stream[F, JValue] = 21 | Stream.eval(KafkaConfig.fromConfig[F](config)).flatMap { cfg => 22 | val consumerSettings = 23 | ConsumerSettings[F, Unit, String] 24 | .withEnableAutoCommit(true) 25 | .withAutoOffsetReset(AutoOffsetReset.Latest) 26 | .withBootstrapServers(cfg.kafkaServers) 27 | .withGroupId(cfg.groupId) 28 | 29 | fs2.kafka.KafkaConsumer 30 | .stream(consumerSettings) 31 | .evalTap(_.subscribeTo(cfg.topic)) 32 | .flatMap { 33 | _.stream 34 | .parEvalMap(cfg.parseParallelization) { r => 35 | ast.JParser 36 | .parseFromString(r.record.value) 37 | .fold( 38 | e => 39 | log 40 | .error( 41 | s"kafka message json parse error. $e \n json: ${r.record.value}" 42 | ) 43 | .as(none[JValue]), 44 | j => Option(j).pure[F] 45 | ) 46 | } 47 | .flattenOption 48 | 49 | } 50 | 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /bandit/src/main/scala/com/iheart/thomas/bandit/bayesian/BanditSpec.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.bandit 2 | package bayesian 3 | 4 | import com.iheart.thomas.{FeatureName, GroupName} 5 | import com.iheart.thomas.analysis.KPIName 6 | import com.iheart.thomas.analysis.monitor.ExperimentKPIState.{Key, Specialization} 7 | 8 | import scala.concurrent.duration.{DurationInt, FiniteDuration} 9 | 10 | /** @param feature 11 | * @param title 12 | * @param author 13 | * @param kpiName 14 | * @param minimumSizeChange 15 | * the minimum threshold of group size change. to avoid small fluctuation on 16 | * statistics change 17 | * @param initialSampleSize 18 | * the sample size from which the allocation starts. 19 | * @param historyRetention 20 | * for how long retired ab test versions are being kept 21 | * @param maintainExplorationSize 22 | * @param reservedGroups 23 | * reserve some arms from being changed by the bandit alg (useful for A/B tests) 24 | */ 25 | case class BanditSpec( 26 | feature: FeatureName, 27 | title: String, 28 | author: String, 29 | kpiName: KPIName, 30 | arms: Seq[ArmSpec], 31 | minimumSizeChange: Double = 0.001, 32 | historyRetention: Option[FiniteDuration] = None, 33 | initialSampleSize: Int = 0, 34 | stateMonitorEventChunkSize: Int = 1000, 35 | stateMonitorFrequency: FiniteDuration = 1.minute, 36 | updatePolicyStateChunkSize: Int = 100, 37 | updatePolicyFrequency: FiniteDuration = 1.hour) { 38 | lazy val stateKey: Key = Key(feature, kpiName, Specialization.BanditCurrent) 39 | 40 | lazy val reservedGroups: Set[GroupName] = arms.filter(_.reserved).map(_.name).toSet 41 | } 42 | 43 | private[thomas] trait BanditSpecDAO[F[_]] { 44 | def insert( 45 | state: BanditSpec 46 | ): F[BanditSpec] 47 | 48 | def remove(featureName: FeatureName): F[Unit] 49 | 50 | def get(featureName: FeatureName): F[BanditSpec] 51 | 52 | def update( 53 | settings: BanditSpec 54 | ): F[BanditSpec] 55 | } 56 | -------------------------------------------------------------------------------- /lihua/src/main/scala/lihua/EntityDAO.scala: -------------------------------------------------------------------------------- 1 | package lihua 2 | 3 | import cats.Monad 4 | import cats.tagless._ 5 | 6 | /** Final tagless encoding of the DAO Algebra 7 | * @tparam F 8 | * effect Monad 9 | * @tparam T 10 | * type of the domain model 11 | */ 12 | @autoFunctorK 13 | @autoContravariant 14 | trait EntityDAO[F[_], T, Query] { 15 | def get(id: EntityId): F[Entity[T]] 16 | 17 | def insert(t: T): F[Entity[T]] 18 | 19 | def update(entity: Entity[T]): F[Entity[T]] 20 | 21 | def upsert(entity: Entity[T]): F[Entity[T]] 22 | 23 | def find(query: Query): F[Vector[Entity[T]]] 24 | 25 | def all: F[Vector[Entity[T]]] 26 | 27 | def findOne(query: Query): F[Entity[T]] 28 | 29 | def findOneOption(query: Query): F[Option[Entity[T]]] 30 | 31 | def remove(id: EntityId): F[Unit] 32 | 33 | def removeAll(query: Query): F[Int] 34 | 35 | /** update the first entity query finds 36 | * @param query 37 | * search query 38 | * @param entity 39 | * to be updated to 40 | * @param upsert 41 | * whether to insert of nothing is found 42 | * @return 43 | * whether anything is updated 44 | */ 45 | def update( 46 | query: Query, 47 | entity: Entity[T], 48 | upsert: Boolean 49 | ): F[Boolean] 50 | 51 | def upsert( 52 | query: Query, 53 | t: T 54 | ): F[Entity[T]] 55 | 56 | def removeAll(): F[Int] 57 | } 58 | 59 | object EntityDAO { 60 | 61 | /** Provides more default implementation thanks to F being a Monad 62 | * @tparam F 63 | * effect Monad 64 | * @tparam T 65 | * type of the domain model 66 | * @tparam Query 67 | */ 68 | abstract class EntityDAOMonad[F[_]: Monad, T, Query] 69 | extends EntityDAO[F, T, Query] { 70 | import cats.implicits._ 71 | def upsert( 72 | query: Query, 73 | t: T 74 | ): F[Entity[T]] = 75 | findOneOption(query).flatMap( 76 | _.fold(insert(t))(e => update(e.copy(data = t))) 77 | ) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /mongo/src/main/scala/lihua/mongo/Query.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] [iHeartMedia Inc] 3 | * All rights reserved 4 | */ 5 | package lihua 6 | package mongo 7 | 8 | import play.api.libs.json.{Format, JsObject, Json, Writes} 9 | import reactivemongo.api.ReadPreference 10 | 11 | import JsonFormats._ 12 | case class Query( 13 | selector: JsObject, 14 | sort: Option[JsObject] = None, 15 | readPreference: Option[ReadPreference] = None, 16 | projection: Option[JsObject] = None) 17 | 18 | object Query { 19 | def idSelector(id: EntityId): JsObject = Json.obj(idFieldName -> id) 20 | 21 | implicit def fromSelector(selector: JsObject): Query = Query(selector) 22 | 23 | implicit def fromField1[A: Writes](tp: (Symbol, A)): Query = 24 | Query(Json.obj(tp._1.name -> Json.toJson(tp._2))) 25 | 26 | implicit def fromFields2[A: Writes, B: Writes]( 27 | p: ((Symbol, A), (Symbol, B)) 28 | ): Query = p match { 29 | case ((s1, a), (s2, b)) => Query(Json.obj(s1.name -> a, s2.name -> b)) 30 | } 31 | 32 | implicit def fromFields3[A: Writes, B: Writes, C: Writes]( 33 | p: ((Symbol, A), (Symbol, B), (Symbol, C)) 34 | ): Query = p match { 35 | case ((s1, a), (s2, b), (s3, c)) => 36 | Query(Json.obj(s1.name -> a, s2.name -> b, s3.name -> c)) 37 | } 38 | 39 | implicit def fromFields4[A: Writes, B: Writes, C: Writes, D: Writes]( 40 | p: ((Symbol, A), (Symbol, B), (Symbol, C), (Symbol, D)) 41 | ): Query = p match { 42 | case ((s1, a), (s2, b), (s3, c), (s4, d)) => 43 | Query( 44 | Json.obj(s1.name -> a, s2.name -> b, s3.name -> c, s4.name -> d) 45 | ) 46 | } 47 | 48 | implicit def fromId(id: EntityId): Query = idSelector(id) 49 | 50 | implicit def fromIds(ids: List[EntityId]): Query = 51 | Json.obj(idFieldName -> Json.obj("$in" -> ids.map(_.value))) 52 | 53 | def byProperty[PT: Format]( 54 | propertyName: String, 55 | propertyValue: PT 56 | ) = 57 | Query(Json.obj(propertyName -> propertyValue)) 58 | } 59 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/BusAlg.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.stream 2 | 3 | import cats.effect._ 4 | import cats.effect.kernel.Concurrent 5 | import cats.implicits._ 6 | import fs2.concurrent.Topic 7 | import fs2.{Pipe, Stream} 8 | import org.typelevel.log4cats.Logger 9 | 10 | 11 | 12 | trait BusAlg[F[_]] { 13 | def subscribe[T](f: PartialFunction[AdminEvent, F[Option[T]]]): Stream[F, T] 14 | 15 | /** 16 | * Always do something when partial function matches 17 | */ 18 | def subscribe_(f: PartialFunction[AdminEvent, F[Unit]]): Stream[F, Unit] 19 | 20 | def subscribe[A, T](preProcess: Pipe[F, AdminEvent, A])(f: A => F[Option[T]]): Stream[F, T] 21 | 22 | def publish(e: AdminEvent): F[Unit] 23 | } 24 | 25 | object BusAlg { 26 | def resource[F[_]: Temporal]( 27 | queueSize: Int 28 | )(implicit F: Concurrent[F], 29 | errorReporter: Logger[F]): Resource[F, BusAlg[F]] = 30 | Resource.make(Topic[F, AdminEvent])(_.close.void).map { topic => 31 | new BusAlg[F] { 32 | def subscribe[T]( 33 | f: PartialFunction[AdminEvent, F[Option[T]]] 34 | ): Stream[F, T] = { 35 | subscribe(identity(_))(f.lift.map(_.getOrElse(F.pure(none[T])))) 36 | } 37 | 38 | def subscribe[A, T](preProcess: Pipe[F, AdminEvent, A])(f: A => F[Option[T]]): Stream[F, T] = 39 | topic 40 | .subscribe(queueSize) 41 | .through(preProcess) 42 | .evalMap( 43 | f.map(_.handleErrorWith(t => { 44 | errorReporter.error(t)("failed to handle event").as(none) 45 | })) 46 | ) 47 | .flattenOption 48 | 49 | def publish(e: AdminEvent): F[Unit] = topic.publish1(e).void 50 | 51 | def subscribe_(f: PartialFunction[AdminEvent, F[Unit]]): Stream[F, Unit] = 52 | subscribe(f.andThen(_.map(Option(_)))) 53 | 54 | 55 | 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/abtest/TestsDataSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package abtest 3 | 4 | import com.iheart.thomas.utils.time._ 5 | import org.scalacheck.Arbitrary 6 | import org.scalacheck.Gen._ 7 | import org.scalatest.freespec.AnyFreeSpec 8 | import org.scalatest.matchers.should.Matchers 9 | import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks 10 | 11 | import java.time.Instant 12 | import scala.concurrent.duration._ 13 | 14 | class TestsDataSuite 15 | extends AnyFreeSpec 16 | with ScalaCheckDrivenPropertyChecks 17 | with Matchers { 18 | 19 | "TestsData.withinTolerance" - { 20 | 21 | "indicate if the target date is within the tolerance band" in { 22 | forAll { 23 | ( 24 | testsData: TestsData, 25 | toleranceR: FiniteDuration, 26 | offset: Long 27 | ) => 28 | val tolerance = toleranceR.toNanos.abs.nanos 29 | val target = testsData.at.plusNanos(offset) 30 | val cutOffTimeBegin = testsData.at.minusNanos(tolerance.toNanos.abs) 31 | 32 | val endTime = 33 | testsData.duration.fold(testsData.at)(testsData.at.plusDuration) 34 | val cutOffTimeEnd = endTime.plusNanos(tolerance.toNanos.abs) 35 | 36 | val withinRange = !target.isBefore(cutOffTimeBegin) && !target.isAfter( 37 | cutOffTimeEnd 38 | ) 39 | 40 | testsData.withinTolerance(tolerance, target) shouldBe withinRange 41 | } 42 | } 43 | 44 | "return true if the band is zero and the time is the same" in { 45 | val t = Instant.now 46 | TestsData(at = t, Vector.empty, None) 47 | .withinTolerance(Duration.Zero, t) shouldBe true 48 | } 49 | } 50 | 51 | implicit val arbTestsData: Arbitrary[TestsData] = Arbitrary { 52 | for { 53 | at <- choose(-1000000000L, 10000000000L) 54 | duration <- option(choose(Duration.Zero, 1000000000000L.nanos)) 55 | } yield TestsData(Instant.now.plusMillis(at), Vector.empty, duration) 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/analysis/bayesian/ModelEvaluatorSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package analysis 3 | package bayesian 4 | 5 | import cats.Id 6 | import com.iheart.thomas.analysis.bayesian.models.LogNormalModel 7 | import org.scalatest.freespec.AnyFreeSpecLike 8 | import org.scalatest.matchers.should.Matchers 9 | 10 | class ModelEvaluatorSuite extends AnyFreeSpecLike with Matchers { 11 | 12 | "LogNormal Evaluator" - { 13 | implicit val rdb = breeze.stats.distributions.RandBasis.mt0 14 | val evaluator = 15 | implicitly[ModelEvaluator[Id, LogNormalModel, PerUserSamples.LnSummary]] 16 | "compare two data samples" in { 17 | val n = 10000 18 | val dist1 = breeze.stats.distributions.LogNormal(mu = 0d, sigma = 0.3d) 19 | val dist2 = breeze.stats.distributions.LogNormal(mu = 0.01d, sigma = 0.3d) 20 | val data1 = PerUserSamples(dist1.sample(n).toArray).lnSummary 21 | 22 | val data2 = PerUserSamples(dist2.sample(n).toArray).lnSummary 23 | val model = LogNormalModel(0, 1, 1, 1) 24 | 25 | val result = evaluator.evaluate( 26 | Map( 27 | ("data1", (data1, model)), 28 | ("data2", (data2, model)) 29 | ) 30 | ) 31 | 32 | result("data2").p shouldBe >(result("data1").p) 33 | } 34 | 35 | "compare more realistic samples" in { 36 | val dataC = 37 | PerUserSamplesLnSummary(mean = -0.3106, variance = 3.8053, count = 1989360) 38 | val dataB = 39 | PerUserSamplesLnSummary(mean = -0.3115, variance = 3.8159, count = 1990479) 40 | val dataA = 41 | PerUserSamplesLnSummary(mean = -0.3089, variance = 3.8009, count = 2051370) 42 | 43 | val model = 44 | LogNormalModel(miu0 = -0.3050, n0 = 49095, alpha = 24547, beta = 94820) 45 | 46 | val result = evaluator.evaluate( 47 | Map( 48 | ("C", (dataC, model)), 49 | ("B", (dataB, model)), 50 | ("A", (dataA, model)) 51 | ) 52 | ) 53 | 54 | result("A").p shouldBe <(result("B").p) // due to the higher variance. 55 | } 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /monitor/src/main/scala/com/iheart/thomas/monitor/DatadogClient.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.monitor 2 | 3 | import cats.effect.{Async, Resource} 4 | import org.http4s.client.{Client => Http4sClient} 5 | import org.http4s.play._ 6 | import org.http4s.client.dsl.Http4sClientDsl 7 | import _root_.play.api.libs.json.{JsObject, Json} 8 | import cats.implicits._ 9 | import com.typesafe.config.Config 10 | import pureconfig.ConfigSource 11 | import pureconfig.module.catseffect.syntax._ 12 | import DatadogClient.ErrorResponseFromDataDogService 13 | import org.http4s.blaze.client.BlazeClientBuilder 14 | 15 | import scala.util.control.NoStackTrace 16 | 17 | class DatadogClient[F[_]]( 18 | c: Http4sClient[F], 19 | apiKey: String 20 | )(implicit 21 | F: Async[F]) 22 | extends Http4sClientDsl[F] { 23 | import org.http4s.{Method, Uri, EntityEncoder} 24 | import Method._ 25 | 26 | implicit def jsObjectEncoder: EntityEncoder[F, JsObject] = 27 | jsonEncoder[F].narrow 28 | 29 | def send(e: MonitorEvent)(errorHandler: Throwable => F[Unit]): F[Unit] = { 30 | F.start( 31 | c.successful( 32 | POST( 33 | Json.toJson(e), 34 | Uri 35 | .unsafeFromString( 36 | "https://api.datadoghq.com/api/v1/events" 37 | ) 38 | .withQueryParam("api_key", apiKey) 39 | ) 40 | ).ensure(ErrorResponseFromDataDogService)(identity) 41 | .void 42 | .handleErrorWith(errorHandler) 43 | ).void 44 | } 45 | } 46 | 47 | object DatadogClient { 48 | 49 | case object ErrorResponseFromDataDogService 50 | extends RuntimeException 51 | with NoStackTrace 52 | 53 | def resource[F[_]: Async]( 54 | apiKey: String 55 | ): Resource[F, DatadogClient[F]] = { 56 | BlazeClientBuilder[F].resource 57 | .map(new DatadogClient[F](_, apiKey)) 58 | } 59 | 60 | def fromConfig[F[_]: Async]( 61 | cfg: Config 62 | ): Resource[F, DatadogClient[F]] = 63 | Resource 64 | .eval( 65 | ConfigSource 66 | .fromConfig(cfg) 67 | .at("thomas.datadog.api-key") 68 | .loadF[F, String]() 69 | ) 70 | .flatMap(resource(_)) 71 | } 72 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/Formatters.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s 2 | 3 | import java.time.{Instant, OffsetDateTime, ZoneId} 4 | import java.time.format.DateTimeFormatter 5 | import com.iheart.thomas.abtest.model.{Abtest, GroupSize, UserMetaCriterion} 6 | import com.iheart.thomas.abtest.model.Abtest.Status 7 | import lihua.Entity 8 | import _root_.play.api.libs.json.{JsObject, Json, Writes} 9 | import com.iheart.thomas.abtest.json.play.Formats._ 10 | 11 | object Formatters { 12 | val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z") 13 | val dateTimeFormatterShort = DateTimeFormatter.ofPattern("MMM dd HH:mm") 14 | val dateTimeFormatterMid = DateTimeFormatter.ofPattern("M/d/yy HH:mm") 15 | 16 | def formatPercentage(d: Double): String = f"${d * 100}%.2f%%" 17 | 18 | def formatDate(date: OffsetDateTime): String = 19 | formatDate(date.toInstant) 20 | 21 | def dateTimeMid(instant: Instant): String = 22 | formatDate(instant, dateTimeFormatterMid) 23 | 24 | def formatDate( 25 | date: Instant, 26 | formatter: DateTimeFormatter = dateTimeFormatter 27 | ): String = { 28 | date.atZone(ZoneId.systemDefault).format(formatter) 29 | } 30 | 31 | def formatJSON[A](a: A)(implicit w: Writes[A]): String = { 32 | Json.prettyPrint(w.writes(a)) 33 | } 34 | 35 | //writes all criteria in c into json objects and merge them into one. 36 | def formatUserMetaCriteria(c: UserMetaCriterion.And): String = { 37 | val jsArray = userMetaCriteriaFormat.writes(c) 38 | val jsObject = jsArray.as[List[JsObject]].reduceLeft(_ deepMerge _) 39 | 40 | Json.prettyPrint( 41 | if(c.criteria.size > jsObject.fields.size) Json.obj("%and" -> jsArray) else jsObject 42 | ) 43 | } 44 | 45 | def formatArmSize(size: GroupSize): String = { 46 | "%.3f".format(size) 47 | } 48 | 49 | def formatStatus(test: Entity[Abtest]): (String, String) = { 50 | test.data.statusAsOf(OffsetDateTime.now) match { 51 | case Status.Expired => ("Stopped", "secondary") 52 | case Status.InProgress => ("Running", "success") 53 | case Status.Scheduled => ("Scheduled", "warning") 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/topNav.scala.html: -------------------------------------------------------------------------------- 1 | 2 | @import com.iheart.thomas.http4s.UIEnv 3 | @import _root_.play.twirl.api.Html 4 | @import com.iheart.thomas.admin.Authorization._ 5 | @import com.iheart.thomas.BuildInfo 6 | @(title: String, active: String)(content: Html)(implicit env: UIEnv) 7 | 8 | 9 | @navItem(name: String, link: String) = { 10 |
  • 15 | @name 21 |
    • 22 | } 23 | 24 | 25 | @main(title){ 26 | 62 | 63 | 64 | @content 65 | 66 | } -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/stream/background.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.stream.Job 2 | @import com.iheart.thomas.html._ 3 | 4 | @import com.iheart.thomas.http4s.{UIEnv, Formatters}, Formatters._ 5 | 6 | 7 | @(jobs: Vector[Job])(implicit env: UIEnv) 8 | 9 | 10 | @topNav("Background Processes", "Background") { 11 |
    12 | Background Processes 13 |
    14 |
    15 |
    16 | @for(job <- jobs) { 17 | 18 |
    19 |
    20 |
    21 |
    22 | 23 | @job.key 24 | @jobStatusBadge(job.started) 25 |
    26 |
    27 | @job.spec.description 28 |
    29 |
    30 | 31 |
    32 | 33 | @for(checkedOut <- job.checkedOut) { 34 |
    35 | 36 | Last Check: 37 | 38 | @formatDate(checkedOut) 39 | 40 | 41 |
    42 | } 43 | 44 | 45 |
    46 |
    47 | 50 | 51 | 52 |
    53 | 54 |
    55 |
    56 | } 57 | 58 | @if(jobs.isEmpty) { 59 | 62 | } 63 | 64 |
    65 |
    66 | 67 | } -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/bandit/ManagerAlg.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.http4s.bandit 2 | 3 | import cats.Monad 4 | import com.iheart.thomas.FeatureName 5 | import com.iheart.thomas.bandit.BanditStatus 6 | import com.iheart.thomas.bandit.bayesian.{BanditSpec, BayesianMABAlg} 7 | import com.iheart.thomas.stream.{Job, JobAlg} 8 | import com.iheart.thomas.stream.JobSpec.RunBandit 9 | import cats.implicits._ 10 | import com.iheart.thomas.bandit.bayesian.BayesianMABAlg.BanditAbtestSpec 11 | 12 | trait ManagerAlg[F[_]] { 13 | def status(feature: FeatureName): F[BanditStatus] 14 | def pause(feature: FeatureName): F[Unit] 15 | def update(bs: BanditSpec, bas: BanditAbtestSpec): F[BanditSpec] 16 | def start(feature: FeatureName): F[Option[Job]] 17 | def create(bs: BanditSpec): F[Bandit] 18 | def allBandits: F[Seq[Bandit]] 19 | def get(feature: FeatureName): F[Bandit] 20 | } 21 | 22 | object ManagerAlg { 23 | implicit def apply[F[_]]( 24 | implicit alg: BayesianMABAlg[F], 25 | jobAlg: JobAlg[F], 26 | F: Monad[F] 27 | ): ManagerAlg[F] = new ManagerAlg[F] { 28 | 29 | def update(bs: BanditSpec, bas: BanditAbtestSpec): F[BanditSpec] = { 30 | for { 31 | s <- status(bs.feature) 32 | running = s === BanditStatus.Running 33 | _ <- if (running) pause(bs.feature) else F.unit 34 | r <- alg.update(bs, bas) 35 | _ <- if (running) start(bs.feature) else F.unit 36 | } yield r 37 | } 38 | 39 | def status(feature: FeatureName): F[BanditStatus] = { 40 | jobAlg 41 | .find(RunBandit(feature)) 42 | .map(_.fold(BanditStatus.Paused: BanditStatus)(_ => BanditStatus.Running)) 43 | } 44 | 45 | def pause(feature: FeatureName): F[Unit] = jobAlg.stop(RunBandit(feature)) 46 | 47 | def start(feature: FeatureName): F[Option[Job]] = 48 | jobAlg.schedule(RunBandit(feature)) 49 | 50 | def allBandits: F[Seq[Bandit]] = 51 | alg.getAll.flatMap(_.traverse(b => status(b.feature).map(Bandit(b, _)))).widen 52 | 53 | def create(bs: BanditSpec): F[Bandit] = 54 | alg.init(bs).map(Bandit(_, BanditStatus.Paused)) 55 | 56 | def get(feature: FeatureName): F[Bandit] = 57 | (alg.get(feature), status(feature)).mapN(Bandit.apply) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /tests/src/it/scala/com/iheart/thomas/abtest/EndpointSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.abtest 2 | 3 | import cats.effect.IO 4 | import cats.effect.testing.scalatest.AsyncIOSpec 5 | import com.iheart.thomas.abtest.TestUtils.withAlg 6 | import com.iheart.thomas.http4s.abtest.AbtestService 7 | import com.iheart.thomas.tracking.EventLogger 8 | import fs2._ 9 | import org.http4s.Status._ 10 | import org.http4s.{Method, Request} 11 | import org.scalatest.freespec.AsyncFreeSpec 12 | import org.scalatest.matchers.should.Matchers 13 | import org.typelevel.log4cats.slf4j.Slf4jLogger 14 | import org.http4s.implicits.http4sLiteralsSyntax 15 | 16 | class EndpointSuite extends AsyncFreeSpec with AsyncIOSpec with Matchers { 17 | implicit val logger: EventLogger[IO] = EventLogger.catsLogger(Slf4jLogger.getLogger[IO]) 18 | 19 | "/users/groups/query endpoint should handle different request bodies appropriately" - { 20 | "good request body json should return OK" in { 21 | withAlg { alg => 22 | val data: String = """{"meta":{"country":"us"},"userId":"123"}""" 23 | val body: Stream[IO, Byte] = Stream.emit(data).through(text.utf8.encode) 24 | new AbtestService(alg).public.orNotFound.run( 25 | Request(method = Method.POST, uri = uri"/users/groups/query", body = body) 26 | ) 27 | }.asserting { response => 28 | response.status shouldBe Ok 29 | } 30 | } 31 | 32 | "empty request body json should return BadRequest" in { 33 | withAlg { alg => 34 | new AbtestService(alg).public.orNotFound.run( 35 | Request(method = Method.POST, uri = uri"/users/groups/query") 36 | ) 37 | }.asserting { response => 38 | response.status shouldBe BadRequest 39 | } 40 | } 41 | 42 | "bad request body json should return BadRequest" in { 43 | withAlg { alg => 44 | val data: String = """{"meta":{"country":"us", "deviceId": {}},"userId":"123"}""" 45 | val body: Stream[IO, Byte] = Stream.emit(data).through(text.utf8.encode) 46 | new AbtestService(alg).public.orNotFound.run( 47 | Request(method = Method.POST, uri = uri"/users/groups/query", body = body) 48 | ) 49 | }.asserting { response => 50 | response.status shouldBe BadRequest 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /stream/src/main/scala/com/iheart/thomas/stream/JobSpec.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package stream 3 | import com.iheart.thomas.analysis.KPIName 4 | 5 | import java.time.Instant 6 | import scala.concurrent.duration.FiniteDuration 7 | 8 | sealed trait JobSpec extends Serializable with Product { 9 | def key: String 10 | def description: String 11 | } 12 | 13 | object JobSpec { 14 | type ErrorMsg = String 15 | 16 | /** Update a KPI's prior based a sample 17 | * @param kpiName 18 | * @param sampleSize 19 | */ 20 | case class UpdateKPIPrior( 21 | kpiName: KPIName, 22 | processSettings: ProcessSettingsOptional) 23 | extends JobSpec { 24 | val key = UpdateKPIPrior.keyOf(kpiName) 25 | val description = 26 | s"Update the prior for KPI $kpiName using ongoing data}" 27 | } 28 | 29 | object UpdateKPIPrior { 30 | def keyOf(kpiName: KPIName) = "Update_KPI_Prior_For_" + kpiName.n 31 | } 32 | 33 | case class MonitorTest( 34 | feature: FeatureName, 35 | kpiName: KPIName, 36 | processSettings: ProcessSettingsOptional) 37 | extends JobSpec { 38 | val key = MonitorTest.jobKey(feature, kpiName) 39 | val description = 40 | s"Real time monitor for A/B tests on feature $feature using KPI $kpiName." 41 | } 42 | 43 | object MonitorTest { 44 | def jobKey( 45 | feature: FeatureName, 46 | kpi: KPIName 47 | ) = "Monitor_Test_" + feature + "_With_KPI_" + kpi 48 | } 49 | 50 | case class RunBandit( 51 | featureName: FeatureName) 52 | extends JobSpec { 53 | val key = "Run_Bandit_" + featureName 54 | 55 | val description = 56 | s"Running Multi Arm Bandit $featureName" 57 | 58 | } 59 | 60 | case class ProcessSettings( 61 | frequency: FiniteDuration, 62 | eventChunkSize: Int, 63 | expiration: Option[Instant]) 64 | 65 | case class ProcessSettingsOptional( 66 | frequency: Option[FiniteDuration], 67 | eventChunkSize: Option[Int], 68 | expiration: Option[Instant]) { 69 | def withDefault(defaultSettings: ProcessSettings): ProcessSettings = 70 | ProcessSettings( 71 | frequency.getOrElse(defaultSettings.frequency), 72 | eventChunkSize.getOrElse(defaultSettings.eventChunkSize), 73 | expiration orElse defaultSettings.expiration 74 | ) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/bandit/banditView.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | @import com.iheart.thomas.http4s.UIEnv 3 | @import com.iheart.thomas.http4s.bandit.Bandit 4 | @import com.iheart.thomas.analysis.KPIName 5 | @import com.iheart.thomas.bandit._ 6 | @import com.iheart.thomas.analysis.html._ 7 | @import com.iheart.thomas.abtest.admin.formParts.html._ 8 | 9 | @(bandit: Bandit, kpis: Seq[KPIName])(implicit env: UIEnv) 10 | 11 | 12 | @topNav("Bandit " + bandit.feature, "Bandit") { 13 | 14 | 15 |
    16 | Bandit 17 | @bandit.feature 18 |
    19 |
    20 |
    Current Status: @bandit.status
    21 | 22 | @bandit.status match { 23 | case BanditStatus.Running => { 24 | 25 | Pause 26 | 27 | } 28 | case BanditStatus.Paused => { 29 | 30 | Resume 31 | 32 | } 33 | case BanditStatus.Stopped => { 34 | Start 35 | } 36 | } 37 | 38 |
    39 |
    40 | @banditSpecForm(Some(bandit), kpis) 41 |
    42 |
    43 | @eligibilityControl(Some(bandit.abtest.data.toSpec)) 44 |
    45 |
    46 | @mutualExclusivity(Some(bandit.abtest.data.toSpec)) 47 |
    48 |
    49 |
    50 |
    51 | 52 | 53 |
    54 |
    55 |
    56 | 57 | 58 |
    Current State
    59 | @for(state <- bandit.state) { 60 | @kpiState(state, showReset= false) 61 | } 62 | 63 | @if( bandit.state.isEmpty ) { 64 | No state reported yet. 65 | } 66 | 67 | << Back to bandits 68 | } 69 | 70 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/bayesian/Posterior.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | package bayesian 3 | 4 | import breeze.stats.meanAndVariance.MeanAndVariance 5 | import com.iheart.thomas.analysis.bayesian.models.{ 6 | BetaModel, 7 | LogNormalModel, 8 | NormalModel 9 | } 10 | import com.iheart.thomas.analysis.{Conversions, PerUserSamples} 11 | import henkan.convert.Syntax._ 12 | 13 | trait Posterior[Model, Measurement] { 14 | def apply( 15 | model: Model, 16 | data: Measurement 17 | ): Model 18 | } 19 | 20 | object Posterior { 21 | 22 | def update[Model, Measurement]( 23 | model: Model, 24 | measurement: Measurement 25 | )(implicit 26 | posterior: Posterior[Model, Measurement] 27 | ): Model = posterior(model, measurement) 28 | 29 | implicit def conversionsPosterior: Posterior[ConversionKPI, Conversions] = 30 | (k: ConversionKPI, data: Conversions) => k.copy(model = update(k.model, data)) 31 | 32 | implicit def accumulativePosterior 33 | : Posterior[QueryAccumulativeKPI, PerUserSamplesLnSummary] = 34 | (k: QueryAccumulativeKPI, data: PerUserSamplesLnSummary) => 35 | k.copy(model = update(k.model, data)) 36 | 37 | implicit val betaConversion: Posterior[BetaModel, Conversions] = 38 | (model: BetaModel, data: Conversions) => { 39 | val postAlpha = model.alpha + data.converted 40 | val postBeta = model.beta + data.total - data.converted 41 | BetaModel(postAlpha, postBeta) 42 | } 43 | 44 | /** Ref: 45 | * https://people.eecs.berkeley.edu/~jordan/courses/260-spring10/lectures/lecture5.pdf 46 | */ 47 | implicit val normalSamples: Posterior[NormalModel, MeanAndVariance] = 48 | (model: NormalModel, data: MeanAndVariance) => { 49 | import model._ 50 | val n = data.count.toDouble 51 | NormalModel( 52 | miu0 = ((n0 * miu0) + (n * data.mean)) / (n + n0), 53 | n0 = n0 + n, 54 | alpha = alpha + (n / 2d), 55 | beta = beta + (data.variance * (n - 1d) / 2d) + 56 | (n * model.n0) * Math.pow(data.mean - miu0, 2) / (2 * (n + n0)) 57 | ) 58 | } 59 | 60 | implicit val logNormalSamplesSummary 61 | : Posterior[LogNormalModel, PerUserSamples.LnSummary] = 62 | (model: LogNormalModel, data: PerUserSamples.LnSummary) => { 63 | LogNormalModel(update(model.inner, data.to[MeanAndVariance]())) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/auth/users.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.html._ 2 | @import com.iheart.thomas.admin.User 3 | @import com.iheart.thomas.admin.Role 4 | @import com.iheart.thomas.http4s.UIEnv 5 | 6 | @(allUsers: Vector[User], 7 | msgO: Option[String] 8 | )(implicit env: UIEnv) 9 | 10 | @topNav("All Users", "Users") { 11 | 12 | @for(msg <- msgO) { 13 | 16 | } 17 | 18 |
    19 |
    20 |
    21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | @for(user <- allUsers.sortBy(_.username)) { 32 | 33 | 34 | 50 | 54 | 60 | 61 | } 62 | 63 |
    UsernameRoleReset Password
    @user.username 35 |
    36 | 37 | 47 | 48 |
    49 |     		51 | Generate Password Reset Link 52 | 53 | 55 | 57 | 58 | 59 |
    64 |
    65 |
    66 |
    67 | } -------------------------------------------------------------------------------- /mongo/src/main/scala/lihua/crypt/CryptTsec.scala: -------------------------------------------------------------------------------- 1 | package lihua 2 | package crypt 3 | 4 | import cats.effect.{ExitCode, IO, IOApp, Sync} 5 | import lihua.mongo.Crypt 6 | import tsec.cipher.symmetric.PlainText 7 | import tsec.common._ 8 | import tsec.cipher.symmetric._ 9 | import tsec.cipher.symmetric.jca._ 10 | 11 | import scala.io.StdIn 12 | import cats.implicits._ 13 | import lihua.crypt.CryptTsec.Base64Error 14 | 15 | class CryptTsec[F[_]](key: String)(implicit F: Sync[F]) extends Crypt[F] { 16 | 17 | private val ekeyF: F[SecretKey[AES128CTR]] = 18 | b64(key).flatMap(AES128CTR.buildKey[F]) 19 | 20 | def b64(s: String): F[Array[Byte]] = 21 | s.b64Bytes.liftTo[F](Base64Error) 22 | 23 | implicit val ctrStrategy: IvGen[F, AES128CTR] = AES128CTR.defaultIvStrategy[F] 24 | 25 | def encrypt(value: String): F[String] = 26 | for { 27 | ekey <- ekeyF 28 | encrypted <- AES128CTR 29 | .genEncryptor[F] 30 | .encrypt(PlainText(value.utf8Bytes), ekey) 31 | } yield (encrypted.content ++ encrypted.nonce).toB64String 32 | 33 | def decrypt(value: String): F[String] = 34 | for { 35 | ekey <- ekeyF 36 | vb <- b64(value) 37 | cypherText <- AES128CTR.ciphertextFromConcat(vb).liftTo[F] 38 | decrypted <- AES128CTR.genEncryptor[F].decrypt(cypherText, ekey) 39 | } yield decrypted.toUtf8String 40 | 41 | } 42 | 43 | object CryptTsec extends IOApp { 44 | def apply[F[_]: Sync](key: String): Crypt[F] = new CryptTsec[F](key) 45 | 46 | def genKey[F[_]: Sync]: F[String] = 47 | AES128CTR.generateKey[F].map(_.getEncoded.toB64String) 48 | 49 | case object Base64Error extends RuntimeException 50 | 51 | def run(args: List[String]): IO[ExitCode] = { 52 | val command = args.headOption match { 53 | case Some("genKey") => genKey[IO] 54 | case Some("encrypt") => 55 | for { 56 | key <- IO(StdIn.readLine("Enter your key:")) 57 | pass <- IO(StdIn.readLine("Enter your text:")) 58 | r <- CryptTsec[IO](key).encrypt(pass) 59 | } yield r 60 | 61 | case Some("decrypt") => 62 | for { 63 | key <- IO(StdIn.readLine("Enter your key:")) 64 | pass <- IO(StdIn.readLine("Enter your text:")) 65 | r <- CryptTsec[IO](key).decrypt(pass) 66 | } yield r 67 | case _ => IO.pure("usage: [genKey|encrypt]") 68 | } 69 | 70 | command.attempt 71 | .flatMap(_.fold(s => IO(println(s)), s => IO(println(s)))) 72 | .as(ExitCode.Success) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/updatePrior.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.analysis._ 2 | @import com.iheart.thomas.stream.html._ 3 | @import com.iheart.thomas.stream._, JobSpec._ 4 | @import com.iheart.thomas.http4s.Formatters, Formatters._ 5 | 6 | @( 7 | kpiName: KPIName, 8 | runningJobO: Option[JobInfo[UpdateKPIPrior]] 9 | ) 10 | 11 |
    12 |
    13 |
    14 |
    15 | Update prior using ongoing data 16 |
    17 |
    18 | @if(runningJobO.isEmpty) { 19 | 20 |
    21 |
    22 | 23 | 29 |
    30 |
    31 |
    32 | 35 |
    36 |
    37 |
    38 | } else { 39 | @for(job <- runningJobO) { 40 | 41 | Updating using ongoing data 42 | @for(exp <- job.spec.processSettings.expiration) { 43 | until @dateTimeMid(exp) 44 | } 45 | 46 | @jobStatusBadge(job.started) 47 | 48 | } 49 | } 50 | 51 |
    52 |
    53 | 54 |
    55 |
    56 | 57 | @if(runningJobO.isEmpty) { 58 | 67 | } 68 | 69 | -------------------------------------------------------------------------------- /spark/src/main/scala/com/iheart/thomas/spark/Assigner.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package spark 3 | 4 | import java.time.Instant 5 | import cats.effect.{Async, IO} 6 | import com.iheart.thomas.abtest.{AssignGroups, TestsData} 7 | import com.iheart.thomas.abtest.model.UserGroupQuery 8 | import com.iheart.thomas.client.AbtestClient 9 | import org.apache.spark.sql.DataFrame 10 | import org.apache.spark.sql.functions.{col, udf} 11 | import cats.implicits._ 12 | import com.iheart.thomas.abtest.AssignGroups.AssignmentResult 13 | 14 | import scala.concurrent.ExecutionContext 15 | import scala.concurrent.duration.{Duration, DurationInt} 16 | 17 | class Assigner(data: TestsData) extends Serializable { 18 | 19 | def assignUdf(feature: FeatureName) = 20 | udf { (userId: String) => 21 | assign(feature, userId).getOrElse(null) 22 | } 23 | 24 | def assign( 25 | feature: FeatureName, 26 | userId: String 27 | ): Option[GroupName] = { 28 | implicit val nowF = IO.delay(Instant.now) 29 | import cats.effect.unsafe.implicits.global 30 | AssignGroups 31 | .assign[IO]( 32 | data, 33 | UserGroupQuery(Some(userId), None, features = List(feature)), 34 | Duration.Zero 35 | ) 36 | .unsafeRunSync() 37 | .get(feature) 38 | .collect { case AssignmentResult(groupName, _) => 39 | groupName 40 | } 41 | } 42 | 43 | def assignments( 44 | userIds: DataFrame, 45 | feature: FeatureName, 46 | idColumn: String 47 | ): DataFrame = { 48 | userIds.withColumn("assignment", assignUdf(feature)(col(idColumn))) 49 | } 50 | } 51 | 52 | object Assigner { 53 | def create(url: String): Assigner = create(url, None) 54 | def create( 55 | url: String, 56 | asOf: Long 57 | ): Assigner = create(url, Some(asOf)) 58 | 59 | def create( 60 | url: String, 61 | asOf: Option[Long] 62 | ): Assigner = apply(url, asOf) 63 | 64 | def apply( 65 | url: String, 66 | asOf: Option[Long] 67 | ): Assigner = { 68 | import cats.effect.unsafe.implicits.global 69 | implicit val ex: ExecutionContext = global.compute 70 | 71 | create[IO](url, asOf).unsafeRunSync() 72 | } 73 | 74 | def create[F[_]: Async]( 75 | url: String, 76 | asOf: Option[Long] 77 | )(implicit ec: ExecutionContext 78 | ): F[Assigner] = { 79 | val time = asOf.map(Instant.ofEpochSecond).getOrElse(Instant.now) 80 | val durationOfTestsToUse = 12.hours 81 | AbtestClient.testsData[F](url, time, Some(durationOfTestsToUse)).map { 82 | new Assigner(_) 83 | } 84 | 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /client/src/main/scala/com/iheart/thomas/client/JavaAbtestAssignments.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2018] [iHeartMedia Inc] 3 | * All rights reserved 4 | */ 5 | 6 | package com.iheart.thomas 7 | package client 8 | 9 | import java.time.{Instant, ZoneOffset} 10 | import cats.effect.IO 11 | import cats.effect.unsafe.implicits.global 12 | import com.iheart.thomas.abtest.AssignGroups 13 | import com.iheart.thomas.abtest.AssignGroups.AssignmentResult 14 | import com.iheart.thomas.abtest.model.UserGroupQuery 15 | 16 | import scala.jdk.CollectionConverters._ 17 | import scala.concurrent.duration.{Duration, DurationInt} 18 | 19 | class JavaAbtestAssignments private ( 20 | serviceUrl: String, 21 | asOf: Option[Long]) { 22 | private val time = asOf.map(Instant.ofEpochSecond).getOrElse(Instant.now) 23 | implicit val nowF: IO[Instant] = IO.delay(Instant.now) 24 | implicit val ex: concurrent.ExecutionContext = global.compute 25 | val testData = 26 | AbtestClient.testsData[IO](serviceUrl, time, Some(12.hours)).unsafeRunSync() 27 | 28 | def assignments( 29 | userId: String, 30 | tags: java.util.List[String], 31 | meta: java.util.Map[String, String], 32 | features: java.util.List[String] 33 | ): java.util.Map[FeatureName, GroupName] = { 34 | AssignGroups 35 | .assign[IO]( 36 | testData, 37 | UserGroupQuery( 38 | Some(userId), 39 | Some(time.atOffset(ZoneOffset.UTC)), 40 | tags.asScala.toList, 41 | meta.asScala.toMap, 42 | features = features.asScala.toList 43 | ), 44 | Duration.Zero 45 | ) 46 | .map(_.collect { case (fn, AssignmentResult(gn, _)) => (fn, gn) }.asJava) 47 | .unsafeRunSync() 48 | 49 | } 50 | 51 | def assignments(userId: String): java.util.Map[FeatureName, GroupName] = 52 | assignments( 53 | userId, 54 | new java.util.ArrayList[String](), 55 | new java.util.HashMap[String, String](), 56 | new java.util.ArrayList[String]() 57 | ) 58 | 59 | def assignments( 60 | userId: String, 61 | features: java.util.List[String] 62 | ): java.util.Map[FeatureName, GroupName] = 63 | assignments( 64 | userId, 65 | new java.util.ArrayList[String](), 66 | new java.util.HashMap[String, String](), 67 | features 68 | ) 69 | } 70 | 71 | object JavaAbtestAssignments { 72 | def create(serviceUrl: String): JavaAbtestAssignments = 73 | new JavaAbtestAssignments(serviceUrl, None) 74 | def create( 75 | serviceUrl: String, 76 | asOf: Long 77 | ): JavaAbtestAssignments = 78 | new JavaAbtestAssignments(serviceUrl, Some(asOf)) 79 | } 80 | -------------------------------------------------------------------------------- /tests/src/it/scala/com/iheart/thomas/abtest/TestUtils.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package abtest 3 | 4 | import cats.MonadError 5 | import cats.effect.IO 6 | import com.iheart.thomas.abtest.model._ 7 | 8 | import java.time.{Instant, OffsetDateTime, ZoneOffset} 9 | import scala.util.Random 10 | 11 | object TestUtils { 12 | type F[A] = IO[A] 13 | val F = MonadError[IO, Throwable] 14 | def fakeAb: AbtestSpec = fakeAb() 15 | 16 | def group( 17 | name: GroupName, 18 | size: GroupSize 19 | ) = 20 | Group(name, size, None) 21 | 22 | def fakeAb( 23 | start: Int = 0, 24 | end: Int = 100, 25 | feature: String = "AMakeUpFeature" + Random.alphanumeric.take(5).mkString, 26 | alternativeIdName: Option[MetaFieldName] = None, 27 | groups: List[Group] = List(Group("A", 0.5, None), Group("B", 0.5, None)), 28 | userMetaCriteria: UserMetaCriteria = None, 29 | segRanges: List[GroupRange] = Nil, 30 | requiredTags: List[Tag] = Nil 31 | ): AbtestSpec = 32 | AbtestSpec( 33 | name = "test", 34 | author = "kai", 35 | feature = feature, 36 | start = OffsetDateTime.now.plusDays(start.toLong), 37 | end = Some(OffsetDateTime.now.plusDays(end.toLong)), 38 | groups = groups, 39 | alternativeIdName = alternativeIdName, 40 | userMetaCriteria = userMetaCriteria, 41 | segmentRanges = segRanges, 42 | requiredTags = requiredTags 43 | ) 44 | 45 | def randomUserId = Random.alphanumeric.take(10).mkString 46 | 47 | lazy val tomorrow = Some(OffsetDateTime.now.plusDays(1)) 48 | 49 | implicit def fromInstantToOffset(instant: Instant): OffsetDateTime = 50 | instant.atOffset(ZoneOffset.UTC) 51 | 52 | implicit def fromInstantOToOffset( 53 | instant: Option[Instant] 54 | ): Option[OffsetDateTime] = 55 | instant.map(fromInstantToOffset) 56 | 57 | implicit def fromOffsetDateTimeToInstant(offsetDateTime: OffsetDateTime): Instant = 58 | offsetDateTime.toInstant 59 | 60 | implicit def fromOffsetDateTimeOToInstant( 61 | offsetDateTime: Option[OffsetDateTime] 62 | ): Option[Instant] = 63 | offsetDateTime.map(fromOffsetDateTimeToInstant) 64 | 65 | def withAlg[A](f: AbtestAlg[F] => F[A]): F[A] = 66 | testkit.Resources.apis.map(_._2).use(f) 67 | 68 | def q( 69 | userId: UserId, 70 | at: Option[OffsetDateTime] = None, 71 | meta: UserMeta = Map(), 72 | eligibilityControlFilter: EligibilityControlFilter = 73 | EligibilityControlFilter.All 74 | ) = 75 | UserGroupQuery( 76 | Some(userId), 77 | at = at, 78 | meta = meta, 79 | eligibilityControlFilter = eligibilityControlFilter 80 | ) 81 | } 82 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/abtest/admin/formParts/mutualExclusivity.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.abtest.model._ 2 | 3 | @( 4 | draft: Option[AbtestSpec], 5 | readonly: Boolean = false, 6 | operatingSettingsOnly: Boolean = false 7 | ) 8 | 9 | @range(rg: Option[GroupRange]) = { 10 |
    11 |
    12 |
    13 |
    14 | 15 | 19 |
    20 |
    21 |
    22 |
    23 | 24 | 28 |
    29 |
    30 |
    31 |
    32 | 36 |
    37 |
    38 | } 39 | 40 | 41 |
    42 |
    43 | Mutual Exclusivity 44 |
    45 |
    46 |
    47 | @for( g <- draft.map(_.segmentRanges).getOrElse(Nil)) { 48 | @range(Some(g)) 49 | } 50 |
    51 | @if((readonly || operatingSettingsOnly) && draft.fold(false)(_.segmentRanges.isEmpty)) { 52 | This test is not mutually exclusive with other tests. 53 | } 54 |
    55 | 59 |
    60 | 63 |
    64 |
    -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/auth/AuthedEndpointsUtils.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package http4s 3 | package auth 4 | 5 | import cats.data.{Kleisli, OptionT} 6 | import cats.implicits._ 7 | import cats.{Applicative, Monad, MonadThrow} 8 | import com.iheart.thomas.admin.{Role, User} 9 | import org.http4s.dsl.Http4sDsl 10 | import org.http4s.headers.Location 11 | import org.http4s.twirl._ 12 | import org.http4s.{HttpRoutes, Request, Response, Uri} 13 | import tsec.authentication.{SecuredRequest, TSecAuthService, TSecMiddleware} 14 | import tsec.authorization.{AuthGroup, AuthorizationInfo, BasicRBAC} 15 | 16 | trait AuthedEndpointsUtils[F[_], Auth] { 17 | self: Http4sDsl[F] => 18 | 19 | type AuthService = TSecAuthService[User, Token[Auth], F] 20 | 21 | type AuthEndpoint = 22 | PartialFunction[SecuredRequest[F, User, Token[Auth]], F[ 23 | Response[F] 24 | ]] 25 | 26 | type Authenticator = tsec.authentication.Authenticator[ 27 | F, 28 | Username, 29 | User, 30 | Token[Auth] 31 | ] 32 | 33 | implicit def authorizationInfoForUserRole( 34 | implicit F: Applicative[F] 35 | ): AuthorizationInfo[F, Role, User] = 36 | (u: User) => F.pure(u.role) 37 | 38 | def liftService( 39 | service: AuthService 40 | )(implicit authenticator: Authenticator, 41 | reverseRoutes: ReverseRoutes, 42 | F: Monad[F] 43 | ): HttpRoutes[F] = { 44 | val middleWare = TSecMiddleware( 45 | Kleisli(authenticator.extractAndValidate), 46 | (req: Request[F]) => 47 | SeeOther(reverseRoutes.login(req.uri.renderString).location) 48 | ) 49 | middleWare(service) 50 | } 51 | 52 | def redirectTo(location: String)(implicit F: Applicative[F]) = 53 | SeeOther( 54 | Location(Uri.unsafeFromString(location)) 55 | ) 56 | 57 | def roleBasedService( 58 | authGroup: AuthGroup[Role] 59 | )(pf: AuthEndpoint 60 | )(implicit 61 | F: MonadThrow[F] 62 | ): AuthService = { 63 | val auth = BasicRBAC.fromGroup[F, Role, User, Token[Auth]](authGroup) 64 | val onUnauthorized = BadRequest( 65 | html.errorMsg( 66 | s"Sorry, you do not have sufficient access." 67 | ) 68 | ) 69 | 70 | Kleisli { req: SecuredRequest[F, User, Token[Auth]] => 71 | if (pf.isDefinedAt(req)) { 72 | OptionT( 73 | auth 74 | .isAuthorized(req) 75 | .fold(onUnauthorized)(pf) 76 | .flatten 77 | .map(Option(_)) 78 | ) 79 | } else 80 | OptionT.none[F, Response[F]] 81 | 82 | } 83 | } 84 | 85 | def roleBasedService( 86 | roles: Seq[Role] 87 | )(pf: AuthEndpoint 88 | )(implicit 89 | F: MonadThrow[F] 90 | ): AuthService = roleBasedService(AuthGroup.fromSeq(roles))(pf) 91 | 92 | } 93 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/KPIStats.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package analysis 3 | 4 | import breeze.stats.meanAndVariance.MeanAndVariance 5 | import cats.UnorderedFoldable 6 | import cats.kernel.{CommutativeMonoid, Monoid} 7 | import cats.implicits._ 8 | import henkan.convert.Syntax._ 9 | 10 | sealed trait KPIStats 11 | 12 | case class Conversions( 13 | converted: Long, 14 | total: Long) 15 | extends KPIStats { 16 | def rate = converted.toDouble / total.toDouble 17 | 18 | def sampleSize: Long = total 19 | 20 | override def toString: String = 21 | s"Conversions(converted: $converted, total: $total, rate: ${"%.2f".format(rate * 100)}%)" 22 | } 23 | 24 | object Conversions { 25 | implicit val monoidInstance: Monoid[Conversions] = new Monoid[Conversions] { 26 | def empty: Conversions = Conversions(0, 0) 27 | 28 | def combine( 29 | x: Conversions, 30 | y: Conversions 31 | ): Conversions = 32 | Conversions( 33 | x.converted + y.converted, 34 | x.total + y.total 35 | ) 36 | } 37 | 38 | def apply[C[_]: UnorderedFoldable]( 39 | events: C[ConversionEvent] 40 | ): Conversions = { 41 | val converted = events.count(identity) 42 | val init = events.size - converted 43 | Conversions(converted, init) 44 | } 45 | } 46 | 47 | case class PerUserSamplesLnSummary( 48 | mean: Double, 49 | variance: Double, 50 | count: Long) 51 | extends KPIStats 52 | 53 | object PerUserSamplesLnSummary { 54 | def fromSamples(samples: PerUserSamples): PerUserSamplesLnSummary = 55 | samples.lnSummary 56 | 57 | def apply(samples: PerUserSamples): PerUserSamplesLnSummary = fromSamples(samples) 58 | 59 | implicit val instances: CommutativeMonoid[PerUserSamplesLnSummary] = 60 | new CommutativeMonoid[PerUserSamplesLnSummary] { 61 | def empty: PerUserSamplesLnSummary = PerUserSamplesLnSummary(0d, 0d, 0L) 62 | 63 | def combine( 64 | x: PerUserSamplesLnSummary, 65 | y: PerUserSamplesLnSummary 66 | ): PerUserSamplesLnSummary = 67 | (x.to[MeanAndVariance]() + y.to[MeanAndVariance]()) 68 | .to[PerUserSamplesLnSummary]() 69 | } 70 | } 71 | 72 | trait Aggregation[Event, KS <: KPIStats] { 73 | def apply[C[_]: UnorderedFoldable](events: C[Event]): KS 74 | } 75 | 76 | object Aggregation { 77 | implicit val conversionsAggregation: Aggregation[ConversionEvent, Conversions] = 78 | new Aggregation[ConversionEvent, Conversions] { 79 | def apply[C[_]: UnorderedFoldable](events: C[ConversionEvent]) = 80 | Conversions(events) 81 | } 82 | 83 | implicit val accumulativeAggregation 84 | : Aggregation[PerUserSamples, PerUserSamplesLnSummary] = 85 | new Aggregation[PerUserSamples, PerUserSamplesLnSummary] { 86 | def apply[C[_]: UnorderedFoldable](events: C[PerUserSamples]) = 87 | events.unorderedFold.lnSummary 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /dynamo/src/main/scala/com/iheart/thomas/dynamo/AdminDAOs.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.dynamo 2 | 3 | import cats.effect.Async 4 | import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient 5 | import com.iheart.thomas.admin.{AuthRecord, AuthRecordDAO, User, UserDAO} 6 | import DynamoFormats._ 7 | import cats.implicits._ 8 | import com.iheart.thomas.stream.{Job, JobDAO} 9 | import org.scanamo.syntax._ 10 | 11 | import java.time.Instant 12 | 13 | object AdminDAOs extends ScanamoManagement { 14 | val authTableName = "ds-abtest-auth" 15 | val authKeyName = "id" 16 | val authKey = ScanamoDAOHelperStringKey.keyOf(authKeyName) 17 | 18 | val userTableName = "ds-abtest-user" 19 | val userKeyName = "username" 20 | val userKey = ScanamoDAOHelperStringKey.keyOf(userKeyName) 21 | 22 | val streamJobTableName = "ds-abtest-stream-job" 23 | val streamJobKeyName = "key" 24 | val streamJobKey = ScanamoDAOHelperStringKey.keyOf(streamJobKeyName) 25 | 26 | val tables = List( 27 | (authTableName, authKey), 28 | (userTableName, userKey), 29 | (streamJobTableName, streamJobKey) 30 | ) 31 | 32 | def ensureAuthTables[F[_]: Async]( 33 | readCapacity: Long, 34 | writeCapacity: Long 35 | )(implicit dc: DynamoDbAsyncClient 36 | ): F[Unit] = ensureTables(tables, readCapacity, writeCapacity) 37 | 38 | implicit def authRecordDAO[F[_]: Async]( 39 | implicit dynamoClient: DynamoDbAsyncClient 40 | ): AuthRecordDAO[F] = 41 | new ScanamoDAOHelperStringKey[F, AuthRecord]( 42 | authTableName, 43 | authKeyName, 44 | dynamoClient 45 | ) with AuthRecordDAO[F] 46 | 47 | implicit def userDAO[F[_]: Async]( 48 | implicit dynamoClient: DynamoDbAsyncClient 49 | ): UserDAO[F] = 50 | new ScanamoDAOHelperStringKey[F, User]( 51 | userTableName, 52 | userKeyName, 53 | dynamoClient 54 | ) with UserDAO[F] 55 | 56 | implicit def streamJobDAO[F[_]: Async]( 57 | implicit dynamoClient: DynamoDbAsyncClient 58 | ): JobDAO[F] = 59 | new ScanamoDAOHelperStringKey[F, Job]( 60 | streamJobTableName, 61 | streamJobKeyName, 62 | dynamoClient 63 | ) with JobDAO[F] { 64 | 65 | def updateCheckedOut( 66 | job: Job, 67 | at: Instant 68 | ): F[Option[Job]] = { 69 | val cond = streamJobKeyName === job.key 70 | val setV = set("checkedOut", Some(at)) 71 | sc.exec( 72 | job.checkedOut 73 | .fold( 74 | table 75 | .when(attributeNotExists("checkedOut")) 76 | .update(cond, setV) 77 | )(c => 78 | table 79 | .when("checkedOut" === c) 80 | .update(cond, setV) 81 | ) 82 | ).map(_.toOption) 83 | } 84 | 85 | def setStarted( 86 | job: Job, 87 | at: Instant 88 | ): F[Job] = update(job.key, set("started", Some(at))) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /analysis/src/main/scala/com/iheart/thomas/analysis/KPIRepo.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | 3 | import cats.data.ValidatedNel 4 | import cats.implicits._ 5 | import cats.{FlatMap, MonadThrow} 6 | import com.iheart.thomas.abtest.Error.NotFound 7 | import com.iheart.thomas.analysis.KPIRepo.{InvalidKPI, validate} 8 | import com.iheart.thomas.analysis.bayesian.models.{BetaModel, LogNormalModel} 9 | 10 | import scala.util.control.NoStackTrace 11 | 12 | trait KPIRepo[F[_], K <: KPI] { 13 | 14 | protected def insert(newKpi: K): F[K] 15 | 16 | def create( 17 | newKpi: K 18 | )(implicit F: MonadThrow[F] 19 | ): F[K] = // todo: add AllKPIRepo deps to ensure kpi name uniqueness. 20 | validate(newKpi) 21 | .leftMap(es => InvalidKPI(es.mkString_("; "))) 22 | .liftTo[F] *> insert(newKpi) 23 | 24 | def update(k: K): F[K] 25 | 26 | def remove(name: KPIName): F[Unit] 27 | 28 | def find(name: KPIName): F[Option[K]] 29 | 30 | def all: F[Vector[K]] 31 | 32 | def get(name: KPIName): F[K] 33 | 34 | def update(name: KPIName)(f: K => K)(implicit F: FlatMap[F]): F[K] = 35 | get(name).flatMap(k => update(f(k))) 36 | } 37 | 38 | object KPIRepo { 39 | 40 | case class InvalidKPI(override val getMessage: String) 41 | extends RuntimeException 42 | with NoStackTrace 43 | 44 | def validate(newKpi: KPI): ValidatedNel[String, Unit] = 45 | KPIName.fromString(newKpi.name.n).toValidatedNel.void *> { 46 | newKpi match { 47 | case ck: ConversionKPI => 48 | BetaModel.validate(ck.model).void 49 | case ak: AccumulativeKPI => 50 | LogNormalModel.validate(ak.model).void 51 | } 52 | } 53 | } 54 | 55 | trait AllKPIRepo[F[_]] { 56 | def all: F[Vector[KPI]] 57 | def find(name: KPIName): F[Option[KPI]] 58 | def get(name: KPIName): F[KPI] 59 | def delete(name: KPIName): F[Option[Unit]] 60 | } 61 | 62 | object AllKPIRepo { 63 | implicit def default[F[_]: MonadThrow]( 64 | implicit cRepo: KPIRepo[F, ConversionKPI], 65 | aRepo: KPIRepo[F, QueryAccumulativeKPI] 66 | ): AllKPIRepo[F] = 67 | new AllKPIRepo[F] { 68 | def all: F[Vector[KPI]] = 69 | for { 70 | cs <- cRepo.all 71 | as <- aRepo.all 72 | } yield (cs.widen[KPI] ++ as 73 | .widen[KPI]) 74 | 75 | def find(name: KPIName): F[Option[KPI]] = 76 | cRepo.find(name).flatMap { r => 77 | r.fold(aRepo.find(name).widen[Option[KPI]])(_ => r.pure[F].widen) 78 | } 79 | 80 | def delete(name: KPIName): F[Option[Unit]] = { 81 | find(name).flatMap(_.traverse { 82 | case c: ConversionKPI => cRepo.remove(c.name) 83 | case a: QueryAccumulativeKPI => aRepo.remove(a.name) 84 | }) 85 | } 86 | 87 | def get(name: KPIName): F[KPI] = 88 | find(name).flatMap( 89 | _.liftTo[F]( 90 | NotFound("Cannot find KPI named " + name + " is not found in DB") 91 | ) 92 | ) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /testkit/src/main/scala/com/iheart/thomas/testkit/TestMessageKafkaProducer.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.testkit 2 | 3 | import cats.effect.{ExitCode, IO, IOApp} 4 | import com.iheart.thomas.{FeatureName, GroupName} 5 | import com.iheart.thomas.kafka.KafkaConfig 6 | import com.typesafe.config.ConfigFactory 7 | import fs2.kafka.{KafkaProducer, ProducerRecord, ProducerRecords, ProducerSettings} 8 | import fs2.Stream 9 | 10 | import java.time.Instant 11 | import concurrent.duration._ 12 | import scala.util.{Random, Try} 13 | 14 | object TestMessageKafkaProducer extends IOApp { 15 | 16 | def run(args: List[String]): IO[ExitCode] = { 17 | val duration = 18 | args.headOption.flatMap(h => Try(h.toInt).toOption).getOrElse(3).seconds 19 | (Stream.eval { 20 | IO.delay { 21 | println("=====================================") 22 | println(s"Running kafka producer for $duration") 23 | println("=====================================") 24 | } 25 | } ++ 26 | Stream 27 | .eval(KafkaConfig.fromConfig[IO](ConfigFactory.load)) 28 | .flatMap { cfg => 29 | val producerSettings = 30 | ProducerSettings[IO, String, String] 31 | .withBootstrapServers("127.0.0.1:9092") 32 | Stream 33 | .fixedDelay[IO](100.millis) 34 | .flatMap { _ => 35 | Stream.fromIterator[IO](messages.iterator, 1).map { value => 36 | val record = ProducerRecord(cfg.topic, "k", value) 37 | ProducerRecords.one(record) 38 | } 39 | } 40 | .through(KafkaProducer.pipe(producerSettings)) 41 | 42 | } 43 | .interruptAfter(duration)).compile.drain 44 | .as(ExitCode.Success) 45 | 46 | } 47 | 48 | def message( 49 | eventString: String, 50 | groups: Seq[(FeatureName, GroupName)] 51 | ) = { 52 | val groupValues = groups 53 | .map { case (fn, gn) => 54 | s""" "$fn" : "$gn" """ 55 | } 56 | .mkString(""", 57 | | """.stripMargin) 58 | 59 | s""" 60 | |{ 61 | | $eventString, 62 | | "treatment-groups": { 63 | | $groupValues, 64 | | timeStamp: ${Instant.now.toEpochMilli} 65 | | } 66 | |} 67 | |""".stripMargin 68 | } 69 | 70 | val initEvent = 71 | """ "page_shown": "front_page" """ 72 | 73 | val clickEvent = 74 | """ "click": "front_page_recommendation" """ 75 | 76 | val groups = List("A", "B", "C", "D", "E", "F", "G", "H", "J") 77 | val features = List("A_Feature", "Another_Feature", "Third_Feature") 78 | 79 | def randomFG(n: Int) = 80 | List.fill(n)((Random.shuffle(features).head, Random.shuffle(groups).head)) 81 | 82 | def randomMessage( 83 | n: Int, 84 | event: String 85 | ): List[String] = 86 | List.fill(n)(message(event, randomFG(Random.nextInt(5) + 1))) 87 | 88 | val messages: List[String] = 89 | randomMessage(100, initEvent) ++ randomMessage(40, clickEvent) 90 | } 91 | -------------------------------------------------------------------------------- /http4s/src/main/scala/com/iheart/thomas/http4s/CommonFormDecoders.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package http4s 3 | 4 | import java.time.{Instant, OffsetDateTime, ZonedDateTime} 5 | import _root_.play.api.libs.json.{JsObject, Json, Reads} 6 | import cats.data.NonEmptyList 7 | import cats.implicits._ 8 | import org.http4s.FormDataDecoder._ 9 | import org.http4s.{ 10 | FormDataDecoder, 11 | ParseFailure, 12 | QueryParamDecoder, 13 | QueryParameterValue 14 | } 15 | import io.estatico.newtype.Coercible 16 | import io.estatico.newtype.ops._ 17 | 18 | import scala.concurrent.duration.{Duration, FiniteDuration} 19 | import scala.util.Try 20 | 21 | trait CommonQueryParamDecoders { 22 | implicit val offsetDateTimeQueryParamDecoder: QueryParamDecoder[OffsetDateTime] = { 23 | QueryParamDecoder.fromUnsafeCast(qp => 24 | ZonedDateTime.parse(qp.value, Formatters.dateTimeFormatter).toOffsetDateTime 25 | )("OffsetDateTime") 26 | } 27 | 28 | implicit val instantQueryParamDecoder: QueryParamDecoder[Instant] = { 29 | offsetDateTimeQueryParamDecoder.map(_.toInstant) 30 | } 31 | 32 | implicit val finiteDurationQueryParamDecoder: QueryParamDecoder[FiniteDuration] = { 33 | QueryParamDecoder.fromUnsafeCast(qp => 34 | Duration(qp.value).asInstanceOf[FiniteDuration] 35 | )("FiniteDuration") 36 | } 37 | 38 | implicit val bigDecimalQPD: QueryParamDecoder[BigDecimal] = 39 | QueryParamDecoder.fromUnsafeCast[BigDecimal](qp => 40 | BigDecimal(qp.value.toDouble) 41 | )("BigDecimal") 42 | 43 | implicit def coercibleQueryParamDecoder[A, B]( 44 | implicit coercible: Coercible[A, B], 45 | qpd: QueryParamDecoder[A] 46 | ): QueryParamDecoder[B] = qpd.map(_.coerce) 47 | 48 | def jsonEntityQueryParamDecoder[A](implicit A: Reads[A]): QueryParamDecoder[A] = 49 | (qp: QueryParameterValue) => 50 | Try(Json.parse(qp.value)).toEither 51 | .leftMap(e => ParseFailure("Invalid Json", e.getMessage)) 52 | .toValidatedNel 53 | .andThen { json => 54 | A.reads(json) 55 | .asEither 56 | .leftMap { e => 57 | NonEmptyList.fromListUnsafe(e.map { case (path, errors) => 58 | ParseFailure( 59 | "Json field parse failed", 60 | s"$path: ${errors.map(_.message).mkString(";")}" 61 | ) 62 | }.toList) 63 | } 64 | .toValidated 65 | } 66 | 67 | implicit val jsObjectQueryParamDecoder: QueryParamDecoder[JsObject] = 68 | jsonEntityQueryParamDecoder 69 | 70 | def mapQueryParamDecoder: QueryParamDecoder[Map[String, String]] = 71 | jsonEntityQueryParamDecoder[Map[String, String]] 72 | } 73 | 74 | trait CommonFormDecoders extends CommonQueryParamDecoders { 75 | 76 | implicit def tuple2[A, B]( 77 | implicit A: QueryParamDecoder[A], 78 | B: QueryParamDecoder[B] 79 | ): FormDataDecoder[(A, B)] = 80 | ( 81 | field[A]("_1"), 82 | field[B]("_2") 83 | ).tupled 84 | 85 | } 86 | 87 | object CommonFormDecoders extends CommonFormDecoders 88 | -------------------------------------------------------------------------------- /mongo/src/main/scala/lihua/mongo/JsonFormats.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] [iHeartMedia Inc] 3 | * All rights reserved 4 | */ 5 | package lihua 6 | package mongo 7 | 8 | import cats.Invariant 9 | import play.api.libs.json._ 10 | 11 | import scala.reflect.ClassTag 12 | 13 | object JsonFormats extends playJson.Formats { 14 | 15 | implicit val invariantFormat: Invariant[Format] = new Invariant[Format] { 16 | def imap[A, B](fa: Format[A])(f: A => B)(g: B => A): Format[B] = new Format[B] { 17 | override def reads(json: JsValue): JsResult[B] = fa.reads(json).map(f) 18 | override def writes(o: B): JsValue = fa.writes(g(o)) 19 | } 20 | } 21 | 22 | object StringBooleanFormat extends Format[Boolean] { 23 | 24 | override def reads(json: JsValue): JsResult[Boolean] = 25 | json.validate[String].map(_.toLowerCase == "true") 26 | 27 | override def writes(o: Boolean): JsValue = JsString(o.toString) 28 | } 29 | 30 | object IntBooleanFormat extends Format[Boolean] { 31 | 32 | override def reads(json: JsValue): JsResult[Boolean] = 33 | json.validate[Int].map(_ != 0) 34 | 35 | override def writes(o: Boolean): JsValue = JsNumber(if (o) 1 else 0) 36 | } 37 | 38 | implicit class JsPathMongoDBOps(val self: JsPath) extends AnyVal { 39 | def formatEntityId = OFormat[String]( 40 | self.read[EntityId].map(_.value), 41 | OWrites[String] { s => self.write[EntityId].writes(EntityId(s)) } 42 | ) 43 | } 44 | 45 | implicit def mapFormat[KT: StringParser, VT: Format]: Format[Map[KT, VT]] = 46 | new Format[Map[KT, VT]] { 47 | def writes(o: Map[KT, VT]): JsValue = 48 | JsObject(o.toSeq.map { case (k, v) => (k.toString, Json.toJson(v)) }) 49 | 50 | def reads(json: JsValue): JsResult[Map[KT, VT]] = 51 | json 52 | .validate[JsObject] 53 | .map(_.fields.map { case (ks, vValue) => 54 | (implicitly[StringParser[KT]].parse(ks), vValue.as[VT]) 55 | }.toMap) 56 | } 57 | 58 | private def myClassOf[T: ClassTag] = 59 | implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]] 60 | 61 | implicit def javaEnumWrites[ET <: Enum[ET]]: Writes[ET] = Writes { (r: Enum[_]) => 62 | JsString(r.name()) 63 | } 64 | 65 | implicit def javaEnumReads[ET <: Enum[ET]: ClassTag]: Reads[ET] = Reads { 66 | case JsString(name) => 67 | JsSuccess(Enum.valueOf(myClassOf[ET], name)) 68 | // TODO: improve error 69 | case _ => JsError("unrecognized format") 70 | } 71 | 72 | implicit def javaEnumFormats[ET <: Enum[ET]: ClassTag]: Format[ET] = 73 | Format(javaEnumReads[ET], javaEnumWrites[ET]) 74 | 75 | trait StringParser[T] { 76 | def parse(s: String): T 77 | } 78 | 79 | object StringParser { 80 | implicit val intStringParser: StringParser[Int] = new StringParser[Int] { 81 | def parse(s: String): Int = java.lang.Integer.parseInt(s) 82 | } 83 | 84 | implicit val stringStringParser: StringParser[String] = 85 | new StringParser[String] { 86 | def parse(s: String): String = s 87 | } 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /testkit/src/main/scala/com/iheart/thomas/testkit/Resources.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package testkit 3 | 4 | import java.time.Instant 5 | import cats.effect.{IO, Resource} 6 | import cats.implicits._ 7 | import com.iheart.thomas.abtest.AbtestAlg 8 | import com.iheart.thomas.bandit.bayesian.BayesianMABAlg 9 | import com.iheart.thomas.{dynamo, mongo} 10 | import com.typesafe.config.ConfigFactory 11 | import _root_.play.api.libs.json.Json 12 | import com.iheart.thomas.analysis.monitor.ExperimentKPIStateDAO 13 | import com.iheart.thomas.analysis.{ConversionKPI, Conversions, KPIRepo} 14 | import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient 15 | import com.iheart.thomas.http4s.AuthImp 16 | import com.iheart.thomas.http4s.auth.AuthenticationAlg 17 | import com.iheart.thomas.tracking.EventLogger 18 | 19 | import scala.concurrent.duration._ 20 | import dynamo.AnalysisDAOs._ 21 | import dynamo.BanditsDAOs._ 22 | 23 | object Resources { 24 | 25 | import concurrent.ExecutionContext.Implicits.global 26 | 27 | val defaultNowF = IO.delay(Instant.now) 28 | lazy val mangoDAOs = 29 | Resource.eval(IO(ConfigFactory.load(getClass.getClassLoader))).flatMap { 30 | config => 31 | mongo.daosResource[IO](config).flatMap { case daos => 32 | Resource 33 | .make(IO.pure(daos)) { case (abtestDAO, featureDAO) => 34 | List(abtestDAO, featureDAO) 35 | .traverse(_.removeAll(Json.obj())) 36 | .void 37 | } 38 | } 39 | } 40 | 41 | val tables = 42 | dynamo.BanditsDAOs.tables ++ dynamo.AdminDAOs.tables ++ dynamo.AnalysisDAOs.tables 43 | 44 | lazy val localDynamoR = 45 | LocalDynamo 46 | .clientWithTables[IO]( 47 | tables.map(_.map(Seq(_))): _* 48 | ) 49 | 50 | /** An ConversionAPI resource that cleans up after 51 | */ 52 | def apis( 53 | implicit logger: EventLogger[IO] = EventLogger.noop[IO] 54 | ): Resource[ 55 | IO, 56 | ( 57 | BayesianMABAlg[IO], 58 | AbtestAlg[IO], 59 | KPIRepo[IO, ConversionKPI], 60 | ExperimentKPIStateDAO[IO, Conversions] 61 | ) 62 | ] = 63 | (mangoDAOs, localDynamoR).tupled 64 | .flatMap { deps => 65 | implicit val ((abtestDAO, featureDAO), dynamoDb) = deps 66 | lazy val refreshPeriod = 0.seconds 67 | AbtestAlg.defaultResource[IO](refreshPeriod).map { implicit abtestAlg => 68 | ( 69 | implicitly, 70 | abtestAlg, 71 | implicitly, 72 | implicitly 73 | ) 74 | } 75 | } 76 | 77 | def authAlg( 78 | implicit dc: DynamoDbAsyncClient 79 | ): Resource[IO, AuthenticationAlg[IO, AuthImp]] = 80 | Resource.eval(AuthenticationAlg.default[IO](sys.env("THOMAS_ADMIN_KEY"))) 81 | 82 | def abtestAlg: Resource[ 83 | IO, 84 | AbtestAlg[IO] 85 | ] = 86 | mangoDAOs.flatMap { daos => 87 | implicit val (abtestDAO, featureDAO) = daos 88 | lazy val refreshPeriod = 0.seconds 89 | AbtestAlg.defaultResource[IO](refreshPeriod) 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /testkit/src/main/scala/com/iheart/thomas/testkit/MockQueryAccumulativeKPIAlg.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.testkit 2 | 3 | import cats.Applicative 4 | import cats.effect.IO 5 | import cats.implicits._ 6 | import com.iheart.thomas.analysis.{ 7 | KPI, 8 | KPIEventQuery, 9 | KPIName, 10 | PerUserSamples, 11 | PerUserSamplesQuery, 12 | QueryAccumulativeKPI, 13 | AccumulativeKPIQueryRepo, 14 | QueryName 15 | } 16 | import com.iheart.thomas.{ArmName, FeatureName} 17 | 18 | import java.time.Instant 19 | import scala.concurrent.duration._ 20 | 21 | object MockQueryAccumulativeKPIAlg { 22 | implicit val rdb = breeze.stats.distributions.RandBasis.mt0 23 | implicit val nullAlg = AccumulativeKPIQueryRepo.unsupported[IO] 24 | type MockData[E] = (FeatureName, ArmName, KPIName, Instant, Instant, E) 25 | val mockQueryName = QueryName("Mock Query with preset data") 26 | def mockLogNormalData(kpiName: KPIName) = { 27 | val dist = breeze.stats.distributions.LogNormal(1d, 0.3d) 28 | val begin = Instant.now.minusSeconds(20000) 29 | val end = Instant.now.plusSeconds(20000) 30 | List( 31 | ( 32 | "A_Feature", 33 | "A", 34 | kpiName, 35 | begin, 36 | end, 37 | PerUserSamples(dist.sample(5000).toArray) 38 | ), 39 | ( 40 | "A_Feature", 41 | "B", 42 | kpiName, 43 | begin, 44 | end, 45 | PerUserSamples(dist.sample(5000).toArray) 46 | ) 47 | ) 48 | } 49 | 50 | def apply[F[_]: Applicative]( 51 | data: List[MockData[PerUserSamples]] = mockLogNormalData( 52 | KPIName("testAccumulativeKPI") 53 | ), 54 | freq: FiniteDuration = 500.millis 55 | ): AccumulativeKPIQueryRepo[F] = 56 | new AccumulativeKPIQueryRepo[F] { 57 | 58 | def queries: F[List[PerUserSamplesQuery[F]]] = 59 | List( 60 | new MockQuery[F, QueryAccumulativeKPI, PerUserSamples](data) 61 | with PerUserSamplesQuery[F] { 62 | val name: QueryName = mockQueryName 63 | 64 | def frequency: FiniteDuration = freq 65 | } 66 | ).pure[F].widen 67 | } 68 | 69 | abstract class MockQuery[F[_]: Applicative, K <: KPI, E]( 70 | data: List[MockData[E]]) 71 | extends KPIEventQuery[F, K, E] { 72 | 73 | def find( 74 | k: K, 75 | at: Instant 76 | ): List[(FeatureName, ArmName, E)] = 77 | data 78 | .collect { 79 | case (fn, am, kpiName, start, end, data) 80 | if kpiName == k.name && start.isBefore(at) && end.isAfter(at) => 81 | (fn, am, data) 82 | } 83 | def apply( 84 | k: K, 85 | at: Instant 86 | ): F[List[E]] = 87 | find(k, at).map(_._3).pure[F] 88 | 89 | def apply( 90 | k: K, 91 | feature: FeatureName, 92 | at: Instant 93 | ): F[List[(ArmName, E)]] = { 94 | find(k, at) 95 | .collect { 96 | case (fn, am, data) if (fn == feature) => (am, data) 97 | } 98 | .pure[F] 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/index.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.analysis._ 2 | @import com.iheart.thomas.analysis.monitor._ 3 | @import com.iheart.thomas.html._ 4 | @import com.iheart.thomas.admin.Authorization._ 5 | 6 | @import com.iheart.thomas.http4s.UIEnv 7 | 8 | @( 9 | states: Vector[ExperimentKPIState[KPIStats]], 10 | kpis: Vector[KPI] 11 | )(implicit env: UIEnv) 12 | 13 | 14 | @topNav("Analysis", "Analysis") { 15 |
    16 |
    17 |
    18 | Monitored A/B tests 19 |
    20 | @for(state <- states) { 21 |
    22 | @kpiState(state) 23 |
    24 | } 25 | 26 |
    27 |
    28 |
    29 | KPIs 30 |
    31 |
    32 |
    33 | @for(kpi <- kpis) { 34 |
    35 |
    36 |
    37 | 42 |
    43 | @kpi.description 44 |
    45 |
    46 | 47 |
    48 | Author: 49 | @kpi.author 50 | 51 |
    52 |
    53 | @if(env.currentUser.has(ManageAnalysis)) { 54 | 57 | 58 | 59 | } 60 |
    61 |
    62 |
    63 | } 64 | 65 |
    66 |
    67 | 68 |
    69 |
    70 | 71 | 74 | 75 | 76 | 79 | 80 |
    81 | 82 |
    83 |
    84 | 85 | 86 |
    87 | 88 | } -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/analysis/kpiState.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.ArmName 2 | @import com.iheart.thomas.analysis._ 3 | @import com.iheart.thomas.analysis.monitor._ 4 | @import com.iheart.thomas.http4s.{UIEnv, Formatters}, Formatters._ 5 | 6 | @( 7 | state: ExperimentKPIState[KPIStats], 8 | showCommands: Boolean = true, 9 | includedArms: Set[ArmName] = Set.empty, 10 | showReset: Boolean = true 11 | )(implicit env: UIEnv) 12 | 13 |
    14 |
    15 | KPI stats for @state.key.feature on @state.key.kpi 17 | from @formatDate(state.dataPeriod.from) to @formatDate(state.dataPeriod.to) 18 |
    19 |
    20 | @if(showCommands) { 21 |
    22 | } 23 |
    24 | 25 | @for(arm <- state.arms.sortBy(_.name).toList) { 26 |
    27 | 500L && includedArms.isEmpty) || includedArms.contains(arm.name)) { 33 | checked 34 | } 35 | value="@arm.name"> 36 | 39 | @arm.kpiStats match { 40 | case (c: Conversions) => { 41 | @conversionsStats(c) 42 | } 43 | case (s: PerUserSamplesLnSummary) => { 44 | @perUserSamplesSummary(s) 45 | } 46 | } 47 |
    48 | } 49 |
    50 | @if(showCommands) { 51 |
    52 | 54 | @if(showReset){ 55 | 59 | 60 | Reset 61 | } 62 |
    63 |
    64 | } 65 | 66 |
    67 |
    68 | 69 | 70 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/bandit/bayesian/BayesianMABAlgSuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas 2 | package bandit 3 | package bayesian 4 | 5 | import cats.implicits._ 6 | import com.iheart.thomas.analysis.{KPIName, Probability} 7 | import org.scalacheck.Arbitrary 8 | import org.scalatest.funsuite.AnyFunSuiteLike 9 | import org.scalatest.matchers.should.Matchers 10 | import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks 11 | 12 | import java.time.OffsetDateTime 13 | import scala.util.Try 14 | 15 | class BayesianMABAlgSuite 16 | extends AnyFunSuiteLike 17 | with Matchers 18 | with ScalaCheckDrivenPropertyChecks { 19 | 20 | import com.iheart.thomas.abtest.BucketingTests.groupsGen 21 | 22 | implicit val distributionGen: Arbitrary[Map[GroupName, Probability]] = Arbitrary { 23 | groupsGen(3).map(_.map { group => 24 | group.name -> Probability(group.size.doubleValue) 25 | }.toMap) 26 | } 27 | 28 | def createSpec(arms: Seq[ArmSpec]) = 29 | BanditSpec( 30 | "feature", 31 | "title", 32 | "author", 33 | KPIName("kpi"), 34 | arms, 35 | stateMonitorEventChunkSize = 1, 36 | updatePolicyStateChunkSize = 1 37 | ) 38 | 39 | test("allocateGroupSize allocates using available size") { 40 | forAll { (distribution: Map[GroupName, Probability]) => 41 | val precision = BigDecimal(0.01) 42 | val availableSize = BigDecimal(0.8) 43 | val groups = BayesianMABAlg 44 | .allocateGroupSize(distribution, precision, availableSize) 45 | 46 | groups.size shouldBe distribution.size 47 | val totalSize = groups.foldMap(_.size) 48 | 49 | totalSize should be( 50 | BigDecimal( 51 | distribution.values.toList.foldMap(_.p) 52 | ) * availableSize +- precision 53 | ) 54 | 55 | totalSize should be <= availableSize 56 | 57 | groups 58 | .foreach { group => 59 | group.size shouldBe BigDecimal( 60 | distribution(group.name).p 61 | ) * availableSize +- precision 62 | 63 | (group.size % precision) should be(BigDecimal(0)) 64 | 65 | } 66 | } 67 | } 68 | 69 | test("abtestSpecFromBanditSpec distributes sizes evenly") { 70 | 71 | val result = BayesianMABAlg 72 | .createTestSpec[Try]( 73 | createSpec(arms = List(ArmSpec("A"), ArmSpec("B"))), 74 | OffsetDateTime.now 75 | ) 76 | 77 | result.isSuccess shouldBe true 78 | result.get.groups.map(_.size).sum shouldBe BigDecimal(1) 79 | } 80 | 81 | test("abtestSpecFromBanditSpec allocate based on initial sizes") { 82 | 83 | val result = BayesianMABAlg 84 | .createTestSpec[Try]( 85 | createSpec(arms = 86 | List(ArmSpec("A", initialSize = Some(0.3)), ArmSpec("B"), ArmSpec("C")) 87 | ), 88 | OffsetDateTime.now 89 | ) 90 | 91 | result.isSuccess shouldBe true 92 | 93 | def sizeOf(gn: GroupName) = result.get.groups.find(_.name == gn).get.size 94 | 95 | sizeOf("A") shouldBe BigDecimal(0.3) 96 | sizeOf("B") shouldBe sizeOf("C") 97 | result.get.groups.map(_.size).sum shouldBe BigDecimal(1) 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /tests/src/test/scala/com/iheart/thomas/analysis/bayesian/models/BetaKPISuite.scala: -------------------------------------------------------------------------------- 1 | package com.iheart.thomas.analysis 2 | package bayesian 3 | package models 4 | 5 | import cats.effect.IO 6 | import cats.effect.testing.scalatest.AsyncIOSpec 7 | import cats.implicits._ 8 | import com.iheart.thomas.abtest.model.Abtest 9 | import com.stripe.rainier.sampler.{RNG, SamplerConfig} 10 | import org.scalatest.freespec.AsyncFreeSpec 11 | import org.scalatest.matchers.should.Matchers 12 | 13 | class BetaKPISuite extends AsyncFreeSpec with AsyncIOSpec with Matchers { 14 | implicit val rng = RNG.default 15 | implicit val sampler = SamplerConfig.default 16 | 17 | val mockAb: Abtest = null 18 | val alg: ModelEvaluator[IO, BetaModel, Conversions] = 19 | implicitly 20 | 21 | "BetaKPI Assessment Alg" - { 22 | "can evaluation optimal group distribution" in { 23 | alg 24 | .evaluate( 25 | BetaModel(200d, 300d), 26 | Map( 27 | "B" -> Conversions(250L, 300L), 28 | "A" -> Conversions(200L, 300L), 29 | "C" -> Conversions(265L, 300L), 30 | "D" -> Conversions(230L, 300L) 31 | ), 32 | None 33 | ) 34 | .asserting { dist => 35 | dist 36 | .sortBy(_.probabilityBeingOptimal.p) 37 | .map(_.name) shouldBe List("A", "D", "B", "C") 38 | } 39 | 40 | } 41 | 42 | "can evaluate optimal group distribution with large sample size" in { 43 | alg 44 | .evaluate( 45 | BetaModel(0d, 0d), 46 | Map( 47 | "A" -> Conversions(7000L, 10000L), 48 | "B" -> Conversions(6800, 10000L), 49 | "C" -> Conversions(6700L, 10000L), 50 | "D" -> Conversions(7500L, 10000L) 51 | ), 52 | None 53 | ) 54 | .asserting { dist => 55 | dist 56 | .sortBy(_.probabilityBeingOptimal.p) 57 | .map(_.name) 58 | .last shouldBe "D" 59 | } 60 | } 61 | 62 | "can evaluate optimal group distribution with asymmetric sample size" in { 63 | alg 64 | .evaluate( 65 | BetaModel(0d, 0d), 66 | Map( 67 | "A" -> Conversions(70L, 100L), 68 | "B" -> Conversions(680, 1000L), 69 | "C" -> Conversions(6700L, 10000L), 70 | "D" -> Conversions(750L, 1000L) 71 | ), 72 | None 73 | ) 74 | .asserting { dist => 75 | dist 76 | .sortBy(_.probabilityBeingOptimal.p) 77 | .map(_.name) 78 | .last shouldBe "D" 79 | } 80 | } 81 | 82 | "can evaluate optimal group distribution with real data" in { 83 | alg 84 | .evaluate( 85 | BetaModel(1414L, 500L), 86 | Map( 87 | "A" -> Conversions(1414L, 1973L), 88 | "C" -> Conversions(20985L, 31267L) 89 | ), 90 | None 91 | ) 92 | .asserting { dist => 93 | dist 94 | .sortBy(_.probabilityBeingOptimal.p) 95 | .map(_.name) 96 | .last shouldBe "A" 97 | } 98 | } 99 | 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /http4s/src/main/twirl/com/iheart/thomas/abtest/admin/assignments.scala.html: -------------------------------------------------------------------------------- 1 | @import com.iheart.thomas.abtest.model._ 2 | @import com.iheart.thomas.http4s.UIEnv 3 | @import com.iheart.thomas.html._ 4 | 5 | @import play.api.libs.json._ 6 | @import com.iheart.thomas.http4s.Formatters._ 7 | 8 | 9 | @(r: Option[(UserGroupQuery, UserGroupQueryResult)])(implicit env: UIEnv) 10 | 11 | @topNav("Test Assignments", "Assignments") { 12 |
    13 |
    14 |
    15 |
    Test A/B Test Assignment
    16 | All fields optional 17 |
    18 | 19 | 21 |
    22 |
    23 | 24 | 26 |
    27 |
    28 | 29 | 30 |
    31 | 32 |
    33 | 34 | 37 |
    38 |
    39 | 40 |
    41 |
    42 | @for(result <- r.map(_._2)){ 43 |
    Assignments
    44 | 45 | At: @formatDate(result.at) 46 | 47 |
    48 | 49 | 50 | 53 | 56 | 59 | 60 | 61 | @for( ga <- result.groups.toList) { 62 | 63 | 64 | 65 | 66 | 67 | } 68 | 69 |
    51 | Feature 52 | 54 | Group Assigned 55 | 57 | Group Meta 58 |
    @ga._1@ga._2@result.metas.get(ga._1)
    70 |
    71 | 72 | } 73 |
    74 |
    75 | 76 | 88 | } -------------------------------------------------------------------------------- /docs/docs/core.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Core concepts" 4 | section: "Core" 5 | position: 70 6 | --- 7 | 8 | 9 | 10 | ### Core concepts 11 | 12 | 13 | * Feature - a string representing a feature in the product to be A/B tested. 14 | A/B tests in Thomas are organized by the feature they are testing against. 15 | For each feature there is at most one ongoing test at a specific time. 16 | A/B test treatment group assignments are by features, no by specific tests. 17 | That is, when Thomas receives an inquiry about the treatment group assignments for a user, 18 | it returns a list of pairs of feature and group name. 19 | Specific test ids are hidden from such inquiries. 20 | For a client to determine which treatment to give for a feature, 21 | all it needs is the assigned group name for that feature. 22 | 23 | * A/B test - represents a single experiment that gives different treatments 24 | to different groups of users for a certain feature during a period. 25 | It's often that people run multiple rounds of A/B tests to determine the optimal 26 | treatment for a feature. In Thomas, users can create one A/B test after another 27 | testing a specific feature. This series of A/B tests can be deemed as evolving 28 | versions of the experiment. The A/B test's group setup becomes immutable after it starts. 29 | To revise a running A/B test experiment, user has to terminate it and start a new one. 30 | This immutability is required to guarantee the correctness of the history as well 31 | as enabling the distributed assignment computation. 32 | For more details about the metadata in an A/B test, check [the API documentation of AbtestSpec](https://iheartradio.github.io/thomas/api/com/iheart/thomas/model/AbtestSpec.html) 33 | 34 | 35 | * Eligibility Control - 36 | Mechanism to determine which users are eligible for which experiments. 37 | When a feature is not available to certain users, it is desirable to avoid 38 | providing any group assignment for these users. 39 | For one thing, it may become a source of false report in analytic data. 40 | This feature also allows targeting a subset of all users for a certain experiment. 41 | 42 | * User Meta - When client send a group 43 | assignment inquiry request, it can include a Json object as the meta data for the user. 44 | This then can be used for eligibility control. A/B tests can be set with a 45 | filtering criteria to determine which users are eligible based on 46 | the user meta passed in. These criteria is written in query language in JSON format. 47 | 48 | 49 | * Group Meta - a meta data associated with a treatment group that is included in group 50 | assignment response. This metadata could potentially be used to determine the treatment 51 | end-user receives. Client could, instead of hard code different behavior based on group 52 | names, use some configuration to control the behavior. 53 | The group meta can provide such configurations. Thus different group assignment can result 54 | in different treatment thanks to the different configurations in their group meta. 55 | 56 | * Overrides - to fix group assignment for certain users. User assignment is stochastic 57 | which could be tricky when QAing an A/B tests. Overrides allow users to fix group assignments 58 | for certain users so that QA can test expected behavior from the product for those users. --------------------------------------------------------------------------------