├── .git-blame-ignore-revs ├── .github └── workflows │ ├── ci.yml │ ├── dependency-graph.yml │ └── release.yml ├── .gitignore ├── .jvmopts ├── .scalafix.conf ├── .scalafmt.conf ├── LICENSE ├── README.md ├── build.sbt ├── core-it-tests └── src │ └── test │ └── scala │ └── com │ └── evolutiongaming │ └── kafka │ └── flow │ ├── ForAllKafkaSuite.scala │ └── ShutdownSpec.scala ├── core └── src │ ├── main │ ├── scala-3 │ │ └── com │ │ │ └── evolutiongaming │ │ │ └── kafka │ │ │ └── flow │ │ │ └── persistence │ │ │ └── compression │ │ │ └── CodecOps.scala │ └── scala │ │ └── com │ │ └── evolutiongaming │ │ └── kafka │ │ └── flow │ │ ├── AdditionalStatePersist.scala │ │ ├── AdditionalStatePersistOf.scala │ │ ├── ConsumerFlow.scala │ │ ├── ConsumerFlowConfig.scala │ │ ├── ConsumerFlowOf.scala │ │ ├── EnhancedFold.scala │ │ ├── Fold.scala │ │ ├── FoldOption.scala │ │ ├── FoldToState.scala │ │ ├── KafkaFlow.scala │ │ ├── KafkaKey.scala │ │ ├── KeyContext.scala │ │ ├── KeyFlow.scala │ │ ├── KeyFlowExtras.scala │ │ ├── KeyFlowOf.scala │ │ ├── KeyState.scala │ │ ├── KeyStateOf.scala │ │ ├── LogPrefix.scala │ │ ├── LogResource.scala │ │ ├── PartitionFlow.scala │ │ ├── PartitionFlowConfig.scala │ │ ├── PartitionFlowOf.scala │ │ ├── RebalanceListener.scala │ │ ├── RemapKey.scala │ │ ├── Tick.scala │ │ ├── TickOption.scala │ │ ├── TickToState.scala │ │ ├── ToOffset.scala │ │ ├── TopicFlow.scala │ │ ├── TopicFlowOf.scala │ │ ├── effect │ │ └── CatsEffectMtlInstances.scala │ │ ├── journal │ │ ├── JournalDatabase.scala │ │ ├── Journals.scala │ │ └── JournalsOf.scala │ │ ├── kafka │ │ ├── Codecs.scala │ │ ├── Consumer.scala │ │ ├── ConsumerOf.scala │ │ ├── EmptyRebalanceConsumer.scala │ │ ├── KafkaModule.scala │ │ ├── OffsetToCommit.scala │ │ ├── PendingCommits.scala │ │ └── ScheduleCommit.scala │ │ ├── key │ │ ├── KeyDatabase.scala │ │ ├── Keys.scala │ │ └── KeysOf.scala │ │ ├── package.scala │ │ ├── persistence │ │ ├── Persistence.scala │ │ ├── PersistenceModule.scala │ │ ├── PersistenceOf.scala │ │ └── compression │ │ │ ├── Compression.scala │ │ │ ├── CompressionError.scala │ │ │ ├── Compressor.scala │ │ │ ├── CompressorSyntax.scala │ │ │ ├── Header.scala │ │ │ └── HeaderAndPayload.scala │ │ ├── registry │ │ └── EntityRegistry.scala │ │ ├── snapshot │ │ ├── KafkaSnapshot.scala │ │ ├── SnapshotDatabase.scala │ │ ├── SnapshotFold.scala │ │ ├── Snapshots.scala │ │ └── SnapshotsOf.scala │ │ └── timer │ │ ├── KafkaTimer.scala │ │ ├── TimerContext.scala │ │ ├── TimerFlow.scala │ │ ├── TimerFlowOf.scala │ │ ├── TimerType.scala │ │ ├── TimerWindow.scala │ │ ├── Timers.scala │ │ ├── TimersOf.scala │ │ ├── Timestamp.scala │ │ └── Timestamps.scala │ └── test │ └── scala │ └── com │ └── evolutiongaming │ └── kafka │ └── flow │ ├── AdditionalPersistSpec.scala │ ├── ConsumerFlowSpec.scala │ ├── ExplodingRebalanceConsumer.scala │ ├── FoldSpec.scala │ ├── FoldToStateSpec.scala │ ├── KafkaFlowSpec.scala │ ├── KeyFlowSpec.scala │ ├── MonadStateHelper.scala │ ├── PartitionFlowSpec.scala │ ├── RebalanceListenerSpec.scala │ ├── journal │ ├── JournalDatabaseSpec.scala │ └── JournalsSpec.scala │ ├── key │ └── KeysSpec.scala │ ├── persistence │ ├── PersistenceSpec.scala │ └── compression │ │ └── CompressorSpec.scala │ ├── registry │ └── EntityRegistryTest.scala │ ├── snapshot │ ├── SnapshotFoldSpec.scala │ └── SnapshotsSpec.scala │ └── timer │ └── TimerFlowOfSpec.scala ├── docs ├── faq.md ├── overview.md ├── persistence.md ├── setup.md └── styleguide.md ├── kafka-journal └── src │ ├── main │ └── scala │ │ └── com │ │ └── evolutiongaming │ │ └── kafka │ │ └── flow │ │ └── journal │ │ ├── JournalFold.scala │ │ └── JournalParser.scala │ └── test │ └── scala │ └── com │ └── evolutiongaming │ └── kafka │ └── flow │ └── journal │ ├── JournalFoldSpec.scala │ └── JournalParserSpec.scala ├── metrics └── src │ ├── main │ └── scala │ │ └── com │ │ └── evolutiongaming │ │ └── kafka │ │ └── flow │ │ ├── FlowMetrics.scala │ │ ├── FoldMetrics.scala │ │ ├── KeyStateMetrics.scala │ │ ├── PartitionFlowMetrics.scala │ │ ├── TopicFlowMetrics.scala │ │ ├── compression │ │ └── CompressorMetrics.scala │ │ ├── journal │ │ └── JournalDatabaseMetrics.scala │ │ ├── key │ │ └── KeyDatabaseMetrics.scala │ │ ├── metrics │ │ ├── Metrics.scala │ │ ├── MetricsK.scala │ │ └── syntax │ │ │ └── package.scala │ │ ├── persistence │ │ └── PersistenceModuleMetrics.scala │ │ └── snapshot │ │ └── SnapshotDatabaseMetrics.scala │ └── test │ └── scala │ └── com │ └── evolutiongaming │ └── kafka │ └── flow │ ├── FoldMetricsSpec.scala │ └── metrics │ └── SyntaxSpec.scala ├── persistence-cassandra-it-tests └── src │ └── test │ └── scala │ └── com │ └── evolutiongaming │ └── kafka │ └── flow │ ├── CassandraContainerResource.scala │ ├── CassandraSessionStub.scala │ ├── CassandraSpec.scala │ ├── FlowSpec.scala │ ├── journal │ ├── JournalSchemaSpec.scala │ └── JournalSpec.scala │ ├── key │ ├── KeySchemaSpec.scala │ └── KeySpec.scala │ └── snapshot │ ├── SnapshotSchemaSpec.scala │ └── SnapshotSpec.scala ├── persistence-cassandra └── src │ └── main │ └── scala │ └── com │ ├── evolution │ └── kafka │ │ └── flow │ │ └── cassandra │ │ ├── CassandraHealthCheckOf.scala │ │ └── CassandraModule.scala │ └── evolutiongaming │ └── kafka │ └── flow │ ├── cassandra │ ├── CassandraCodecs.scala │ ├── CassandraConfig.scala │ ├── CassandraPersistence.scala │ ├── ConsistencyOverrides.scala │ ├── RecordExpiration.scala │ ├── SessionHelper.scala │ └── StatementHelper.scala │ ├── journal │ ├── CassandraJournals.scala │ ├── JournalSchema.scala │ └── conversions │ │ ├── HeaderToTuple.scala │ │ └── TupleToHeader.scala │ ├── key │ ├── CassandraKeys.scala │ ├── KeySchema.scala │ ├── KeySegments.scala │ └── SegmentNr.scala │ └── snapshot │ ├── CassandraSnapshots.scala │ └── SnapshotSchema.scala ├── persistence-kafka-it-tests └── src │ └── test │ ├── resources │ └── logback-test.xml │ └── scala │ └── com │ └── evolutiongaming │ └── kafka │ └── flow │ ├── ForAllKafkaSuite.scala │ ├── RemapKeySpec.scala │ └── StatefulProcessingWithKafkaSpec.scala ├── persistence-kafka └── src │ └── main │ └── scala │ └── com │ └── evolutiongaming │ └── kafka │ └── flow │ └── kafkapersistence │ ├── KafkaPartitionPersistence.scala │ ├── KafkaPersistenceModule.scala │ ├── KafkaPersistenceModuleOf.scala │ ├── KafkaPersistencePartitionMapper.scala │ ├── KafkaSnapshotReadDatabase.scala │ ├── KafkaSnapshotWriteDatabase.scala │ └── package.scala ├── project ├── Dependencies.scala ├── build.properties └── plugins.sbt └── website ├── blog ├── 2016-03-11-blog-post.md ├── 2017-04-10-blog-post-two.md ├── 2017-09-25-testing-rss.md ├── 2017-09-26-adding-rss.md └── 2017-10-24-new-version-1.0.0.md ├── core └── Footer.js ├── i18n └── en.json ├── package.json ├── pages └── en │ ├── help.js │ ├── index.js │ └── users.js ├── sidebars.json ├── siteConfig.js ├── static ├── css │ └── custom.css └── img │ ├── favicon.ico │ ├── oss_logo.png │ ├── undraw_code_review.svg │ ├── undraw_fast_loading_0lbh.svg │ ├── undraw_fishing_hoxa.svg │ ├── undraw_floating_61u6.svg │ ├── undraw_monitor.svg │ ├── undraw_note_list.svg │ ├── undraw_online.svg │ ├── undraw_open_source.svg │ ├── undraw_operating_system.svg │ ├── undraw_react.svg │ ├── undraw_server_cluster_jwwq.svg │ ├── undraw_tweetstorm.svg │ └── undraw_youtube_tutorial.svg └── yarn.lock /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Format all code with scalafmt 2 | 1b13f44fefe2363b2fca626e136879f08f4d4a5f 3 | 4 | # Format all code with scalafmt 3.7.3 5 | 7d16f65e44aa2b7bf1ac0cd031ea762dea0bb5e9 6 | 7 | # Scala Steward: Reformat with scalafmt 3.7.4 8 | 3fdc4ecbe2c83dd36ed03eb55dc7f3966abb410b 9 | 10 | # Scala Steward: Reformat with scalafmt 3.7.6 11 | 9812bdced2f50b3f4782eadfff9250d2f6fe1f59 12 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | strategy: 11 | matrix: 12 | scala: 13 | - 2.13.16 14 | - 3.3.5 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - uses: coursier/cache-action@v6 20 | 21 | - name: setup Java 17 22 | uses: actions/setup-java@v4 23 | with: 24 | java-version: '17' 25 | distribution: 'oracle' 26 | cache: 'sbt' 27 | 28 | - name: setup SBT 29 | uses: sbt/setup-sbt@v1 30 | 31 | - name: Check formatting 32 | run: sbt scalafmtCheckAll 33 | 34 | - name: Run tests ${{ matrix.scala }} 35 | if: success() 36 | run: sbt clean coverage "++${{ matrix.scala }} test" docs/mdoc "++${{ matrix.scala }} versionPolicyCheck" 37 | 38 | - name: Report test coverage 39 | if: success() && github.repository == 'evolution-gaming/kafka-flow' 40 | env: 41 | COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} 42 | run: sbt "++${{ matrix.scala }} coverageReport" coverageAggregate coveralls 43 | 44 | - name: Publish documentation / Setup Node 45 | if: success() 46 | uses: actions/setup-node@v3 47 | with: 48 | node-version: '12.x' 49 | 50 | - name: Publish documentation / Get yarn cache 51 | if: success() 52 | id: yarn-cache 53 | run: echo "::set-output name=dir::$(yarn cache dir)" 54 | 55 | - name: Publish documentation / Cache dependencies 56 | if: success() 57 | uses: actions/cache@v3 58 | with: 59 | path: ${{ steps.yarn-cache.outputs.dir }} 60 | key: ${{ runner.os }}-website-${{ hashFiles('**/yarn.lock') }} 61 | restore-keys: | 62 | ${{ runner.os }}-website- 63 | 64 | - name: Publish documentation / Install dependencies 65 | if: success() 66 | working-directory: ./website 67 | run: yarn install --frozen-lockfile 68 | 69 | - name: Publish documentation / Build site 70 | if: success() 71 | working-directory: ./website 72 | run: yarn build 73 | 74 | - name: Publish documentation / Deploy 75 | if: success() && github.ref == 'refs/heads/master' && github.repository == 'evolution-gaming/kafka-flow' 76 | uses: peaceiris/actions-gh-pages@v3 77 | with: 78 | github_token: ${{ secrets.GITHUB_TOKEN }} 79 | publish_dir: ./website/build/kafka-flow 80 | 81 | - name: Slack Notification 82 | uses: homoluctus/slatify@master 83 | if: failure() && github.ref == 'refs/heads/master' && github.repository == 'evolution-gaming/kafka-flow' 84 | with: 85 | type: ${{ job.status }} 86 | job_name: Build 87 | url: ${{ secrets.SLACK_WEBHOOK }} 88 | -------------------------------------------------------------------------------- /.github/workflows/dependency-graph.yml: -------------------------------------------------------------------------------- 1 | name: Update Dependency Graph 2 | on: 3 | push: 4 | branches: 5 | - master 6 | jobs: 7 | dependency-graph: 8 | name: Update Dependency Graph 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | 13 | - name: setup Java 17 14 | uses: actions/setup-java@v4 15 | with: 16 | java-version: '17' 17 | distribution: 'oracle' 18 | cache: 'sbt' 19 | 20 | - name: setup SBT 21 | uses: sbt/setup-sbt@v1 22 | 23 | - uses: scalacenter/sbt-dependency-submission@v2 24 | with: 25 | modules-ignore: > 26 | docs_2.13 27 | kafka-flow-core-it-tests_2.13 28 | kafka-flow-persistence-kafka-it-tests_2.13 29 | kafka-flow-persistence-cassandra-it-tests_2.13 30 | configs-ignore: test integration-test scala-tool scala-doc-tool 31 | permissions: 32 | contents: write 33 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Test and publish a new release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | release: 10 | uses: evolution-gaming/scala-github-actions/.github/workflows/release.yml@v3 11 | secrets: inherit 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | .bsp 6 | .cache 7 | .history 8 | .lib/ 9 | dist/* 10 | target/ 11 | lib_managed/ 12 | src_managed/ 13 | project/boot/ 14 | project/plugins/project/ 15 | 16 | # Scala-IDE specific 17 | .scala_dependencies 18 | .worksheet 19 | 20 | # Idea 21 | .idea/ 22 | *.iml 23 | *.ipr 24 | 25 | # Metals 26 | .bloop 27 | .vscode 28 | .history 29 | .metals 30 | settings.json 31 | metals.sbt 32 | 33 | # Docusaurus 34 | website/translated_docs 35 | website/build/ 36 | website/node_modules 37 | website/i18n/* 38 | !website/i18n/en.json 39 | 40 | # Mac 41 | .DS_Store 42 | -------------------------------------------------------------------------------- /.jvmopts: -------------------------------------------------------------------------------- 1 | -Dcats.effect.traceBufferLogSize=5 2 | -------------------------------------------------------------------------------- /.scalafix.conf: -------------------------------------------------------------------------------- 1 | OrganizeImports { 2 | targetDialect = Scala3 3 | blankLines = Auto 4 | coalesceToWildcardImportThreshold = 5 5 | expandRelative = false 6 | groupExplicitlyImportedImplicitsSeparately = false 7 | groupedImports = Merge 8 | groups = [ 9 | "*" 10 | "re:(javax?|scala)\\." 11 | ] 12 | importSelectorsOrder = Ascii 13 | importsOrder = Ascii 14 | preset = INTELLIJ_2020_3 15 | removeUnused = true 16 | } -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | // Trial version of the config 2 | // https://scalameta.org/scalafmt/docs/configuration.html 3 | project.git = true 4 | 5 | runner.dialect = scala213source3 6 | 7 | version = 3.7.14 8 | 9 | maxColumn = 120 10 | 11 | trailingCommas = preserve 12 | 13 | continuationIndent { 14 | callSite = 2 15 | defnSite = 2 16 | } 17 | 18 | align.preset = some 19 | align.tokenCategory { 20 | LeftArrow = Assign 21 | Equals = Assign 22 | } 23 | # Mostly subset of `align.preset=more`, but with extra settings for `=` 24 | # For all settings see `AlignToken#default` in 25 | # https://github.com/scalameta/scalafmt/blob/master/scalafmt-core/shared/src/main/scala/org/scalafmt/config/AlignToken.scala 26 | align.tokens.add = [ 27 | { code = "%", owner = "Term.ApplyInfix" } # This is for Dependencies.scala… 28 | { code = "%%", owner = "Term.ApplyInfix" } # … and this as well. 29 | { code = "%%%", owner = "Term.ApplyInfix" } # … and this as well. 30 | { code = "=>", owner = "(Case|Term.Function)" } 31 | { code = "<-", owner = "Enumerator.Generator" } 32 | { code = "=", owner = "(Defn.Val|Defn.Var|Type|Def|Enumerator.Val|Assign|Term.Param)" } # Defn.Val is mostly for Dependencies.scala, 33 | { code = "->", owner = "Term.ApplyInfix" } 34 | ] 35 | 36 | newlines.implicitParamListModifierPrefer = before 37 | newlines.beforeCurlyLambdaParams = multilineWithCaseOnly 38 | 39 | indentOperator.topLevelOnly = true 40 | 41 | docstrings.blankFirstLine = keep 42 | 43 | includeCurlyBraceInSelectChains = true 44 | includeNoParensInSelectChains = true 45 | optIn.breakChainOnFirstMethodDot = true 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Evolution Gaming 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-flow 2 | [![Build Status](https://github.com/evolution-gaming/kafka-flow/workflows/CI/badge.svg)](https://github.com/evolution-gaming/kafka-flow/actions?query=workflow%3ACI) 3 | [![Coverage Status](https://coveralls.io/repos/github/evolution-gaming/kafka-flow/badge.svg?branch=master)](https://coveralls.io/github/evolution-gaming/kafka-flow?branch=master) 4 | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/9d1c127fef824d4faaf775a1a5c5fb0d)](https://app.codacy.com/gh/evolution-gaming/kafka-flow/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) 5 | [![Version](https://img.shields.io/badge/version-click-blue)](https://evolution.jfrog.io/artifactory/api/search/latestVersion?g=com.evolutiongaming&a=kafka-flow_2.13&repos=public) 6 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellowgreen.svg)](https://opensource.org/licenses/MIT) 7 | 8 | ## Microsite 9 | 10 | https://evolution-gaming.github.io/kafka-flow 11 | 12 | ## Scala 3 compatibility 13 | Starting from version `6.1.0` all of the modules are cross-compiled to Scala 3 except `kafka-flow-kafka-journal` which doesn't support Scala 3 yet. 14 | 15 | ## Setup 16 | 17 | ```scala 18 | addSbtPlugin("com.evolution" % "sbt-artifactory-plugin" % "0.0.2") 19 | 20 | lazy val version = "" // see the latest one in the badge above or in Releases page 21 | 22 | libraryDependencies ++= Seq( 23 | "com.evolutiongaming" %% "kafka-flow" % version, 24 | // if you want to use Cassandra for storing persistent state 25 | "com.evolutiongaming" %% "kafka-flow-persistence-cassandra" % version, 26 | // if you want to use Kafka compact topic for storing persistent state 27 | "com.evolutiongaming" %% "kafka-flow-persistence-kafka" % version, 28 | // if you want to use predefined metrics 29 | "com.evolutiongaming" %% "kafka-flow-metrics" % version, 30 | // if you want to use kafka-journal integration 31 | "com.evolutiongaming" %% "kafka-flow-kafka-journal" % version, 32 | ) 33 | ``` 34 | 35 | ## Release process 36 | The release process is based on Git tags and makes use of [evolution-gaming/scala-github-actions](https://github.com/evolution-gaming/scala-github-actions) which uses [sbt-dynver](https://github.com/sbt/sbt-dynver) to automatically obtain the version from the latest Git tag. The flow is defined in `.github/workflows/release.yml`. 37 | A typical release process is as follows: 38 | 1. Create and push a new Git tag. The version should be in the format `vX.Y.Z` (example: `v4.1.0`). Example: `git tag v4.1.0 && git push origin v4.1.0` 39 | 2. On success, a new GitHub release is automatically created with a calculated diff and auto-generated release notes. You can see it on `Releases` page, change the description if needed 40 | 3. On failure, the tag is deleted from the remote repository. Please note that your local tag isn't deleted, so if the failure is recoverable then you can delete the local tag and try again (an example of *unrecoverable* failure is successfully publishing only a few of the artifacts to Artifactory which means a new attempt would fail since Artifactory doesn't allow overwriting its contents) 41 | -------------------------------------------------------------------------------- /core-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/ForAllKafkaSuite.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.data.NonEmptyList 4 | import cats.effect.IO 5 | import com.dimafeng.testcontainers.KafkaContainer 6 | import com.dimafeng.testcontainers.munit.fixtures.TestContainersFixtures 7 | import com.evolutiongaming.catshelper.LogOf 8 | import com.evolutiongaming.kafka.flow.kafka.KafkaModule 9 | import com.evolutiongaming.skafka.CommonConfig 10 | import com.evolutiongaming.skafka.consumer.ConsumerConfig 11 | import com.evolutiongaming.smetrics.CollectorRegistry 12 | import munit.FunSuite 13 | 14 | import java.util.concurrent.atomic.AtomicReference 15 | 16 | abstract class ForAllKafkaSuite extends FunSuite with TestContainersFixtures { 17 | import cats.effect.unsafe.implicits.global 18 | 19 | val kafka = ForAllContainerFixture(KafkaContainer()) 20 | 21 | val kafkaModule = new Fixture[KafkaModule[IO]]("KafkaModule") { 22 | private val moduleRef = new AtomicReference[(KafkaModule[IO], IO[Unit])]() 23 | 24 | override def apply(): KafkaModule[IO] = moduleRef.get()._1 25 | 26 | override def beforeAll(): Unit = { 27 | val config = 28 | ConsumerConfig(common = CommonConfig(bootstrapServers = NonEmptyList.one(kafka.container.bootstrapServers))) 29 | implicit val logOf: LogOf[IO] = LogOf.slf4j[IO].unsafeRunSync() 30 | val result = KafkaModule.of[IO]("KafkaSuite", config, CollectorRegistry.empty[IO]).allocated.unsafeRunSync() 31 | moduleRef.set(result) 32 | } 33 | 34 | override def afterAll(): Unit = Option(moduleRef.get()).foreach { case (_, finalizer) => finalizer.unsafeRunSync() } 35 | } 36 | 37 | override def munitFixtures: Seq[Fixture[_]] = List(kafka, kafkaModule) 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/scala-3/com/evolutiongaming/kafka/flow/persistence/compression/CodecOps.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence.compression 2 | 3 | import scodec.Codec 4 | import scodec.Codec.inlineImplementations 5 | 6 | // format: off 7 | extension[A, B](codecA: Codec[A]) 8 | /** Combines this Codec with another one, added for compatibility with Scala 2.13 version of scodec-core. 9 | * @param codecB 10 | * Codec for B 11 | * @return 12 | * Codec for tuple of A and B 13 | */ 14 | def ~(codecB: Codec[B]): Codec[(A, B)] = 15 | codecA :: codecB 16 | // format: on 17 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/AdditionalStatePersistOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Applicative 4 | import cats.effect.{Clock, MonadCancelThrow, Ref} 5 | import com.evolutiongaming.kafka.flow.persistence.Persistence 6 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 7 | import scodec.bits.ByteVector 8 | 9 | import scala.concurrent.duration.FiniteDuration 10 | 11 | /** A factory of `AdditionalStatePersist`. It's invoked when a key is recovered, either from a persistence layer or from 12 | * a source topic. 13 | * 14 | * @see 15 | * [[com.evolutiongaming.kafka.flow.KeyStateOf]] for usage during recovery of a key 16 | */ 17 | trait AdditionalStatePersistOf[F[_], S] { 18 | def apply( 19 | persistence: Persistence[F, S, ConsumerRecord[String, ByteVector]], 20 | keyContext: KeyContext[F] 21 | ): F[AdditionalStatePersist[F, S, ConsumerRecord[String, ByteVector]]] 22 | } 23 | 24 | object AdditionalStatePersistOf { 25 | def empty[F[_]: Applicative, S]: AdditionalStatePersistOf[F, S] = 26 | new AdditionalStatePersistOf[F, S] { 27 | override def apply( 28 | persistence: Persistence[F, S, ConsumerRecord[String, ByteVector]], 29 | keyContext: KeyContext[F] 30 | ): F[AdditionalStatePersist[F, S, ConsumerRecord[String, ByteVector]]] = 31 | Applicative[F].pure(AdditionalStatePersist.empty[F, S, ConsumerRecord[String, ByteVector]]) 32 | } 33 | 34 | def of[F[_]: MonadCancelThrow: Ref.Make: Clock, S]( 35 | cooldown: FiniteDuration, 36 | ignorePersistErrors: Boolean = false, 37 | ): AdditionalStatePersistOf[F, S] = { 38 | new AdditionalStatePersistOf[F, S] { 39 | def apply( 40 | persistence: Persistence[F, S, ConsumerRecord[String, ByteVector]], 41 | keyContext: KeyContext[F] 42 | ): F[AdditionalStatePersist[F, S, ConsumerRecord[String, ByteVector]]] = { 43 | AdditionalStatePersist.of(persistence, keyContext, cooldown, ignorePersistErrors) 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/ConsumerFlowConfig.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import scala.concurrent.duration.* 4 | 5 | /** Configuration of `ConsumerFlow`. 6 | * 7 | * @param pollTimeout 8 | * See `KafkaConsumer#poll(timout)` for more details. 9 | */ 10 | case class ConsumerFlowConfig( 11 | pollTimeout: FiniteDuration = 10.milliseconds 12 | ) 13 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/ConsumerFlowOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.MonadThrow 4 | import cats.data.NonEmptySet 5 | import cats.effect.Resource 6 | import com.evolutiongaming.catshelper.LogOf 7 | import com.evolutiongaming.skafka.Topic 8 | import kafka.Consumer 9 | 10 | /** Factory which creates `ConsumerFlow` instances */ 11 | trait ConsumerFlowOf[F[_]] { 12 | 13 | def apply(consumer: Consumer[F]): Resource[F, ConsumerFlow[F]] 14 | 15 | } 16 | object ConsumerFlowOf { 17 | 18 | /** Constructs a consumer flow for specific topic. 19 | * 20 | * Note, that topic specified by an appropriate parameter should contain a journal in the format of `Kafka Journal` 21 | * library. 22 | */ 23 | def apply[F[_]: MonadThrow: LogOf]( 24 | topic: Topic, 25 | flowOf: TopicFlowOf[F], 26 | config: ConsumerFlowConfig = ConsumerFlowConfig() 27 | ): ConsumerFlowOf[F] = { consumer => 28 | ConsumerFlow.of(consumer, topic, flowOf, config) 29 | } 30 | 31 | /** Constructs a consumer flow for specific topics. 32 | * 33 | * Note, that topics specified by an appropriate parameter should contain a journal in the format of `Kafka Journal` 34 | * library. 35 | */ 36 | def apply[F[_]: MonadThrow: LogOf]( 37 | topics: NonEmptySet[Topic], 38 | flowOf: TopicFlowOf[F], 39 | ): ConsumerFlowOf[F] = ConsumerFlowOf(topics, flowOf, ConsumerFlowConfig()) 40 | 41 | /** Constructs a consumer flow for specific topics. 42 | * 43 | * Note, that topics specified by an appropriate parameter should contain a journal in the format of `Kafka Journal` 44 | * library. 45 | */ 46 | def apply[F[_]: MonadThrow: LogOf]( 47 | topics: NonEmptySet[Topic], 48 | flowOf: TopicFlowOf[F], 49 | config: ConsumerFlowConfig 50 | ): ConsumerFlowOf[F] = { consumer => 51 | ConsumerFlow.of(consumer, topics, flowOf, config) 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/EnhancedFold.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.syntax.all.* 4 | import cats.{Applicative, ApplicativeError, Monad} 5 | 6 | /** Given an optional state `S` and an incoming event `E`, produces a resulting optional state `S`. This is the main 7 | * user API to define stateful processing. In contrast to `FoldOption`, it also accepts `KeyFlowExtras`, providing an 8 | * access to some additional framework APIs. 9 | * 10 | * This abstraction is not expressed in terms of `FoldOption` to avoid unnecessary allocations of `FoldOption` when the 11 | * logic involves multiple nested `EnhancedFold` instances invoking each other. 12 | * 13 | * @tparam F 14 | * computation effect 15 | * @tparam S 16 | * state 17 | * @tparam E 18 | * incoming event 19 | */ 20 | trait EnhancedFold[F[_], S, E] { 21 | 22 | def apply(extras: KeyFlowExtras[F], s: Option[S], e: E): F[Option[S]] 23 | 24 | final def flatMap(f: S => EnhancedFold[F, S, E])(implicit F: Monad[F]): EnhancedFold[F, S, E] = 25 | (extras, state0, event) => { 26 | apply(extras, state0, event).flatMap { 27 | case s @ Some(state1) => f(state1).apply(extras, s, event) 28 | case None => F.pure(None) 29 | } 30 | } 31 | 32 | final def filter(f: (S, E) => Boolean)(implicit F: Applicative[F]): EnhancedFold[F, S, E] = 33 | (extras, state0, event) => 34 | state0 match { 35 | case Some(state) => if (f(state, event)) apply(extras, state0, event) else F.pure(state0) 36 | case None => apply(extras, state0, event) 37 | } 38 | 39 | final def filterM(f: (S, E) => F[Boolean])(implicit F: Monad[F]): EnhancedFold[F, S, E] = 40 | (extras, state0, event) => 41 | state0 match { 42 | case Some(state) => F.ifM(f(state, event))(ifTrue = apply(extras, state0, event), ifFalse = F.pure(state0)) 43 | case None => apply(extras, state0, event) 44 | } 45 | 46 | final def handleErrorWith[E1](f: (S, E1) => F[S])(implicit F: ApplicativeError[F, E1]): EnhancedFold[F, S, E] = 47 | (extras, state0, event) => apply(extras, state0, event).handleErrorWith(e => state0.traverse(state => f(state, e))) 48 | 49 | } 50 | 51 | object EnhancedFold { 52 | 53 | def empty[F[_], S, E](implicit F: Applicative[F]): EnhancedFold[F, S, E] = (_, _, _) => F.pure(None) 54 | 55 | /** Creates an instance of `EnhancedFold` from a function; similar to `FoldOption.of` */ 56 | def of[F[_], S, E](f: (KeyFlowExtras[F], Option[S], E) => F[Option[S]]): EnhancedFold[F, S, E] = 57 | (extras, state, event) => f(extras, state, event) 58 | 59 | /** Convenience method to transform a non-enhanced `FoldOption` into an enhanced one */ 60 | def fromFold[F[_], S, E](fold: FoldOption[F, S, E]): EnhancedFold[F, S, E] = 61 | EnhancedFold.of((_, s, e) => fold(s, e)) 62 | } 63 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/KafkaFlow.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.effect.{Concurrent, Resource} 4 | import cats.effect.implicits.* 5 | import cats.effect.kernel.Outcome 6 | import cats.syntax.all.* 7 | import com.evolutiongaming.catshelper.{BracketThrowable, LogOf} 8 | import com.evolutiongaming.kafka.flow.kafka.Consumer 9 | import com.evolutiongaming.random.Random 10 | import com.evolutiongaming.retry.{OnError, Retry, Sleep, Strategy} 11 | import com.evolutiongaming.sstream.Stream 12 | 13 | import scala.concurrent.duration.* 14 | import scodec.bits.ByteVector 15 | import com.evolutiongaming.skafka.consumer.ConsumerRecords 16 | 17 | object KafkaFlow { 18 | 19 | /** Process records from consumer with default retry strategy 20 | * 21 | * Returns records already processed by the `KafkaFlow`. 22 | * 23 | * Note, that returned record does not guarantee that commit to Kafka happened, i.e. that the record will not be 24 | * processsed for the second time. 25 | * 26 | * WARNING: Do not forget to `flatMap` returned `F[Unit]` or the potential errors may be lost. 27 | */ 28 | def retryOnError[F[_]: Concurrent: Sleep: LogOf]( 29 | consumer: Resource[F, Consumer[F]], 30 | flowOf: ConsumerFlowOf[F] 31 | ): Resource[F, F[Unit]] = { 32 | 33 | val retry = for { 34 | random <- Random.State.fromClock[F]() 35 | log <- LogOf[F].apply(KafkaFlow.getClass) 36 | } yield Retry( 37 | strategy = Strategy 38 | .exponential(100.millis) 39 | .jitter(random) 40 | .cap(1.minute) 41 | .resetAfter(5.minutes), 42 | onError = OnError.fromLog(log) 43 | ) 44 | 45 | Resource.eval(retry) flatMap { implicit retry => 46 | resource(consumer, flowOf) 47 | } 48 | 49 | } 50 | 51 | /** Process records from consumer with given flow and retry strategy 52 | * 53 | * Returns records already processed by the `KafkaFlow`. 54 | * 55 | * Note, that returned record does not guarantee that commit to Kafka happened, i.e. that the record will not be 56 | * processsed for the second time. 57 | */ 58 | def stream[F[_]: BracketThrowable: Retry]( 59 | consumer: Resource[F, Consumer[F]], 60 | flowOf: ConsumerFlowOf[F] 61 | ): Stream[F, ConsumerRecords[String, ByteVector]] = 62 | for { 63 | _ <- Stream.around(Retry[F].toFunctionK) 64 | consumer <- Stream.fromResource(consumer) 65 | flow <- Stream.fromResource(flowOf(consumer)) 66 | records <- flow.stream 67 | } yield records 68 | 69 | /** Process records from consumer with given flow and retry strategy 70 | * 71 | * Tears down if cancelled or retry strategy failed. 72 | * 73 | * WARNING: Do not forget to `flatMap` returned `F[Unit]` or the potential errors may be lost. 74 | */ 75 | def resource[F[_]: Concurrent: Retry]( 76 | consumer: Resource[F, Consumer[F]], 77 | flowOf: ConsumerFlowOf[F] 78 | ): Resource[F, F[Unit]] = 79 | stream(consumer, flowOf) 80 | .drain 81 | .background 82 | .map(_.flatMap { 83 | case Outcome.Succeeded(fa) => fa 84 | case Outcome.Errored(e) => e.raiseError[F, Unit] 85 | case Outcome.Canceled() => Concurrent[F].canceled 86 | }) 87 | 88 | } 89 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/KafkaKey.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.kernel.Hash 4 | import com.evolutiongaming.skafka.TopicPartition 5 | 6 | final case class KafkaKey( 7 | applicationId: String, 8 | groupId: String, 9 | topicPartition: TopicPartition, 10 | key: String 11 | ) 12 | 13 | object KafkaKey { 14 | implicit val hashKafkaKey: Hash[KafkaKey] = Hash.fromUniversalHashCode 15 | implicit val logPrefix: LogPrefix[KafkaKey] = LogPrefix.function(key => s"${key.topicPartition} ${key.key}") 16 | } 17 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/KeyContext.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.effect.{Ref, Resource} 4 | import cats.mtl.Stateful 5 | import cats.syntax.all.* 6 | import cats.{Applicative, Monad} 7 | import com.evolutiongaming.catshelper.Log 8 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 9 | import com.evolutiongaming.skafka.Offset 10 | 11 | /** Key specific metainformation inside of parititon. 12 | * 13 | * Recreated each time `KeyState` is created or loaded from the storage. 14 | */ 15 | trait KeyContext[F[_]] { 16 | def holding: F[Option[Offset]] 17 | def hold(offset: Offset): F[Unit] 18 | def remove: F[Unit] 19 | def log: Log[F] 20 | } 21 | object KeyContext { 22 | 23 | def apply[F[_]](implicit F: KeyContext[F]): KeyContext[F] = F 24 | 25 | def empty[F[_]: Applicative]: KeyContext[F] = new KeyContext[F] { 26 | def log = Log.empty 27 | def holding = none[Offset].pure[F] 28 | def hold(offset: Offset) = ().pure[F] 29 | def remove = ().pure[F] 30 | } 31 | 32 | def of[F[_]: Ref.Make: Monad: Log](removeFromCache: F[Unit]): F[KeyContext[F]] = 33 | Ref.of[F, Option[Offset]](None) map { storage => 34 | KeyContext(storage.stateInstance, removeFromCache) 35 | } 36 | 37 | def apply[F[_]: Monad: Log]( 38 | storage: Stateful[F, Option[Offset]], 39 | removeFromCache: F[Unit] 40 | ): KeyContext[F] = new KeyContext[F] { 41 | def holding = storage.get 42 | def hold(offset: Offset) = storage.set(Some(offset)) 43 | def remove = storage.set(None) *> removeFromCache 44 | def log = Log[F] 45 | } 46 | 47 | def resource[F[_]: Ref.Make: Monad]( 48 | removeFromCache: F[Unit], 49 | log: Log[F] 50 | ): Resource[F, KeyContext[F]] = { 51 | implicit val _log = log 52 | Resource.eval(of(removeFromCache)) 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/KeyFlowExtras.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Applicative 4 | 5 | /** Provides an access to some additional internal functionality of the framework. 6 | * 7 | * It's passed as an argument to a user-defined code of `EnhancedFold`, allowing one to use this functionality when 8 | * processing incoming events. 9 | * 10 | * An instance is created per key when the key is recovered (either from persistence or when encountered the first time 11 | * in a source Kafka topic) 12 | * 13 | * @see 14 | * [[com.evolutiongaming.kafka.flow.EnhancedFold]] for the user API allowing construction of an enhanced fold 15 | */ 16 | trait KeyFlowExtras[F[_]] { 17 | 18 | /** Requests to persist a current state of the key. Calling this function doesn't guarantee that the state will be 19 | * persisted immediately; it persists the state after the fold is executed 20 | * 21 | * @see 22 | * See [[com.evolutiongaming.kafka.flow.AdditionalStatePersist]] for the underlying implementation of persisting 23 | */ 24 | def requestAdditionalPersist: F[Unit] 25 | } 26 | 27 | object KeyFlowExtras { 28 | def empty[F[_]](implicit F: Applicative[F]): KeyFlowExtras[F] = new KeyFlowExtras[F] { 29 | def requestAdditionalPersist: F[Unit] = F.unit 30 | } 31 | 32 | def of[F[_]](requestPersist: F[Unit]): KeyFlowExtras[F] = new KeyFlowExtras[F] { 33 | def requestAdditionalPersist: F[Unit] = requestPersist 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/KeyFlowOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Monad 4 | import cats.effect.{Ref, Resource} 5 | import com.evolutiongaming.kafka.flow.persistence.Persistence 6 | import com.evolutiongaming.kafka.flow.registry.EntityRegistry 7 | import com.evolutiongaming.kafka.flow.timer.{TimerContext, TimerFlowOf} 8 | 9 | trait KeyFlowOf[F[_], S, A] { 10 | 11 | def apply( 12 | key: KafkaKey, 13 | context: KeyContext[F], 14 | persistence: Persistence[F, S, A], 15 | timers: TimerContext[F], 16 | additionalPersist: AdditionalStatePersist[F, S, A], 17 | registry: EntityRegistry[F, KafkaKey, S], 18 | ): Resource[F, KeyFlow[F, A]] 19 | 20 | } 21 | object KeyFlowOf { 22 | 23 | /** Construct `KeyFlow` from the premade components. This version doesn't have a notion of `EnhancedFold` thus it 24 | * can't use any additional functionality of `KeyFlowExtras`. 25 | * 26 | * @param timerFlowOf 27 | * storage / factory of timers flows, usually configures how often the timer ticks etc. 28 | * @param fold 29 | * defines how to change the state on incoming records. 30 | * @param tick 31 | * defines what to do when the timer ticks. 32 | */ 33 | def apply[F[_]: Monad: Ref.Make, S, A]( 34 | timerFlowOf: TimerFlowOf[F], 35 | fold: FoldOption[F, S, A], 36 | tick: TickOption[F, S], 37 | ): KeyFlowOf[F, S, A] = apply(timerFlowOf, EnhancedFold.fromFold(fold), tick) 38 | 39 | /** Construct `KeyFlow` from the premade components. This version accepts `EnhancedFold` which can use an additional 40 | * functionality provided by `KeyFlowExtras` 41 | * 42 | * @param timerFlowOf 43 | * storage / factory of timers flows, usually configures how often the timer ticks etc. 44 | * @param fold 45 | * defines how to change the state on incoming records 46 | * @param tick 47 | * defines what to do when the timer ticks 48 | */ 49 | def apply[F[_]: Monad: Ref.Make, S, A]( 50 | timerFlowOf: TimerFlowOf[F], 51 | fold: EnhancedFold[F, S, A], 52 | tick: TickOption[F, S], 53 | ): KeyFlowOf[F, S, A] = { (key, context, persistence, timers, additionalPersist, registry) => 54 | implicit val _context = context 55 | timerFlowOf(context, persistence, timers) flatMap { timerFlow => 56 | KeyFlow.of(key, fold, tick, persistence, additionalPersist, timerFlow, registry) 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/KeyState.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import timer.TimerContext 4 | 5 | /** Key specific state inside of parititon. 6 | * 7 | * Might be persisted to an external storage. 8 | */ 9 | final case class KeyState[F[_], E]( 10 | flow: KeyFlow[F, E], 11 | timers: TimerContext[F] 12 | ) 13 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/LogPrefix.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | /** Extract a common prefix from `A` that is suitable for being logged in order to correlate a specific log message with 4 | * `A` 5 | */ 6 | trait LogPrefix[A] { 7 | def extract(value: A): String 8 | } 9 | 10 | object LogPrefix { 11 | def apply[A](implicit instance: LogPrefix[A]): LogPrefix[A] = instance 12 | 13 | def function[A](f: A => String): LogPrefix[A] = value => f(value) 14 | 15 | implicit val stringPrefix: LogPrefix[String] = function(identity) 16 | } 17 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/LogResource.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.FlatMap 4 | import cats.effect.Resource 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.catshelper.{Log, LogOf} 7 | 8 | object LogResource { 9 | 10 | def apply[F[_]: FlatMap: LogOf](source: Class[_], prefix: String): Resource[F, Log[F]] = { 11 | val result = for { 12 | log0 <- LogOf[F].apply(source) 13 | log = log0.prefixed(prefix) 14 | _ <- log.info("starting") 15 | } yield { 16 | val release = log.info("stopping") 17 | (log, release) 18 | } 19 | Resource(result) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/PartitionFlowOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.effect.Resource 4 | import cats.effect.kernel.Async 5 | import com.evolutiongaming.catshelper.LogOf 6 | import com.evolutiongaming.kafka.flow.PartitionFlow.FilterRecord 7 | import com.evolutiongaming.kafka.flow.kafka.ScheduleCommit 8 | import com.evolutiongaming.skafka.{Offset, TopicPartition} 9 | 10 | trait PartitionFlowOf[F[_]] { 11 | 12 | /** Creates partition record handler for assigned partition */ 13 | def apply( 14 | topicPartition: TopicPartition, 15 | assignedAt: Offset, 16 | scheduleCommit: ScheduleCommit[F] 17 | ): Resource[F, PartitionFlow[F]] 18 | 19 | } 20 | object PartitionFlowOf { 21 | 22 | /** Creates `PartitionFlowOf` for specific application with optional filtering of events 23 | * 24 | * @param filter 25 | * determines whether an incoming consumer record should be processed or skipped. Skipping a record means that (1) 26 | * no state will be restored for that key; (2) no fold will be executed for that event. It doesn't affect 27 | * committing consumer offsets, thus, even if all records in a batch are skipped, new offsets will still be 28 | * committed if necessary 29 | * @param remapKey 30 | * allows to remap the key of a record before it is processed by the flow. Remapping is done before the record is 31 | * processed by the flow. Thus, the next steps in the flow (such as `FilterRecord` and `FoldOption`) will see the 32 | * remapped key 33 | */ 34 | def apply[F[_]: Async: LogOf]( 35 | keyStateOf: KeyStateOf[F], 36 | config: PartitionFlowConfig = PartitionFlowConfig(), 37 | filter: Option[FilterRecord[F]] = None, 38 | remapKey: Option[RemapKey[F]] = None, 39 | ): PartitionFlowOf[F] = { (topicPartition, assignedAt, scheduleCommit) => 40 | PartitionFlow.resource(topicPartition, assignedAt, keyStateOf, config, filter, remapKey, scheduleCommit) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/RebalanceListener.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Monad 4 | import cats.data.NonEmptySet 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.catshelper.LogOf 7 | import com.evolutiongaming.kafka.flow.ConsumerFlow.log 8 | import com.evolutiongaming.skafka.consumer.RebalanceCallback.* 9 | import com.evolutiongaming.skafka.consumer.RebalanceCallback.syntax.* 10 | import com.evolutiongaming.skafka.consumer.{ 11 | RebalanceCallback, 12 | RebalanceListener1 => SRebalanceListener1, 13 | RebalanceListener1WithConsumer 14 | } 15 | import com.evolutiongaming.skafka.{Partition, Topic, TopicPartition} 16 | 17 | object RebalanceListener { 18 | 19 | def apply[F[_]: Monad: LogOf]( 20 | flows: Map[Topic, TopicFlow[F]] 21 | ): SRebalanceListener1[F] = new RebalanceListener1WithConsumer[F] { 22 | 23 | override def onPartitionsAssigned(topicPartitions: NonEmptySet[TopicPartition]): RebalanceCallback[F, Unit] = 24 | groupByTopic(topicPartitions) traverse_ { 25 | case (topic, flow, partitions) => 26 | for { 27 | log <- log[F].flatTap(log => log.prefixed(topic).info(s"$partitions assigned")).lift 28 | partitions <- partitions.toNonEmptyList traverse { partition => 29 | consumer.position(TopicPartition(topic, partition)) map (partition -> _) 30 | } 31 | _ <- (log.prefixed(topic).info(s"committed offsets: $partitions") >> flow.add(partitions.toNes)).lift 32 | } yield () 33 | } 34 | 35 | override def onPartitionsRevoked(topicPartitions: NonEmptySet[TopicPartition]): RebalanceCallback[F, Unit] = 36 | groupByTopic(topicPartitions) traverse_ { 37 | case (topic, flow, partitions) => 38 | (for { 39 | log <- log[F] 40 | _ <- log.prefixed(topic).info(s"$partitions revoked, removing from topic flow") 41 | _ <- flow.remove(partitions) 42 | } yield ()).lift 43 | } 44 | 45 | override def onPartitionsLost(topicPartitions: NonEmptySet[TopicPartition]): RebalanceCallback[F, Unit] = 46 | groupByTopic(topicPartitions) traverse_ { 47 | case (topic, flow, partitions) => 48 | (for { 49 | log <- log[F] 50 | _ <- log.prefixed(topic).info(s"$partitions lost, removing from topic flow") 51 | _ <- flow.remove(partitions) 52 | } yield ()).lift 53 | } 54 | 55 | def groupByTopic[A]( 56 | topicPartitions: NonEmptySet[TopicPartition] 57 | ): List[(Topic, TopicFlow[F], NonEmptySet[Partition])] = 58 | flows.toList.flatMap { 59 | case (topic, flow) => 60 | topicPartitions 61 | .filter(_.topic == topic) 62 | .map(_.partition) 63 | .toNes 64 | .map(partition => (topic, flow, partition)) 65 | } 66 | 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/RemapKey.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Applicative 4 | import cats.syntax.applicative.* 5 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 6 | import scodec.bits.ByteVector 7 | 8 | trait RemapKey[F[_]] { 9 | 10 | /** Derive a new key for the consumer record based on the current key (if there is one) and the record itself. 11 | * Deriving is done before the record is processed by the flow. Thus, the next steps in the flow (such as 12 | * `FilterRecord` and `FoldOption`) will see the remapped key in the consumer record. 13 | */ 14 | def remap(key: String, record: ConsumerRecord[String, ByteVector]): F[String] 15 | } 16 | 17 | object RemapKey { 18 | def of[F[_]](f: (String, ConsumerRecord[String, ByteVector]) => F[String]): RemapKey[F] = (key, record) => 19 | f(key, record) 20 | 21 | def empty[F[_]: Applicative]: RemapKey[F] = (key, _) => key.pure[F] 22 | } 23 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/Tick.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Applicative 4 | import cats.syntax.all.* 5 | import cats.Functor 6 | 7 | case class Tick[F[_], S](run: S => F[S]) { 8 | 9 | /** Alias for `run` */ 10 | def apply(s: S): F[S] = run(s) 11 | 12 | /** Transforms the state `S` of the `Tick` to something else. 13 | * 14 | * It is possible to use additional information from previous `T` to build a new state. 15 | * 16 | * The common use for this method is to augument original state with some metainformation, i.e. offset or sequence 17 | * number. 18 | * 19 | * See also `StateT#transformS` for more details. 20 | */ 21 | def expand[T](f: T => S)(g: (S, T) => T)(implicit F: Functor[F]): Tick[F, T] = 22 | Tick { t => 23 | run(f(t)) map (g(_, t)) 24 | } 25 | 26 | /** Constructs `Fold` which ignores all incoming `A` elements and just triggers underlying `run` */ 27 | def toFold[A]: Fold[F, S, A] = Fold { (s, _) => 28 | run(s) 29 | } 30 | 31 | } 32 | object Tick { 33 | 34 | /** Does nothing to the state */ 35 | def id[F[_]: Applicative, S]: Tick[F, S] = Tick[F, S](_.pure[F]) 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/TickOption.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Applicative 4 | import cats.Functor 5 | 6 | case class TickOption[F[_], S](value: Tick[F, Option[S]]) { 7 | 8 | /** Alias for `value.run` */ 9 | def apply(s: Option[S]): F[Option[S]] = value(s) 10 | 11 | /** Transforms the state `S` of the `Tick` to something else. 12 | * 13 | * It is possible to use additional information from previous `T` to build a new state. 14 | * 15 | * The common use for this method is to augument original state with some metainformation, i.e. offset or sequence 16 | * number. 17 | * 18 | * See also `StateT#transformS` for more details. 19 | * 20 | * If any of `S` or `T` is `None` then the output will also be `None`, though underlying `run` will be called anyway. 21 | */ 22 | def expand[T](f: T => S)(g: (S, T) => T)(implicit F: Functor[F]): TickOption[F, T] = 23 | TickOption { 24 | value.expand[Option[T]](_ map f) { (s, t) => 25 | for { 26 | s <- s 27 | t <- t 28 | } yield g(s, t) 29 | } 30 | } 31 | 32 | /** Constructs `FoldOption` which ignores all incoming `A` elements and just triggers underlying `value.run` */ 33 | def toFold[A]: FoldOption[F, S, A] = FoldOption(value.toFold[A]) 34 | 35 | } 36 | object TickOption { 37 | 38 | def of[F[_], S](run: Option[S] => F[Option[S]]): TickOption[F, S] = 39 | TickOption(Tick(run)) 40 | 41 | /** Does nothing to the state */ 42 | def id[F[_]: Applicative, S]: TickOption[F, S] = TickOption(Tick.id) 43 | 44 | } 45 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/TickToState.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Monad 4 | import cats.effect.Ref 5 | import cats.mtl.Stateful 6 | import cats.syntax.all.* 7 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 8 | import com.evolutiongaming.kafka.flow.persistence.Persistence 9 | 10 | /** Calls the stateful routine stored inside */ 11 | trait TickToState[F[_]] { 12 | 13 | def run: F[Unit] 14 | 15 | } 16 | 17 | object TickToState { 18 | 19 | def of[F[_]: Monad: Ref.Make: KeyContext, S]( 20 | initialState: Option[S], 21 | tick: TickOption[F, S], 22 | persistence: Persistence[F, S, _] 23 | ): F[TickToState[F]] = Ref.of(initialState) map { storage => 24 | TickToState(storage.stateInstance, tick, persistence) 25 | } 26 | 27 | /** Uses `tick` to call the effect on a state stored inside of `storage`. 28 | * 29 | * Performs the necessary actions upon the state being changes, i.e. sends it to persistence, or removes the key if 30 | * the flow processing is finished. 31 | */ 32 | def apply[F[_]: Monad: KeyContext, S]( 33 | storage: Stateful[F, Option[S]], 34 | tick: TickOption[F, S], 35 | persistence: Persistence[F, S, _] 36 | ): TickToState[F] = new TickToState[F] { 37 | def run = for { 38 | state <- storage.get 39 | state <- tick(state) 40 | _ <- state traverse_ persistence.replaceState 41 | _ <- storage set state 42 | _ <- 43 | if (state.isEmpty) { 44 | persistence.delete *> KeyContext[F].remove 45 | } else { 46 | ().pure[F] 47 | } 48 | } yield () 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/ToOffset.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafka 2 | 3 | import com.evolutiongaming.skafka.Offset 4 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 5 | 6 | /** Provides kafka offset of the record or a snapshot */ 7 | trait ToOffset[T] { 8 | 9 | def offset(event: T): Offset 10 | 11 | } 12 | object ToOffset { 13 | 14 | implicit def consumerRecordToOffset[K, V]: ToOffset[ConsumerRecord[K, V]] = 15 | new ToOffset[ConsumerRecord[K, V]] { 16 | def offset(event: ConsumerRecord[K, V]): Offset = event.offset 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/TopicFlowOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Parallel 4 | import cats.effect.{Concurrent, Resource} 5 | import com.evolutiongaming.catshelper.{LogOf, Runtime} 6 | import com.evolutiongaming.kafka.flow.kafka.Consumer 7 | import com.evolutiongaming.skafka.Topic 8 | 9 | trait TopicFlowOf[F[_]] { 10 | 11 | def apply(consumer: Consumer[F], topic: Topic): Resource[F, TopicFlow[F]] 12 | 13 | } 14 | object TopicFlowOf { 15 | 16 | def apply[F[_]: Concurrent: Runtime: Parallel: LogOf]( 17 | partitionFlowOf: PartitionFlowOf[F] 18 | ): TopicFlowOf[F] = { (consumer, topic) => 19 | TopicFlow.of(consumer, topic, partitionFlowOf) 20 | } 21 | 22 | def route[F[_]](f: Topic => TopicFlowOf[F]): TopicFlowOf[F] = { (consumer, topic) => 23 | val flowOf = f(topic) 24 | flowOf(consumer, topic) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/effect/CatsEffectMtlInstances.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.effect 2 | 3 | import cats.Monad 4 | import cats.effect.Ref 5 | import cats.mtl.Stateful 6 | import cats.syntax.functor.* 7 | 8 | /** meow-mtl library is not updated and supported anymore, this is just a straight copypasta from there */ 9 | private[flow] object CatsEffectMtlInstances { 10 | implicit class RefEffects[F[_], A](val self: Ref[F, A]) extends AnyVal { 11 | def stateInstance(implicit F: Monad[F]): Stateful[F, A] = new RefStateful[F, A](self) 12 | } 13 | 14 | class RefStateful[F[_], S](ref: Ref[F, S])(implicit F: Monad[F]) extends Stateful[F, S] { 15 | val monad: Monad[F] = F 16 | def get: F[S] = ref.get 17 | def set(s: S): F[Unit] = ref.set(s) 18 | override def inspect[A](f: S => A): F[A] = ref.get.map(f) 19 | override def modify(f: S => S): F[Unit] = ref.update(f) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/journal/JournalDatabase.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal 2 | 3 | import cats.effect.{Ref, Sync} 4 | import cats.mtl.Stateful 5 | import cats.syntax.all.* 6 | import cats.{Applicative, Monad} 7 | import com.evolutiongaming.catshelper.LogOf 8 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 9 | import com.evolutiongaming.kafka.flow.kafka.ToOffset 10 | import com.evolutiongaming.skafka.Offset 11 | import com.evolutiongaming.sstream.Stream 12 | 13 | import scala.collection.immutable.SortedMap 14 | 15 | trait JournalDatabase[F[_], K, R] { 16 | 17 | /** Adds or replaces the event in a database */ 18 | def persist(key: K, event: R): F[Unit] 19 | 20 | /** Restores journal for the key, if any */ 21 | def get(key: K): Stream[F, R] 22 | 23 | /** Deletes journal for they key, if any */ 24 | def delete(key: K): F[Unit] 25 | 26 | def journalsOf(implicit F: Sync[F], logOf: LogOf[F]): F[JournalsOf[F, K, R]] = 27 | logOf(JournalDatabase.getClass) map { implicit log => key => 28 | Journals.of(key, this) 29 | } 30 | 31 | } 32 | object JournalDatabase { 33 | 34 | /** Creates in-memory database implementation. 35 | * 36 | * The data will survive destruction of specific `Journals` instance, but will not survive destruction of specific 37 | * `JournalDatabase` instance. 38 | */ 39 | def memory[F[_]: Monad: Ref.Make, K, E](implicit E: ToOffset[E]): F[JournalDatabase[F, K, E]] = 40 | Ref.of[F, Map[K, SortedMap[Offset, E]]](Map.empty) map { storage => 41 | memory(storage.stateInstance) 42 | } 43 | 44 | /** Creates in-memory database implementation. 45 | * 46 | * The data will survive destruction of specific `Journals` instance, but will not survive destruction of specific 47 | * `JournalDatabase` instance. 48 | */ 49 | def memory[F[_]: Monad, K, E]( 50 | storage: Stateful[F, Map[K, SortedMap[Offset, E]]] 51 | )(implicit E: ToOffset[E]): JournalDatabase[F, K, E] = 52 | new JournalDatabase[F, K, E] { 53 | 54 | def persist(key: K, event: E) = storage modify { storage => 55 | val existingEvents = storage.getOrElse(key, SortedMap.empty[Offset, E]) 56 | val updatedEvents = existingEvents + (E.offset(event) -> event) 57 | storage + (key -> updatedEvents) 58 | } 59 | 60 | def get(key: K) = Stream.lift(storage.get) flatMap { storage => 61 | val events = storage get key map (_.values.toList) getOrElse Nil 62 | Stream.from(events) 63 | } 64 | 65 | def delete(key: K) = storage modify (_ - key) 66 | 67 | } 68 | 69 | def empty[F[_]: Applicative, K, R]: JournalDatabase[F, K, R] = 70 | new JournalDatabase[F, K, R] { 71 | def persist(key: K, event: R) = ().pure 72 | def get(key: K) = Stream.empty 73 | def delete(key: K) = ().pure 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/journal/Journals.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal 2 | 3 | import cats.{Applicative, Monad} 4 | import cats.effect.Ref 5 | import cats.mtl.Stateful 6 | import cats.syntax.all.* 7 | import com.evolutiongaming.catshelper.Log 8 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 9 | import com.evolutiongaming.sstream.Stream 10 | 11 | trait Journals[F[_], E] extends JournalReader[F, E] with JournalWriter[F, E] 12 | 13 | /** Allows to read a previously saved journal */ 14 | trait JournalReader[F[_], E] { 15 | 16 | /** Restores a journal */ 17 | def read: Stream[F, E] 18 | 19 | } 20 | 21 | /** Provides a persistence for a specific key */ 22 | trait JournalWriter[F[_], E] { 23 | 24 | /** Saves the next event to a buffer. 25 | * 26 | * Note, that completing the append does not guarantee that the state will be persisted. I.e. persistence might 27 | * choose to do the updates in batches. 28 | */ 29 | def append(event: E): F[Unit] 30 | 31 | /** Flush buffer to a database */ 32 | def flush: F[Unit] 33 | 34 | /** Removes state from the buffers and optionally also from persistence. 35 | * 36 | * @param persist 37 | * if `true` then also calls underlying database, only clears buffers otherwise. 38 | */ 39 | def delete(persist: Boolean): F[Unit] 40 | 41 | } 42 | object Journals { 43 | 44 | /** Creates a buffer for a given database */ 45 | private[journal] def of[F[_]: Monad: Ref.Make: Log, K, E]( 46 | key: K, 47 | database: JournalDatabase[F, K, E] 48 | ): F[Journals[F, E]] = 49 | Ref.of[F, List[E]](List.empty) map { buffer => 50 | Journals(key, database, buffer.stateInstance) 51 | } 52 | 53 | private[journal] def apply[F[_]: Monad: Log, K, E]( 54 | key: K, 55 | database: JournalDatabase[F, K, E], 56 | buffer: Stateful[F, List[E]], 57 | ): Journals[F, E] = new Journals[F, E] { 58 | 59 | def read = database.get(key) 60 | 61 | def append(event: E) = buffer modify (event :: _) 62 | 63 | def flush = for { 64 | events <- buffer.get 65 | _ <- events.reverse traverse_ { event => 66 | database.persist(key, event) 67 | } 68 | } yield () 69 | 70 | def delete(persist: Boolean) = { 71 | val delete = if (persist) { 72 | database.delete(key) *> 73 | Log[F].info("deleted journal") 74 | } else { 75 | ().pure[F] 76 | } 77 | buffer.set(Nil) *> delete 78 | } 79 | 80 | } 81 | 82 | def empty[F[_]: Applicative, E]: Journals[F, E] = 83 | new Journals[F, E] { 84 | def read = Stream.empty[F, E] 85 | def append(event: E) = ().pure[F] 86 | def flush = ().pure[F] 87 | def delete(persist: Boolean) = ().pure[F] 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/journal/JournalsOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal 2 | 3 | import cats.effect.Sync 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.catshelper.Log 6 | import com.evolutiongaming.kafka.flow.kafka.ToOffset 7 | 8 | trait JournalsOf[F[_], K, E] { 9 | 10 | def apply(key: K): F[Journals[F, E]] 11 | 12 | } 13 | object JournalsOf { 14 | 15 | def memory[F[_]: Sync: Log, K, E: ToOffset]: F[JournalsOf[F, K, E]] = 16 | JournalDatabase.memory[F, K, E] map { database => key => 17 | Journals.of(key, database) 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/kafka/Codecs.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafka 2 | 3 | import cats.Applicative 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.skafka.{FromBytes, ToBytes, Topic} 6 | import scodec.bits.ByteVector 7 | 8 | private[flow] object Codecs { 9 | implicit def skafkaFromBytes[F[_]: Applicative]: FromBytes[F, ByteVector] = { (a: Array[Byte], _: Topic) => 10 | ByteVector.view(a).pure[F] 11 | } 12 | 13 | implicit def skafkaToBytes[F[_]: Applicative]: ToBytes[F, ByteVector] = { (a: ByteVector, _: Topic) => 14 | a.toArray.pure[F] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/kafka/ConsumerOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafka 2 | 3 | import cats.effect.Resource 4 | 5 | trait ConsumerOf[F[_]] { 6 | 7 | def apply(groupId: String): Resource[F, Consumer[F]] 8 | 9 | } 10 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/kafka/EmptyRebalanceConsumer.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafka 2 | 3 | import java.time.Instant 4 | 5 | import cats.data.{NonEmptyMap, NonEmptySet} 6 | import com.evolutiongaming.skafka.consumer.{ConsumerGroupMetadata, OffsetAndTimestamp, RebalanceConsumer} 7 | import com.evolutiongaming.skafka.* 8 | 9 | import scala.concurrent.duration.FiniteDuration 10 | import scala.util.Try 11 | 12 | class EmptyRebalanceConsumer extends RebalanceConsumer { 13 | def assignment(): Try[Set[TopicPartition]] = Try(Set.empty) 14 | 15 | def beginningOffsets(partitions: NonEmptySet[TopicPartition]): Try[Map[TopicPartition, Offset]] = Try(Map.empty) 16 | 17 | def beginningOffsets( 18 | partitions: NonEmptySet[TopicPartition], 19 | timeout: FiniteDuration 20 | ): Try[Map[TopicPartition, Offset]] = Try(Map.empty) 21 | 22 | def commit(): Try[Unit] = Try(()) 23 | 24 | def commit(timeout: FiniteDuration): Try[Unit] = Try(()) 25 | 26 | def commit(offsets: NonEmptyMap[TopicPartition, OffsetAndMetadata]): Try[Unit] = Try(()) 27 | 28 | def commit(offsets: NonEmptyMap[TopicPartition, OffsetAndMetadata], timeout: FiniteDuration): Try[Unit] = Try(()) 29 | 30 | def committed(partitions: NonEmptySet[TopicPartition]): Try[Map[TopicPartition, OffsetAndMetadata]] = Try(Map.empty) 31 | 32 | def committed( 33 | partitions: NonEmptySet[TopicPartition], 34 | timeout: FiniteDuration 35 | ): Try[Map[TopicPartition, OffsetAndMetadata]] = Try(Map.empty) 36 | 37 | def endOffsets(partitions: NonEmptySet[TopicPartition]): Try[Map[TopicPartition, Offset]] = Try(Map.empty) 38 | 39 | def endOffsets(partitions: NonEmptySet[TopicPartition], timeout: FiniteDuration): Try[Map[TopicPartition, Offset]] = 40 | Try(Map.empty) 41 | 42 | def groupMetadata(): Try[ConsumerGroupMetadata] = Try(ConsumerGroupMetadata.Empty) 43 | 44 | def topics(): Try[Map[Topic, List[PartitionInfo]]] = Try(Map.empty) 45 | 46 | def topics(timeout: FiniteDuration): Try[Map[Topic, List[PartitionInfo]]] = Try(Map.empty) 47 | 48 | def offsetsForTimes( 49 | timestampsToSearch: NonEmptyMap[TopicPartition, Instant] 50 | ): Try[Map[TopicPartition, Option[OffsetAndTimestamp]]] = Try(Map.empty) 51 | 52 | def offsetsForTimes( 53 | timestampsToSearch: NonEmptyMap[TopicPartition, Instant], 54 | timeout: FiniteDuration 55 | ): Try[Map[TopicPartition, Option[OffsetAndTimestamp]]] = 56 | Try(Map.empty) 57 | 58 | def partitionsFor(topic: Topic): Try[List[PartitionInfo]] = Try(List.empty) 59 | 60 | def partitionsFor(topic: Topic, timeout: FiniteDuration): Try[List[PartitionInfo]] = Try(List.empty) 61 | 62 | def paused(): Try[Set[TopicPartition]] = Try(Set.empty) 63 | 64 | def position(partition: TopicPartition): Try[Offset] = Try(Offset.min) 65 | 66 | def position(partition: TopicPartition, timeout: FiniteDuration): Try[Offset] = Try(Offset.min) 67 | 68 | def seek(partition: TopicPartition, offset: Offset): Try[Unit] = Try(()) 69 | 70 | def seek(partition: TopicPartition, offsetAndMetadata: OffsetAndMetadata): Try[Unit] = Try(()) 71 | 72 | def seekToBeginning(partitions: NonEmptySet[TopicPartition]): Try[Unit] = Try(()) 73 | 74 | def seekToEnd(partitions: NonEmptySet[TopicPartition]): Try[Unit] = Try(()) 75 | 76 | def subscription(): Try[Set[Topic]] = Try(Set.empty) 77 | } 78 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/kafka/KafkaModule.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafka 2 | 3 | import cats.effect.{Async, Clock, Resource} 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.catshelper.* 6 | import com.evolutiongaming.kafka.flow.LogResource 7 | import com.evolutiongaming.kafka.flow.kafka.Codecs.* 8 | import com.evolutiongaming.skafka.KafkaHealthCheck 9 | import com.evolutiongaming.skafka.consumer.{ 10 | AutoOffsetReset, 11 | ConsumerConfig, 12 | ConsumerMetrics, 13 | ConsumerOf => RawConsumerOf 14 | } 15 | import com.evolutiongaming.skafka.producer.{ProducerConfig, ProducerMetrics, ProducerOf => RawProducerOf} 16 | import com.evolutiongaming.smetrics.CollectorRegistry 17 | import scodec.bits.ByteVector 18 | 19 | trait KafkaModule[F[_]] { 20 | 21 | def healthCheck: KafkaHealthCheck[F] 22 | 23 | def consumerOf: ConsumerOf[F] 24 | def producerOf: RawProducerOf[F] 25 | } 26 | 27 | object KafkaModule { 28 | 29 | /** Creates kafka consumer and producer builders, and additionally launches kafka healthcheck mechanism which 30 | * repeatedly sends and consumes messages to/from topic named 'healthcheck' (refer to 31 | * `KafkaHealthCheck.Config.default`) 32 | */ 33 | def of[F[_]: Async: FromTry: ToTry: ToFuture: LogOf]( 34 | applicationId: String, 35 | config: ConsumerConfig, 36 | registry: CollectorRegistry[F] 37 | ): Resource[F, KafkaModule[F]] = { 38 | implicit val measureDuration: MeasureDuration[F] = MeasureDuration.fromClock[F](Clock[F]) 39 | for { 40 | producerMetrics <- ProducerMetrics.of(registry) 41 | consumerMetrics <- ConsumerMetrics.of(registry) 42 | _producerOf = RawProducerOf.apply1[F](producerMetrics(applicationId).some) 43 | _consumerOf = RawConsumerOf.apply1[F](consumerMetrics(applicationId).some) 44 | 45 | _healthCheck <- { 46 | implicit val randomIdOf: RandomIdOf[F] = RandomIdOf.uuid[F] 47 | implicit val consumerOf: RawConsumerOf[F] = _consumerOf 48 | implicit val producerOf: RawProducerOf[F] = _producerOf 49 | 50 | val commonConfig = config.common.copy(clientId = config.common.clientId.map(id => s"$id-HealthCheck")) 51 | 52 | val healthCheck = KafkaHealthCheck.of( 53 | KafkaHealthCheck.Config.default, 54 | ConsumerConfig(common = commonConfig, saslSupport = config.saslSupport, sslSupport = config.sslSupport), 55 | ProducerConfig(common = commonConfig, saslSupport = config.saslSupport, sslSupport = config.sslSupport) 56 | ) 57 | LogResource[F](KafkaModule.getClass, "KafkaHealthCheck") *> healthCheck 58 | } 59 | } yield new KafkaModule[F] { 60 | 61 | def healthCheck = _healthCheck 62 | 63 | def consumerOf = { (groupId: String) => 64 | LogResource[F](KafkaModule.getClass, s"Consumer($groupId)") *> 65 | _consumerOf[String, ByteVector]( 66 | config.copy( 67 | groupId = groupId.some, 68 | autoCommit = false, 69 | autoOffsetReset = AutoOffsetReset.Earliest 70 | ) 71 | ) evalMap { consumer => 72 | LogOf[F].apply(Consumer.getClass) map { log => 73 | Consumer(consumer.withLogging(log)) 74 | } 75 | } 76 | } 77 | 78 | def producerOf = _producerOf 79 | 80 | } 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/kafka/OffsetToCommit.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafka 2 | 3 | import cats.ApplicativeThrow 4 | import com.evolutiongaming.skafka.Offset 5 | 6 | /** Constructs an offset to commit to Kafka. 7 | * 8 | * From KafkaClient documentation: "The committed offset should be the next message your application will consume, i.e. 9 | * lastProcessedMessageOffset + 1" 10 | */ 11 | object OffsetToCommit { 12 | 13 | def apply[F[_]: ApplicativeThrow](offset: Offset): F[Offset] = Offset.of[F](offset.value + 1) 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/kafka/PendingCommits.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafka 2 | 3 | import cats.syntax.all.* 4 | import cats.data.NonEmptySet 5 | import cats.effect.Ref 6 | import com.evolutiongaming.skafka.{OffsetAndMetadata, Partition, TopicPartition} 7 | 8 | /** A storage of offsets that were requested to be committed by each individual partition 9 | */ 10 | private[flow] trait PendingCommits[F[_]] { 11 | 12 | /** Clear the storage and return the previous values */ 13 | def clear: F[Map[TopicPartition, OffsetAndMetadata]] 14 | 15 | /** Remove stored offsets for a given set of partitions */ 16 | def remove(topicPartitions: NonEmptySet[TopicPartition]): F[Unit] 17 | 18 | /** Create a new instance of [[ScheduleCommit]] allowing individual partitions to request ("schedule") an offset to be 19 | * committed during the next commit attempt 20 | */ 21 | def newScheduleCommit(topic: String, partition: Partition): ScheduleCommit[F] 22 | } 23 | 24 | private[flow] object PendingCommits { 25 | 26 | /** An in-memory implementation, using [[cats.effect.Ref]] as a storage */ 27 | private final class FromRef[F[_]](pendingCommits: Ref[F, Map[TopicPartition, OffsetAndMetadata]]) 28 | extends PendingCommits[F] { 29 | 30 | /** @inheritdoc */ 31 | override def clear: F[Map[TopicPartition, OffsetAndMetadata]] = 32 | pendingCommits.getAndSet(Map.empty) 33 | 34 | /** @inheritdoc */ 35 | override def remove(topicPartitions: NonEmptySet[TopicPartition]): F[Unit] = 36 | pendingCommits.update(_ -- topicPartitions.toList) 37 | 38 | /** @inheritdoc */ 39 | override def newScheduleCommit(topic: String, partition: Partition): ScheduleCommit[F] = 40 | ScheduleCommit.fromRef(topic, partition, pendingCommits) 41 | } 42 | 43 | def fromRef[F[_]](pendingCommits: Ref[F, Map[TopicPartition, OffsetAndMetadata]]): PendingCommits[F] = 44 | new FromRef[F](pendingCommits) 45 | } 46 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/kafka/ScheduleCommit.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafka 2 | 3 | import cats.Applicative 4 | import cats.effect.Ref 5 | import com.evolutiongaming.skafka.{Offset, OffsetAndMetadata, Partition, TopicPartition} 6 | 7 | trait ScheduleCommit[F[_]] { 8 | 9 | /** Request ("schedule") an offset to be committed for a partition during the next commit attempt */ 10 | def schedule(offset: Offset): F[Unit] 11 | } 12 | 13 | object ScheduleCommit { 14 | 15 | def empty[F[_]: Applicative]: ScheduleCommit[F] = new Empty[F] 16 | 17 | def fromRef[F[_]]( 18 | topic: String, 19 | partition: Partition, 20 | pendingCommits: Ref[F, Map[TopicPartition, OffsetAndMetadata]] 21 | ): ScheduleCommit[F] = 22 | new FromRef[F](topic, partition, pendingCommits) 23 | 24 | private final class Empty[F[_]: Applicative] extends ScheduleCommit[F] { 25 | override def schedule(offset: Offset): F[Unit] = Applicative[F].unit 26 | } 27 | 28 | private final class FromRef[F[_]]( 29 | topic: String, 30 | partition: Partition, 31 | pendingCommits: Ref[F, Map[TopicPartition, OffsetAndMetadata]] 32 | ) extends ScheduleCommit[F] { 33 | override def schedule(offset: Offset): F[Unit] = pendingCommits.update { pendingCommits => 34 | pendingCommits + (TopicPartition(topic, partition) -> OffsetAndMetadata(offset)) 35 | } 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/key/KeyDatabase.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.effect.{Ref, Sync} 4 | import cats.mtl.Stateful 5 | import cats.syntax.all.* 6 | import cats.{Applicative, Monad} 7 | import com.evolutiongaming.catshelper.LogOf 8 | import com.evolutiongaming.kafka.flow.LogPrefix 9 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 10 | import com.evolutiongaming.skafka.TopicPartition 11 | import com.evolutiongaming.sstream.Stream 12 | 13 | trait KeyDatabase[F[_], K] { 14 | 15 | /** Adds the key to the database if it exists */ 16 | def persist(key: K): F[Unit] 17 | 18 | /** Deletes snapshot for they key, if any */ 19 | def delete(key: K): F[Unit] 20 | 21 | def all(applicationId: String, groupId: String, topicPartition: TopicPartition): Stream[F, K] 22 | 23 | @deprecated("Use `toKeysOf` instead", "5.0.6") 24 | def keysOf(implicit F: Monad[F], logOf: LogOf[F]): F[KeysOf[F, K]] = 25 | logOf(KeyDatabase.getClass) map { implicit log => KeysOf(this) } 26 | 27 | def toKeysOf(implicit F: Monad[F], logOf: LogOf[F], logPrefix: LogPrefix[K]): F[KeysOf[F, K]] = 28 | logOf(KeyDatabase.getClass) map { implicit log => KeysOf.of(this) } 29 | } 30 | object KeyDatabase { 31 | 32 | /** Creates in-memory database implementation */ 33 | def memory[F[_]: Sync, K]: F[KeyDatabase[F, K]] = 34 | Ref.of[F, Set[K]](Set.empty[K]) map { storage => 35 | memory(storage.stateInstance) 36 | } 37 | 38 | /** Creates in-memory database implementation */ 39 | def memory[F[_]: Monad, K](storage: Stateful[F, Set[K]]): KeyDatabase[F, K] = 40 | new KeyDatabase[F, K] { 41 | 42 | def persist(key: K) = 43 | storage modify (_ + key) 44 | 45 | def delete(key: K) = 46 | storage modify (_ - key) 47 | 48 | def all(applicationId: String, groupId: String, topicPartition: TopicPartition) = 49 | Stream.lift(storage.get) flatMap { keys => 50 | Stream.from(keys.toList) 51 | } 52 | 53 | } 54 | 55 | def empty[F[_]: Applicative, K]: KeyDatabase[F, K] = 56 | new KeyDatabase[F, K] { 57 | def persist(key: K) = ().pure 58 | def delete(key: K) = ().pure 59 | def all(applicationId: String, groupId: String, topicPartition: TopicPartition) = Stream.empty 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/key/Keys.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.syntax.all.* 4 | import cats.{Applicative, Monad} 5 | import com.evolutiongaming.catshelper.Log 6 | import com.evolutiongaming.kafka.flow.LogPrefix 7 | 8 | trait Keys[F[_]] extends KeyWriter[F] 9 | 10 | /** Provided a persistence for a specific key */ 11 | trait KeyWriter[F[_]] { 12 | 13 | /** Flushes buffer to a database */ 14 | def flush: F[Unit] 15 | 16 | /** Removes state from the buffers and optionally also from persistence. 17 | * 18 | * @param persist 19 | * if `true` then also calls underlying database, flushes buffers only otherwise. 20 | */ 21 | def delete(persist: Boolean): F[Unit] 22 | 23 | } 24 | object Keys { 25 | 26 | /** Creates a buffer for a given writer */ 27 | @deprecated("Use `of` instead", "5.0.6") 28 | private[key] def apply[F[_]: Monad: Log, K]( 29 | key: K, 30 | database: KeyDatabase[F, K] 31 | ): Keys[F] = new Keys[F] { 32 | 33 | def flush: F[Unit] = database.persist(key) 34 | 35 | def delete(persist: Boolean): F[Unit] = 36 | if (persist) { 37 | database.delete(key) *> Log[F].info("deleted key") 38 | } else { 39 | ().pure[F] 40 | } 41 | 42 | } 43 | 44 | private[key] def of[F[_]: Monad: Log, K: LogPrefix]( 45 | key: K, 46 | database: KeyDatabase[F, K] 47 | ): Keys[F] = new Keys[F] { 48 | 49 | private val prefixedLog = Log[F].prefixed(s"[${LogPrefix[K].extract(key)}]") 50 | 51 | def flush: F[Unit] = database.persist(key) 52 | 53 | def delete(persist: Boolean): F[Unit] = 54 | if (persist) { 55 | database.delete(key) *> prefixedLog.info("deleted key") 56 | } else { 57 | ().pure[F] 58 | } 59 | 60 | } 61 | 62 | def empty[F[_]: Applicative]: Keys[F] = new Keys[F] { 63 | def flush: F[Unit] = ().pure[F] 64 | def delete(persist: Boolean): F[Unit] = ().pure[F] 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/key/KeysOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.Monad 4 | import cats.effect.Sync 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.catshelper.Log 7 | import com.evolutiongaming.kafka.flow.LogPrefix 8 | import com.evolutiongaming.skafka.TopicPartition 9 | import com.evolutiongaming.sstream.Stream 10 | 11 | trait KeysOf[F[_], K] { 12 | 13 | def apply(key: K): Keys[F] 14 | 15 | def all(applicationId: String, groupId: String, topicPartition: TopicPartition): Stream[F, K] 16 | 17 | } 18 | object KeysOf { 19 | 20 | @deprecated("Use another `memory1` instead", "5.0.6") 21 | def memory[F[_]: Sync: Log, K]: F[KeysOf[F, K]] = 22 | KeyDatabase.memory[F, K].map(database => KeysOf.apply(database)) 23 | 24 | def memory1[F[_]: Sync: Log, K: LogPrefix]: F[KeysOf[F, K]] = 25 | KeyDatabase.memory[F, K].map(database => KeysOf.of(database)) 26 | 27 | /** Creates `KeysOf` with a passed logger */ 28 | @deprecated("Use `of` instead", "5.0.6") 29 | def apply[F[_]: Monad: Log, K]( 30 | database: KeyDatabase[F, K] 31 | ): KeysOf[F, K] = new KeysOf[F, K] { 32 | def apply(key: K) = Keys(key, database) 33 | def all(applicationId: String, groupId: String, topicPartition: TopicPartition) = 34 | database.all(applicationId, groupId, topicPartition) 35 | } 36 | 37 | /** Creates `KeysOf` with a passed logger */ 38 | def of[F[_]: Monad: Log, K: LogPrefix]( 39 | database: KeyDatabase[F, K] 40 | ): KeysOf[F, K] = new KeysOf[F, K] { 41 | def apply(key: K) = Keys.of(key, database) 42 | def all(applicationId: String, groupId: String, topicPartition: TopicPartition) = 43 | database.all(applicationId, groupId, topicPartition) 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/package.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka 2 | 3 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 4 | import scodec.bits.ByteVector 5 | 6 | package object flow { 7 | type FoldCons[F[_], S] = Fold[F, S, ConsumerRecord[String, ByteVector]] 8 | type FoldOptionCons[F[_], S] = FoldOption[F, S, ConsumerRecord[String, ByteVector]] 9 | } 10 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/persistence/PersistenceModule.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence 2 | 3 | import cats.Applicative 4 | import cats.effect.{Resource, Sync} 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.catshelper.LogOf 7 | import com.evolutiongaming.kafka.flow.KafkaKey 8 | import com.evolutiongaming.kafka.flow.journal.JournalDatabase 9 | import com.evolutiongaming.kafka.flow.key.KeyDatabase 10 | import com.evolutiongaming.kafka.flow.snapshot.{KafkaSnapshot, SnapshotDatabase} 11 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 12 | import scodec.bits.ByteVector 13 | 14 | /** Convenience methods to create most common persistence setups */ 15 | trait PersistenceModule[F[_], S] { 16 | 17 | def keys: KeyDatabase[F, KafkaKey] 18 | def journals: JournalDatabase[F, KafkaKey, ConsumerRecord[String, ByteVector]] 19 | def snapshots: SnapshotDatabase[F, KafkaKey, KafkaSnapshot[S]] 20 | 21 | /** Saves both events and snapshots, restores state from events */ 22 | def restoreEvents( 23 | implicit F: Sync[F], 24 | logOf: LogOf[F] 25 | ): Resource[F, PersistenceOf[F, KafkaKey, KafkaSnapshot[S], ConsumerRecord[String, ByteVector]]] = for { 26 | keysOf <- Resource.eval(keys.toKeysOf) 27 | journalsOf <- Resource.eval(journals.journalsOf) 28 | snapshotsOf <- Resource.eval(snapshots.snapshotsOf) 29 | persistenceOf <- PersistenceOf.restoreEvents(keysOf, journalsOf, snapshotsOf) 30 | } yield persistenceOf 31 | 32 | /** Saves both events and snapshots, restores state from snapshots */ 33 | def restoreSnapshots( 34 | implicit F: Sync[F], 35 | logOf: LogOf[F] 36 | ): F[SnapshotPersistenceOf[F, KafkaKey, KafkaSnapshot[S], ConsumerRecord[String, ByteVector]]] = for { 37 | keysOf <- keys.toKeysOf 38 | journalsOf <- journals.journalsOf 39 | snapshotsOf <- snapshots.snapshotsOf 40 | } yield PersistenceOf.restoreSnapshots(keysOf, journalsOf, snapshotsOf) 41 | 42 | /** Saves snapshots only, restores state from snapshots */ 43 | def snapshotsOnly( 44 | implicit F: Sync[F], 45 | logOf: LogOf[F] 46 | ): F[SnapshotPersistenceOf[F, KafkaKey, KafkaSnapshot[S], ConsumerRecord[String, ByteVector]]] = for { 47 | keysOf <- keys.toKeysOf 48 | snapshotsOf <- snapshots.snapshotsOf 49 | } yield PersistenceOf.snapshotsOnly(keysOf, snapshotsOf) 50 | 51 | } 52 | 53 | object PersistenceModule { 54 | 55 | def empty[F[_]: Applicative, S]: PersistenceModule[F, S] = 56 | new PersistenceModule[F, S] { 57 | def keys = KeyDatabase.empty 58 | def journals = JournalDatabase.empty 59 | def snapshots = SnapshotDatabase.empty 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/persistence/compression/Compression.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence.compression 2 | 3 | import cats.MonadThrow 4 | import cats.syntax.all.* 5 | import net.jpountz.lz4.LZ4Factory 6 | import scodec.Attempt 7 | import scodec.bits.ByteVector 8 | import scodec.codecs.* 9 | 10 | import java.nio.ByteBuffer 11 | 12 | /** Internal implementation of compression */ 13 | private[flow] trait Compression[F[_]] { 14 | def compress(bytes: ByteVector): F[ByteVector] 15 | def decompress(bytes: ByteVector): F[ByteVector] 16 | } 17 | 18 | private[flow] object Compression { 19 | def lz4[F[_]: MonadThrow](): F[Compression[F]] = { 20 | for { 21 | factory <- MonadThrow[F].catchNonFatal(LZ4Factory.fastestInstance()) 22 | } yield new Lz4Compression[F](factory) 23 | } 24 | } 25 | 26 | /** LZ4-based implementation. The layout of the resulting byte vector is as follows: 27 | * {{{ 28 | * |---------------|------------------| 29 | * | p_len (int32) | payload (byte[]) | 30 | * |---------------|------------------| 31 | * }}} 32 | * where: 33 | * - `p_len` is a length of an uncompressed input byte vector. It's required to allocate the buffer of a proper size 34 | * when decompressing is done 35 | * - `payload` is a compressed version of the input byte vector 36 | * @param factory 37 | * a factory instance which is reusable 38 | */ 39 | private[flow] class Lz4Compression[F[_]: MonadThrow](factory: LZ4Factory) extends Compression[F] { 40 | private val codec = int32 ~ bytes 41 | 42 | private val compressor = factory.fastCompressor() 43 | private val decompressor = factory.fastDecompressor() 44 | 45 | override def compress(bytes0: ByteVector): F[ByteVector] = { 46 | val bytes = bytes0.toArray 47 | val lengthDecompressed = bytes.length 48 | for { 49 | bytesCompressed <- MonadThrow[F].catchNonFatal { 50 | val maxLengthCompressed = compressor.maxCompressedLength(lengthDecompressed) 51 | val compressed = new Array[Byte](maxLengthCompressed) 52 | val lengthCompressed = compressor.compress(bytes, compressed) 53 | ByteVector.view(compressed, 0, lengthCompressed) 54 | } 55 | bits <- codec.encode((lengthDecompressed, bytesCompressed)) match { 56 | case Attempt.Successful(value) => value.pure[F] 57 | case Attempt.Failure(e) => CompressionError(e.message).raiseError 58 | } 59 | } yield bits.bytes 60 | } 61 | 62 | override def decompress(bytes: ByteVector): F[ByteVector] = { 63 | for { 64 | decoded <- codec.decode(bytes.bits) match { 65 | case Attempt.Successful(value) => value.pure[F] 66 | case Attempt.Failure(e) => CompressionError(e.message).raiseError 67 | } 68 | (lengthDecompressed, byteVector) = decoded.value 69 | buffer = byteVector.toByteBuffer 70 | decompressed <- MonadThrow[F].catchNonFatal { 71 | val decompressed = ByteBuffer.allocate(lengthDecompressed) 72 | decompressor.decompress(buffer, decompressed) 73 | decompressed 74 | } 75 | } yield ByteVector.view(decompressed.array()) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/persistence/compression/CompressionError.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence.compression 2 | 3 | import scala.util.control.NoStackTrace 4 | 5 | final case class CompressionError(message: String) extends RuntimeException(message) with NoStackTrace 6 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/persistence/compression/CompressorSyntax.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence.compression 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.skafka.{Bytes, FromBytes, ToBytes, Topic} 6 | import scodec.bits.ByteVector 7 | 8 | object CompressorSyntax { 9 | implicit class ToBytesWithCompressionOps[F[_], A](val self: ToBytes[F, A]) extends AnyVal { 10 | def withCompression(compressor: Compressor[F])(implicit m: Monad[F]): ToBytes[F, A] = 11 | new ToBytesWithCompression[F, A](self, compressor) 12 | } 13 | 14 | implicit class FromBytesWithCompressionOps[F[_], A](val self: FromBytes[F, A]) extends AnyVal { 15 | def withCompression(compressor: Compressor[F])(implicit m: Monad[F]): FromBytes[F, A] = 16 | new FromBytesWithCompression[F, A](self, compressor) 17 | } 18 | 19 | private final class ToBytesWithCompression[F[_]: Monad, A]( 20 | self: ToBytes[F, A], 21 | compressor: Compressor[F] 22 | ) extends ToBytes[F, A] { 23 | def apply(a: A, topic: Topic): F[Bytes] = 24 | for { 25 | bytes <- self(a, topic) 26 | compressedBytes <- compressor.to(ByteVector.view(bytes)) 27 | } yield compressedBytes.toArray 28 | } 29 | 30 | private final class FromBytesWithCompression[F[_]: Monad, A]( 31 | self: FromBytes[F, A], 32 | compressor: Compressor[F] 33 | ) extends FromBytes[F, A] { 34 | def apply(bytes: Bytes, topic: Topic): F[A] = 35 | for { 36 | decompressed <- compressor.from(ByteVector.view(bytes)) 37 | value <- self(decompressed.toArray, topic) 38 | } yield value 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/persistence/compression/Header.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence.compression 2 | 3 | /** Meta-information about a compressed user state */ 4 | final case class Header(compressed: Boolean) 5 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/persistence/compression/HeaderAndPayload.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence.compression 2 | 3 | import cats.MonadThrow 4 | import cats.syntax.all.* 5 | import scodec.Attempt 6 | import scodec.bits.ByteVector 7 | import scodec.codecs.* 8 | 9 | private[compression] object HeaderAndPayload { 10 | 11 | private val codec = variableSizeBytes(int32, bytes) ~ bytes 12 | 13 | def toBytes[F[_]: MonadThrow](header: ByteVector, payload: ByteVector): F[ByteVector] = { 14 | codec.encode((header, payload)) match { 15 | case Attempt.Successful(value) => value.bytes.pure[F] 16 | case Attempt.Failure(e) => CompressionError(e.message).raiseError 17 | } 18 | } 19 | 20 | def fromBytes[F[_]: MonadThrow](bytes: ByteVector): F[(ByteVector, ByteVector)] = { 21 | codec.decode(bytes.bits) match { 22 | case Attempt.Successful(value) => value.value.pure[F] 23 | case Attempt.Failure(e) => CompressionError(e.message).raiseError 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/snapshot/KafkaSnapshot.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import com.evolutiongaming.kafka.flow.kafka.ToOffset 4 | import com.evolutiongaming.skafka.Offset 5 | 6 | /** Snapshot of the current state. 7 | * 8 | * We want to have it parameterized by `T`, because of performance reasons. 9 | * 10 | * I.e. we want to store snapshot metadata on every step of our folds, but we do not want to serialize it to 11 | * `ByteVector` each time unless we actually decided to save snapshot into persistence. 12 | */ 13 | final case class KafkaSnapshot[T]( 14 | offset: Offset, 15 | // Reserved field 16 | metadata: String = "", 17 | value: T 18 | ) 19 | object KafkaSnapshot { 20 | implicit def toOffsets[T]: ToOffset[KafkaSnapshot[T]] = { snapshot => 21 | snapshot.offset 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/snapshot/SnapshotDatabase.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.effect.{Ref, Sync} 4 | import cats.mtl.Stateful 5 | import cats.syntax.all.* 6 | import cats.{Applicative, Functor, Monad} 7 | import com.evolutiongaming.catshelper.LogOf 8 | import com.evolutiongaming.kafka.flow.LogPrefix 9 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 10 | 11 | trait SnapshotDatabase[F[_], K, S] extends SnapshotReadDatabase[F, K, S] with SnapshotWriteDatabase[F, K, S] 12 | 13 | trait SnapshotReadDatabase[F[_], K, S] { 14 | 15 | /** Restores snapshot for the key, if any */ 16 | def get(key: K): F[Option[S]] 17 | } 18 | 19 | trait SnapshotWriteDatabase[F[_], K, S] { self => 20 | 21 | /** Adds or replaces the snapshot in a database */ 22 | def persist(key: K, snapshot: S): F[Unit] 23 | 24 | /** Deletes snapshot for they key, if any */ 25 | def delete(key: K): F[Unit] 26 | 27 | } 28 | 29 | object SnapshotDatabase { 30 | 31 | /** Creates in-memory database implementation. 32 | * 33 | * The data will survive destruction of specific `Snapshots` instance, but will not survive destruction of specific 34 | * `SnapshotDatabase` instance. 35 | */ 36 | def memory[F[_]: Ref.Make: Monad, K, S]: F[SnapshotDatabase[F, K, S]] = 37 | Ref.of[F, Map[K, S]](Map.empty).map(storage => memory(storage.stateInstance)) 38 | 39 | /** Creates in-memory database implementation. 40 | * 41 | * The data will survive destruction of specific `Snapshots` instance, but will not survive destruction of specific 42 | * `SnapshotDatabase` instance. 43 | */ 44 | def memory[F[_]: Functor, K, S](storage: Stateful[F, Map[K, S]]): SnapshotDatabase[F, K, S] = 45 | new SnapshotDatabase[F, K, S] { 46 | 47 | def persist(key: K, snapshot: S) = 48 | storage modify (_ + (key -> snapshot)) 49 | 50 | def get(key: K) = 51 | storage.get map (_ get key) 52 | 53 | def delete(key: K) = 54 | storage modify (_ - key) 55 | 56 | } 57 | 58 | // TODO: clean up this code (coming from `kafka-flow-persistence-kafka`?) 59 | def apply[F[_], K, S]( 60 | read: SnapshotReadDatabase[F, K, S], 61 | write: SnapshotWriteDatabase[F, K, S] 62 | ): SnapshotDatabase[F, K, S] = 63 | new SnapshotDatabase[F, K, S] { 64 | override def persist(key: K, snapshot: S): F[Unit] = write.persist(key, snapshot) 65 | 66 | override def delete(key: K): F[Unit] = write.delete(key) 67 | 68 | override def get(key: K): F[Option[S]] = read.get(key) 69 | } 70 | 71 | def empty[F[_]: Applicative, K, S]: SnapshotDatabase[F, K, S] = 72 | new SnapshotDatabase[F, K, S] { 73 | def get(key: K) = none[S].pure 74 | def persist(key: K, snapshot: S) = ().pure 75 | def delete(key: K) = ().pure 76 | } 77 | 78 | implicit class SnapshotDatabaseKafkaSnapshotOps[F[_], K, S]( 79 | val self: SnapshotDatabase[F, K, KafkaSnapshot[S]] 80 | ) extends AnyVal { 81 | 82 | def snapshotsOf( 83 | implicit F: Sync[F], 84 | logOf: LogOf[F], 85 | logPrefix: LogPrefix[K] 86 | ): F[SnapshotsOf[F, K, KafkaSnapshot[S]]] = 87 | logOf(SnapshotDatabase.getClass) map { implicit log => key => Snapshots.of(key, self) } 88 | 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/snapshot/SnapshotFold.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.Applicative 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.kafka.flow.FoldOption 6 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 7 | import scodec.bits.ByteVector 8 | 9 | /** Wraps state into `KafkaSnapshot` and deduplicates by offset */ 10 | object SnapshotFold { 11 | 12 | /** Creates `SnapshotFold` without metrics */ 13 | def apply[F[_]: Applicative, S]( 14 | fold: FoldOption[F, S, ConsumerRecord[String, ByteVector]] 15 | ): FoldOption[F, KafkaSnapshot[S], ConsumerRecord[String, ByteVector]] = 16 | fold 17 | .transformState[KafkaSnapshot[S]](_.value) { (state, record) => 18 | KafkaSnapshot(value = state, offset = record.offset) 19 | } 20 | .filter { (snapshot, record) => 21 | record.offset > snapshot.offset 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/snapshot/Snapshots.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.effect.Ref 4 | import cats.mtl.Stateful 5 | import cats.syntax.all.* 6 | import cats.{Applicative, Monad} 7 | import com.evolutiongaming.catshelper.Log 8 | import com.evolutiongaming.kafka.flow.LogPrefix 9 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 10 | 11 | trait Snapshots[F[_], S] extends SnapshotReader[F, S] with SnapshotWriter[F, S] 12 | 13 | /** Allows to read a previously saved snapshot */ 14 | trait SnapshotReader[F[_], S] { 15 | 16 | /** Restores a snapshot */ 17 | def read: F[Option[S]] 18 | 19 | } 20 | 21 | /** Provides a persistence for a specific key */ 22 | trait SnapshotWriter[F[_], S] { 23 | 24 | /** Saves the next snapshot to a buffer. 25 | * 26 | * Note, that completing the append does not guarantee that the state will be persisted. I.e. persistence might 27 | * choose to do the updates in batches. 28 | */ 29 | def append(snapshot: S): F[Unit] 30 | 31 | /** Flushes buffer to a database */ 32 | def flush: F[Unit] 33 | 34 | /** Removes state from the buffers and optionally also from persistence. 35 | * 36 | * @param persist 37 | * if `true` then also calls underlying database, flushes buffers only otherwise. 38 | */ 39 | def delete(persist: Boolean): F[Unit] 40 | 41 | } 42 | object Snapshots { 43 | 44 | /** Creates a buffer for a given writer */ 45 | private[flow] def of[F[_]: Ref.Make: Monad, K: LogPrefix, S]( 46 | key: K, 47 | database: SnapshotDatabase[F, K, S] 48 | )(implicit log: Log[F]): F[Snapshots[F, S]] = 49 | Ref.of[F, Option[Snapshot[S]]](None).map(buffer => Snapshots(key, database, buffer.stateInstance)) 50 | 51 | private[snapshot] def apply[F[_]: Monad, K: LogPrefix, S]( 52 | key: K, 53 | database: SnapshotDatabase[F, K, S], 54 | buffer: Stateful[F, Option[Snapshot[S]]] 55 | )(implicit log: Log[F]): Snapshots[F, S] = new Snapshots[F, S] { 56 | private val prefixLog: Log[F] = log.prefixed(LogPrefix[K].extract(key)) 57 | 58 | def read = database.get(key) 59 | 60 | def append(snapshot: S) = { 61 | buffer.modify { 62 | case Some(s) => s.updateValue(snapshot).some 63 | case None => Snapshot.init(snapshot).some 64 | } 65 | } 66 | 67 | def flush = { 68 | for { 69 | snapshot <- buffer.get 70 | _ <- snapshot traverse_ { snapshot => 71 | if (!snapshot.persisted) { 72 | for { 73 | _ <- database.persist(key, snapshot.value) 74 | _ <- buffer.set(snapshot.copy(persisted = true).some) 75 | } yield () 76 | } else ().pure[F] 77 | } 78 | } yield () 79 | } 80 | 81 | def delete(persist: Boolean) = { 82 | val delete = if (persist) { 83 | database.delete(key) *> prefixLog.info("deleted snapshot") 84 | } else { 85 | ().pure[F] 86 | } 87 | buffer.set(None) *> delete 88 | } 89 | 90 | } 91 | 92 | def empty[F[_]: Applicative, S]: Snapshots[F, S] = new Snapshots[F, S] { 93 | def read = none[S].pure[F] 94 | def append(event: S) = ().pure[F] 95 | def flush = ().pure[F] 96 | def delete(persist: Boolean) = ().pure[F] 97 | } 98 | 99 | final case class Snapshot[S](value: S, persisted: Boolean) { self => 100 | def updateValue(newValue: S): Snapshot[S] = 101 | if (value == newValue) self 102 | else copy(value = newValue, persisted = false) 103 | } 104 | 105 | object Snapshot { 106 | def init[S](value: S): Snapshot[S] = Snapshot(value, persisted = false) 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/snapshot/SnapshotsOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.Monad 4 | import cats.effect.Ref 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.catshelper.Log 7 | import com.evolutiongaming.kafka.flow.LogPrefix 8 | 9 | trait SnapshotsOf[F[_], K, S] { 10 | 11 | def apply(key: K): F[Snapshots[F, S]] 12 | 13 | } 14 | object SnapshotsOf { 15 | 16 | def memory[F[_]: Ref.Make: Monad: Log, K: LogPrefix, S]: F[SnapshotsOf[F, K, S]] = 17 | SnapshotDatabase.memory[F, K, S].map(database => backedBy(database)) 18 | 19 | def backedBy[F[_]: Ref.Make: Monad: Log, K: LogPrefix, S](db: SnapshotDatabase[F, K, S]): SnapshotsOf[F, K, S] = { 20 | key => 21 | Snapshots.of(key, db) 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/KafkaTimer.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | import cats.ApplicativeThrow 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.skafka.{Offset => KafkaOffset} 6 | import java.time.Instant 7 | import scala.concurrent.duration.* 8 | 9 | sealed trait KafkaTimer { 10 | def valueType: String 11 | def toLong: Long 12 | def toWindow: TimerWindow 13 | } 14 | object KafkaTimer { 15 | 16 | sealed trait InstantTimer extends KafkaTimer { 17 | def value: Instant 18 | def toLong: Long = value.toEpochMilli 19 | def toWindow: TimerWindow = TimerWindow.of(value, 1.day) 20 | } 21 | final case class Clock(value: Instant) extends InstantTimer { 22 | def valueType: String = "clock" 23 | } 24 | object Clock { 25 | def ofEpochMilli(value: Long): Clock = Clock(Instant.ofEpochMilli(value)) 26 | } 27 | final case class Watermark(value: Instant) extends InstantTimer { 28 | def valueType: String = "watermark" 29 | } 30 | object Watermark { 31 | def ofEpochMilli(value: Long): Watermark = Watermark(Instant.ofEpochMilli(value)) 32 | } 33 | final case class Offset(value: KafkaOffset) extends KafkaTimer { 34 | def valueType: String = "offset" 35 | def toLong: Long = value.value 36 | def toWindow: TimerWindow = TimerWindow.of(value, 100000) 37 | } 38 | 39 | def of[F[_]: ApplicativeThrow](valueType: String, value: Long): F[KafkaTimer] = 40 | valueType match { 41 | case "clock" => Clock.ofEpochMilli(value).pure[F].widen 42 | case "watermark" => Watermark.ofEpochMilli(value).pure[F].widen 43 | case "offset" => KafkaOffset.of[F](value) map Offset.apply 44 | case other => new IllegalArgumentException(s"Unknown timer code: $other").raiseError[F, KafkaTimer] 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/TimerContext.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | import cats.Monad 4 | import cats.effect.Ref 5 | import cats.syntax.all.* 6 | 7 | import java.time.Instant 8 | 9 | trait TimerContext[F[_]] extends Timers[F] with Timestamps[F] 10 | object TimerContext { 11 | 12 | def apply[F[_]](implicit F: TimerContext[F]): TimerContext[F] = F 13 | 14 | def memory[F[_]: Monad: Ref.Make](createdAt: Timestamp): F[TimerContext[F]] = 15 | Timestamps.of[F](createdAt) flatMap { implicit timestamps => 16 | Timers.memory[F].map(timers => TimerContext(timers, timestamps)) 17 | } 18 | 19 | def apply[F[_]](timers: Timers[F], timestamps: Timestamps[F]): TimerContext[F] = new TimerContext[F] { 20 | 21 | def current: F[Timestamp] = timestamps.current 22 | def persistedAt: F[Option[Timestamp]] = timestamps.persistedAt 23 | def processedAt: F[Option[Timestamp]] = timestamps.processedAt 24 | 25 | def set(timestamp: Timestamp): F[Unit] = timestamps.set(timestamp) 26 | def onPersisted: F[Unit] = timestamps.onPersisted 27 | def onProcessed: F[Unit] = timestamps.onProcessed 28 | 29 | def registerProcessing(timestamp: Instant): F[Unit] = timers.registerProcessing(timestamp) 30 | def trigger(flow: TimerFlow[F]): F[Unit] = timers.trigger(flow) 31 | 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/TimerFlow.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | import cats.Applicative 4 | import cats.syntax.all.* 5 | 6 | /** Processes the timer trigger event. 7 | * 8 | * Provides an additional callback in addition to the one caused by incoming data for the specific key. 9 | * 10 | * I.e., if one needs to react to the events incoming from Kafka, one just builds an appropriate `FoldOption`. But, if 11 | * the event must be triggered even if there is no specific key encountered in Kafka (i.e. for session expiration) then 12 | * `TimerFlow` could be used instead. 13 | */ 14 | trait TimerFlow[F[_]] { 15 | 16 | def onTimer: F[Unit] 17 | 18 | } 19 | object TimerFlow { 20 | 21 | def apply[F[_]](implicit F: TimerFlow[F]): TimerFlow[F] = F 22 | 23 | def empty[F[_]: Applicative]: TimerFlow[F] = new TimerFlow[F] { 24 | def onTimer = ().pure[F] 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/TimerType.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | private[timer] sealed trait TimerType 4 | private[timer] object TimerType { 5 | case object Clock extends TimerType 6 | case object Watermark extends TimerType 7 | case object Offset extends TimerType 8 | } 9 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/TimerWindow.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | import com.evolutiongaming.skafka.Offset 4 | import java.time.Instant 5 | import scala.concurrent.duration.FiniteDuration 6 | 7 | /** Represents next window for which persistent timers should be loaded */ 8 | private[timer] final case class TimerWindow(value: Long, size: Long) { 9 | def next: TimerWindow = this.copy(value = value + size) 10 | } 11 | private[timer] object TimerWindow { 12 | 13 | private def unsafe(value: Long, size: Long): TimerWindow = 14 | TimerWindow(value - (value % size), size) 15 | 16 | def of(value: Instant, size: FiniteDuration): TimerWindow = 17 | unsafe(value.toEpochMilli, size.toMillis) 18 | 19 | def of(value: Offset, size: Long): TimerWindow = 20 | unsafe(value.value, size) 21 | 22 | } 23 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/Timers.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | import cats.effect.Ref 4 | import cats.mtl.Stateful 5 | import cats.syntax.all.* 6 | import cats.{Applicative, Monad} 7 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 8 | 9 | import java.time.Instant 10 | 11 | /** Contains the scheduled timers for the key. */ 12 | trait Timers[F[_]] { 13 | 14 | /** Timer which takes the current time from the clock */ 15 | def registerProcessing(timestamp: Instant): F[Unit] 16 | 17 | /** Triggers all the timers which were registered to trigger at or before current timestamp */ 18 | def trigger(F: TimerFlow[F]): F[Unit] 19 | } 20 | 21 | object Timers { 22 | 23 | final case class TimerState(processing: Set[Instant] = Set.empty) { 24 | def registerProcessing(timestamp: Instant): TimerState = 25 | this.copy(processing = processing + timestamp) 26 | 27 | /** Remove expired timers */ 28 | def expire(timestamp: Timestamp): TimerState = TimerState(processing.filter(_.isAfter(timestamp.clock))) 29 | 30 | } 31 | 32 | def memory[F[_]: Ref.Make: Monad: ReadTimestamps]: F[Timers[F]] = 33 | Ref.of(TimerState()).map(storage => Timers(storage.stateInstance)) 34 | 35 | def transient[F[_]: Monad]( 36 | buffer: Stateful[F, TimerState], 37 | timestamps: ReadTimestamps[F] 38 | ): Timers[F] = { 39 | implicit val _timestamps = timestamps 40 | Timers(buffer) 41 | } 42 | 43 | def apply[F[_]: Monad: ReadTimestamps]( 44 | buffer: Stateful[F, TimerState] 45 | ): Timers[F] = new Timers[F] { 46 | def registerProcessing(timestamp: Instant): F[Unit] = 47 | buffer modify (_.registerProcessing(timestamp)) 48 | 49 | def trigger(timerFlow: TimerFlow[F]): F[Unit] = for { 50 | timestamp <- ReadTimestamps[F].current 51 | beforeExpiration <- buffer.get 52 | afterExpiration = beforeExpiration expire timestamp 53 | _ <- 54 | if (beforeExpiration != afterExpiration) { 55 | buffer.set(afterExpiration) *> timerFlow.onTimer 56 | } else { 57 | ().pure[F] 58 | } 59 | } yield () 60 | } 61 | 62 | def empty[F[_]: Applicative]: Timers[F] = new Timers[F] { 63 | def registerProcessing(timestamp: Instant) = ().pure[F] 64 | def trigger(timerFlow: TimerFlow[F]) = ().pure[F] 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/TimersOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | import cats.Monad 4 | import cats.effect.Ref 5 | 6 | trait TimersOf[F[_], K] { 7 | 8 | def apply(key: K, createdAt: Timestamp): F[TimerContext[F]] 9 | 10 | } 11 | 12 | object TimersOf { 13 | 14 | def memory[F[_]: Monad: Ref.Make, K]: F[TimersOf[F, K]] = { 15 | val timersOf = new TimersOf[F, K] { 16 | override def apply(key: K, createdAt: Timestamp): F[TimerContext[F]] = { 17 | TimerContext.memory[F](createdAt) 18 | } 19 | } 20 | 21 | Monad[F].pure(timersOf) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/Timestamp.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | import com.evolutiongaming.skafka.Offset 4 | import java.time.Instant 5 | 6 | final case class Timestamp( 7 | clock: Instant, 8 | watermark: Option[Instant], 9 | offset: Offset 10 | ) 11 | -------------------------------------------------------------------------------- /core/src/main/scala/com/evolutiongaming/kafka/flow/timer/Timestamps.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.timer 2 | 3 | import cats.{Functor, Monad} 4 | import cats.effect.Ref 5 | import cats.mtl.Stateful 6 | import cats.syntax.all.* 7 | import com.evolutiongaming.kafka.flow.effect.CatsEffectMtlInstances.* 8 | 9 | /** Contains timestamp related to a specific key. 10 | * 11 | * I.e. when the key was persisted, processed etc. 12 | */ 13 | trait Timestamps[F[_]] extends ReadTimestamps[F] with WriteTimestamps[F] 14 | trait ReadTimestamps[F[_]] { 15 | 16 | /** When the current event happened, i.e. batch of records came in or timer triggered */ 17 | def current: F[Timestamp] 18 | 19 | /** Value of timer when the state was persisted last time */ 20 | def persistedAt: F[Option[Timestamp]] 21 | 22 | /** Value of timer when the state was processed last time */ 23 | def processedAt: F[Option[Timestamp]] 24 | 25 | } 26 | trait WriteTimestamps[F[_]] { 27 | 28 | /** Set the current event timestamp (before processing or triggering the timer) */ 29 | def set(timestamp: Timestamp): F[Unit] 30 | 31 | /** Use the `current` timestamp to record that persisting an event just happened */ 32 | def onPersisted: F[Unit] 33 | 34 | /** Use the `current` timestamp to record that processing an event just happened */ 35 | def onProcessed: F[Unit] 36 | 37 | } 38 | object Timestamps { 39 | 40 | final case class TimestampState( 41 | current: Timestamp, 42 | persisted: Option[Timestamp] = None, 43 | processed: Option[Timestamp] = None 44 | ) 45 | 46 | def apply[F[_]](implicit F: Timestamps[F]): Timestamps[F] = F 47 | 48 | /** Creates a timestamp storage for a key. 49 | * 50 | * @param createdAt 51 | * Current timestamp at the time the key was encountered. 52 | */ 53 | def of[F[_]: Monad: Ref.Make](createdAt: Timestamp): F[Timestamps[F]] = 54 | Ref.of(TimestampState(createdAt)) map { storage => 55 | Timestamps(storage.stateInstance) 56 | } 57 | 58 | /** Creates a timestamp storage for a key */ 59 | def apply[F[_]: Functor]( 60 | storage: Stateful[F, TimestampState] 61 | ): Timestamps[F] = new Timestamps[F] { 62 | 63 | def current = storage.get map (_.current) 64 | def persistedAt = storage.get map (_.persisted) 65 | def processedAt = storage.get map (_.processed) 66 | 67 | def set(timestamp: Timestamp) = storage modify (_.copy(current = timestamp)) 68 | def onPersisted = storage modify { state => 69 | state.copy(persisted = Some(state.current)) 70 | } 71 | def onProcessed = storage modify { state => 72 | state.copy(processed = Some(state.current)) 73 | } 74 | 75 | } 76 | 77 | } 78 | object ReadTimestamps { 79 | def apply[F[_]](implicit F: ReadTimestamps[F]): ReadTimestamps[F] = F 80 | } 81 | object WriteTimestamps { 82 | def apply[F[_]](implicit F: WriteTimestamps[F]): WriteTimestamps[F] = F 83 | } 84 | -------------------------------------------------------------------------------- /core/src/test/scala/com/evolutiongaming/kafka/flow/FoldSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Id 4 | import munit.FunSuite 5 | 6 | class FoldSpec extends FunSuite { 7 | 8 | test("Fold#productR combines two folds correctly") { 9 | val subtract = Fold[Id, Int, Int] { (s, a) => s - a } 10 | val multiply = Fold[Id, Int, Int] { (s, a) => s * a } 11 | 12 | val subtractAndMultiply = subtract *> multiply 13 | val multiplyAndSubtract = multiply *> subtract 14 | 15 | // (0 - 10) * 10 = -100 16 | assert(subtractAndMultiply(0, 10) == -100) 17 | 18 | // 0 * 10 - 10 = -10 19 | assert(multiplyAndSubtract(0, 10) == -10) 20 | } 21 | 22 | test("Fold#handleErrorWith keeps the state from the first fold if used early") { 23 | 24 | type F[T] = Either[String, T] 25 | 26 | val add = Fold[F, Int, Int] { (s, a) => Right(s + a) } 27 | val fail = Fold[F, Int, Int] { (_, _) => Left("failed") } 28 | 29 | val addAndFail = add *> fail 30 | val failAndRecover = fail.handleErrorWith[String] { (s, _) => Right(s) } 31 | 32 | val recoverEarly = add *> failAndRecover 33 | val recoverLate = addAndFail.handleErrorWith[String] { (s, _) => Right(s) } 34 | 35 | // 1 + 2 = 3 36 | assertEquals(recoverEarly(1, 2), Right(3)) 37 | 38 | // 1 = 1 39 | assertEquals(recoverLate(1, 2), Right(1)) 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /core/src/test/scala/com/evolutiongaming/kafka/flow/MonadStateHelper.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Functor 4 | import cats.mtl.Stateful 5 | import cats.syntax.all.* 6 | import monocle.Lens 7 | 8 | object MonadStateHelper { 9 | 10 | implicit class MonadStateOps[F[_]: Functor, A, B](val self: Stateful[F, A]) { 11 | def focus(lens: Lens[A, B]): Stateful[F, B] = new Stateful[F, B] { 12 | val monad = self.monad 13 | def get = self.get.map(lens.get(_)) 14 | def set(b: B) = self.modify(lens.replace(b)) 15 | } 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/test/scala/com/evolutiongaming/kafka/flow/journal/JournalDatabaseSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal 2 | 3 | import cats.data.State 4 | import cats.mtl.Stateful 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.kafka.flow.journal.JournalDatabaseSpec.* 7 | import com.evolutiongaming.kafka.flow.kafka.ToOffset 8 | import com.evolutiongaming.skafka.Offset 9 | import munit.FunSuite 10 | 11 | import scala.collection.immutable.SortedMap 12 | 13 | class JournalDatabaseSpec extends FunSuite { 14 | 15 | test("JournalDatabase.memory stores records correctly") { 16 | 17 | val f = new ConstFixture 18 | 19 | // Given("empty database") 20 | val database = JournalDatabase.memory(f.database) 21 | 22 | // When("update is performed") 23 | val program = 24 | database.persist("key1", 100 -> "event1") *> 25 | database.persist("key1", 101 -> "event2") *> 26 | database.persist("key1", 102 -> "event3") *> 27 | database.get("key1").toList 28 | 29 | val result = program.runA(Map.empty).value 30 | 31 | // Then("records go to the database") 32 | assert( 33 | result == 34 | List(100 -> "event1", 101 -> "event2", 102 -> "event3") 35 | ) 36 | 37 | } 38 | 39 | test("JournalDatabase.memory does not delete a wrong key") { 40 | 41 | val f = new ConstFixture 42 | 43 | // Given("database with a key1") 44 | val database = JournalDatabase.memory(f.database) 45 | val context = Map( 46 | "key1" -> SortedMap( 47 | Offset.unsafe(100) -> ((100, "event1")), 48 | Offset.unsafe(101) -> ((101, "event2")), 49 | Offset.unsafe(102) -> ((102, "event3")) 50 | ) 51 | ) 52 | 53 | // When("wrong key gets deleted") 54 | val program = 55 | database.delete("key2") *> 56 | database.get("key1").toList 57 | 58 | // Then("records do not disappear the database") 59 | val result = program.runA(context).value 60 | assert(result.nonEmpty) 61 | 62 | } 63 | 64 | test("JournalDatabase.memory deletes a right key") { 65 | 66 | val f = new ConstFixture 67 | 68 | // Given("database with a key1") 69 | val database = JournalDatabase.memory(f.database) 70 | val context = Map( 71 | "key1" -> SortedMap( 72 | Offset.unsafe(100) -> ((100, "event1")), 73 | Offset.unsafe(101) -> ((101, "event2")), 74 | Offset.unsafe(102) -> ((102, "event3")) 75 | ) 76 | ) 77 | 78 | // When("right key gets deleted") 79 | val program = 80 | database.delete("key1") *> 81 | database.get("key1").toList 82 | 83 | // Then("records disappear from the database") 84 | val result = program.runA(context).value 85 | assert(result.isEmpty) 86 | 87 | } 88 | 89 | } 90 | object JournalDatabaseSpec { 91 | 92 | type F[T] = State[Context, T] 93 | 94 | type K = String 95 | type E = (Int, String) 96 | type Context = Map[K, SortedMap[Offset, E]] 97 | 98 | class ConstFixture { 99 | val database = Stateful[F, Context] 100 | } 101 | 102 | implicit val withOffset: ToOffset[(Int, String)] = { 103 | case (offset, _) => 104 | Offset.unsafe(offset) 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /core/src/test/scala/com/evolutiongaming/kafka/flow/key/KeysSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.data.State 4 | import cats.mtl.Stateful 5 | import com.evolutiongaming.catshelper.Log 6 | import com.evolutiongaming.kafka.flow.key.KeysSpec.* 7 | import munit.FunSuite 8 | 9 | class KeysSpec extends FunSuite { 10 | 11 | test("Keys add key to a database on flush") { 12 | 13 | val f = new ConstFixture 14 | 15 | // Given("empty database") 16 | val database = KeyDatabase.memory(f.database) 17 | val keys = Keys.of("key1", database) 18 | 19 | // When("Keys is flushed") 20 | val program = keys.flush 21 | 22 | val result = program.runS(Set.empty).value 23 | 24 | // Then("state gets into database") 25 | assert(result == Set("key1")) 26 | 27 | } 28 | 29 | test("Keys delete a key from a database when requested") { 30 | 31 | val f = new ConstFixture 32 | 33 | // Given("database with contents") 34 | val database = KeyDatabase.memory(f.database) 35 | val snapshots = Keys.of("key1", database) 36 | val context = Set("key1") 37 | 38 | // When("delete is requested") 39 | val program = snapshots.delete(true) 40 | val result = program.runS(context).value 41 | 42 | // Then("key is deleted") 43 | assert(result.isEmpty) 44 | 45 | } 46 | 47 | test("Keys do not delete a key from a database when not requested") { 48 | 49 | val f = new ConstFixture 50 | 51 | // Given("database with contents") 52 | val database = KeyDatabase.memory(f.database) 53 | val snapshots = Keys.of("key1", database) 54 | val context = Set("key1") 55 | 56 | // When("delete is requested") 57 | val program = snapshots.delete(false) 58 | val result = program.runS(context).value 59 | 60 | // Then("key is not deleted") 61 | assert(result.nonEmpty) 62 | 63 | } 64 | 65 | } 66 | 67 | object KeysSpec { 68 | 69 | type F[T] = State[Set[String], T] 70 | 71 | class ConstFixture { 72 | val database = Stateful[F, Set[String]] 73 | } 74 | 75 | implicit val log: Log[F] = Log.empty[F] 76 | 77 | } 78 | -------------------------------------------------------------------------------- /core/src/test/scala/com/evolutiongaming/kafka/flow/persistence/compression/CompressorSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence.compression 2 | 3 | import com.evolutiongaming.skafka.{FromBytes, ToBytes} 4 | import munit.FunSuite 5 | import scodec.bits.{BitVector, ByteVector} 6 | 7 | import java.nio.charset.Charset 8 | import scala.util.Try 9 | 10 | class CompressorSpec extends FunSuite { 11 | import scodec.codecs.* 12 | 13 | implicit val headerToBytes: ToBytes[Try, Header] = 14 | (h, _) => bool.encode(h.compressed).toTry.map(_.toByteArray) 15 | implicit val headerFromBytes: FromBytes[Try, Header] = 16 | (bytes, _) => bool.decode(BitVector(bytes)).map(dr => Header(dr.value)).toTry 17 | 18 | val compressor = Compressor.of[Try](compressionThreshold = 1).get 19 | 20 | test("compression below threshold") { 21 | val t = for { 22 | compressor <- Compressor.of[Try](compressionThreshold = 10000) 23 | bytes <- ByteVector.encodeString("test")(Charset.defaultCharset()).toTry 24 | compressed <- compressor.to(bytes) 25 | uncompressed <- compressor.from(compressed) 26 | uncompressedString <- uncompressed.decodeString(Charset.defaultCharset()).toTry 27 | } yield { 28 | assertEquals(compressed.length, 9L) // 4-byte string + metainformation 29 | assertEquals(uncompressed, bytes) 30 | assertEquals(uncompressedString, "test") 31 | } 32 | 33 | t.get 34 | } 35 | 36 | test("compression above threshold") { 37 | val t = for { 38 | compressor <- Compressor.of[Try](compressionThreshold = 1) 39 | bytes <- ByteVector.encodeString("test")(Charset.defaultCharset()).toTry 40 | compressed <- compressor.to(bytes) 41 | uncompressed <- compressor.from(compressed) 42 | uncompressedString <- uncompressed.decodeString(Charset.defaultCharset()).toTry 43 | } yield { 44 | assertEquals(uncompressed.length, bytes.length) 45 | assertEquals(uncompressedString, "test") 46 | 47 | assertEquals(compressed.length, 14L) 48 | assertNotEquals(compressed.length, bytes.length) 49 | assertNotEquals(compressed, bytes) 50 | } 51 | 52 | t.get 53 | } 54 | 55 | test("backward-compatibility support") { 56 | val t = for { 57 | compressor <- Compressor.of[Try](compressionThreshold = 1) 58 | bytes <- ByteVector.encodeString("""{"key":"value"}""")(Charset.defaultCharset()).toTry 59 | uncompressed <- compressor.from(bytes) 60 | } yield { 61 | assertEquals(uncompressed, bytes) 62 | } 63 | 64 | t.get 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /core/src/test/scala/com/evolutiongaming/kafka/flow/snapshot/SnapshotFoldSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.Id 4 | import com.evolutiongaming.kafka.flow.FoldOption 5 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 6 | import com.evolutiongaming.skafka.{Offset, TopicPartition} 7 | import munit.FunSuite 8 | import scodec.bits.ByteVector 9 | 10 | import SnapshotFoldSpec.* 11 | 12 | class SnapshotFoldSpec extends FunSuite { 13 | 14 | test("SnapshotFold updates KafkaSnapshot when there is no state") { 15 | val f = new ConstFixture 16 | val state = f.fold( 17 | s = None, 18 | a = ConsumerRecord[String, ByteVector](TopicPartition.empty, Offset.unsafe(1), None) 19 | ) 20 | assert(state == Some(KafkaSnapshot(offset = Offset.unsafe(1), value = 100))) 21 | } 22 | 23 | test("SnapshotFold updates KafkaSnapshot when there is an existing state") { 24 | val f = new ConstFixture 25 | val state = f.fold( 26 | s = Some(KafkaSnapshot(offset = Offset.unsafe(1), value = 100)), 27 | a = ConsumerRecord[String, ByteVector](TopicPartition.empty, Offset.unsafe(2), None) 28 | ) 29 | assert(state == Some(KafkaSnapshot(offset = Offset.unsafe(2), value = 200))) 30 | } 31 | 32 | test("SnapshotFold ignores duplicate update") { 33 | val f = new ConstFixture 34 | val state1 = f.fold( 35 | s = None, 36 | a = ConsumerRecord[String, ByteVector](TopicPartition.empty, Offset.unsafe(1), None) 37 | ) 38 | val state2 = f.fold( 39 | s = state1, 40 | a = ConsumerRecord[String, ByteVector](TopicPartition.empty, Offset.unsafe(1), None) 41 | ) 42 | assert(state1 == Some(KafkaSnapshot(offset = Offset.unsafe(1), value = 100))) 43 | assert(state2 == state1) 44 | } 45 | 46 | } 47 | object SnapshotFoldSpec { 48 | 49 | class ConstFixture { 50 | val fold = SnapshotFold[Id, Int]( 51 | fold = FoldOption.modifyFold { state => state map (_ + 100) orElse Some(100) } 52 | ) 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: faq 3 | title: FAQ 4 | sidebar_label: FAQ 5 | --- 6 | 7 | # Why the library eats a bit of CPU despite lack of incoming messages? 8 | 9 | Kafka Flow is validating if any of the timers need to be triggered from time to time 10 | during Kafka poll. If there are a lot of timers registered, the CPU usage could be 11 | noticable even if there are no incoming messages. 12 | 13 | If CPU usage is too high, one might opt for configuring `PartitionFlow` to do such 14 | a triggering less often. See the `PartitionFlow` documentation in overview section 15 | for more details. -------------------------------------------------------------------------------- /docs/setup.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: setup 3 | title: Setup 4 | sidebar_label: Setup 5 | --- 6 | 7 | To use Kafka Flow in your project, add the following lines to our `build.sbt` 8 | file. 9 | 10 | ```scala 11 | addSbtPlugin("com.evolution" % "sbt-artifactory-plugin" % "0.0.2") 12 | 13 | lazy val version = "1.0.4" // For cats-effect 3 - compatible version 14 | // lazy val version = "0.6.7" // For cats-effect 2 - compatible version 15 | 16 | libraryDependencies ++= Seq( 17 | "com.evolutiongaming" %% "kafka-flow" % version, 18 | // if you want to use Cassandra for storing persistent state 19 | "com.evolutiongaming" %% "kafka-flow-persistence-cassandra" % version, 20 | // if you want to use Kafka compact topic for storing persistent state 21 | "com.evolutiongaming" %% "kafka-flow-persistence-kafka" % version, 22 | // if you want to use predefined metrics 23 | "com.evolutiongaming" %% "kafka-flow-metrics" % version 24 | ) 25 | ``` 26 | -------------------------------------------------------------------------------- /docs/styleguide.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: styleguide 3 | title: Style Guide 4 | sidebar_label: Style Guide 5 | --- 6 | 7 | Overall coding style is so called Tagless Final. 8 | 9 | "Factory" pattern is renamed to "Of" pattern for sake of breivity. I.e. if you 10 | see `ConsumerFlowOf` class it is the same as `ConsumerFlowFactory`. 11 | 12 | Metric names are using Prometheus naming style guide: 13 | https://prometheus.io/docs/practices/naming/ -------------------------------------------------------------------------------- /kafka-journal/src/main/scala/com/evolutiongaming/kafka/flow/journal/JournalFold.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.catshelper.LogOf 6 | import com.evolutiongaming.kafka.flow.FoldOption 7 | import com.evolutiongaming.kafka.flow.snapshot.{KafkaSnapshot, SnapshotFold} 8 | import com.evolutiongaming.kafka.journal.SeqNr 9 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 10 | import scodec.bits.ByteVector 11 | 12 | /** Wraps state into `KafkaSnapshot` and deduplicates by sequence number in addition to offsets */ 13 | object JournalFold { 14 | 15 | // TODO: introduce new state wrapper to not force library user to store `SeqNr` in state 16 | def explicitSeqNr[F[_]: Monad: JournalParser: LogOf, S]( 17 | fold: FoldOption[F, S, ConsumerRecord[String, ByteVector]], 18 | )(stateToSeqNr: S => SeqNr): F[FoldOption[F, KafkaSnapshot[S], ConsumerRecord[String, ByteVector]]] = 19 | LogOf[F].apply(JournalFold.getClass) map { log => 20 | SnapshotFold(fold) filterM { (snapshot, record) => 21 | for { 22 | seqRange <- JournalParser[F].toSeqRange(record) 23 | stateSeqNr = stateToSeqNr(snapshot.value) 24 | // we ignore records without sequence numbers silently, but warn about the actual duplicates 25 | condition <- seqRange.fold(false.pure[F]) { seqRange => 26 | if (seqRange.from > stateSeqNr) 27 | true.pure[F] 28 | else 29 | log.info(s"skipping ($stateSeqNr, $seqRange): $record") as false 30 | } 31 | } yield condition 32 | } 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/KeyStateMetrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Monad 4 | import cats.effect.Resource 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.kafka.flow.metrics.MetricsOf 7 | import com.evolutiongaming.kafka.flow.timer.Timestamp 8 | import com.evolutiongaming.skafka.TopicPartition 9 | import com.evolutiongaming.smetrics.LabelNames 10 | 11 | object KeyStateMetrics { 12 | 13 | implicit def keyStateOfMetricsOf[F[_]: Monad]: MetricsOf[F, KeyStateOf[F]] = { registry => 14 | registry.gauge( 15 | name = "key_flow_count", 16 | help = "The number of active key flows", 17 | labels = LabelNames("topic") 18 | ) map { countGauge => 19 | def count(topic: String) = { 20 | val count = countGauge.labels(topic) 21 | Resource.make(count.inc()) { _ => count.dec() } 22 | } 23 | 24 | keyStateOf => 25 | new KeyStateOf[F] { 26 | def apply( 27 | topicPartition: TopicPartition, 28 | key: String, 29 | createdAt: Timestamp, 30 | context: KeyContext[F] 31 | ) = count(topicPartition.topic) *> keyStateOf(topicPartition, key, createdAt, context) 32 | 33 | def all(topicPartition: TopicPartition) = 34 | keyStateOf.all(topicPartition) 35 | } 36 | 37 | } 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/PartitionFlowMetrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.catshelper.MeasureDuration 6 | import com.evolutiongaming.kafka.flow.kafka.ScheduleCommit 7 | import com.evolutiongaming.kafka.flow.metrics.MetricsOf 8 | import com.evolutiongaming.kafka.flow.metrics.syntax.* 9 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 10 | import com.evolutiongaming.skafka.{Offset, TopicPartition} 11 | import com.evolutiongaming.smetrics.MetricsHelper.* 12 | import com.evolutiongaming.smetrics.{LabelNames, Quantile, Quantiles} 13 | import scodec.bits.ByteVector 14 | 15 | object PartitionFlowMetrics { 16 | 17 | implicit def partitionFlowMetricsOf[F[_]: Monad: MeasureDuration]: MetricsOf[F, PartitionFlow[F]] = { registry => 18 | for { 19 | applySummary <- registry.summary( 20 | name = "partition_flow_apply_duration_seconds", 21 | help = "Time required to apply a batch coming to partition flow", 22 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 23 | labels = LabelNames("topic", "partition") 24 | ) 25 | triggerTimersSummary <- registry.summary( 26 | name = "partition_flow_triggerTimers_duration_seconds", 27 | help = "Time required to apply an empty batch coming to partition flow", 28 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 29 | labels = LabelNames() 30 | ) 31 | } yield { partitionFlow => 32 | new PartitionFlow[F] { 33 | def apply(records: List[ConsumerRecord[String, ByteVector]]) = { 34 | val processRecords = partitionFlow(records) 35 | // if there are no records incoming, we are triggering timers 36 | records.headOption map { head => 37 | val topicPartition = head.topicPartition 38 | processRecords measureDuration { duration => 39 | applySummary 40 | .labels(topicPartition.topic, topicPartition.partition.show) 41 | .observe(duration.toNanos.nanosToSeconds) 42 | } 43 | } getOrElse { 44 | processRecords measureDuration { duration => 45 | triggerTimersSummary 46 | .observe(duration.toNanos.nanosToSeconds) 47 | } 48 | } 49 | } 50 | } 51 | } 52 | } 53 | 54 | implicit def partitionFlowOfMetricsOf[F[_]: Monad: MeasureDuration]: MetricsOf[F, PartitionFlowOf[F]] = 55 | partitionFlowMetricsOf[F] transform { implicit metrics => partitionFlowOf => 56 | new PartitionFlowOf[F] { 57 | def apply(topicPartition: TopicPartition, assignedAt: Offset, scheduleCommit: ScheduleCommit[F]) = 58 | partitionFlowOf(topicPartition, assignedAt, scheduleCommit) map (_.withMetrics) 59 | } 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/TopicFlowMetrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.Monad 4 | import cats.data.NonEmptySet 5 | import com.evolutiongaming.catshelper.MeasureDuration 6 | import com.evolutiongaming.kafka.flow.metrics.MetricsOf 7 | import com.evolutiongaming.kafka.flow.metrics.syntax.* 8 | import com.evolutiongaming.skafka.consumer.ConsumerRecords 9 | import com.evolutiongaming.skafka.{Offset, Partition, Topic} 10 | import com.evolutiongaming.smetrics.MetricsHelper.* 11 | import com.evolutiongaming.smetrics.{LabelNames, Quantile, Quantiles} 12 | import scodec.bits.ByteVector 13 | 14 | import kafka.Consumer 15 | 16 | object TopicFlowMetrics { 17 | 18 | implicit def topicFlowMetricsOf[F[_]: Monad: MeasureDuration]: MetricsOf[F, TopicFlow[F]] = { registry => 19 | for { 20 | applySummary <- registry.summary( 21 | name = "topic_flow_apply_duration_seconds", 22 | help = "Time required to process the records from the poll", 23 | quantiles = Quantiles(Quantile(1.0, 0.0001)), 24 | labels = LabelNames() 25 | ) 26 | addSummary <- registry.summary( 27 | name = "topic_flow_add_duration_seconds", 28 | help = "Time required to add all assigned partitions to topic flow", 29 | quantiles = Quantiles(Quantile(1.0, 0.0001)), 30 | labels = LabelNames() 31 | ) 32 | } yield { topicFlow => 33 | new TopicFlow[F] { 34 | def apply(records: ConsumerRecords[String, ByteVector]) = 35 | topicFlow.apply(records) measureDuration { duration => 36 | applySummary.observe(duration.toNanos.nanosToSeconds) 37 | } 38 | def add(partitions: NonEmptySet[(Partition, Offset)]) = 39 | topicFlow.add(partitions) measureDuration { duration => 40 | addSummary.observe(duration.toNanos.nanosToSeconds) 41 | } 42 | def remove(partitions: NonEmptySet[Partition]) = 43 | topicFlow.remove(partitions) 44 | } 45 | } 46 | } 47 | 48 | implicit def topicFlowOfMetricsOf[F[_]: Monad: MeasureDuration]: MetricsOf[F, TopicFlowOf[F]] = 49 | topicFlowMetricsOf[F] transform { implicit metrics => topicFlowOf => 50 | new TopicFlowOf[F] { 51 | def apply(consumer: Consumer[F], topic: Topic) = 52 | topicFlowOf(consumer, topic) map (_.withMetrics) 53 | } 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/compression/CompressorMetrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.compression 2 | 3 | import cats.Monad 4 | import cats.effect.Resource 5 | import cats.implicits.* 6 | import com.evolutiongaming.kafka.flow.metrics.Metrics 7 | import com.evolutiongaming.kafka.flow.persistence.compression.Compressor 8 | import com.evolutiongaming.smetrics.* 9 | import scodec.bits.ByteVector 10 | 11 | object CompressorMetrics { 12 | private[flow] class CompressorMetricsOf[F[_]: Monad]( 13 | rawBytes: LabelValues.`1`[Summary[F]], 14 | compressedBytes: LabelValues.`1`[Summary[F]] 15 | ) { 16 | def make(component: String): Metrics[Compressor[F]] = 17 | new Metrics[Compressor[F]] { 18 | override def withMetrics(compressor: Compressor[F]): Compressor[F] = new Compressor[F] { 19 | override def to(payload: ByteVector): F[ByteVector] = 20 | for { 21 | _ <- rawBytes.labels(component).observe(payload.size.toDouble) 22 | compressed <- compressor.to(payload) 23 | _ <- compressedBytes.labels(component).observe(compressed.size.toDouble) 24 | } yield compressed 25 | 26 | override def from(bytes: ByteVector): F[ByteVector] = 27 | compressor.from(bytes) 28 | } 29 | } 30 | } 31 | 32 | def compressorMetricsOf[F[_]: Monad](registry: CollectorRegistry[F]): Resource[F, CompressorMetricsOf[F]] = { 33 | val rawBytesSummary = registry.summary( 34 | name = "kafka_flow_compressor_raw_bytes", 35 | help = "Payload size in bytes before compression", 36 | quantiles = Quantiles(Quantile(value = 0.9, error = 0.05), Quantile(value = 0.99, error = 0.005)), 37 | labels = LabelNames("component") 38 | ) 39 | 40 | val compressedSummary = registry.summary( 41 | name = "kafka_flow_compressor_compressed_bytes", 42 | help = "Payload size in bytes after compression", 43 | quantiles = Quantiles(Quantile(value = 0.9, error = 0.05), Quantile(value = 0.99, error = 0.005)), 44 | labels = LabelNames("component") 45 | ) 46 | 47 | for { 48 | rawBytesSummary <- rawBytesSummary 49 | compressedSummary <- compressedSummary 50 | } yield new CompressorMetricsOf[F](rawBytesSummary, compressedSummary) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/journal/JournalDatabaseMetrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.catshelper.MeasureDuration 6 | import com.evolutiongaming.kafka.flow.KafkaKey 7 | import com.evolutiongaming.kafka.flow.metrics.MetricsOf 8 | import com.evolutiongaming.kafka.flow.metrics.syntax.* 9 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 10 | import com.evolutiongaming.smetrics.MetricsHelper.* 11 | import com.evolutiongaming.smetrics.{LabelNames, Quantile, Quantiles} 12 | import scodec.bits.ByteVector 13 | 14 | object JournalDatabaseMetrics { 15 | 16 | def of[F[_]: Monad: MeasureDuration]: MetricsOf[F, JournalDatabase[F, KafkaKey, ConsumerRecord[String, ByteVector]]] = 17 | journalDatabaseMetricsOf 18 | 19 | implicit def journalDatabaseMetricsOf[F[_]: Monad: MeasureDuration] 20 | : MetricsOf[F, JournalDatabase[F, KafkaKey, ConsumerRecord[String, ByteVector]]] = { registry => 21 | for { 22 | persistSummary <- registry.summary( 23 | name = "journal_database_persist_duration_seconds", 24 | help = "Time required to persist a single record to a database", 25 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 26 | labels = LabelNames("topic", "partition") 27 | ) 28 | getSummary <- registry.summary( 29 | name = "journal_database_get_duration_seconds", 30 | help = "Time required to get a whole journal from a database", 31 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 32 | labels = LabelNames("topic", "partition") 33 | ) 34 | deleteSummary <- registry.summary( 35 | name = "journal_database_delete_duration_seconds", 36 | help = "Time required to delete all journal records for the key from a database", 37 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 38 | labels = LabelNames("topic", "partition") 39 | ) 40 | } yield database => 41 | new JournalDatabase[F, KafkaKey, ConsumerRecord[String, ByteVector]] { 42 | def persist(key: KafkaKey, journal: ConsumerRecord[String, ByteVector]) = 43 | database.persist(key, journal) measureDuration { duration => 44 | persistSummary 45 | .labels(key.topicPartition.topic, key.topicPartition.partition.show) 46 | .observe(duration.toNanos.nanosToSeconds) 47 | } 48 | def get(key: KafkaKey) = 49 | database.get(key) measureTotalDuration { duration => 50 | getSummary 51 | .labels(key.topicPartition.topic, key.topicPartition.partition.show) 52 | .observe(duration.toNanos.nanosToSeconds) 53 | } 54 | def delete(key: KafkaKey) = 55 | database.delete(key) measureDuration { duration => 56 | deleteSummary 57 | .labels(key.topicPartition.topic, key.topicPartition.partition.show) 58 | .observe(duration.toNanos.nanosToSeconds) 59 | } 60 | 61 | } 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/key/KeyDatabaseMetrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.Monad 4 | import com.evolutiongaming.catshelper.MeasureDuration 5 | import com.evolutiongaming.kafka.flow.KafkaKey 6 | import com.evolutiongaming.kafka.flow.metrics.MetricsOf 7 | import com.evolutiongaming.kafka.flow.metrics.syntax.* 8 | import com.evolutiongaming.smetrics.LabelNames 9 | import com.evolutiongaming.smetrics.MetricsHelper.* 10 | import com.evolutiongaming.smetrics.Quantile 11 | import com.evolutiongaming.smetrics.Quantiles 12 | import com.evolutiongaming.skafka.TopicPartition 13 | 14 | object KeyDatabaseMetrics { 15 | 16 | def of[F[_]: Monad: MeasureDuration]: MetricsOf[F, KeyDatabase[F, KafkaKey]] = 17 | keyDatabaseMetricsOf 18 | 19 | implicit def keyDatabaseMetricsOf[F[_]: Monad: MeasureDuration]: MetricsOf[F, KeyDatabase[F, KafkaKey]] = { 20 | registry => 21 | for { 22 | persistSummary <- registry.summary( 23 | name = "key_database_persist_duration_seconds", 24 | help = "Time required to persist a single record to a database", 25 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 26 | labels = LabelNames() 27 | ) 28 | deleteSummary <- registry.summary( 29 | name = "key_database_delete_duration_seconds", 30 | help = "Time required to delete a key from a database", 31 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 32 | labels = LabelNames() 33 | ) 34 | allSummary <- registry.summary( 35 | name = "key_database_get_duration_seconds", 36 | help = "Time required to get all keys from a database", 37 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 38 | labels = LabelNames() 39 | ) 40 | } yield database => 41 | new KeyDatabase[F, KafkaKey] { 42 | def persist(key: KafkaKey) = 43 | database.persist(key) measureDuration { duration => 44 | persistSummary 45 | .observe(duration.toNanos.nanosToSeconds) 46 | } 47 | def all(applicationId: String, groupId: String, topicPartition: TopicPartition) = 48 | database.all(applicationId, groupId, topicPartition) measureTotalDuration { duration => 49 | allSummary 50 | .observe(duration.toNanos.nanosToSeconds) 51 | } 52 | def delete(key: KafkaKey) = 53 | database.delete(key) measureDuration { duration => 54 | deleteSummary 55 | .observe(duration.toNanos.nanosToSeconds) 56 | } 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/metrics/Metrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.metrics 2 | 3 | import cats.effect.Resource 4 | import com.evolutiongaming.smetrics.CollectorRegistry 5 | 6 | /** Enriches existing `A` instance with metrics */ 7 | trait Metrics[A] { 8 | 9 | def withMetrics(a: A): A 10 | 11 | } 12 | 13 | object Metrics { 14 | 15 | def empty[A]: Metrics[A] = (a: A) => a 16 | 17 | } 18 | 19 | /** Creates `Metrics` for specific `CollectorRegistry` */ 20 | trait MetricsOf[F[_], A] { 21 | self => 22 | 23 | def apply(collectorRegistry: CollectorRegistry[F]): Resource[F, Metrics[A]] 24 | 25 | /** Use metrics from `A` for another type `B`. 26 | * 27 | * Useful to create metrics for factory classes. 28 | * 29 | * The signature makes it easier to pass metrics as implicit value. I.e. run it as `metrics.transform { implicit 30 | * metrics => b => ... }. 31 | */ 32 | def transform[B](f: Metrics[A] => B => B): MetricsOf[F, B] = { registry => 33 | self(registry) map { metrics => b => f(metrics)(b) } 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/metrics/MetricsK.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.metrics 2 | 3 | import cats.effect.Resource 4 | import cats.~> 5 | import com.evolutiongaming.smetrics.CollectorRegistry 6 | 7 | /** Enriches existing `F[A]` instance with metrics. 8 | * 9 | * Useful when single instance of metrics is used with several different class instances only different with some time 10 | * parameter `A`. 11 | * 12 | * I.e. if one have `Repository[User]` and `Repository[Wallet]` and wants to have a single metrics instance for both of 13 | * them. 14 | */ 15 | trait MetricsK[F[_]] { 16 | 17 | def withMetrics[A](fa: F[A]): F[A] 18 | 19 | } 20 | 21 | object MetricsK { 22 | 23 | def empty[F[_]]: MetricsK[F] = new MetricsK[F] { 24 | override def withMetrics[A](fa: F[A]): F[A] = fa 25 | } 26 | 27 | } 28 | 29 | /** Creates `MetricsK` for specific `CollectorRegistry` */ 30 | trait MetricsKOf[F[_], G[_]] { self => 31 | 32 | def apply(collectorRegistry: CollectorRegistry[F]): Resource[F, MetricsK[G]] 33 | 34 | /** Use metrics from `A` for another type `B`. 35 | * 36 | * Useful to create metrics for factory classes. 37 | */ 38 | def transform[H[_]](f: MetricsK[G] => H ~> H): MetricsKOf[F, H] = { registry => 39 | self(registry) map { metrics => 40 | new MetricsK[H] { 41 | def withMetrics[A](ha: H[A]) = f(metrics)(ha) 42 | } 43 | } 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/metrics/syntax/package.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.metrics 2 | 3 | import cats.{FlatMap, Monad} 4 | import cats.effect.Resource 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.catshelper.MeasureDuration 7 | import com.evolutiongaming.smetrics.CollectorRegistry 8 | import com.evolutiongaming.sstream.Stream 9 | 10 | import scala.concurrent.duration.FiniteDuration 11 | 12 | package object syntax { 13 | 14 | implicit class MetricsOps[A](val a: A) extends AnyVal { 15 | def withMetrics(implicit metrics: Metrics[A]): A = 16 | metrics.withMetrics(a) 17 | } 18 | implicit class MetricsOfOps[F[_], A](a: A)(implicit metricsOf: MetricsOf[F, A]) { 19 | def withCollectorRegistry(registry: CollectorRegistry[F]): Resource[F, A] = 20 | metricsOf(registry) map { implicit metrics => 21 | a.withMetrics 22 | } 23 | } 24 | implicit class MetricsKOps[F[_], A](val fa: F[A]) extends AnyVal { 25 | def withMetricsK(implicit metrics: MetricsK[F]): F[A] = 26 | metrics.withMetrics(fa) 27 | } 28 | implicit class MetricsKOfOps[F[_], G[_], A](ga: G[A])(implicit metricsOf: MetricsKOf[F, G]) { 29 | def withCollectorRegistry(registry: CollectorRegistry[F]): Resource[F, G[A]] = 30 | metricsOf(registry) map { implicit metrics => 31 | ga.withMetricsK 32 | } 33 | } 34 | 35 | implicit class MetricsFlatMapOps[F[_], A](val fa: F[A]) extends AnyVal { 36 | 37 | /** Measures how long the expensive operation took. 38 | * 39 | * Allows the following usage: 40 | * ``` 41 | * def record(duration: FiniteDuration): F[Unit] = ??? 42 | * def expensiveOperation: F[Unit] 43 | * 44 | * expensiveOperation.measure(record) 45 | * ``` 46 | */ 47 | def measureDuration( 48 | onFinish: FiniteDuration => F[Unit] 49 | )(implicit F: FlatMap[F], measure: MeasureDuration[F]): F[A] = 50 | for { 51 | duration <- measure.start 52 | a <- fa 53 | duration <- duration 54 | _ <- onFinish(duration) 55 | } yield a 56 | 57 | } 58 | 59 | implicit class MetricsStreamOps[F[_], A](val self: Stream[F, A]) extends AnyVal { 60 | 61 | /** Measures how long the stream processing took. 62 | * 63 | * Allows the following usage: 64 | * ``` 65 | * def record(duration: FiniteDuration): F[Unit] = ??? 66 | * def expensiveStream: Stream[F, Unit] 67 | * 68 | * expensiveStream.measure(record) 69 | * ``` 70 | */ 71 | def measureTotalDuration( 72 | onFinish: FiniteDuration => F[Unit] 73 | )(implicit F: Monad[F], measureDuration: MeasureDuration[F]): Stream[F, A] = { 74 | new Stream[F, A] { 75 | def foldWhileM[L, R](l: L)(f: (L, A) => F[Either[L, R]]) = { 76 | for { 77 | duration <- measureDuration.start 78 | result <- self.foldWhileM(l)(f) 79 | duration <- duration 80 | _ <- onFinish(duration) 81 | } yield result 82 | } 83 | } 84 | } 85 | 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/persistence/PersistenceModuleMetrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.persistence 2 | 3 | import cats.Monad 4 | import com.evolutiongaming.catshelper.MeasureDuration 5 | import com.evolutiongaming.kafka.flow.journal.JournalDatabaseMetrics 6 | import com.evolutiongaming.kafka.flow.key.KeyDatabaseMetrics 7 | import com.evolutiongaming.kafka.flow.metrics.MetricsK 8 | import com.evolutiongaming.kafka.flow.metrics.MetricsKOf 9 | import com.evolutiongaming.kafka.flow.metrics.syntax.* 10 | import com.evolutiongaming.kafka.flow.snapshot.SnapshotDatabaseMetrics 11 | 12 | object PersistenceModuleMetrics { 13 | 14 | implicit def persistenceModuleMetricsKOf[F[_]: Monad: MeasureDuration]: MetricsKOf[F, PersistenceModule[F, *]] = 15 | registry => 16 | for { 17 | keyMetrics <- KeyDatabaseMetrics.of[F].apply(registry) 18 | journalMetrics <- JournalDatabaseMetrics.of[F].apply(registry) 19 | snapshotMetrics <- SnapshotDatabaseMetrics.of[F].apply(registry) 20 | } yield new MetricsK[PersistenceModule[F, *]] { 21 | def withMetrics[S](module: PersistenceModule[F, S]) = new PersistenceModule[F, S] { 22 | def keys = module.keys.withMetrics(keyMetrics) 23 | def journals = module.journals.withMetrics(journalMetrics) 24 | def snapshots = module.snapshots.withMetricsK(snapshotMetrics) 25 | } 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /metrics/src/main/scala/com/evolutiongaming/kafka/flow/snapshot/SnapshotDatabaseMetrics.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.catshelper.MeasureDuration 6 | import com.evolutiongaming.kafka.flow.KafkaKey 7 | import com.evolutiongaming.kafka.flow.metrics.MetricsK 8 | import com.evolutiongaming.kafka.flow.metrics.MetricsKOf 9 | import com.evolutiongaming.kafka.flow.metrics.syntax.* 10 | import com.evolutiongaming.smetrics.LabelNames 11 | import com.evolutiongaming.smetrics.MetricsHelper.* 12 | import com.evolutiongaming.smetrics.Quantile 13 | import com.evolutiongaming.smetrics.Quantiles 14 | 15 | object SnapshotDatabaseMetrics { 16 | 17 | def of[F[_]: Monad: MeasureDuration]: MetricsKOf[F, SnapshotDatabase[F, KafkaKey, *]] = 18 | snapshotDatabaseMetricsOf 19 | 20 | implicit def snapshotDatabaseMetricsOf[F[_]: Monad: MeasureDuration] 21 | : MetricsKOf[F, SnapshotDatabase[F, KafkaKey, *]] = { registry => 22 | for { 23 | persistSummary <- registry.summary( 24 | name = "snapshot_database_persist_duration_seconds", 25 | help = "Time required to persist a single snapshot to a database", 26 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 27 | labels = LabelNames("topic", "partition") 28 | ) 29 | getSummary <- registry.summary( 30 | name = "snapshot_database_get_duration_seconds", 31 | help = "Time required to get a single snapshot from a database", 32 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 33 | labels = LabelNames("topic", "partition") 34 | ) 35 | deleteSummary <- registry.summary( 36 | name = "snapshot_database_delete_duration_seconds", 37 | help = "Time required to delete all snapshots for the key from a database", 38 | quantiles = Quantiles(Quantile(0.9, 0.05), Quantile(0.99, 0.005)), 39 | labels = LabelNames("topic", "partition") 40 | ) 41 | } yield new MetricsK[SnapshotDatabase[F, KafkaKey, *]] { 42 | def withMetrics[S](database: SnapshotDatabase[F, KafkaKey, S]) = new SnapshotDatabase[F, KafkaKey, S] { 43 | def persist(key: KafkaKey, snapshot: S) = 44 | database.persist(key, snapshot) measureDuration { duration => 45 | persistSummary 46 | .labels(key.topicPartition.topic, key.topicPartition.partition.show) 47 | .observe(duration.toNanos.nanosToSeconds) 48 | } 49 | def get(key: KafkaKey) = 50 | database.get(key) measureDuration { duration => 51 | getSummary 52 | .labels(key.topicPartition.topic, key.topicPartition.partition.show) 53 | .observe(duration.toNanos.nanosToSeconds) 54 | } 55 | def delete(key: KafkaKey) = 56 | database.delete(key) measureDuration { duration => 57 | deleteSummary 58 | .labels(key.topicPartition.topic, key.topicPartition.partition.show) 59 | .observe(duration.toNanos.nanosToSeconds) 60 | } 61 | } 62 | } 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /metrics/src/test/scala/com/evolutiongaming/kafka/flow/FoldMetricsSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import com.evolutiongaming.catshelper.MeasureDuration 4 | import com.evolutiongaming.skafka.consumer.ConsumerRecord 5 | import com.evolutiongaming.smetrics.CollectorRegistry 6 | import munit.FunSuite 7 | import scodec.bits.ByteVector 8 | 9 | import FoldMetrics.* 10 | import metrics.syntax.* 11 | 12 | class FoldMetricsSpec extends FunSuite { 13 | 14 | type F[T] = Option[T] 15 | 16 | test("having MetricsKOf enables withCollectorRegistry syntax") { 17 | implicit val measureDuration: MeasureDuration[F] = MeasureDuration.empty[F] 18 | val fold: FoldOptionCons[F, Int] = FoldOption.empty[F, Int, ConsumerRecord[String, ByteVector]] 19 | fold.withCollectorRegistry(CollectorRegistry.empty[F]) 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /metrics/src/test/scala/com/evolutiongaming/kafka/flow/metrics/SyntaxSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.metrics 2 | 3 | import cats.data.State 4 | import cats.effect.Resource 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.catshelper.MeasureDuration 7 | import com.evolutiongaming.smetrics.CollectorRegistry 8 | import com.evolutiongaming.sstream.Stream 9 | import munit.FunSuite 10 | 11 | import scala.concurrent.duration.FiniteDuration 12 | import syntax.* 13 | 14 | class SyntaxSpec extends FunSuite { 15 | 16 | type F[T] = State[Option[FiniteDuration], T] 17 | 18 | def save(duration: FiniteDuration): F[Unit] = State.set(Some(duration)) 19 | 20 | implicit val stateMeasureDuration: MeasureDuration[F] = 21 | MeasureDuration.empty 22 | 23 | test("having MetricsKOf enables withCollectorRegistry syntax") { 24 | class Service[T] 25 | implicit val metricsKOf: MetricsKOf[F, Service] = { _ => 26 | Resource.pure[F, MetricsK[Service]] { 27 | new MetricsK[Service] { 28 | def withMetrics[T](service: Service[T]) = service 29 | } 30 | } 31 | } 32 | val service = new Service[Int] 33 | service.withCollectorRegistry(CollectorRegistry.empty) 34 | } 35 | 36 | test("measureTotalDuration on a stream of numbers") { 37 | val stream = Stream.from[F, List, Int](List(1, 2, 3, 4, 5)).measureTotalDuration(save) 38 | val (duration, list) = stream.toList.run(None).value 39 | assert(duration.nonEmpty) 40 | assertEquals(list, List(1, 2, 3, 4, 5)) 41 | } 42 | 43 | test("measureTotalDuration on an empty stream") { 44 | val stream = Stream.empty[F, Unit].measureTotalDuration(save) 45 | val (duration, list) = stream.toList.run(None).value 46 | assert(duration.nonEmpty) 47 | assert(list.isEmpty) 48 | } 49 | 50 | test("measureDuration on an effect") { 51 | val effect = 1.some.pure[F].measureDuration(save) 52 | val (duration, value) = effect.run(None).value 53 | assert(duration.nonEmpty) 54 | assertEquals(value, Some(1)) 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /persistence-cassandra-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/CassandraContainerResource.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import com.dimafeng.testcontainers.CassandraContainer 4 | 5 | /** Cassandra container, shared among multiple test classes. Manual stop would be preferable but not necessary as Ryuk 6 | * guarantees to collect it after tests pass 7 | */ 8 | object CassandraContainerResource { 9 | val cassandra: CassandraContainer = { 10 | val container = new CassandraContainer() 11 | container.start() 12 | container 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /persistence-cassandra-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/CassandraSessionStub.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.MonadThrow 4 | import cats.effect.Ref 5 | import cats.syntax.all.* 6 | import com.datastax.driver.core.{Host, PreparedStatement, RegularStatement, ResultSet, Statement} 7 | import com.evolutiongaming.scassandra.CassandraSession 8 | 9 | object CassandraSessionStub { 10 | 11 | def injectFailures[F[_]]( 12 | session: CassandraSession[F], 13 | failAfter: Ref[F, Int] 14 | )(implicit F: MonadThrow[F]): CassandraSession[F] = new CassandraSession[F] { 15 | def fail[T](query: String): F[T] = F.raiseError { 16 | new RuntimeException(s"CassandraSessionStub: failing after proper calls exhausted: $query") 17 | } 18 | 19 | val failed = failAfter modify { failAfter => 20 | (failAfter - 1, failAfter <= 0) 21 | } 22 | 23 | override def loggedKeyspace: F[Option[String]] = F.pure(None) 24 | override def init: F[Unit] = F.unit 25 | override def execute(query: String): F[ResultSet] = failed.ifM(fail(query), session.execute(query)) 26 | 27 | override def execute(query: String, values: Any*): F[ResultSet] = 28 | failed.ifM(fail(query), session.execute(query, values: _*)) 29 | 30 | override def execute(query: String, values: Map[String, AnyRef]): F[ResultSet] = 31 | failed.ifM(fail(query), session.execute(query, values)) 32 | 33 | override def execute(statement: Statement): F[ResultSet] = 34 | failed.ifM(fail(statement.toString), session.execute(statement)) 35 | 36 | override def prepare(query: String): F[PreparedStatement] = 37 | failed.ifM(fail(query), session.prepare(query)) 38 | 39 | override def prepare(statement: RegularStatement): F[PreparedStatement] = 40 | failed.ifM(fail(statement.toString), session.prepare(statement)) 41 | 42 | override def state: CassandraSession.State[F] = new CassandraSession.State[F] { 43 | override def connectedHosts: F[Iterable[Host]] = F.pure(Iterable.empty) 44 | override def openConnections(host: Host): F[Int] = F.pure(0) 45 | override def trashedConnections(host: Host): F[Int] = F.pure(0) 46 | override def inFlightQueries(host: Host): F[Int] = F.pure(0) 47 | } 48 | 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /persistence-cassandra-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/CassandraSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.effect.IO 4 | import cats.effect.unsafe.IORuntime 5 | import com.evolution.kafka.flow.cassandra.CassandraModule 6 | import com.evolutiongaming.catshelper.LogOf 7 | import com.evolutiongaming.kafka.flow.cassandra.CassandraConfig 8 | import com.evolutiongaming.nel.Nel 9 | import com.evolutiongaming.scassandra 10 | import munit.FunSuite 11 | 12 | import java.util.concurrent.atomic.AtomicReference 13 | 14 | abstract class CassandraSpec extends FunSuite { 15 | implicit val ioRuntime: IORuntime = IORuntime.global 16 | 17 | override def munitFixtures: Seq[Fixture[_]] = List(cassandra) 18 | 19 | val cassandra: Fixture[CassandraModule[IO]] = new Fixture[CassandraModule[IO]]("CassandraModule") { 20 | private val moduleRef = new AtomicReference[(CassandraModule[IO], IO[Unit])]() 21 | 22 | override def apply(): CassandraModule[IO] = moduleRef.get()._1 23 | 24 | override def beforeAll(): Unit = { 25 | implicit val logOf: LogOf[IO] = LogOf.slf4j[IO].unsafeRunSync() 26 | 27 | val container = CassandraContainerResource.cassandra.cassandraContainer 28 | val result: (CassandraModule[IO], IO[Unit]) = 29 | CassandraModule 30 | .of[IO]( 31 | CassandraConfig(client = 32 | scassandra.CassandraConfig(contactPoints = Nel(container.getHost), port = container.getFirstMappedPort) 33 | ) 34 | ) 35 | .allocated 36 | .unsafeRunSync() 37 | 38 | moduleRef.set(result) 39 | } 40 | 41 | override def afterAll(): Unit = { 42 | Option(moduleRef.get()).foreach { case (_, finalizer) => finalizer.unsafeRunSync() } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /persistence-cassandra-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/FlowSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.data.NonEmptyList 4 | import cats.effect.unsafe.IORuntime 5 | import cats.effect.{IO, Ref, Resource} 6 | import com.evolutiongaming.catshelper.LogOf 7 | import com.evolutiongaming.kafka.flow.cassandra.{CassandraPersistence, ConsistencyOverrides} 8 | import com.evolutiongaming.kafka.flow.kafka.Consumer 9 | import com.evolutiongaming.kafka.flow.key.CassandraKeys 10 | import com.evolutiongaming.kafka.flow.registry.EntityRegistry 11 | import com.evolutiongaming.kafka.flow.snapshot.KafkaSnapshot 12 | import com.evolutiongaming.kafka.flow.timer.{TimerFlowOf, TimersOf} 13 | import com.evolutiongaming.retry.Retry 14 | import com.evolutiongaming.skafka.consumer.{ConsumerRecord, ConsumerRecords, WithSize} 15 | import com.evolutiongaming.skafka.{Offset, TopicPartition} 16 | import scodec.bits.ByteVector 17 | 18 | import scala.concurrent.duration.* 19 | 20 | class FlowSpec extends CassandraSpec { 21 | 22 | test("flow fails when Cassandra insert fails") { 23 | val flow = for { 24 | failAfter <- Resource.eval(Ref.of[IO, Int](10000)) 25 | session = CassandraSessionStub.injectFailures(cassandra().session, failAfter) 26 | storage <- Resource.eval( 27 | CassandraPersistence 28 | .withSchema[IO, String]( 29 | session, 30 | cassandra().sync, 31 | ConsistencyOverrides.none, 32 | CassandraKeys.DefaultSegments 33 | ) 34 | ) 35 | timersOf <- Resource.eval(TimersOf.memory[IO, KafkaKey]) 36 | keysOf <- Resource.eval(storage.keys.toKeysOf) 37 | persistenceOf <- storage.restoreEvents 38 | keyStateOf = KeyStateOf.eagerRecovery[IO, KafkaSnapshot[String]]( 39 | applicationId = "FlowSpec", 40 | groupId = "integration-tests-1", 41 | keysOf = keysOf, 42 | persistenceOf = persistenceOf, 43 | timersOf = timersOf, 44 | timerFlowOf = TimerFlowOf.unloadOrphaned[IO]( 45 | fireEvery = 10.minutes, 46 | maxIdle = 30.minutes, 47 | flushOnRevoke = true 48 | ), 49 | fold = FoldOption.empty[IO, KafkaSnapshot[String], ConsumerRecord[String, ByteVector]], 50 | tick = TickOption.id[IO, KafkaSnapshot[String]], 51 | registry = EntityRegistry.empty[IO, KafkaKey, KafkaSnapshot[String]] 52 | ) 53 | partitionFlowOf = PartitionFlowOf( 54 | keyStateOf = keyStateOf, 55 | config = PartitionFlowConfig( 56 | triggerTimersInterval = 1.minute, 57 | commitOnRevoke = true 58 | ) 59 | ) 60 | topicFlowOf = TopicFlowOf(partitionFlowOf) 61 | records = NonEmptyList.of( 62 | ConsumerRecord[String, ByteVector]( 63 | topicPartition = TopicPartition.empty, 64 | offset = Offset.min, 65 | timestampAndType = None, 66 | key = Some(WithSize("key")) 67 | ) 68 | ) 69 | consumer = Consumer.repeat[IO] { 70 | ConsumerRecords(Map(TopicPartition.empty -> records)) 71 | } 72 | join <- { 73 | implicit val retry = Retry.empty[IO] 74 | KafkaFlow.resource( 75 | consumer = Resource.eval(consumer), 76 | flowOf = ConsumerFlowOf(topic = "", flowOf = topicFlowOf) 77 | ) 78 | } 79 | } yield join 80 | 81 | val test: IO[Unit] = flow use { join => 82 | join.attempt map { result => 83 | assert(clue(result.isLeft)) 84 | } 85 | } 86 | 87 | test.unsafeRunSync() 88 | } 89 | 90 | implicit val log: LogOf[IO] = LogOf.slf4j[IO].unsafeRunSync()(IORuntime.global) 91 | 92 | } 93 | -------------------------------------------------------------------------------- /persistence-cassandra-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/journal/JournalSchemaSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal 2 | 3 | import cats.effect.IO 4 | import com.evolutiongaming.kafka.flow.CassandraSpec 5 | import com.evolutiongaming.scassandra.CassandraSession 6 | 7 | import scala.concurrent.duration.* 8 | 9 | class JournalSchemaSpec extends CassandraSpec { 10 | override def munitTimeout: Duration = 2.minutes 11 | 12 | test("table is created using scassandra session API") { 13 | val session = cassandra().session 14 | val sync = cassandra().sync 15 | val schema = JournalSchema.of(session, sync, CassandraJournals.DefaultTableName) 16 | 17 | val test = for { 18 | _ <- schema.create 19 | _ <- validateTableExists(session) 20 | } yield () 21 | 22 | test.unsafeRunSync() 23 | } 24 | 25 | test("table is truncated using scassandra session API") { 26 | val session = cassandra().session 27 | val sync = cassandra().sync 28 | 29 | val schema = JournalSchema.of(session, sync, CassandraJournals.DefaultTableName) 30 | 31 | val test = for { 32 | _ <- schema.create 33 | _ <- insertRecord(session) 34 | _ <- schema.truncate 35 | _ <- validateTableIsEmpty(session) 36 | } yield () 37 | 38 | test.unsafeRunSync() 39 | } 40 | 41 | private def insertRecord(session: CassandraSession[IO]): IO[Unit] = { 42 | session 43 | .execute( 44 | """ 45 | INSERT INTO records (application_id, group_id, topic, partition, key, offset, created, timestamp, timestamp_type, headers, metadata, value) 46 | VALUES ('app', 'group', 'topic', 1, 'key', 1, toTimestamp(now()), toTimestamp(now()), 'create', {'header': 'value'}, 'metadata', textAsBlob('value')) 47 | """ 48 | ) 49 | .void 50 | } 51 | 52 | private def validateTableExists(session: CassandraSession[IO]): IO[Unit] = { 53 | for { 54 | resultSet <- session.execute( 55 | "select table_name from system_schema.tables where table_name = 'records' allow filtering" 56 | ) 57 | maybeRow <- IO.delay(Option(resultSet.one())) 58 | _ = maybeRow.fold(fail("Table 'records' not found in system_schema.tables")) { row => 59 | val name = row.getString("table_name") 60 | assert( 61 | name == "records", 62 | s"Unexpected table name '$name' in system_schema.tables, expected 'records'" 63 | ) 64 | } 65 | } yield () 66 | } 67 | 68 | private def validateTableIsEmpty(session: CassandraSession[IO]): IO[Unit] = { 69 | for { 70 | resultSet <- session.execute("select count(*) from records allow filtering") 71 | row <- IO.delay(resultSet.one()) 72 | count <- IO.delay(row.getLong(0)) 73 | _ = assert(count == 0, s"Expected 0 rows in 'records' table, found $count") 74 | } yield () 75 | } 76 | 77 | override def afterEach(context: AfterEach): Unit = { 78 | super.afterEach(context) 79 | cassandra().session.execute("DROP TABLE IF EXISTS records").void.unsafeRunSync() 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /persistence-cassandra-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/key/KeySchemaSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.effect.IO 4 | import com.evolutiongaming.kafka.flow.CassandraSpec 5 | import com.evolutiongaming.scassandra.CassandraSession 6 | 7 | import scala.concurrent.duration.* 8 | 9 | class KeySchemaSpec extends CassandraSpec { 10 | override def munitTimeout: FiniteDuration = 2.minutes 11 | 12 | test("table is created using scassandra session API") { 13 | val session = cassandra().session 14 | val sync = cassandra().sync 15 | 16 | val keySchema = KeySchema.of(session, sync, CassandraKeys.DefaultTableName) 17 | 18 | val test = for { 19 | _ <- keySchema.create 20 | _ <- validateTableExists(session) 21 | } yield () 22 | 23 | test.unsafeRunSync() 24 | } 25 | 26 | test("table is truncated using scassandra session API") { 27 | val session = cassandra().session 28 | val sync = cassandra().sync 29 | 30 | val keySchema = KeySchema.of(session, sync, CassandraKeys.DefaultTableName) 31 | 32 | val test = for { 33 | _ <- keySchema.create 34 | _ <- insertKey(session) 35 | _ <- keySchema.truncate 36 | _ <- validateTableIsEmpty(session) 37 | } yield () 38 | 39 | test.unsafeRunSync() 40 | } 41 | 42 | private def insertKey(session: CassandraSession[IO]): IO[Unit] = { 43 | session 44 | .execute( 45 | """ 46 | INSERT INTO keys (application_id, group_id, segment, topic, partition, key, created, created_date, metadata) 47 | VALUES ('app', 'group', 1, 'topic', 1, 'key', toTimestamp(now()), toDate(now()), '{}') 48 | """ 49 | ) 50 | .void 51 | } 52 | 53 | private def validateTableExists(session: CassandraSession[IO]): IO[Unit] = { 54 | for { 55 | resultSet <- session.execute( 56 | "select table_name from system_schema.tables where table_name = 'keys' allow filtering" 57 | ) 58 | maybeRow <- IO.delay(Option(resultSet.one())) 59 | _ = maybeRow.fold(fail("Table 'keys' not found in system_schema.tables")) { row => 60 | val name = row.getString("table_name") 61 | assert(name == "keys", s"Unexpected table name '$name' in system_schema.tables, expected 'keys'") 62 | } 63 | } yield () 64 | } 65 | 66 | private def validateTableIsEmpty(session: CassandraSession[IO]): IO[Unit] = { 67 | for { 68 | resultSet <- session.execute("select count(*) from keys allow filtering") 69 | row <- IO.delay(resultSet.one()) 70 | count <- IO.delay(row.getLong(0)) 71 | _ = assert(count == 0, s"Expected 0 rows in 'keys' table, found $count") 72 | } yield () 73 | } 74 | 75 | override def afterEach(context: AfterEach): Unit = { 76 | cassandra().session.execute("DROP TABLE IF EXISTS keys").void.unsafeRunSync() 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /persistence-cassandra-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/snapshot/SnapshotSchemaSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.effect.IO 4 | import com.evolutiongaming.kafka.flow.CassandraSpec 5 | import com.evolutiongaming.scassandra.CassandraSession 6 | 7 | import scala.concurrent.duration.* 8 | 9 | class SnapshotSchemaSpec extends CassandraSpec { 10 | override def munitTimeout: Duration = 2.minutes 11 | 12 | test("table is created using scassandra session API") { 13 | val session = cassandra().session 14 | val sync = cassandra().sync 15 | val schema = SnapshotSchema.of(session, sync, CassandraSnapshots.DefaultTableName) 16 | 17 | val test = for { 18 | _ <- schema.create 19 | _ <- validateTableExists(session) 20 | } yield () 21 | 22 | test.unsafeRunSync() 23 | } 24 | 25 | test("table is truncated using scassandra session API") { 26 | val session = cassandra().session 27 | val sync = cassandra().sync 28 | 29 | val schema = SnapshotSchema.of(session, sync, CassandraSnapshots.DefaultTableName) 30 | 31 | val test = for { 32 | _ <- schema.create 33 | _ <- insertSnapshot(session) 34 | _ <- schema.truncate 35 | _ <- validateTableIsEmpty(session) 36 | } yield () 37 | 38 | test.unsafeRunSync() 39 | } 40 | 41 | private def insertSnapshot(session: CassandraSession[IO]): IO[Unit] = { 42 | session 43 | .execute( 44 | """ 45 | INSERT INTO snapshots_v2 (application_id, group_id, topic, partition, key, offset, created, metadata, value) 46 | VALUES ('app_id', 'group_id', 'topic', 1, 'key', 1, toTimestamp(now()), '{}', textAsBlob('value')) 47 | """ 48 | ) 49 | .void 50 | } 51 | 52 | private def validateTableExists(session: CassandraSession[IO]): IO[Unit] = { 53 | for { 54 | resultSet <- session.execute( 55 | "select table_name from system_schema.tables where table_name = 'snapshots_v2' allow filtering" 56 | ) 57 | maybeRow <- IO.delay(Option(resultSet.one())) 58 | _ = maybeRow.fold(fail("Table 'snapshots_v2' not found in system_schema.tables")) { row => 59 | val name = row.getString("table_name") 60 | assert( 61 | name == "snapshots_v2", 62 | s"Unexpected table name '$name' in system_schema.tables, expected 'snapshots_v2'" 63 | ) 64 | } 65 | } yield () 66 | } 67 | 68 | private def validateTableIsEmpty(session: CassandraSession[IO]): IO[Unit] = { 69 | for { 70 | resultSet <- session.execute("select count(*) from snapshots_v2 allow filtering") 71 | row <- IO.delay(resultSet.one()) 72 | count <- IO.delay(row.getLong(0)) 73 | _ = assert(count == 0, s"Expected 0 rows in 'snapshots_v2' table, found $count") 74 | } yield () 75 | } 76 | 77 | override def afterEach(context: AfterEach): Unit = { 78 | super.afterEach(context) 79 | cassandra().session.execute("DROP TABLE IF EXISTS snapshots_v2").void.unsafeRunSync() 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /persistence-cassandra-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/snapshot/SnapshotSpec.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.effect.{IO, Ref} 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.kafka.flow.{CassandraSessionStub, CassandraSpec, KafkaKey} 6 | import com.evolutiongaming.scassandra.syntax.* 7 | import com.evolutiongaming.skafka.{Offset, TopicPartition} 8 | 9 | import scala.concurrent.duration.* 10 | import scala.jdk.CollectionConverters.* 11 | 12 | class SnapshotSpec extends CassandraSpec { 13 | 14 | test("queries") { 15 | val key = KafkaKey("SnapshotSpec", "integration-tests-1", TopicPartition.empty, "queries") 16 | val snapshot = KafkaSnapshot(offset = Offset.min, value = "snapshot-contents") 17 | val test: IO[Unit] = for { 18 | snapshots <- CassandraSnapshots.withSchema[IO, String](cassandra().session, cassandra().sync) 19 | snapshotBeforeTest <- snapshots.get(key) 20 | _ <- snapshots.persist(key, snapshot) 21 | snapshotAfterPersist <- snapshots.get(key) 22 | ttls <- getTtls(key) 23 | _ <- snapshots.delete(key) 24 | snapshotAfterDelete <- snapshots.get(key) 25 | } yield { 26 | assert(clue(snapshotBeforeTest.isEmpty)) 27 | assertEquals(clue(snapshotAfterPersist), Some(snapshot)) 28 | assert(clue(snapshotAfterDelete.isEmpty)) 29 | assertEquals(clue(ttls), List(none)) 30 | } 31 | 32 | test.unsafeRunSync() 33 | } 34 | 35 | test("failures") { 36 | val key = KafkaKey("SnapshotSpec", "integration-tests-1", TopicPartition.empty, "queries") 37 | val test: IO[Unit] = for { 38 | failAfter <- Ref.of[IO, Int](100) 39 | session = CassandraSessionStub.injectFailures(cassandra().session, failAfter) 40 | snapshots <- CassandraSnapshots.withSchema[IO, String](session, cassandra().sync) 41 | _ <- failAfter.set(1) 42 | snapshots <- snapshots.get(key).attempt 43 | } yield assert(clue(snapshots.isLeft)) 44 | 45 | test.unsafeRunSync() 46 | } 47 | 48 | test("ttl") { 49 | val key = KafkaKey("SnapshotSpec", "integration-tests-1", TopicPartition.empty, "queries") 50 | val snapshot = KafkaSnapshot(offset = Offset.min, value = "snapshot-contents") 51 | val test: IO[Unit] = for { 52 | snapshots <- CassandraSnapshots.withSchema[IO, String](cassandra().session, cassandra().sync, ttl = 1.hour.some) 53 | _ <- snapshots.persist(key, snapshot) 54 | snapshotAfterPersist <- snapshots.get(key) 55 | ttls <- getTtls(key) 56 | } yield { 57 | assertEquals(clue(snapshotAfterPersist), snapshot.some) 58 | assertEquals(clue(ttls.size), 1) 59 | assert(clue(ttls.head.isDefined)) 60 | } 61 | 62 | test.unsafeRunSync() 63 | } 64 | 65 | private def getTtls(key: KafkaKey): IO[List[Option[Int]]] = { 66 | val session = cassandra().session 67 | for { 68 | prepared <- session.prepare( 69 | s"""SELECT TTL(value) FROM ${CassandraSnapshots.DefaultTableName} WHERE 70 | | application_id = :application_id 71 | | AND group_id = :group_id 72 | | AND topic = :topic 73 | | AND partition = :partition 74 | | AND key = :key""".stripMargin 75 | ) 76 | bound = prepared 77 | .bind() 78 | .encode("application_id", key.applicationId) 79 | .encode("group_id", key.groupId) 80 | .encode("topic", key.topicPartition.topic) 81 | .encode("partition", key.topicPartition.partition.value) 82 | .encode("key", key.key) 83 | ttls <- session.execute(bound) 84 | } yield ttls.all().asScala.map(row => row.decodeAt[Option[Int]](0)).toList 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolution/kafka/flow/cassandra/CassandraHealthCheckOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolution.kafka.flow.cassandra 2 | 3 | import cats.Parallel 4 | import cats.effect.{Async, Resource} 5 | import com.datastax.driver.core.ConsistencyLevel 6 | import com.evolutiongaming.catshelper.LogOf 7 | import com.evolutiongaming.kafka.flow.cassandra.CassandraConfig 8 | import com.evolutiongaming.kafka.flow.cassandra.SessionHelper.* 9 | import com.evolutiongaming.scassandra.{CassandraHealthCheck, CassandraSession} 10 | 11 | private[cassandra] object CassandraHealthCheckOf { 12 | 13 | def apply[F[_]: Async: Parallel: LogOf]( 14 | cassandraSession: CassandraSession[F], 15 | config: CassandraConfig 16 | ): Resource[F, CassandraHealthCheck[F]] = { 17 | for { 18 | cassandraSession <- cassandraSession.enhanceError.cachePrepared.map(_.withRetries(config.retries)) 19 | cassandraHealthCheck <- CassandraHealthCheck.of( 20 | Resource.pure[F, CassandraSession[F]](cassandraSession), 21 | ConsistencyLevel.LOCAL_QUORUM 22 | ) 23 | } yield { 24 | cassandraHealthCheck 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolution/kafka/flow/cassandra/CassandraModule.scala: -------------------------------------------------------------------------------- 1 | package com.evolution.kafka.flow.cassandra 2 | 3 | import cats.Parallel 4 | import cats.effect.{Async, Resource, Sync} 5 | import cats.syntax.all.* 6 | import com.evolutiongaming.cassandra.sync.{AutoCreate, CassandraSync} 7 | import com.evolutiongaming.catshelper.{Log, LogOf} 8 | import com.evolutiongaming.kafka.flow.LogResource 9 | import com.evolutiongaming.kafka.flow.cassandra.CassandraConfig 10 | import com.evolutiongaming.scassandra.util.FromGFuture 11 | import com.evolutiongaming.scassandra.{CassandraClusterOf, CassandraHealthCheck, CassandraSession} 12 | import com.google.common.util.concurrent.ListenableFuture 13 | 14 | trait CassandraModule[F[_]] { 15 | def session: CassandraSession[F] 16 | def sync: CassandraSync[F] 17 | def healthCheck: CassandraHealthCheck[F] 18 | } 19 | object CassandraModule { 20 | 21 | def log[F[_]: LogOf]: F[Log[F]] = LogOf[F].apply(CassandraModule.getClass) 22 | 23 | def clusterOf[F[_]: Sync]( 24 | fromGFuture: FromGFuture[F] 25 | ): F[CassandraClusterOf[F]] = { 26 | implicit val _fromGFuture = fromGFuture 27 | CassandraClusterOf.of[F] 28 | } 29 | 30 | /** Creates connection, synchronization and health check routines 31 | * 32 | * @param config 33 | * Connection parameters. 34 | */ 35 | def of[F[_]: Async: Parallel: LogOf]( 36 | config: CassandraConfig 37 | ): Resource[F, CassandraModule[F]] = { 38 | import com.evolutiongaming.kafka.flow.cassandra.SessionHelper.* 39 | for { 40 | log <- Resource.eval(log[F]) 41 | // this is required to log all Cassandra errors before popping them up, 42 | // which is useful because popped up errors might be lost in some cases 43 | // while kafka-flow is accessing Cassandra in bracket/resource release 44 | // routine 45 | fromGFuture = new FromGFuture[F] { 46 | val self = FromGFuture.lift1[F] 47 | def apply[A](future: => ListenableFuture[A]) = { 48 | self(future).onError { case e => log.error("Cassandra request failed", e) } 49 | } 50 | } 51 | clusterOf <- Resource.eval(clusterOf[F](fromGFuture)) 52 | cluster <- clusterOf(config.client) 53 | keyspace = config.schema.keyspace 54 | globalSession = { 55 | LogResource[F](CassandraModule.getClass, "CassandraGlobal") *> 56 | cluster.connect 57 | } 58 | keyspaceSession = { 59 | LogResource[F](CassandraModule.getClass, "Cassandra") *> 60 | cluster.connect(keyspace.name) 61 | } 62 | // we need globally scoped session as connecting with non-existing keyspace will fail 63 | syncSession <- if (keyspace.autoCreate) globalSession else keyspaceSession 64 | _sync <- Resource.eval( 65 | CassandraSync.of[F]( 66 | session = syncSession, 67 | keyspace = keyspace.name, 68 | autoCreate = if (keyspace.autoCreate) AutoCreate.KeyspaceAndTable.Default else AutoCreate.None 69 | ) 70 | ) 71 | // `syncSession` is `keyspaceSession` if `autoCreate` was disabled, 72 | // no need to reconnect 73 | unsafeSession <- if (keyspace.autoCreate) keyspaceSession else Resource.eval(syncSession.pure[F]) 74 | 75 | _session <- unsafeSession.enhanceError.cachePrepared 76 | _healthCheck <- CassandraHealthCheckOf(unsafeSession, config) 77 | } yield new CassandraModule[F] { 78 | def session = _session 79 | def sync = _sync 80 | def healthCheck = _healthCheck 81 | } 82 | 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/cassandra/CassandraCodecs.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.cassandra 2 | 3 | import com.datastax.driver.core.{GettableByNameData, SettableData} 4 | import com.evolutiongaming.scassandra.{DecodeByName, EncodeByName} 5 | import com.evolutiongaming.skafka.{Offset, Partition, TimestampType} 6 | import scodec.bits.ByteVector 7 | 8 | import scala.jdk.CollectionConverters.* 9 | import scala.util.Try 10 | 11 | private[flow] object CassandraCodecs { 12 | 13 | implicit val byteVectorEncodeByName: EncodeByName[ByteVector] = new EncodeByName[ByteVector] { 14 | def apply[B <: SettableData[B]](data: B, name: String, value: ByteVector) = 15 | data.setBytes(name, value.toByteBuffer) 16 | } 17 | implicit val byteVectorDecodeByName: DecodeByName[ByteVector] = DecodeByName[ByteVector] { (data, name) => 18 | ByteVector(data.getBytes(name)) 19 | } 20 | 21 | implicit val timestampTypeEncodeByName: EncodeByName[TimestampType] = new EncodeByName[TimestampType] { 22 | def apply[B <: SettableData[B]](data: B, name: String, value: TimestampType) = { 23 | val text = value match { 24 | case TimestampType.Create => "C" 25 | case TimestampType.Append => "A" 26 | } 27 | data.setString(name, text) 28 | } 29 | } 30 | implicit val timestampTypeDecodeByName: DecodeByName[TimestampType] = DecodeByName[TimestampType] { (data, name) => 31 | data.getString(name) match { 32 | case "C" => TimestampType.Create 33 | case "A" => TimestampType.Append 34 | } 35 | } 36 | 37 | implicit val listStringEncodeByName: EncodeByName[List[String]] = { 38 | new EncodeByName[List[String]] { 39 | def apply[B <: SettableData[B]](data: B, name: String, values: List[String]) = { 40 | data.setList(name, values.asJava, classOf[String]) 41 | } 42 | } 43 | } 44 | 45 | implicit val mapTextEncodeByName: EncodeByName[Map[String, String]] = { 46 | val text = classOf[String] 47 | new EncodeByName[Map[String, String]] { 48 | def apply[B <: SettableData[B]](data: B, name: String, value: Map[String, String]) = { 49 | data.setMap(name, value.asJava, text, text) 50 | } 51 | } 52 | } 53 | 54 | implicit val mapTextDecodeByName: DecodeByName[Map[String, String]] = { 55 | val text = classOf[String] 56 | (data: GettableByNameData, name: String) => { 57 | data.getMap(name, text, text).asScala.toMap 58 | } 59 | } 60 | 61 | implicit val encodeByNamePartition: EncodeByName[Partition] = EncodeByName[Int].contramap { (a: Partition) => 62 | a.value 63 | } 64 | 65 | implicit val decodeByNamePartition: DecodeByName[Partition] = DecodeByName[Int].map { a => Partition.of[Try](a).get } 66 | 67 | implicit val encodeByNameOffset: EncodeByName[Offset] = EncodeByName[Long].contramap { (a: Offset) => a.value } 68 | 69 | implicit val decodeByNameOffset: DecodeByName[Offset] = DecodeByName[Long].map { a => Offset.of[Try](a).get } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/cassandra/CassandraConfig.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.cassandra 2 | 3 | import com.evolutiongaming.scassandra 4 | import pureconfig.ConfigReader 5 | import pureconfig.generic.semiauto.deriveReader 6 | 7 | final case class CassandraConfig( 8 | schema: CassandraConfig.Schema = CassandraConfig.Schema.default, 9 | retries: Int = 100, 10 | client: scassandra.CassandraConfig, 11 | consistencyOverrides: ConsistencyOverrides = ConsistencyOverrides.none 12 | ) 13 | 14 | object CassandraConfig { 15 | 16 | implicit val cassandraConfigReader: ConfigReader[CassandraConfig] = deriveReader 17 | 18 | final case class Schema(keyspace: Keyspace = Keyspace.default, autoCreate: Boolean = true) 19 | 20 | object Schema { 21 | 22 | implicit val schemaConfigReader: ConfigReader[Schema] = deriveReader 23 | 24 | val default: Schema = Schema() 25 | } 26 | 27 | final case class Keyspace(name: String = "kafka_flow", autoCreate: Boolean = true) 28 | 29 | object Keyspace { 30 | 31 | implicit val keyspaceConfigReader: ConfigReader[Keyspace] = deriveReader 32 | 33 | val default: Keyspace = Keyspace() 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/cassandra/ConsistencyOverrides.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.cassandra 2 | 3 | import com.datastax.driver.core.ConsistencyLevel 4 | import pureconfig.ConfigReader 5 | import pureconfig.generic.semiauto.deriveReader 6 | 7 | final case class ConsistencyOverrides( 8 | read: Option[ConsistencyLevel] = None, 9 | write: Option[ConsistencyLevel] = None 10 | ) 11 | 12 | object ConsistencyOverrides { 13 | val none: ConsistencyOverrides = ConsistencyOverrides(None, None) 14 | 15 | implicit val configReader: ConfigReader[ConsistencyOverrides] = deriveReader 16 | } 17 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/cassandra/RecordExpiration.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.cassandra 2 | 3 | import scala.concurrent.duration.FiniteDuration 4 | 5 | final case class RecordExpiration( 6 | journals: Option[FiniteDuration] = None, 7 | keys: Option[FiniteDuration] = None, 8 | snapshots: Option[FiniteDuration] = None, 9 | ) 10 | 11 | object RecordExpiration { 12 | val default: RecordExpiration = RecordExpiration() 13 | } 14 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/cassandra/StatementHelper.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.cassandra 2 | 3 | import com.datastax.driver.core.{ConsistencyLevel, Statement} 4 | 5 | import scala.concurrent.duration.FiniteDuration 6 | 7 | object StatementHelper { 8 | implicit final class StatementOps(val self: Statement) extends AnyVal { 9 | def withConsistencyLevel(level: Option[ConsistencyLevel]): Statement = 10 | level.map(self.setConsistencyLevel).getOrElse(self) 11 | } 12 | 13 | def ttlFragment(ttl: Option[FiniteDuration]): String = 14 | ttl.map(ttl => s"USING TTL ${ttl.toSeconds}").getOrElse("") 15 | } 16 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/journal/JournalSchema.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.cassandra.sync.CassandraSync 6 | import com.evolutiongaming.scassandra.CassandraSession 7 | 8 | trait JournalSchema[F[_]] { 9 | def create: F[Unit] 10 | 11 | def truncate: F[Unit] 12 | } 13 | 14 | object JournalSchema { 15 | @deprecated( 16 | "Use the version with an explicit table name. This exists to preserve binary compatibility until the next major release", 17 | since = "6.1.3" 18 | ) 19 | def of[F[_]: Monad]( 20 | session: CassandraSession[F], 21 | synchronize: CassandraSync[F], 22 | ): JournalSchema[F] = of(session, synchronize, CassandraJournals.DefaultTableName) 23 | 24 | def of[F[_]: Monad]( 25 | session: CassandraSession[F], 26 | synchronize: CassandraSync[F], 27 | tableName: String, 28 | ): JournalSchema[F] = new JournalSchema[F] { 29 | def create: F[Unit] = synchronize("JournalSchema") { 30 | session 31 | .execute( 32 | s""" 33 | |CREATE TABLE IF NOT EXISTS $tableName ( 34 | | application_id TEXT, 35 | | group_id TEXT, 36 | | topic TEXT, 37 | | partition INT, 38 | | key TEXT, 39 | | offset BIGINT, 40 | | created TIMESTAMP, 41 | | timestamp TIMESTAMP, 42 | | timestamp_type TEXT, 43 | | headers MAP, 44 | | metadata TEXT, 45 | | value BLOB, 46 | | PRIMARY KEY((application_id, group_id, topic, partition, key), offset) 47 | |) 48 | |""".stripMargin 49 | ) 50 | .void 51 | } 52 | 53 | def truncate: F[Unit] = synchronize("JournalSchema") { 54 | session.execute(s"TRUNCATE $tableName").void 55 | } 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/journal/conversions/HeaderToTuple.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal.conversions 2 | 3 | import cats.ApplicativeThrow 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.skafka.Header 6 | import scodec.bits.BitVector 7 | import scodec.codecs 8 | 9 | object HeaderToTuple { 10 | 11 | def convert[F[_]: ApplicativeThrow](header: Header): F[(String, String)] = { 12 | codecs 13 | .utf8 14 | .decode(BitVector.view(header.value)) 15 | .fold( 16 | err => 17 | new RuntimeException(s"HeaderToTuple failed for $header: scodec error: ${err.messageWithContext}") 18 | .raiseError[F, (String, String)], 19 | decoded => (header.key, decoded.value).pure[F] 20 | ) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/journal/conversions/TupleToHeader.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.journal.conversions 2 | 3 | import cats.ApplicativeThrow 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.skafka.Header 6 | import scodec.codecs 7 | 8 | object TupleToHeader { 9 | 10 | def convert[F[_]: ApplicativeThrow](key: String, value: String): F[Header] = { 11 | codecs 12 | .utf8 13 | .encode(value) 14 | .fold( 15 | err => 16 | new RuntimeException(s"TupleToHeader failed for $key:$value: scodec error: ${err.messageWithContext}") 17 | .raiseError[F, Header], 18 | bitVector => Header(key, bitVector.toByteArray).pure[F] 19 | ) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/key/KeySchema.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.cassandra.sync.CassandraSync 6 | import com.evolutiongaming.scassandra.CassandraSession 7 | 8 | trait KeySchema[F[_]] { 9 | def create: F[Unit] 10 | 11 | def truncate: F[Unit] 12 | } 13 | 14 | object KeySchema { 15 | 16 | def of[F[_]: Monad]( 17 | session: CassandraSession[F], 18 | synchronize: CassandraSync[F], 19 | tableName: String, 20 | ): KeySchema[F] = new KeySchema[F] { 21 | def create: F[Unit] = synchronize("KeySchema") { 22 | session 23 | .execute( 24 | s""" 25 | |CREATE TABLE IF NOT EXISTS $tableName ( 26 | | application_id TEXT, 27 | | group_id TEXT, 28 | | segment BIGINT, 29 | | topic TEXT, 30 | | partition INT, 31 | | key TEXT, 32 | | created TIMESTAMP, 33 | | created_date DATE, 34 | | metadata TEXT, 35 | | PRIMARY KEY((application_id, group_id, segment), topic, partition, key) 36 | |) 37 | |WITH compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} 38 | |""".stripMargin 39 | ) >> 40 | session 41 | .execute( 42 | s"CREATE INDEX IF NOT EXISTS ${tableName}_created_date_idx ON $tableName(created_date)" 43 | ) 44 | .void 45 | } 46 | 47 | def truncate: F[Unit] = synchronize("KeySchema") { 48 | session.execute(s"TRUNCATE $tableName").void 49 | } 50 | } 51 | 52 | @deprecated( 53 | "Use the version with an explicit table name. This exists to preserve binary compatibility until the next major release", 54 | since = "6.1.3" 55 | ) 56 | def of[F[_]: Monad]( 57 | session: CassandraSession[F], 58 | synchronize: CassandraSync[F], 59 | ): KeySchema[F] = of(session, synchronize, CassandraKeys.DefaultTableName) 60 | 61 | } 62 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/key/KeySegments.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.Show 4 | import cats.kernel.{Eq, Order} 5 | import cats.syntax.all.* 6 | 7 | /** A maximum number of segments in Cassandra table. 8 | * 9 | * When [[KeySegments]] is used, the value of 'segment' column is determined by consistent hashing of the key column. 10 | * I.e. there always no more than [[KeySegments#value]] different values. 11 | * 12 | * The only place where such approach is used right now is [[com.evolutiongaming.kafka.flow.key.CassandraKeys]]. This 13 | * allows fair distribution of 'key' records between the Cassandra partitions. 14 | */ 15 | sealed abstract case class KeySegments(value: Int) { 16 | 17 | override def toString: String = value.toString 18 | } 19 | 20 | object KeySegments { 21 | 22 | val min: KeySegments = new KeySegments(1) {} 23 | 24 | val max: KeySegments = new KeySegments(Int.MaxValue) {} 25 | 26 | val default: KeySegments = new KeySegments(10000) {} 27 | 28 | implicit val eqKeySegments: Eq[KeySegments] = Eq.fromUniversalEquals 29 | 30 | implicit val showKeySegments: Show[KeySegments] = Show.fromToString 31 | 32 | implicit val orderingKeySegments: Ordering[KeySegments] = Ordering.by(_.value) 33 | 34 | implicit val orderKeySegments: Order[KeySegments] = Order.fromOrdering 35 | 36 | def of(value: Int): Either[String, KeySegments] = { 37 | if (value < min.value) { 38 | Left(s"invalid KeySegments of $value, it must be greater or equal to $min") 39 | } else if (value > max.value) { 40 | Left(s"invalid KeySegments of $value, it must be less or equal to $max") 41 | } else if (value === min.value) { 42 | Right(min) 43 | } else if (value === max.value) { 44 | Right(max) 45 | } else { 46 | Right(new KeySegments(value) {}) 47 | } 48 | } 49 | 50 | def opt(value: Int): Option[KeySegments] = of(value).toOption 51 | 52 | def unsafe[A](value: A)(implicit numeric: Numeric[A]): KeySegments = 53 | of(numeric.toInt(value)).fold(err => throw new RuntimeException(err), identity) 54 | } 55 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/key/SegmentNr.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.key 2 | 3 | import cats.syntax.all.* 4 | import com.evolutiongaming.scassandra.EncodeByName 5 | 6 | private[flow] sealed abstract case class SegmentNr(value: Long) { 7 | override def toString: String = value.toString 8 | } 9 | 10 | private[flow] object SegmentNr { 11 | 12 | val min: SegmentNr = new SegmentNr(0L) {} 13 | 14 | val max: SegmentNr = new SegmentNr(Long.MaxValue) {} 15 | 16 | implicit val encodeByNameSegmentNr: EncodeByName[SegmentNr] = EncodeByName[Long].contramap(_.value) 17 | 18 | def of(value: Long): Either[String, SegmentNr] = { 19 | if (value < min.value) { 20 | s"invalid SegmentNr of $value, it must be greater or equal to $min".asLeft[SegmentNr] 21 | } else if (value > max.value) { 22 | s"invalid SegmentNr of $value, it must be less or equal to $max".asLeft[SegmentNr] 23 | } else if (value === min.value) { 24 | min.asRight[String] 25 | } else if (value === max.value) { 26 | max.asRight[String] 27 | } else { 28 | new SegmentNr(value) {}.asRight[String] 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /persistence-cassandra/src/main/scala/com/evolutiongaming/kafka/flow/snapshot/SnapshotSchema.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.snapshot 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.cassandra.sync.CassandraSync 6 | import com.evolutiongaming.scassandra.CassandraSession 7 | 8 | trait SnapshotSchema[F[_]] { 9 | def create: F[Unit] 10 | 11 | def truncate: F[Unit] 12 | } 13 | 14 | object SnapshotSchema { 15 | 16 | @deprecated( 17 | "Use the version with an explicit table name. This exists to preserve binary compatibility until the next major release", 18 | since = "6.1.3" 19 | ) 20 | def of[F[_]: Monad]( 21 | session: CassandraSession[F], 22 | synchronize: CassandraSync[F], 23 | ): SnapshotSchema[F] = of(session, synchronize, CassandraSnapshots.DefaultTableName) 24 | 25 | def of[F[_]: Monad]( 26 | session: CassandraSession[F], 27 | synchronize: CassandraSync[F], 28 | tableName: String, 29 | ): SnapshotSchema[F] = new SnapshotSchema[F] { 30 | def create: F[Unit] = synchronize("SnapshotSchema") { 31 | session 32 | .execute( 33 | s""" 34 | |CREATE TABLE IF NOT EXISTS $tableName( 35 | | application_id TEXT, 36 | | group_id TEXT, 37 | | topic TEXT, 38 | | partition INT, 39 | | key TEXT, 40 | | offset BIGINT, 41 | | created TIMESTAMP, 42 | | metadata TEXT, 43 | | value BLOB, 44 | | PRIMARY KEY((application_id, group_id, topic, partition, key)) 45 | |) 46 | |""".stripMargin 47 | ) 48 | .void 49 | } 50 | 51 | def truncate: F[Unit] = synchronize("SnapshotSchema") { 52 | session.execute(s"TRUNCATE $tableName").void 53 | } 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /persistence-kafka-it-tests/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /persistence-kafka-it-tests/src/test/scala/com/evolutiongaming/kafka/flow/ForAllKafkaSuite.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow 2 | 3 | import cats.data.NonEmptyList 4 | import cats.effect.IO 5 | import com.dimafeng.testcontainers.KafkaContainer 6 | import com.dimafeng.testcontainers.munit.fixtures.TestContainersFixtures 7 | import com.evolutiongaming.catshelper.LogOf 8 | import com.evolutiongaming.kafka.flow.kafka.KafkaModule 9 | import com.evolutiongaming.skafka.CommonConfig 10 | import com.evolutiongaming.skafka.consumer.ConsumerConfig 11 | import com.evolutiongaming.smetrics.CollectorRegistry 12 | import munit.FunSuite 13 | 14 | import java.util.concurrent.atomic.AtomicReference 15 | 16 | abstract class ForAllKafkaSuite extends FunSuite with TestContainersFixtures { 17 | import cats.effect.unsafe.implicits.global 18 | 19 | val kafka = ForAllContainerFixture(KafkaContainer()) 20 | 21 | val kafkaModule = new Fixture[KafkaModule[IO]]("KafkaModule") { 22 | private val moduleRef = new AtomicReference[(KafkaModule[IO], IO[Unit])]() 23 | 24 | override def apply(): KafkaModule[IO] = moduleRef.get()._1 25 | 26 | override def beforeAll(): Unit = { 27 | val config = 28 | ConsumerConfig(common = CommonConfig(bootstrapServers = NonEmptyList.one(kafka.container.bootstrapServers))) 29 | implicit val logOf: LogOf[IO] = LogOf.slf4j[IO].unsafeRunSync() 30 | val result = KafkaModule.of[IO]("KafkaSuite", config, CollectorRegistry.empty[IO]).allocated.unsafeRunSync() 31 | moduleRef.set(result) 32 | } 33 | 34 | override def afterAll(): Unit = Option(moduleRef.get()).foreach { case (_, finalizer) => finalizer.unsafeRunSync() } 35 | } 36 | 37 | override def munitFixtures: Seq[Fixture[_]] = List(kafka, kafkaModule) 38 | } 39 | -------------------------------------------------------------------------------- /persistence-kafka/src/main/scala/com/evolutiongaming/kafka/flow/kafkapersistence/KafkaPersistenceModuleOf.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafkapersistence 2 | 3 | import cats.Parallel 4 | import cats.effect.{Concurrent, Resource} 5 | import com.evolutiongaming.catshelper.{LogOf, Runtime} 6 | import com.evolutiongaming.kafka.flow.FlowMetrics 7 | import com.evolutiongaming.skafka.consumer.{ConsumerConfig, ConsumerOf} 8 | import com.evolutiongaming.skafka.producer.Producer 9 | import com.evolutiongaming.skafka.* 10 | 11 | /** Convenience factory trait to create an instance of [[KafkaPersistenceModule]] for an assigned partition */ 12 | trait KafkaPersistenceModuleOf[F[_], S] { 13 | def make(partition: Partition): Resource[F, KafkaPersistenceModule[F, S]] 14 | } 15 | 16 | object KafkaPersistenceModuleOf { 17 | 18 | /** Create a [[KafkaPersistenceModuleOf]] factory instance which will then produce a caching implementation of 19 | * [[KafkaPersistenceModule]] for an assigned partition. See `KafkaPersistenceModule.caching` documentation for 20 | * further details. Needed mostly as a convenient helper to provide a part of necessary parameters beforehand and 21 | * pass this factory around. 22 | * 23 | * @param consumerOf 24 | * factory of consumers 25 | * @param producer 26 | * producer for writing to a snapshot topic 27 | * @param consumerConfig 28 | * consumer config to be used when creating snapshot topic consumer 29 | * @param snapshotTopic 30 | * snapshot topic name (should be configured as a 'compacted' topic) 31 | * @param metrics 32 | * instance of `FlowMetrics` for [[KafkaPersistenceModule]] 33 | */ 34 | def caching[F[_]: LogOf: Concurrent: Parallel: Runtime, S]( 35 | consumerOf: ConsumerOf[F], 36 | producer: Producer[F], 37 | consumerConfig: ConsumerConfig, 38 | snapshotTopic: Topic, 39 | metrics: FlowMetrics[F], 40 | partitionMapper: KafkaPersistencePartitionMapper = KafkaPersistencePartitionMapper.identity, 41 | )( 42 | implicit fromBytesKey: FromBytes[F, String], 43 | fromBytesState: FromBytes[F, S], 44 | toBytesState: ToBytes[F, S] 45 | ): KafkaPersistenceModuleOf[F, S] = new KafkaPersistenceModuleOf[F, S] { 46 | override def make(partition: Partition): Resource[F, KafkaPersistenceModule[F, S]] = KafkaPersistenceModule.caching( 47 | consumerOf = consumerOf, 48 | producer = producer, 49 | consumerConfig = consumerConfig, 50 | snapshotTopicPartition = TopicPartition(snapshotTopic, partition), 51 | metrics = metrics, 52 | partitionMapper = partitionMapper, 53 | ) 54 | } 55 | 56 | def caching[F[_]: LogOf: Concurrent: Parallel: Runtime, S]( 57 | consumerOf: ConsumerOf[F], 58 | producer: Producer[F], 59 | consumerConfig: ConsumerConfig, 60 | snapshotTopic: Topic 61 | )( 62 | implicit fromBytesKey: FromBytes[F, String], 63 | fromBytesState: FromBytes[F, S], 64 | toBytesState: ToBytes[F, S] 65 | ): KafkaPersistenceModuleOf[F, S] = 66 | caching(consumerOf, producer, consumerConfig, snapshotTopic, FlowMetrics.empty[F]) 67 | 68 | } 69 | -------------------------------------------------------------------------------- /persistence-kafka/src/main/scala/com/evolutiongaming/kafka/flow/kafkapersistence/KafkaPersistencePartitionMapper.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafkapersistence 2 | 3 | import com.evolutiongaming.skafka.Partition 4 | import org.apache.kafka.clients.producer.internals.BuiltInPartitioner 5 | 6 | /** Maps partitions of source Kafka topics into persistence topics. 7 | * 8 | * Please be careful when using this with `com.evolutiongaming.kafka.flow.RemapKey`. Only the identity mapper is 9 | * guaranteed to work properly with an arbitrary `RemapKey`, for other combinations you have to manually ensure that 10 | * the `isStateKeyOwned` implementation is correct and will not allow duplicate KeyFlows. 11 | * 12 | * If the aggregate key depends on the record's contents, then only the identity mapper can be used. 13 | */ 14 | trait KafkaPersistencePartitionMapper { 15 | 16 | /** Called after rebalance or initial partition assignment. 17 | * @param sourcePartition 18 | * partition of the input stream, i.e. the kafka-journal topic. 19 | * @return 20 | * partition of the persistence topic that has snapshots for aggregates built by events from the `sourcePartition`. 21 | */ 22 | def getStatePartition(sourcePartition: Partition): Partition 23 | 24 | /** Checks if the aggregate in the state partition should be initialized as a 25 | * `com.evolutiongaming.kafka.flow.KeyFlow`. 26 | * 27 | * If the aggregate is initialized, it will have timers and ticks started. This is not desirable if the aggregate is 28 | * actually sourced from a different partition, which will also be started concurrently. 29 | * @param stateKey 30 | * the aggregate's key. 31 | * @param sourcePartition 32 | * partition of the input stream. 33 | * @return 34 | * `true` if the aggregate is built from events in `sourcePartition`. 35 | */ 36 | def isStateKeyOwned(stateKey: String, sourcePartition: Partition): Boolean 37 | } 38 | 39 | object KafkaPersistencePartitionMapper { 40 | def identity: KafkaPersistencePartitionMapper = Identity 41 | def modulo(sourcePartitions: Int, statePartitions: Int): KafkaPersistencePartitionMapper = 42 | new Modulo(sourcePartitions, statePartitions) 43 | 44 | private object Identity extends KafkaPersistencePartitionMapper { 45 | override def getStatePartition(sourcePartition: Partition): Partition = sourcePartition 46 | 47 | override def isStateKeyOwned(stateKey: String, sourcePartition: Partition): Boolean = true 48 | } 49 | 50 | private class Modulo(sourcePartitions: Int, statePartitions: Int) extends KafkaPersistencePartitionMapper { 51 | override def getStatePartition(sourcePartition: Partition): Partition = 52 | Partition.unsafe(sourcePartition.value % statePartitions) 53 | 54 | override def isStateKeyOwned(stateKey: String, sourcePartition: Partition): Boolean = 55 | BuiltInPartitioner.partitionForKey(stateKey.getBytes, sourcePartitions) == sourcePartition.value 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /persistence-kafka/src/main/scala/com/evolutiongaming/kafka/flow/kafkapersistence/KafkaSnapshotReadDatabase.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafkapersistence 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.kafka.flow.KafkaKey 6 | import com.evolutiongaming.kafka.flow.snapshot.SnapshotReadDatabase 7 | import com.evolutiongaming.skafka.{FromBytes, Topic} 8 | import scodec.bits.ByteVector 9 | 10 | object KafkaSnapshotReadDatabase { 11 | def of[F[_]: Monad, S: FromBytes[F, *]]( 12 | snapshotTopic: Topic, 13 | getState: String => F[Option[ByteVector]] 14 | ): SnapshotReadDatabase[F, KafkaKey, S] = 15 | key => 16 | for { 17 | state <- getState(key.key) 18 | maybeState <- state.traverse(bytes => FromBytes[F, S].apply(bytes.toArray, snapshotTopic)) 19 | } yield maybeState 20 | } 21 | -------------------------------------------------------------------------------- /persistence-kafka/src/main/scala/com/evolutiongaming/kafka/flow/kafkapersistence/KafkaSnapshotWriteDatabase.scala: -------------------------------------------------------------------------------- 1 | package com.evolutiongaming.kafka.flow.kafkapersistence 2 | 3 | import cats.Monad 4 | import cats.syntax.all.* 5 | import com.evolutiongaming.catshelper.FromTry 6 | import com.evolutiongaming.kafka.flow.KafkaKey 7 | import com.evolutiongaming.kafka.flow.snapshot.SnapshotWriteDatabase 8 | import com.evolutiongaming.skafka.producer.{Producer, ProducerRecord} 9 | import com.evolutiongaming.skafka.{ToBytes, TopicPartition} 10 | 11 | object KafkaSnapshotWriteDatabase { 12 | def of[F[_]: FromTry: Monad, S: ToBytes[F, *]]( 13 | snapshotTopicPartition: TopicPartition, 14 | producer: Producer[F], 15 | partitionMapper: KafkaPersistencePartitionMapper = KafkaPersistencePartitionMapper.identity, 16 | ): SnapshotWriteDatabase[F, KafkaKey, S] = new SnapshotWriteDatabase[F, KafkaKey, S] { 17 | override def persist(key: KafkaKey, snapshot: S): F[Unit] = produce(key, snapshot.some) 18 | 19 | override def delete(key: KafkaKey): F[Unit] = produce(key, none) 20 | 21 | private def produce(key: KafkaKey, snapshot: Option[S]): F[Unit] = { 22 | val targetPartition = partitionMapper.getStatePartition(key.topicPartition.partition) 23 | val record = new ProducerRecord( 24 | topic = snapshotTopicPartition.topic, 25 | partition = targetPartition.some, 26 | key = key.key.some, 27 | value = snapshot 28 | ) 29 | 30 | producer.send(record).flatten.void 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt.* 2 | 3 | object Dependencies { 4 | 5 | val catsHelper = "com.evolutiongaming" %% "cats-helper" % "3.11.3" 6 | val catsHelperLogback = "com.evolutiongaming" %% "cats-helper-logback" % "3.11.3" 7 | val smetrics = "com.evolutiongaming" %% "smetrics" % "2.2.0" 8 | val scache = "com.evolution" %% "scache" % "5.1.3" 9 | val skafka = "com.evolutiongaming" %% "skafka" % "17.1.3" 10 | val sstream = "com.evolutiongaming" %% "sstream" % "1.0.2" 11 | val scassandra = "com.evolutiongaming" %% "scassandra" % "5.3.0" 12 | val cassandraSync = "com.evolutiongaming" %% "cassandra-sync" % "3.1.0" 13 | val random = "com.evolution" %% "random" % "1.0.5" 14 | val retry = "com.evolutiongaming" %% "retry" % "3.1.0" 15 | val playJsonJsoniter = "com.evolution" %% "play-json-jsoniter" % "1.1.1" 16 | 17 | object Cats { 18 | private val version = "2.13.0" 19 | private val effectVersion = "3.5.7" 20 | val core = "org.typelevel" %% "cats-core" % version 21 | val mtl = "org.typelevel" %% "cats-mtl" % "1.5.0" 22 | val effect = "org.typelevel" %% "cats-effect" % effectVersion 23 | val effectTestkit = "org.typelevel" %% "cats-effect-testkit" % effectVersion 24 | } 25 | 26 | object Scodec { 27 | val coreScala213 = "org.scodec" %% "scodec-core" % "1.11.10" 28 | val coreScala3 = "org.scodec" %% "scodec-core" % "2.3.2" 29 | val bits = "org.scodec" %% "scodec-bits" % "1.2.1" 30 | } 31 | 32 | object KafkaJournal { 33 | private val version = "4.3.0" 34 | val journal = "com.evolutiongaming" %% "kafka-journal" % version 35 | val persistence = "com.evolutiongaming" %% "kafka-journal-persistence" % version 36 | } 37 | 38 | object Monocle { 39 | private val version = "3.3.0" 40 | val core = "dev.optics" %% "monocle-core" % version 41 | val `macro` = "dev.optics" %% "monocle-macro" % version 42 | } 43 | 44 | object PureConfig { 45 | private val version = "0.17.8" 46 | lazy val GenericScala3 = "com.github.pureconfig" %% "pureconfig-generic-scala3" % version 47 | } 48 | 49 | object Testing { 50 | val munit = "org.scalameta" %% "munit" % "1.1.0" 51 | 52 | object Testcontainers { 53 | private val version = "0.43.0" 54 | val munit = "com.dimafeng" %% "testcontainers-scala-munit" % version 55 | val kafka = "com.dimafeng" %% "testcontainers-scala-kafka" % version 56 | val cassandra = "com.dimafeng" %% "testcontainers-scala-cassandra" % version 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.10.10 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // sbt-scoverage 2.x.x brings in scala-xml 2.x.x 2 | libraryDependencySchemes ++= Seq( 3 | "org.scala-lang.modules" %% "scala-xml" % "always" 4 | ) 5 | 6 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.3.1") 7 | 8 | addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.15") 9 | 10 | // This sets the 'version' property based on the git tag during release process to publish the right version 11 | addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.1.0") 12 | 13 | addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.6.5") 14 | 15 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.4") 16 | 17 | addSbtPlugin("com.evolution" % "sbt-scalac-opts-plugin" % "0.0.9") 18 | 19 | addSbtPlugin("com.evolution" % "sbt-artifactory-plugin" % "0.0.2") 20 | 21 | addSbtPlugin("ch.epfl.scala" % "sbt-version-policy" % "3.2.1") 22 | -------------------------------------------------------------------------------- /website/blog/2016-03-11-blog-post.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Blog Title 3 | author: Blog Author 4 | authorURL: http://twitter.com/ 5 | authorFBID: 100002976521003 6 | --- 7 | 8 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus elementum massa eget nulla aliquet sagittis. Proin odio tortor, vulputate ut odio in, ultrices ultricies augue. Cras ornare ultrices lorem malesuada iaculis. Etiam sit amet libero tempor, pulvinar mauris sed, sollicitudin sapien. 9 | 10 | 11 | 12 | Mauris vestibulum ullamcorper nibh, ut semper purus pulvinar ut. Donec volutpat orci sit amet mauris malesuada, non pulvinar augue aliquam. Vestibulum ultricies at urna ut suscipit. Morbi iaculis, erat at imperdiet semper, ipsum nulla sodales erat, eget tincidunt justo dui quis justo. Pellentesque dictum bibendum diam at aliquet. Sed pulvinar, dolor quis finibus ornare, eros odio facilisis erat, eu rhoncus nunc dui sed ex. Nunc gravida dui massa, sed ornare arcu tincidunt sit amet. Maecenas efficitur sapien neque, a laoreet libero feugiat ut. 13 | 14 | Nulla facilisi. Maecenas sodales nec purus eget posuere. Sed sapien quam, pretium a risus in, porttitor dapibus erat. Sed sit amet fringilla ipsum, eget iaculis augue. Integer sollicitudin tortor quis ultricies aliquam. Suspendisse fringilla nunc in tellus cursus, at placerat tellus scelerisque. Sed tempus elit a sollicitudin rhoncus. Nulla facilisi. Morbi nec dolor dolor. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras et aliquet lectus. Pellentesque sit amet eros nisi. Quisque ac sapien in sapien congue accumsan. Nullam in posuere ante. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Proin lacinia leo a nibh fringilla pharetra. 15 | 16 | Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Proin venenatis lectus dui, vel ultrices ante bibendum hendrerit. Aenean egestas feugiat dui id hendrerit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Curabitur in tellus laoreet, eleifend nunc id, viverra leo. Proin vulputate non dolor vel vulputate. Curabitur pretium lobortis felis, sit amet finibus lorem suscipit ut. Sed non mollis risus. Duis sagittis, mi in euismod tincidunt, nunc mauris vestibulum urna, at euismod est elit quis erat. Phasellus accumsan vitae neque eu placerat. In elementum arcu nec tellus imperdiet, eget maximus nulla sodales. Curabitur eu sapien eget nisl sodales fermentum. 17 | 18 | Phasellus pulvinar ex id commodo imperdiet. Praesent odio nibh, sollicitudin sit amet faucibus id, placerat at metus. Donec vitae eros vitae tortor hendrerit finibus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Quisque vitae purus dolor. Duis suscipit ac nulla et finibus. Phasellus ac sem sed dui dictum gravida. Phasellus eleifend vestibulum facilisis. Integer pharetra nec enim vitae mattis. Duis auctor, lectus quis condimentum bibendum, nunc dolor aliquam massa, id bibendum orci velit quis magna. Ut volutpat nulla nunc, sed interdum magna condimentum non. Sed urna metus, scelerisque vitae consectetur a, feugiat quis magna. Donec dignissim ornare nisl, eget tempor risus malesuada quis. 19 | -------------------------------------------------------------------------------- /website/blog/2017-04-10-blog-post-two.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: New Blog Post 3 | author: Blog Author 4 | authorURL: http://twitter.com/ 5 | authorFBID: 100002976521003 6 | --- 7 | 8 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus elementum massa eget nulla aliquet sagittis. Proin odio tortor, vulputate ut odio in, ultrices ultricies augue. Cras ornare ultrices lorem malesuada iaculis. Etiam sit amet libero tempor, pulvinar mauris sed, sollicitudin sapien. 9 | 10 | 11 | 12 | Mauris vestibulum ullamcorper nibh, ut semper purus pulvinar ut. Donec volutpat orci sit amet mauris malesuada, non pulvinar augue aliquam. Vestibulum ultricies at urna ut suscipit. Morbi iaculis, erat at imperdiet semper, ipsum nulla sodales erat, eget tincidunt justo dui quis justo. Pellentesque dictum bibendum diam at aliquet. Sed pulvinar, dolor quis finibus ornare, eros odio facilisis erat, eu rhoncus nunc dui sed ex. Nunc gravida dui massa, sed ornare arcu tincidunt sit amet. Maecenas efficitur sapien neque, a laoreet libero feugiat ut. 13 | 14 | Nulla facilisi. Maecenas sodales nec purus eget posuere. Sed sapien quam, pretium a risus in, porttitor dapibus erat. Sed sit amet fringilla ipsum, eget iaculis augue. Integer sollicitudin tortor quis ultricies aliquam. Suspendisse fringilla nunc in tellus cursus, at placerat tellus scelerisque. Sed tempus elit a sollicitudin rhoncus. Nulla facilisi. Morbi nec dolor dolor. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras et aliquet lectus. Pellentesque sit amet eros nisi. Quisque ac sapien in sapien congue accumsan. Nullam in posuere ante. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Proin lacinia leo a nibh fringilla pharetra. 15 | 16 | Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Proin venenatis lectus dui, vel ultrices ante bibendum hendrerit. Aenean egestas feugiat dui id hendrerit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Curabitur in tellus laoreet, eleifend nunc id, viverra leo. Proin vulputate non dolor vel vulputate. Curabitur pretium lobortis felis, sit amet finibus lorem suscipit ut. Sed non mollis risus. Duis sagittis, mi in euismod tincidunt, nunc mauris vestibulum urna, at euismod est elit quis erat. Phasellus accumsan vitae neque eu placerat. In elementum arcu nec tellus imperdiet, eget maximus nulla sodales. Curabitur eu sapien eget nisl sodales fermentum. 17 | 18 | Phasellus pulvinar ex id commodo imperdiet. Praesent odio nibh, sollicitudin sit amet faucibus id, placerat at metus. Donec vitae eros vitae tortor hendrerit finibus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Quisque vitae purus dolor. Duis suscipit ac nulla et finibus. Phasellus ac sem sed dui dictum gravida. Phasellus eleifend vestibulum facilisis. Integer pharetra nec enim vitae mattis. Duis auctor, lectus quis condimentum bibendum, nunc dolor aliquam massa, id bibendum orci velit quis magna. Ut volutpat nulla nunc, sed interdum magna condimentum non. Sed urna metus, scelerisque vitae consectetur a, feugiat quis magna. Donec dignissim ornare nisl, eget tempor risus malesuada quis. 19 | -------------------------------------------------------------------------------- /website/blog/2017-09-25-testing-rss.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Adding RSS Support - RSS Truncation Test 3 | author: Eric Nakagawa 4 | authorURL: http://twitter.com/ericnakagawa 5 | authorFBID: 661277173 6 | --- 7 | 8 | 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 9 | 10 | This should be truncated. 11 | 12 | 13 | 14 | This line should never render in XML. 15 | -------------------------------------------------------------------------------- /website/blog/2017-09-26-adding-rss.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Adding RSS Support 3 | author: Eric Nakagawa 4 | authorURL: http://twitter.com/ericnakagawa 5 | authorFBID: 661277173 6 | --- 7 | 8 | This is a test post. 9 | 10 | A whole bunch of other information. 11 | -------------------------------------------------------------------------------- /website/blog/2017-10-24-new-version-1.0.0.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: New Version 1.0.0 3 | author: Eric Nakagawa 4 | authorURL: http://twitter.com/ericnakagawa 5 | authorFBID: 661277173 6 | --- 7 | 8 | This blog post will test file name parsing issues when periods are present. 9 | -------------------------------------------------------------------------------- /website/core/Footer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Facebook, Inc. and its affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | const React = require('react'); 9 | 10 | class Footer extends React.Component { 11 | docUrl(doc) { 12 | const baseUrl = this.props.config.baseUrl; 13 | const docsUrl = this.props.config.docsUrl; 14 | const docsPart = `${docsUrl ? `${docsUrl}/` : ''}`; 15 | return `${baseUrl}${docsPart}${doc}`; 16 | } 17 | 18 | render() { 19 | return ( 20 | 87 | ); 88 | } 89 | } 90 | 91 | module.exports = Footer; 92 | -------------------------------------------------------------------------------- /website/i18n/en.json: -------------------------------------------------------------------------------- 1 | { 2 | "_comment": "This file is auto-generated by write-translations.js", 3 | "localized-strings": { 4 | "next": "Next", 5 | "previous": "Previous", 6 | "tagline": "Reliable processing of Kafka Journal events", 7 | "docs": { 8 | "faq": { 9 | "title": "FAQ", 10 | "sidebar_label": "FAQ" 11 | }, 12 | "overview": { 13 | "title": "Overview", 14 | "sidebar_label": "Overview" 15 | }, 16 | "persistence": { 17 | "title": "Persistence", 18 | "sidebar_label": "Persistence" 19 | }, 20 | "setup": { 21 | "title": "Setup", 22 | "sidebar_label": "Setup" 23 | }, 24 | "styleguide": { 25 | "title": "Style Guide", 26 | "sidebar_label": "Style Guide" 27 | } 28 | }, 29 | "links": { 30 | "Overview": "Overview", 31 | "Setup": "Setup", 32 | "Help": "Help" 33 | }, 34 | "categories": { 35 | "Kafka-Flow": "Kafka-Flow" 36 | } 37 | }, 38 | "pages-strings": { 39 | "Help Translate|recruit community translators for your project": "Help Translate", 40 | "Edit this Doc|recruitment message asking to edit the doc source": "Edit", 41 | "Translate this Doc|recruitment message asking to translate the docs": "Translate" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /website/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "examples": "docusaurus-examples", 4 | "start": "docusaurus-start", 5 | "build": "docusaurus-build", 6 | "publish-gh-pages": "docusaurus-publish", 7 | "write-translations": "docusaurus-write-translations", 8 | "version": "docusaurus-version", 9 | "rename-version": "docusaurus-rename-version" 10 | }, 11 | "devDependencies": { 12 | "docusaurus": "^1.14.6" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /website/pages/en/help.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Facebook, Inc. and its affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | const React = require('react'); 9 | 10 | const CompLibrary = require('../../core/CompLibrary.js'); 11 | 12 | const Container = CompLibrary.Container; 13 | const GridBlock = CompLibrary.GridBlock; 14 | 15 | function Help(props) { 16 | const {config: siteConfig, language = ''} = props; 17 | const {baseUrl, docsUrl} = siteConfig; 18 | const docsPart = `${docsUrl ? `${docsUrl}/` : ''}`; 19 | const langPart = `${language ? `${language}/` : ''}`; 20 | const docUrl = (doc) => `${baseUrl}${docsPart}${langPart}${doc}`; 21 | 22 | const supportLinks = [ 23 | { 24 | content: `Learn more using the [documentation on this site.](${docUrl( 25 | 'overview.html', 26 | )})`, 27 | title: 'Browse Docs', 28 | }, 29 | { 30 | content: 'Ask questions about the documentation and project', 31 | title: 'Join the community', 32 | }, 33 | { 34 | content: "Find out what's new with this project", 35 | title: 'Stay up to date', 36 | }, 37 | ]; 38 | 39 | return ( 40 |
41 | 42 |
43 |
44 |

Need help?

45 |
46 |

This project is maintained by a dedicated group of people.

47 | 48 |
49 | 50 |
51 | ); 52 | } 53 | 54 | module.exports = Help; 55 | -------------------------------------------------------------------------------- /website/pages/en/users.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Facebook, Inc. and its affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | const React = require('react'); 9 | 10 | const CompLibrary = require('../../core/CompLibrary.js'); 11 | 12 | const Container = CompLibrary.Container; 13 | 14 | class Users extends React.Component { 15 | render() { 16 | const {config: siteConfig} = this.props; 17 | if ((siteConfig.users || []).length === 0) { 18 | return null; 19 | } 20 | 21 | const showcase = siteConfig.users.map((user) => ( 22 | 23 | {user.caption} 24 | 25 | )); 26 | 27 | return ( 28 |
29 | 30 |
31 |
32 |

Who is Using This?

33 |

This project is used by many folks

34 |
35 |
{showcase}
36 | {siteConfig.repoUrl && ( 37 | 38 |

Are you using this project?

39 | 42 | Add your company 43 | 44 | 45 | )} 46 |
47 | 48 |
49 | ); 50 | } 51 | } 52 | 53 | module.exports = Users; 54 | -------------------------------------------------------------------------------- /website/sidebars.json: -------------------------------------------------------------------------------- 1 | { 2 | "docs": { 3 | "Kafka-Flow": ["overview", "setup", "faq", "styleguide", "persistence"] 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /website/static/css/custom.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Facebook, Inc. and its affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | /* your custom css */ 9 | 10 | @media only screen and (min-device-width: 360px) and (max-device-width: 736px) { 11 | } 12 | 13 | @media only screen and (min-width: 1024px) { 14 | } 15 | 16 | @media only screen and (max-width: 1023px) { 17 | } 18 | 19 | @media only screen and (min-width: 1400px) { 20 | } 21 | 22 | @media only screen and (min-width: 1500px) { 23 | } 24 | -------------------------------------------------------------------------------- /website/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evolution-gaming/kafka-flow/f6193e466fe668b86da652b6cea667ca71ebbfb9/website/static/img/favicon.ico -------------------------------------------------------------------------------- /website/static/img/oss_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evolution-gaming/kafka-flow/f6193e466fe668b86da652b6cea667ca71ebbfb9/website/static/img/oss_logo.png --------------------------------------------------------------------------------