├── gradle ├── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties └── libs.versions.toml ├── pubsub-ktor ├── gradle.properties ├── build.gradle.kts ├── README.MD └── src │ ├── test │ └── kotlin │ │ └── io │ │ └── github │ │ └── nomisrev │ │ └── gcp │ │ └── ktor │ │ └── GcpPluginTest.kt │ └── main │ └── kotlin │ └── io │ └── github │ └── nomisrev │ └── gcp │ └── pubsub │ └── ktor │ ├── GcpPubSubSyntax.kt │ └── GcpPubSub.kt ├── common-api ├── gradle.properties ├── build.gradle.kts ├── README.MD └── src │ └── main │ └── kotlin │ └── io │ └── github │ └── nomisrev │ └── gcp │ └── core │ └── ApiFuture.kt ├── pubsub ├── gradle.properties ├── build.gradle.kts ├── src │ ├── main │ │ └── kotlin │ │ │ └── io │ │ │ └── github │ │ │ └── nomisrev │ │ │ └── gcp │ │ │ └── pubsub │ │ │ ├── model.kt │ │ │ ├── GcpPublisher.kt │ │ │ ├── GcpPull.kt │ │ │ ├── GcpSubscriber.kt │ │ │ └── GcpPubsSubAdmin.kt │ └── test │ │ └── kotlin │ │ └── io │ │ └── github │ │ └── nomisrev │ │ └── gcp │ │ └── pubsub │ │ └── PubSubTest.kt └── README.MD ├── .editorconfig ├── pubsub-test ├── gradle.properties ├── build.gradle.kts ├── README.MD └── src │ └── main │ └── kotlin │ └── io │ └── github │ └── nomisrev │ └── gcp │ └── pubsub │ └── test │ └── PubSubEmulator.kt ├── pubsub-kotlinx-serialization-json ├── gradle.properties ├── build.gradle.kts ├── src │ ├── test │ │ └── kotlin │ │ │ └── io │ │ │ └── github │ │ │ └── nomisrev │ │ │ └── gcp │ │ │ └── pubsub │ │ │ └── serialization │ │ │ └── PublishSerializationTest.kt │ └── main │ │ └── kotlin │ │ └── io │ │ └── github │ │ └── nomisrev │ │ └── gcp │ │ └── pubsub │ │ └── serialization │ │ └── KotlinXSerialization.kt └── README.MD ├── pubsub-ktor-kotlinx-serialization-json ├── gradle.properties ├── build.gradle.kts ├── README.MD └── src │ ├── test │ └── kotlin │ │ └── io │ │ └── github │ │ └── nomisrev │ │ └── gcp │ │ └── pubsub │ │ └── ktor │ │ └── serialization │ │ └── PublishSerializationTest.kt │ └── main │ └── kotlin │ └── io │ └── github │ └── nomisrev │ └── gcp │ └── pubsub │ └── ktor │ └── serialization │ └── KotlinXSerialization.kt ├── .github ├── dependabot.yml └── workflows │ ├── junie.yml │ ├── dependabot-auto-merge.yml │ ├── githubpages.yaml │ ├── pr.yaml │ ├── build.yaml │ └── publish.yml ├── renovate.json ├── gradle.properties ├── settings.gradle.kts ├── README.md ├── gradlew.bat ├── .gitignore ├── gradlew └── LICENSE /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nomisRev/gcp-pubsub-kt/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /pubsub-ktor/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=kotlin-gcp-pubsub-ktor 2 | POM_NAME=kotlin-gcp-pubsub-ktor 3 | POM_DESCRIPTION=Kotlin GCP PubSub Ktor integration -------------------------------------------------------------------------------- /common-api/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=kotlin-gcp-common-api 2 | POM_NAME=kotlin-gcp-common-api 3 | POM_DESCRIPTION=Kotlin GCP integration with KotlinX Coroutines -------------------------------------------------------------------------------- /pubsub/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=kotlin-gcp-pubsub 2 | POM_NAME=kotlin-gcp-pubsub-ktor 3 | POM_DESCRIPTION=Kotlin GCP PubSub integration with KotlinX Coroutines -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{kt,kts}] 2 | indent_size=2 3 | insert_final_newline=true 4 | max_line_length=off 5 | disabled_rules=import-ordering,no-unit-return,curly-spacing,indent 6 | -------------------------------------------------------------------------------- /pubsub-test/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=kotlin-gcp-pubsub-test 2 | POM_NAME=kotlin-gcp-pubsub-test 3 | POM_DESCRIPTION=Kotlin GCP PubSub Test library powered by Testcontainers -------------------------------------------------------------------------------- /pubsub-kotlinx-serialization-json/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=kotlin-gcp-pubsub-kotlinx-serialization-json 2 | POM_NAME=kotlin-gcp-pubsub-kotlinx-serialization-json 3 | POM_DESCRIPTION=Kotlin GCP PubSub integration with KotlinX Serialization Json -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /pubsub-ktor-kotlinx-serialization-json/gradle.properties: -------------------------------------------------------------------------------- 1 | POM_ARTIFACT_ID=kotlin-gcp-pubsub-ktor-kotlinx-serialization-json 2 | POM_NAME=kotlin-gcp-pubsub-ktor-kotlinx-serialization-json 3 | POM_DESCRIPTION=Kotlin GCP PubSub Ktor integration with KotlinX Serialization Json -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gradle" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | rebase-strategy: "disabled" 8 | 9 | - package-ecosystem: "github-actions" 10 | directory: "/" 11 | schedule: 12 | interval: "weekly" 13 | rebase-strategy: "disabled" 14 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ], 5 | "commitBodyTable": true, 6 | "baseBranches": ["main", "optics-setup"], 7 | "packageRules": [ 8 | { 9 | "matchPackagePatterns": [ 10 | "*" 11 | ], 12 | "matchUpdateTypes": [ 13 | "major", 14 | "minor", 15 | "patch" 16 | ], 17 | "groupName": "all dependencies", 18 | "groupSlug": "all" 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /common-api/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id(libs.plugins.kotlin.jvm.get().pluginId) 3 | id(libs.plugins.dokka.get().pluginId) 4 | id(libs.plugins.kover.get().pluginId) 5 | alias(libs.plugins.knit) 6 | alias(libs.plugins.publish) 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | 13 | configure { 14 | toolchain { 15 | languageVersion.set(JavaLanguageVersion.of(8)) 16 | } 17 | } 18 | 19 | dependencies { 20 | api(libs.coroutines) 21 | api(libs.google.api) 22 | } 23 | -------------------------------------------------------------------------------- /.github/workflows/junie.yml: -------------------------------------------------------------------------------- 1 | name: Junie 2 | run-name: Junie run ${{ inputs.run_id }} 3 | 4 | permissions: 5 | contents: write 6 | pull-requests: write 7 | 8 | on: 9 | workflow_dispatch: 10 | inputs: 11 | run_id: 12 | description: "id of workflow process" 13 | required: true 14 | workflow_params: 15 | description: "stringified params" 16 | required: true 17 | 18 | jobs: 19 | call-workflow-passing-data: 20 | uses: jetbrains-junie/junie-workflows/.github/workflows/ej-issue.yml@main 21 | with: 22 | workflow_params: ${{ inputs.workflow_params }} 23 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | kotlin.code.style=official 2 | kotlin.mpp.stability.nowarn=true 3 | SONATYPE_HOST=S01 4 | RELEASE_SIGNING_ENABLED=true 5 | GROUP=io.github.nomisrev 6 | POM_URL=https://github.com/nomisrev/kotlin-gcp-pubsub/ 7 | POM_LICENSE_NAME=The Apache Software License, Version 2.0 8 | POM_LICENSE_URL=https://www.apache.org/licenses/LICENSE-2.0.txt 9 | POM_LICENSE_DIST=repo 10 | POM_SCM_URL=https://github.com/nomisrev/kotlin-gcp-pubsub/ 11 | POM_SCM_CONNECTION=scm:git:git://github.com/nomisRev/kotlin-gcp-pubsub.git 12 | POM_SCM_DEV_CONNECTION=scm:git:ssh://git@github.com/nomisRev/kotlin-gcp-pubsub.git 13 | POM_DEVELOPER_ID=nomisRev 14 | POM_DEVELOPER_NAME=Simon Vergauwen 15 | POM_DEVELOPER_URL=https://github.com/nomisRev/ 16 | -------------------------------------------------------------------------------- /pubsub-ktor/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id(libs.plugins.kotlin.jvm.get().pluginId) 3 | id(libs.plugins.dokka.get().pluginId) 4 | id(libs.plugins.kover.get().pluginId) 5 | alias(libs.plugins.spotless) 6 | alias(libs.plugins.knit) 7 | alias(libs.plugins.publish) 8 | } 9 | 10 | repositories { 11 | mavenCentral() 12 | } 13 | 14 | configure { 15 | toolchain { 16 | languageVersion.set(JavaLanguageVersion.of(8)) 17 | } 18 | } 19 | 20 | spotless { 21 | kotlin { 22 | targetExclude("**/build/**") 23 | ktfmt().googleStyle() 24 | } 25 | } 26 | 27 | kotlin { explicitApi() } 28 | 29 | dependencies { 30 | api(projects.gcpPubsub) 31 | api(libs.ktor.server) 32 | 33 | testImplementation(libs.ktor.test) 34 | testImplementation(projects.gcpPubsubTest) 35 | } 36 | -------------------------------------------------------------------------------- /.github/workflows/dependabot-auto-merge.yml: -------------------------------------------------------------------------------- 1 | name: Dependabot Auto-Merge 2 | on: pull_request 3 | 4 | permissions: 5 | contents: write 6 | pull-requests: write 7 | 8 | jobs: 9 | dependabot-auto-merge: 10 | runs-on: ubuntu-latest 11 | if: ${{ github.actor == 'dependabot[bot]' }} 12 | steps: 13 | - name: Dependabot metadata 14 | id: metadata 15 | uses: dependabot/fetch-metadata@v2 16 | with: 17 | github-token: "${{ secrets.GITHUB_TOKEN }}" 18 | 19 | - name: Enable auto-merge for Dependabot PRs 20 | if: ${{ steps.metadata.outputs.update-type != 'version-update:semver-major' }} 21 | run: gh pr merge --auto --merge "$PR_URL" 22 | env: 23 | PR_URL: ${{ github.event.pull_request.html_url }} 24 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 25 | -------------------------------------------------------------------------------- /pubsub-test/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id(libs.plugins.kotlin.jvm.get().pluginId) 3 | id(libs.plugins.dokka.get().pluginId) 4 | id(libs.plugins.kover.get().pluginId) 5 | alias(libs.plugins.spotless) 6 | alias(libs.plugins.knit) 7 | alias(libs.plugins.publish) 8 | } 9 | 10 | repositories { 11 | mavenCentral() 12 | } 13 | 14 | configure { 15 | toolchain { 16 | languageVersion.set(JavaLanguageVersion.of(8)) 17 | } 18 | } 19 | 20 | spotless { 21 | kotlin { 22 | targetExclude("**/build/**") 23 | ktfmt().googleStyle() 24 | } 25 | } 26 | 27 | tasks.withType { 28 | useJUnitPlatform() 29 | } 30 | 31 | kotlin { explicitApi() } 32 | 33 | dependencies { 34 | api(projects.gcpPubsubKtor) 35 | api(libs.ktor.server) 36 | api(libs.testcontainers.gcloud) 37 | api(libs.testcontainers) 38 | } 39 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") 2 | 3 | rootProject.name = "kotlin-gcp-pubsub" 4 | include("google-common-api") 5 | project(":google-common-api").projectDir = file("common-api") 6 | 7 | include("gcp-pubsub") 8 | project(":gcp-pubsub").projectDir = file("pubsub") 9 | 10 | include("gcp-pubsub-kotlinx-serialization-json") 11 | project(":gcp-pubsub-kotlinx-serialization-json").projectDir = file("pubsub-kotlinx-serialization-json") 12 | 13 | include("gcp-pubsub-ktor") 14 | project(":gcp-pubsub-ktor").projectDir = file("pubsub-ktor") 15 | 16 | include("gcp-pubsub-ktor-kotlinx-serialization-json") 17 | project(":gcp-pubsub-ktor-kotlinx-serialization-json").projectDir = file("pubsub-ktor-kotlinx-serialization-json") 18 | 19 | include("gcp-pubsub-test") 20 | project(":gcp-pubsub-test").projectDir = file("pubsub-test") 21 | -------------------------------------------------------------------------------- /pubsub/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id(libs.plugins.kotlin.jvm.get().pluginId) 3 | id(libs.plugins.dokka.get().pluginId) 4 | id(libs.plugins.kover.get().pluginId) 5 | alias(libs.plugins.spotless) 6 | alias(libs.plugins.knit) 7 | alias(libs.plugins.publish) 8 | } 9 | 10 | repositories { 11 | mavenCentral() 12 | } 13 | 14 | configure { 15 | toolchain { 16 | languageVersion.set(JavaLanguageVersion.of(8)) 17 | } 18 | } 19 | 20 | spotless { 21 | kotlin { 22 | targetExclude("**/build/**") 23 | ktfmt().googleStyle() 24 | } 25 | } 26 | 27 | kotlin { explicitApi() } 28 | 29 | dependencies { 30 | implementation(kotlin("stdlib")) 31 | api(libs.coroutines) 32 | api(libs.pubsub) 33 | api(projects.googleCommonApi) 34 | 35 | testImplementation(kotlin("test")) 36 | testImplementation(projects.gcpPubsubTest) 37 | } 38 | -------------------------------------------------------------------------------- /.github/workflows/githubpages.yaml: -------------------------------------------------------------------------------- 1 | name: githubpages 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | branches: [main] 8 | inputs: 9 | version: 10 | description: 'Version' 11 | required: true 12 | type: string 13 | 14 | jobs: 15 | githubpages: 16 | runs-on: ubuntu-latest 17 | timeout-minutes: 20 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | with: 22 | fetch-depth: 0 23 | 24 | - uses: gradle/gradle-build-action@v3 25 | with: 26 | arguments: -Pversion=${{ github.event.release.tag_name }} dokkaHtmlMultiModule 27 | 28 | - name: Deploy to gh-pages 29 | uses: peaceiris/actions-gh-pages@v4 30 | with: 31 | github_token: ${{ secrets.GITHUB_TOKEN }} 32 | publish_dir: ./build/dokka/htmlMultiModule 33 | -------------------------------------------------------------------------------- /pubsub-kotlinx-serialization-json/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id(libs.plugins.kotlin.jvm.get().pluginId) 3 | id(libs.plugins.dokka.get().pluginId) 4 | id(libs.plugins.kover.get().pluginId) 5 | alias(libs.plugins.spotless) 6 | alias(libs.plugins.knit) 7 | alias(libs.plugins.publish) 8 | kotlin("plugin.serialization") version "2.3.0" 9 | } 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | configure { 16 | toolchain { 17 | languageVersion.set(JavaLanguageVersion.of(8)) 18 | } 19 | } 20 | 21 | spotless { 22 | kotlin { 23 | targetExclude("**/build/**") 24 | ktfmt().googleStyle() 25 | } 26 | } 27 | 28 | kotlin { explicitApi() } 29 | 30 | dependencies { 31 | api(kotlin("stdlib")) 32 | api(projects.gcpPubsub) 33 | api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") 34 | 35 | testImplementation(kotlin("test")) 36 | testImplementation(projects.gcpPubsubTest) 37 | } 38 | -------------------------------------------------------------------------------- /.github/workflows/pr.yaml: -------------------------------------------------------------------------------- 1 | name: "Build main" 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: 6 | - 'docs/**' 7 | - '*.md' 8 | 9 | jobs: 10 | check: 11 | runs-on: ubuntu-latest 12 | timeout-minutes: 30 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | 19 | - uses: actions/setup-java@v4 20 | with: 21 | distribution: 'zulu' 22 | java-version: 17 23 | 24 | - uses: gradle/gradle-build-action@v3 25 | with: 26 | arguments: build --scan --full-stacktrace 27 | 28 | - name: Bundle the build report 29 | if: failure() 30 | run: find . -type d -name 'reports' | zip -@ -r build-reports.zip 31 | 32 | - name: Upload the build report 33 | if: failure() 34 | uses: actions/upload-artifact@master 35 | with: 36 | name: error-report 37 | path: build-reports.zip 38 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: "Build" 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - 'docs/**' 7 | - '*.md' 8 | branches: 9 | - main 10 | 11 | jobs: 12 | check: 13 | runs-on: ubuntu-latest 14 | timeout-minutes: 30 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 20 | 21 | - uses: actions/setup-java@v4 22 | with: 23 | distribution: 'zulu' 24 | java-version: 17 25 | 26 | - uses: gradle/gradle-build-action@v3 27 | with: 28 | arguments: build --scan --full-stacktrace 29 | 30 | - name: Bundle the build report 31 | if: failure() 32 | run: find . -type d -name 'reports' | zip -@ -r build-reports.zip 33 | 34 | - name: Upload the build report 35 | if: failure() 36 | uses: actions/upload-artifact@master 37 | with: 38 | name: error-report 39 | path: build-reports.zip 40 | -------------------------------------------------------------------------------- /pubsub-ktor-kotlinx-serialization-json/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id(libs.plugins.kotlin.jvm.get().pluginId) 3 | id(libs.plugins.dokka.get().pluginId) 4 | id(libs.plugins.kover.get().pluginId) 5 | alias(libs.plugins.spotless) 6 | alias(libs.plugins.knit) 7 | alias(libs.plugins.publish) 8 | kotlin("plugin.serialization") version "2.3.0" 9 | } 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | configure { 16 | toolchain { 17 | languageVersion.set(JavaLanguageVersion.of(8)) 18 | } 19 | } 20 | 21 | spotless { 22 | kotlin { 23 | targetExclude("**/build/**") 24 | ktfmt().googleStyle() 25 | } 26 | } 27 | 28 | kotlin { explicitApi() } 29 | 30 | dependencies { 31 | implementation(kotlin("stdlib")) 32 | api(projects.gcpPubsubKtor) 33 | api(projects.gcpPubsubKotlinxSerializationJson) 34 | api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") 35 | 36 | testImplementation(kotlin("test")) 37 | testImplementation(libs.ktor.test) 38 | testImplementation(projects.gcpPubsubTest) 39 | } 40 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: "Publish library" 2 | 3 | on: 4 | workflow_dispatch: 5 | branches: [main] 6 | inputs: 7 | version: 8 | description: 'Version' 9 | required: true 10 | type: string 11 | 12 | env: 13 | ORG_GRADLE_PROJECT_mavenCentralUsername: '${{ secrets.SONATYPE_USER }}' 14 | ORG_GRADLE_PROJECT_mavenCentralPassword: '${{ secrets.SONATYPE_PWD }}' 15 | ORG_GRADLE_PROJECT_signingInMemoryKeyId: '${{ secrets.SIGNING_KEY_ID }}' 16 | ORG_GRADLE_PROJECT_signingInMemoryKey: '${{ secrets.SIGNING_KEY }}' 17 | ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: '${{ secrets.SIGNING_KEY_PASSPHRASE }}' 18 | 19 | jobs: 20 | publish: 21 | timeout-minutes: 30 22 | runs-on: macos-latest 23 | steps: 24 | - uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | 28 | - uses: actions/setup-java@v4 29 | with: 30 | distribution: 'zulu' 31 | java-version: 11 32 | 33 | - uses: gradle/gradle-build-action@v3 34 | with: 35 | arguments: assemble -Pversion=${{ inputs.version }} 36 | 37 | - name: Upload reports 38 | if: failure() 39 | uses: actions/upload-artifact@v4 40 | with: 41 | name: 'reports-${{ matrix.os }}' 42 | path: '**/build/reports/**' 43 | 44 | - name: Publish final version 45 | uses: gradle/gradle-build-action@v3 46 | with: 47 | arguments: -Pversion=${{ inputs.version }} publishAllPublicationsToMavenCentralRepository 48 | -------------------------------------------------------------------------------- /pubsub-ktor-kotlinx-serialization-json/README.MD: -------------------------------------------------------------------------------- 1 | # Module gcp-pubsub-ktor-kotlinx-serialization-json 2 | 3 | An integration module that offers simplified APIs that use KotlinX serialization to subscribe to Json values. 4 | It's an addition to [Gcp PubSub KotlinX Serialization Json](../pubsub-kotlinx-serialization-json/README.MD) for Ktor. 5 | 6 | | **Name** | **Description** 7 | |-----------------------------------|--------------------------------------------------------------------------------------------------------------- 8 | | [subscribe/subscribeDeserialized] | Subscribe to a subscription to process types annotation with `@Serializable` using KotlinX Serialization Json 9 | 10 | ## Example 11 | 12 | ```kotlin 13 | @Serializable 14 | data class Event(val key: String, val message: String) 15 | 16 | fun Application.pubSub(): Job = 17 | pubSub(ProjectId("my-project")) { 18 | subscribe(SubscriptionId("my-subscription")) { (event, record) -> 19 | println("event.key: ${event.key}, event.message: ${event.message}") 20 | record.ack() 21 | } 22 | } 23 | ``` 24 | 25 | ## Using in your projects 26 | 27 | ### Gradle 28 | 29 | Add dependencies (you can also add other modules that you need): 30 | 31 | ```kotlin 32 | dependencies { 33 | implementation("io.github.nomisrev:gcp-pubsub-ktor-kotlinx-serialization-json:1.0.0") 34 | } 35 | ``` 36 | 37 | ### Maven 38 | 39 | Add dependencies (you can also add other modules that you need): 40 | 41 | ```xml 42 | 43 | 44 | io.github.nomisrev 45 | gcp-pubsub-ktor-kotlinx-serialization-json 46 | 1.0.0 47 | 48 | ``` 49 | -------------------------------------------------------------------------------- /pubsub-ktor/README.MD: -------------------------------------------------------------------------------- 1 | # Module gcp-pubsub-ktor 2 | 3 | Ktor plugin offering convenient way of working with GCP PubSub, it offers a simplified way of configuring Gcp PubSub and 4 | automatically takes care of lifecycle to match lifecycle of the Ktor Application. 5 | 6 | Simply `install` the plugin into Ktor Application, and set up the desired configuration. 7 | 8 | ```kotlin 9 | fun Application.setup(): GcpPubSub = 10 | install(GcpPubSub) { 11 | // all configuration is optional 12 | credentialsProvider = GoogleCredentials.fromStream(FileInputStream(PATH_TO_JSON_KEY)) 13 | subscriber { /* configure subscriber */ } 14 | publisher { /* configure publisher */ } 15 | topicAdmin { /* configure topic admin */ } 16 | subscriptionAdmin { /* configure subscription admin */ } 17 | } 18 | ``` 19 | 20 | After you've installed the `GcpPubSub` plugin, and setup according the needs of your application simply open a `pubSub` 21 | block inside the Ktor Application and you'll have access to all GcpPubSub functionality through the DSL. 22 | 23 | ```kotlin 24 | fun Application.setup(topic: TopicId, subscription: SubscriptionId): Job = 25 | pubSub(ProjectId("my-project")) { 26 | publish(topic, messages) 27 | 28 | subscribe(subscription) { record -> 29 | println(record.message.data.toStringUtf8()) 30 | record.ack() 31 | } 32 | } 33 | ``` 34 | 35 | You can also easily access the `GcpPubSub` plugin using the `pubSub` function in case you need access to `publisher` 36 | from a route, or different parts from your application. 37 | 38 | ```kotlin 39 | fun Application.route(): Routing = 40 | routing { 41 | post("/publish/{message}") { 42 | val message = 43 | requireNotNull(call.parameters["message"]) { "Missing parameter message" } 44 | pubSub().publisher(ProjectId("my-project")) 45 | .publish(TopicId("my-topic"), message) 46 | call.respond(HttpStatusCode.Accepted) 47 | } 48 | } 49 | ``` 50 | -------------------------------------------------------------------------------- /pubsub-kotlinx-serialization-json/src/test/kotlin/io/github/nomisrev/gcp/pubsub/serialization/PublishSerializationTest.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub.serialization 2 | 3 | import io.github.nomisrev.gcp.pubsub.ProjectId 4 | import io.github.nomisrev.gcp.pubsub.test.PubSubEmulator 5 | import kotlin.test.Test 6 | import kotlin.test.assertEquals 7 | import kotlinx.coroutines.Dispatchers.Default 8 | import kotlinx.coroutines.flow.map 9 | import kotlinx.coroutines.flow.take 10 | import kotlinx.coroutines.flow.toList 11 | import kotlinx.coroutines.runBlocking 12 | import kotlinx.serialization.Serializable 13 | import org.junit.AfterClass 14 | import org.junit.ClassRule 15 | 16 | class PublishSerializationTest { 17 | @Serializable data class Event(val id: Long, val content: String) 18 | 19 | @Test 20 | fun canSerializeAndDeserialize() = 21 | runBlocking(Default) { 22 | val topic = pubSubEmulator.uniqueTopic() 23 | val subscription = pubSubEmulator.uniqueSubscription() 24 | admin.createTopic(topic) 25 | admin.createSubscription(subscription, topic) 26 | 27 | val expected = listOf(Event(1, "msg1"), Event(1, "msg2")) 28 | val event = Event(1, "msg3") 29 | publisher.publish(topic, expected) 30 | publisher.publish(topic, event) 31 | 32 | val actual = 33 | pubSubEmulator 34 | .subscriber(projectId) 35 | .subscribe(subscription) 36 | .map { (event, record) -> event.also { record.ack() } } 37 | .take(3) 38 | .toList() 39 | 40 | assertEquals((expected + event).toSet(), actual.toSet()) 41 | } 42 | 43 | companion object { 44 | @get:ClassRule @JvmStatic val pubSubEmulator = PubSubEmulator() 45 | 46 | val projectId = ProjectId("my-project-id") 47 | val admin by lazy { pubSubEmulator.admin(projectId) } 48 | val publisher by lazy { pubSubEmulator.publisher(projectId) } 49 | 50 | @AfterClass 51 | @JvmStatic 52 | fun destroy() { 53 | admin.close() 54 | publisher.close() 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /gradle/libs.versions.toml: -------------------------------------------------------------------------------- 1 | [versions] 2 | coroutines = "1.10.2" 3 | dokka = "2.1.0" 4 | kotlin = "2.3.0" 5 | kotest = "5.9.1" 6 | kover = "0.9.4" 7 | detekt = "1.23.8" 8 | kotlinx-knit="0.5.0" 9 | ktor = "3.3.3" 10 | pubsub = "1.144.0" 11 | gcloud = "1.21.3" 12 | publish="0.35.0" 13 | knit="0.5.0" 14 | spotless="7.2.1" 15 | google-api="2.55.2" 16 | logback = "1.5.23" 17 | 18 | [libraries] 19 | coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "coroutines" } 20 | kotest-assertions = { module = "io.kotest:kotest-assertions-core", version.ref = "kotest" } 21 | kotest-framework = { module = "io.kotest:kotest-framework-engine", version.ref = "kotest" } 22 | kotest-junit5 = { module = "io.kotest:kotest-runner-junit5", version.ref = "kotest" } 23 | kotlinx-knit-test = { module = "org.jetbrains.kotlinx:kotlinx-knit-test", version.ref = "kotlinx-knit" } 24 | ktor-server = { module = "io.ktor:ktor-server-core", version.ref = "ktor" } 25 | ktor-test = { module = "io.ktor:ktor-server-test-host", version.ref = "ktor" } 26 | pubsub = { module = "com.google.cloud:google-cloud-pubsub", version.ref = "pubsub" } 27 | google-api = { module = "com.google.api:api-common", version.ref = "google-api" } 28 | testcontainers-gcloud = { module = "org.testcontainers:gcloud", version.ref = "gcloud" } 29 | testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "gcloud" } 30 | logback = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } 31 | 32 | [plugins] 33 | dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } 34 | kover = { id = "org.jetbrains.kotlinx.kover", version.ref = "kover" } 35 | kotest-multiplatform = { id = "io.kotest.multiplatform", version.ref = "kotest" } 36 | kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } 37 | detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" } 38 | publish = { id = "com.vanniktech.maven.publish", version.ref="publish" } 39 | knit = { id = "org.jetbrains.kotlinx.knit", version.ref="knit" } 40 | spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } 41 | -------------------------------------------------------------------------------- /pubsub-test/README.MD: -------------------------------------------------------------------------------- 1 | # Module gcp-pubsub-test 2 | 3 | Testing module build on top of testcontainers for Gcp PubSub exist out of a single type `PubSubEmulator`, 4 | it implements `AutoCloseable`, and `Startable` for the underlying testcontainers. 5 | 6 | In addition to `ExternalResource` such that you can easily use it with JUnit's `ClassRule` and `Rule`, 7 | and it's also a Ktor Plugin such that you can easily install it in your `testApplication` and all configuration is done 8 | for you. 9 | 10 | ```kotlin 11 | class GcpPluginTest { 12 | val projectId = ProjectId("my-project-id") 13 | 14 | @Test 15 | fun emulatorTest() = runBlocking(Default) { 16 | val topic = emulator.uniqueTopic() 17 | val subscription = emulator.uniqueSubscription() 18 | val messages = (1..3).map { "$it" } 19 | val channel = Channel(3) 20 | 21 | testApplication { 22 | install(emulator) 23 | 24 | application { 25 | pubSub(projectId) { 26 | createTopic(topic) 27 | createSubscription(subscription, topic) 28 | launch { publish(topic, messages) } 29 | 30 | subscribe(subscription) { record -> 31 | record.ack().await() 32 | channel.send(record.message.data.toStringUtf8()) 33 | } 34 | } 35 | }.also { startApplication() } 36 | 37 | assertEquals(messages.toSet(), channel.consumeAsFlow().take(3).toSet()) 38 | } 39 | } 40 | 41 | companion object { 42 | @JvmStatic 43 | @get:ClassRule 44 | val emulator = PubSubEmulator() 45 | } 46 | } 47 | ``` 48 | ## Using in your projects 49 | 50 | ### Gradle 51 | 52 | Add dependencies (you can also add other modules that you need): 53 | 54 | ```kotlin 55 | dependencies { 56 | testImplementation("io.github.nomisrev:gcp-pubsub-test:1.0.0") 57 | } 58 | ``` 59 | 60 | ### Maven 61 | 62 | Add dependencies (you can also add other modules that you need): 63 | 64 | ```xml 65 | 66 | 67 | io.github.nomisrev 68 | gcp-pubsub-test 69 | 1.0.0 70 | test 71 | 72 | ``` 73 | 74 | -------------------------------------------------------------------------------- /pubsub-ktor/src/test/kotlin/io/github/nomisrev/gcp/ktor/GcpPluginTest.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.ktor 2 | 3 | import com.google.cloud.pubsub.v1.AckResponse 4 | import io.github.nomisrev.gcp.core.await 5 | import io.github.nomisrev.gcp.pubsub.ProjectId 6 | import io.github.nomisrev.gcp.pubsub.ktor.pubSub 7 | import io.github.nomisrev.gcp.pubsub.publish 8 | import io.github.nomisrev.gcp.pubsub.test.PubSubEmulator 9 | import io.ktor.server.testing.testApplication 10 | import junit.framework.TestCase.assertEquals 11 | import kotlinx.coroutines.Dispatchers.Default 12 | import kotlinx.coroutines.Dispatchers.IO 13 | import kotlinx.coroutines.channels.Channel 14 | import kotlinx.coroutines.flow.consumeAsFlow 15 | import kotlinx.coroutines.flow.take 16 | import kotlinx.coroutines.flow.toSet 17 | import kotlinx.coroutines.launch 18 | import kotlinx.coroutines.runBlocking 19 | import org.junit.ClassRule 20 | import org.junit.Test 21 | 22 | class GcpPluginTest { 23 | val projectId = ProjectId("my-project-id") 24 | 25 | @Test 26 | fun canReceiveAndProcess3Elements() = 27 | runBlocking(Default) { 28 | val topic = pubSubEmulator.uniqueTopic() 29 | val subscription = pubSubEmulator.uniqueSubscription() 30 | 31 | testApplication { 32 | install(pubSubEmulator) 33 | 34 | val messages = (1..3).map { "$it" } 35 | val latch = Channel(3) 36 | 37 | pubSubEmulator.admin(projectId).createTopic(topic) 38 | pubSubEmulator.admin(projectId).createSubscription(subscription, topic) 39 | 40 | application { 41 | pubSub(projectId) { 42 | subscribe(subscription) { record -> 43 | assertEquals(record.ack().await(), AckResponse.SUCCESSFUL) 44 | latch.send(record.message.data.toStringUtf8()) 45 | } 46 | } 47 | } 48 | 49 | startApplication() 50 | launch(IO) { pubSubEmulator.publisher(projectId).publish(topic, messages) } 51 | 52 | assertEquals(messages.toSet(), latch.consumeAsFlow().take(3).toSet()) 53 | } 54 | } 55 | 56 | companion object { 57 | @JvmStatic @get:ClassRule val pubSubEmulator = PubSubEmulator() 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pubsub-ktor/src/main/kotlin/io/github/nomisrev/gcp/pubsub/ktor/GcpPubSubSyntax.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub.ktor 2 | 3 | import com.google.cloud.pubsub.v1.Subscriber 4 | import io.github.nomisrev.gcp.pubsub.GcpPublisher 5 | import io.github.nomisrev.gcp.pubsub.GcpPubsSubAdmin 6 | import io.github.nomisrev.gcp.pubsub.GcpSubscriber 7 | import io.github.nomisrev.gcp.pubsub.PubsubRecord 8 | import io.github.nomisrev.gcp.pubsub.SubscriptionId 9 | import kotlin.coroutines.CoroutineContext 10 | import kotlinx.coroutines.CoroutineDispatcher 11 | import kotlinx.coroutines.CoroutineScope 12 | import kotlinx.coroutines.Dispatchers 13 | import kotlinx.coroutines.FlowPreview 14 | import kotlinx.coroutines.Job 15 | import kotlinx.coroutines.flow.DEFAULT_CONCURRENCY 16 | 17 | public interface GcpPubSubSyntax : GcpPubsSubAdmin, GcpPublisher, CoroutineScope { 18 | 19 | public val subscriber: GcpSubscriber 20 | 21 | /** 22 | * Subscribe to [subscriptionId], and process the received [PubsubRecord] by running [handler] in 23 | * [concurrency] parallel coroutines 24 | * 25 | * ```kotlin 26 | * fun Application.process(): Job = 27 | * pubSub(ProjectId("my-project")) { 28 | * subscribe(SubscriptionId("my-subscription")) { record -> 29 | * println(record.message.data.toStringUtf8()) 30 | * record.ack(). 31 | * } 32 | * } 33 | * ``` 34 | * 35 | * @param subscriptionId the subscription to subscribe to 36 | * @param concurrency the amount of parallel coroutines that will be processing [PubsubRecord] 37 | * @param context the [CoroutineDispatcher] where [handler] will be running 38 | * @param configure additional configuration to use when creating the [Subscriber], in addition to 39 | * the global [GcpPubSub.Configuration.subscriber] configuration. 40 | * @param handler to use when processing received [PubsubRecord] messages 41 | */ 42 | @OptIn(FlowPreview::class) 43 | public fun subscribe( 44 | subscriptionId: SubscriptionId, 45 | concurrency: Int = DEFAULT_CONCURRENCY, 46 | context: CoroutineContext = Dispatchers.Default, 47 | configure: Subscriber.Builder.() -> Unit = {}, 48 | handler: suspend (PubsubRecord) -> Unit, 49 | ): Job 50 | } 51 | -------------------------------------------------------------------------------- /pubsub-ktor-kotlinx-serialization-json/src/test/kotlin/io/github/nomisrev/gcp/pubsub/ktor/serialization/PublishSerializationTest.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub.ktor.serialization 2 | 3 | import com.google.cloud.pubsub.v1.AckResponse 4 | import io.github.nomisrev.gcp.core.await 5 | import io.github.nomisrev.gcp.pubsub.ProjectId 6 | import io.github.nomisrev.gcp.pubsub.ktor.pubSub 7 | import io.github.nomisrev.gcp.pubsub.serialization.publish 8 | import io.github.nomisrev.gcp.pubsub.test.PubSubEmulator 9 | import io.ktor.server.testing.testApplication 10 | import junit.framework.TestCase 11 | import kotlin.test.assertEquals 12 | import kotlinx.coroutines.Dispatchers 13 | import kotlinx.coroutines.Dispatchers.Default 14 | import kotlinx.coroutines.channels.Channel 15 | import kotlinx.coroutines.flow.consumeAsFlow 16 | import kotlinx.coroutines.flow.take 17 | import kotlinx.coroutines.flow.toSet 18 | import kotlinx.coroutines.launch 19 | import kotlinx.coroutines.runBlocking 20 | import kotlinx.serialization.Serializable 21 | import org.junit.ClassRule 22 | 23 | class PublishSerializationTest { 24 | @Serializable data class Event(val id: Long, val content: String) 25 | 26 | val projectId = ProjectId("my-project-id") 27 | 28 | @org.junit.Test 29 | fun canReceiveAndProcess3Elements() = 30 | runBlocking(Default) { 31 | val topic = pubSubEmulator.uniqueTopic() 32 | val subscription = pubSubEmulator.uniqueSubscription() 33 | 34 | val messages = (1..3L).map { Event(1, "msg$it") } 35 | 36 | testApplication { 37 | install(pubSubEmulator) 38 | 39 | val latch = Channel(3) 40 | 41 | pubSubEmulator.admin(projectId).createTopic(topic) 42 | pubSubEmulator.admin(projectId).createSubscription(subscription, topic) 43 | 44 | application { 45 | pubSub(projectId) { 46 | subscribe(subscription) { (event, record) -> 47 | assertEquals(record.ack().await(), AckResponse.SUCCESSFUL) 48 | latch.send(event) 49 | } 50 | } 51 | } 52 | 53 | startApplication() 54 | launch(Dispatchers.IO) { pubSubEmulator.publisher(projectId).publish(topic, messages) } 55 | 56 | TestCase.assertEquals(messages.toSet(), latch.consumeAsFlow().take(3).toSet()) 57 | } 58 | } 59 | 60 | companion object { 61 | @JvmStatic @get:ClassRule val pubSubEmulator: PubSubEmulator = PubSubEmulator() 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kotlin GCP PubSub 2 | 3 | Google Cloud PubSub made easy! Kotlin GCP PubSub offers idiomatic KotlinX & Ktor integration for GCP. 4 | 5 | ```kotlin 6 | @Serializable 7 | data class Event(val key: String, val message: String) 8 | 9 | fun Application.pubSubApp() { 10 | pubSub(ProjectId("my-project")) { 11 | subscribe(SubscriptionId("my-subscription")) { (event, record) -> 12 | println("event.key: ${event.key}, event.message: ${event.message}") 13 | record.ack() 14 | } 15 | } 16 | 17 | routing { 18 | post("/publish/{key}/{message}") { 19 | val event = Event(call.parameters["key"]!!, call.parameters["message"]!!) 20 | 21 | pubSub() 22 | .publisher(ProjectId("my-project")) 23 | .publish(TopicId("my-topic"), event) 24 | 25 | call.respond(HttpStatusCode.Accepted) 26 | } 27 | } 28 | } 29 | ``` 30 | 31 | ## Modules 32 | 33 | - [PubSub Ktor plugin](pubsub-ktor/README.MD) to conveniently consume messages from GCP PubSub, and publish messages to 34 | GCP PubSub 35 | - [PubSub Ktor KotlinX Serialization Json](pubsub-ktor-kotlinx-serialization-json/README.MD) to conveniently consume 36 | messages from GCP PubSub, and publish messages to GCP PubSub using KotlinX Serialization Json 37 | - [PubSub test](pubsub-test/README.MD) one-line testing support powered by testcontainers 38 | - [GCP PubSub](pubsub/README.MD): KotlinX integration for `TopicAdminClient`, `SubscriptionAdminClient`, `Susbcriber` 39 | and `Publisher`. 40 | - [PubSub Ktor KotlinX Serialization Json](pubsub-kotlinx-serialization-json/README.MD) to conveniently consume messages 41 | from GCP PubSub, and publish messages to GCP PubSub 42 | - [Google Common API](api-core/README.MD): KotlinX integration for `ApiFuture` 43 | 44 | ## Using in your projects 45 | 46 | ### Gradle 47 | 48 | Add dependencies (you can also add other modules that you need): 49 | 50 | ```kotlin 51 | dependencies { 52 | implementation("io.github.nomisrev:gcp-pubsub-ktor:1.0.0") 53 | implementation("io.github.nomisrev:gcp-pubsub-ktor-kotlinx-serialization-json:1.0.0") 54 | testImplementation("io.github.nomisrev:gcp-pubsub-test:1.0.0") 55 | } 56 | ``` 57 | 58 | ### Maven 59 | 60 | Add dependencies (you can also add other modules that you need): 61 | 62 | ```xml 63 | 64 | 65 | io.github.nomisrev 66 | gcp-pubsub-ktor 67 | 1.0.0 68 | 69 | ``` 70 | -------------------------------------------------------------------------------- /common-api/README.MD: -------------------------------------------------------------------------------- 1 | # Module common-api 2 | 3 | KotlinX integration module for Google's api-core module. If you need to work with 4 | Google's [`ApiFuture`](https://cloud.google.com/java/docs/reference/api-common/latest/com.google.api.core.ApiFutures). 5 | 6 | Extension functions: 7 | 8 | | **Name** | **Description** 9 | |--------------------------|--------------------------------------------------- 10 | | [ApiFuture.await]() | Awaits for completion of the future (cancellable) 11 | | [ApiFuture.asDeferred]() | Converts a deferred value to the future 12 | 13 | ## Example 14 | 15 | Given the following functions defined in some Java API based on Guava: 16 | 17 | ```java 18 | public ApiFuture ack(); // starts async acknowledging of message 19 | ``` 20 | 21 | We can consume this API from Kotlin coroutine to load two images and combine then asynchronously. 22 | The resulting function returns `ListenableFuture` for ease of use back from Guava-based Java code. 23 | 24 | ```kotlin 25 | suspend fun processMessage(record: PubsubRecord): Unit { 26 | println(record.message.data.toStringUtf8()) 27 | when (val response = record.ack().await()) { 28 | SUCCESSFUL -> println("Message was successfully acknowledged. Will not be redelivered.") 29 | else -> println("Acknowledgment failed, message might be redelivered.") 30 | } 31 | } 32 | ``` 33 | 34 | Note that this module should be used only for integration with existing Java APIs based on `ApiFuture`. 35 | Writing pure-Kotlin code that uses `ApiFuture` is highly not recommended, since the resulting APIs based 36 | on the futures are quite error-prone. See the discussion on 37 | [Asynchronous Programming Styles](https://github.com/Kotlin/kotlin-coroutines/blob/master/kotlin-coroutines-informal.md#asynchronous-programming-styles) 38 | for details on general problems pertaining to any future-based API and keep in mind that `ApiFuture` exposes 39 | a _blocking_ method [get](https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/Future.html#get--) that makes 40 | it especially bad choice for coroutine-based Kotlin code. 41 | 42 | ## Using in your projects 43 | 44 | ### Gradle 45 | 46 | Add dependencies (you can also add other modules that you need): 47 | 48 | ```kotlin 49 | dependencies { 50 | implementation("io.github.nomisrev:google-common-api:1.0.0") 51 | } 52 | ``` 53 | 54 | ### Maven 55 | 56 | Add dependencies (you can also add other modules that you need): 57 | 58 | ```xml 59 | 60 | 61 | io.github.nomisrev 62 | google-common-api 63 | 1.0.0 64 | 65 | ``` 66 | -------------------------------------------------------------------------------- /pubsub-kotlinx-serialization-json/README.MD: -------------------------------------------------------------------------------- 1 | # Module gcp-pubsub-kotlinx-serialization-json 2 | 3 | An integration module that offers simplified APIs that use KotlinX serialization to encode/decode messages to-and-from 4 | Json values. 5 | 6 | This modules exposes following types that correspond to their relevant Gcp PubSub types but expose `suspend` APIs, that 7 | await the operations in a suspending way. 8 | 9 | | **Name** | **Description** 10 | |-----------------------------------|------------------------------------------------------------------------------------------------------------------------ 11 | | [KotlinXJsonEncoder] | An implementation of `MessageEncoder` using `StringFormat` based serializers from KotlinX Serialization 12 | | [KotlinXJsonDecoder] | An implementation of `MessageDecoder` using `StringFormat` based serializers from KotlinX Serialization 13 | | [publish] | Extension on `GcpPublisher` that allows publish types annotation with `@Serializable` using KotlinX Serialization Json 14 | | [deserialized] | Allows deserializing [PubsubMessage.getData] from [PubsubRecord] using KotlinX serialization. 15 | | [subscribe/subscribeDeserialized] | Subscribe to a subscription to process types annotation with `@Serializable` using KotlinX Serialization Json 16 | 17 | ## Example 18 | 19 | ```kotlin 20 | @Serializable 21 | data class Event(val key: String, val message: String) 22 | 23 | val topic = TopicId("my-topic") 24 | val subscription = SubsriptionId("my-subscription") 25 | val publisher: GcpPublisher = TODO("Create publisher") 26 | val subscriber: GcpSubscriber = TODO("Create subscriber") 27 | 28 | publisher.publish(topic, Event("key", "my-message-1")) 29 | publisher.publish(topic, listOf(Event("key", "my-message-2"), Event("key", "my-message-3"))) 30 | 31 | subscriber.subscribe(subscription) 32 | .collect { (event: Event, record: PubsubRecord) -> 33 | println("event.key: ${event.key}, event.message: ${event.message}") 34 | assertEquals(event, record.deserialized()) 35 | record.ack() 36 | } 37 | ``` 38 | 39 | ## Using in your projects 40 | 41 | ### Gradle 42 | 43 | Add dependencies (you can also add other modules that you need): 44 | 45 | ```kotlin 46 | dependencies { 47 | implementation("io.github.nomisrev:gcp-pubsub-kotlinx-serialization-json:1.0.0") 48 | } 49 | ``` 50 | 51 | ### Maven 52 | 53 | Add dependencies (you can also add other modules that you need): 54 | 55 | ```xml 56 | 57 | 58 | io.github.nomisrev 59 | gcp-pubsub-kotlinx-serialization-json 60 | 1.0.0 61 | 62 | ``` 63 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if %ERRORLEVEL% equ 0 goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if %ERRORLEVEL% equ 0 goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | set EXIT_CODE=%ERRORLEVEL% 84 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 85 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 86 | exit /b %EXIT_CODE% 87 | 88 | :mainEnd 89 | if "%OS%"=="Windows_NT" endlocal 90 | 91 | :omega 92 | -------------------------------------------------------------------------------- /pubsub-ktor-kotlinx-serialization-json/src/main/kotlin/io/github/nomisrev/gcp/pubsub/ktor/serialization/KotlinXSerialization.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub.ktor.serialization 2 | 3 | import com.google.cloud.pubsub.v1.Subscriber 4 | import io.github.nomisrev.gcp.pubsub.AcknowledgeableValue 5 | import io.github.nomisrev.gcp.pubsub.GcpSubscriber 6 | import io.github.nomisrev.gcp.pubsub.SubscriptionId 7 | import io.github.nomisrev.gcp.pubsub.ktor.GcpPubSubSyntax 8 | import kotlin.coroutines.CoroutineContext 9 | import kotlinx.coroutines.Dispatchers 10 | import kotlinx.coroutines.FlowPreview 11 | import kotlinx.coroutines.Job 12 | import kotlinx.coroutines.flow.DEFAULT_CONCURRENCY 13 | import kotlinx.coroutines.flow.flattenMerge 14 | import kotlinx.coroutines.flow.flow 15 | import kotlinx.coroutines.flow.launchIn 16 | import kotlinx.coroutines.flow.map 17 | import kotlinx.coroutines.plus 18 | import kotlinx.serialization.KSerializer 19 | import kotlinx.serialization.json.Json 20 | import kotlinx.serialization.serializer 21 | 22 | /** 23 | * This signature looks the same as the regular [GcpPubSubSyntax.subscribe], but instead takes 24 | * allows processing Json values of type `A`. 25 | * 26 | * To use this method you need to explicitly pass the generic argument, otherwise use 27 | * [subscribeDeserialized]. Alternatively you can also use [deserialized]. 28 | * 29 | * ```kotlin 30 | * @Serializable 31 | * data class Event(val key: String, val message: String) 32 | * 33 | * fun Application.pubSubApp() { 34 | * pubSub(ProjectId("my-project")) { 35 | * subscribe(SubscriptionId("my-subscription")) { (event, record) -> 36 | * println("event.key: ${event.key}, event.message: ${event.message}") 37 | * record.ack() 38 | * } 39 | * } 40 | * ``` 41 | * 42 | * @see GcpSubscriber.subscribe for full documentation. 43 | */ 44 | @OptIn(FlowPreview::class) 45 | public inline fun GcpPubSubSyntax.subscribe( 46 | subscriptionId: SubscriptionId, 47 | concurrency: Int = DEFAULT_CONCURRENCY, 48 | context: CoroutineContext = Dispatchers.Default, 49 | json: Json = Json, 50 | serializer: KSerializer = serializer(), 51 | noinline configure: Subscriber.Builder.() -> Unit = {}, 52 | noinline handler: suspend (AcknowledgeableValue) -> Unit, 53 | ): Job = 54 | subscriber 55 | .subscribe(subscriptionId) { configure() } 56 | .map { record -> 57 | flow { 58 | emit( 59 | handler( 60 | AcknowledgeableValue( 61 | json.decodeFromString(serializer, record.message.data.toStringUtf8()), 62 | record, 63 | ) 64 | ) 65 | ) 66 | } 67 | } 68 | .flattenMerge(concurrency) 69 | .launchIn(this@subscribe + context) 70 | 71 | public inline fun GcpPubSubSyntax.subscribeDeserialized( 72 | subscriptionId: SubscriptionId, 73 | concurrency: Int = DEFAULT_CONCURRENCY, 74 | context: CoroutineContext = Dispatchers.Default, 75 | json: Json = Json, 76 | serializer: KSerializer = serializer(), 77 | noinline configure: Subscriber.Builder.() -> Unit = {}, 78 | noinline handler: suspend (AcknowledgeableValue) -> Unit, 79 | ): Job = subscribe(subscriptionId, concurrency, context, json, serializer, configure, handler) 80 | -------------------------------------------------------------------------------- /pubsub/src/main/kotlin/io/github/nomisrev/gcp/pubsub/model.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub 2 | 3 | import com.google.cloud.pubsub.v1.AckReplyConsumerWithResponse 4 | import com.google.protobuf.Empty 5 | import com.google.pubsub.v1.ProjectSubscriptionName 6 | import com.google.pubsub.v1.ProjectTopicName 7 | import com.google.pubsub.v1.PubsubMessage 8 | import com.google.pubsub.v1.PubsubMessageOrBuilder 9 | import com.google.pubsub.v1.SubscriptionName 10 | import com.google.pubsub.v1.TopicName 11 | import kotlinx.coroutines.Deferred 12 | 13 | @JvmInline public value class ProjectId(public val value: String) 14 | 15 | @JvmInline 16 | public value class SubscriptionId(public val value: String) { 17 | public fun toProjectSubscriptionName(projectId: ProjectId?): ProjectSubscriptionName = 18 | // Fully-qualified subscription name in the 19 | // "projects/[project_name]/subscriptions/[subscription_name]" format 20 | if (ProjectSubscriptionName.isParsableFrom(value)) ProjectSubscriptionName.parse(value) 21 | else { 22 | requireNotNull(projectId) { 23 | "The project ID can't be null when using canonical subscription name." 24 | } 25 | ProjectSubscriptionName.of(projectId.value, value) 26 | } 27 | 28 | public fun toSubscriptionName(projectId: ProjectId?): SubscriptionName = 29 | // Fully-qualified subscription name in the 30 | // "projects/[project_name]/subscriptions/[subscription_name]" format 31 | if (SubscriptionName.isParsableFrom(value)) SubscriptionName.parse(value) 32 | else { 33 | requireNotNull(projectId) { 34 | "The project ID can't be null when using canonical subscription name." 35 | } 36 | SubscriptionName.of(projectId.value, value) 37 | } 38 | } 39 | 40 | @JvmInline 41 | public value class TopicId(public val value: String) { 42 | public fun toProjectTopicName(projectId: ProjectId?): ProjectTopicName = 43 | // Fully-qualified topic name in the "projects/[project_name]/topics/[topic_name]" format 44 | if (ProjectTopicName.isParsableFrom(value)) ProjectTopicName.parse(value) 45 | else { 46 | requireNotNull(projectId) { "The project ID can't be null when using canonical topic name." } 47 | ProjectTopicName.of(projectId.value, value) 48 | } 49 | 50 | public fun toTopicName(projectId: ProjectId?): TopicName = 51 | // Fully-qualified topic name in the "projects/[project_name]/topics/[topic_name]" format 52 | if (TopicName.isParsableFrom(value)) TopicName.parse(value) 53 | else { 54 | requireNotNull(projectId) { "The project ID can't be null when using canonical topic name." } 55 | TopicName.of(projectId.value, value) 56 | } 57 | } 58 | 59 | public class PubsubRecord( 60 | public val message: PubsubMessage, 61 | private val consumer: AckReplyConsumerWithResponse, 62 | public val projectId: ProjectId, 63 | public val subscriptionId: SubscriptionId, 64 | ) : PubsubMessageOrBuilder by message, AckReplyConsumerWithResponse by consumer 65 | 66 | public data class AcknowledgeableValue(val value: A, val record: PubsubRecord) 67 | 68 | public interface AckPubSubMessage : PubsubMessageOrBuilder { 69 | 70 | public val projectId: ProjectId 71 | 72 | public val subscriptionId: SubscriptionId 73 | 74 | public val pubSubMessage: PubsubMessage 75 | 76 | public val ackId: String 77 | 78 | public fun ack(): Deferred 79 | 80 | /** 81 | * Modify the ack deadline of the message. Once the ack deadline expires, the message is 82 | * automatically nacked. 83 | * 84 | * @param ackDeadlineSeconds the new ack deadline in seconds A deadline of 0 effectively nacks the 85 | * message. 86 | */ 87 | public fun modifyAckDeadline(ackDeadlineSeconds: Int): Deferred 88 | 89 | public fun nack(): Deferred = modifyAckDeadline(0) 90 | } 91 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/intellij+all,kotlin,gradle,macos 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=intellij+all,kotlin,gradle,macos 3 | 4 | ### Intellij+all ### 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 6 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 7 | 8 | # User-specific stuff 9 | .idea/**/workspace.xml 10 | .idea/**/tasks.xml 11 | .idea/**/usage.statistics.xml 12 | .idea/**/dictionaries 13 | .idea/**/shelf 14 | 15 | # AWS User-specific 16 | .idea/**/aws.xml 17 | 18 | # Generated files 19 | .idea/**/contentModel.xml 20 | 21 | # Sensitive or high-churn files 22 | .idea/**/dataSources/ 23 | .idea/**/dataSources.ids 24 | .idea/**/dataSources.local.xml 25 | .idea/**/sqlDataSources.xml 26 | .idea/**/dynamic.xml 27 | .idea/**/uiDesigner.xml 28 | .idea/**/dbnavigator.xml 29 | 30 | # Gradle 31 | .idea/**/gradle.xml 32 | .idea/**/libraries 33 | 34 | # Gradle and Maven with auto-import 35 | # When using Gradle or Maven with auto-import, you should exclude module files, 36 | # since they will be recreated, and may cause churn. Uncomment if using 37 | # auto-import. 38 | # .idea/artifacts 39 | # .idea/compiler.xml 40 | # .idea/jarRepositories.xml 41 | # .idea/modules.xml 42 | # .idea/*.iml 43 | # .idea/modules 44 | # *.iml 45 | # *.ipr 46 | 47 | # CMake 48 | cmake-build-*/ 49 | 50 | # Mongo Explorer plugin 51 | .idea/**/mongoSettings.xml 52 | 53 | # File-based project format 54 | *.iws 55 | 56 | # IntelliJ 57 | out/ 58 | 59 | # mpeltonen/sbt-idea plugin 60 | .idea_modules/ 61 | 62 | # JIRA plugin 63 | atlassian-ide-plugin.xml 64 | 65 | # Cursive Clojure plugin 66 | .idea/replstate.xml 67 | 68 | # Crashlytics plugin (for Android Studio and IntelliJ) 69 | com_crashlytics_export_strings.xml 70 | crashlytics.properties 71 | crashlytics-build.properties 72 | fabric.properties 73 | 74 | # Editor-based Rest Client 75 | .idea/httpRequests 76 | 77 | # Android studio 3.1+ serialized cache file 78 | .idea/caches/build_file_checksums.ser 79 | 80 | ### Intellij+all Patch ### 81 | # Ignores the whole .idea folder and all .iml files 82 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 83 | 84 | .idea/ 85 | 86 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 87 | 88 | *.iml 89 | modules.xml 90 | .idea/misc.xml 91 | *.ipr 92 | 93 | # Sonarlint plugin 94 | .idea/sonarlint 95 | 96 | ### Kotlin ### 97 | # Compiled class file 98 | *.class 99 | 100 | # Log file 101 | *.log 102 | 103 | # BlueJ files 104 | *.ctxt 105 | 106 | # Mobile Tools for Java (J2ME) 107 | .mtj.tmp/ 108 | 109 | # Package Files # 110 | *.jar 111 | *.war 112 | *.nar 113 | *.ear 114 | *.zip 115 | *.tar.gz 116 | *.rar 117 | 118 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 119 | hs_err_pid* 120 | 121 | ### macOS ### 122 | # General 123 | .DS_Store 124 | .AppleDouble 125 | .LSOverride 126 | 127 | # Icon must end with two \r 128 | Icon 129 | 130 | 131 | # Thumbnails 132 | ._* 133 | 134 | # Files that might appear in the root of a volume 135 | .DocumentRevisions-V100 136 | .fseventsd 137 | .Spotlight-V100 138 | .TemporaryItems 139 | .Trashes 140 | .VolumeIcon.icns 141 | .com.apple.timemachine.donotpresent 142 | 143 | # Directories potentially created on remote AFP share 144 | .AppleDB 145 | .AppleDesktop 146 | Network Trash Folder 147 | Temporary Items 148 | .apdisk 149 | 150 | ### Gradle ### 151 | .gradle 152 | build/ 153 | 154 | # Ignore Gradle GUI config 155 | gradle-app.setting 156 | 157 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 158 | !gradle-wrapper.jar 159 | 160 | # Cache of project 161 | .gradletasknamecache 162 | 163 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 164 | # gradle/wrapper/gradle-wrapper.properties 165 | 166 | ### Gradle Patch ### 167 | **/build/ 168 | 169 | # End of https://www.toptal.com/developers/gitignore/api/intellij+all,kotlin,gradle,macos -------------------------------------------------------------------------------- /pubsub/src/main/kotlin/io/github/nomisrev/gcp/pubsub/GcpPublisher.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub 2 | 3 | import com.google.api.core.ApiFutures 4 | import com.google.cloud.pubsub.v1.Publisher 5 | import com.google.protobuf.ByteString 6 | import com.google.pubsub.v1.PubsubMessage 7 | import io.github.nomisrev.gcp.core.await 8 | import java.nio.ByteBuffer 9 | import java.util.concurrent.ConcurrentHashMap 10 | 11 | public interface MessageEncoder { 12 | public suspend fun encode(value: A): PubsubMessage 13 | } 14 | 15 | public fun GcpPublisher( 16 | projectId: ProjectId, 17 | configure: Publisher.Builder.(topicId: TopicId) -> Unit = {}, 18 | ): GcpPublisher = DefaultGcpPublisher(projectId, configure) 19 | 20 | public interface GcpPublisher : AutoCloseable { 21 | public suspend fun publish(topicId: TopicId, message: PubsubMessage): String 22 | 23 | public suspend fun publish(topicId: TopicId, messages: Iterable): List 24 | 25 | public suspend fun publish( 26 | topicId: TopicId, 27 | messages: Iterable, 28 | encoder: MessageEncoder, 29 | ): List = publish(topicId, messages.map { encoder.encode(it) }) 30 | 31 | public suspend fun publish( 32 | topicId: TopicId, 33 | message: ByteString, 34 | configure: PubsubMessage.Builder.() -> Unit = {}, 35 | ): String = publish(topicId, PubsubMessage.newBuilder().setData(message).apply(configure).build()) 36 | 37 | public suspend fun publish( 38 | topicId: TopicId, 39 | message: String, 40 | configure: PubsubMessage.Builder.() -> Unit = {}, 41 | ): String = 42 | publish( 43 | topicId, 44 | PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8(message)).apply(configure).build(), 45 | ) 46 | 47 | public suspend fun publish( 48 | topicId: TopicId, 49 | message: ByteBuffer, 50 | configure: PubsubMessage.Builder.() -> Unit = {}, 51 | ): String = 52 | publish( 53 | topicId, 54 | PubsubMessage.newBuilder().setData(ByteString.copyFrom(message)).apply(configure).build(), 55 | ) 56 | 57 | public suspend fun publish( 58 | topicId: TopicId, 59 | message: ByteArray, 60 | configure: PubsubMessage.Builder.() -> Unit = {}, 61 | ): String = 62 | publish( 63 | topicId, 64 | PubsubMessage.newBuilder().setData(ByteString.copyFrom(message)).apply(configure).build(), 65 | ) 66 | 67 | public suspend fun publish(topicId: TopicId, message: A, encoder: MessageEncoder): String = 68 | publish(topicId, encoder.encode(message)) 69 | } 70 | 71 | @JvmName("publishByteString") 72 | public suspend fun GcpPublisher.publish( 73 | topicId: TopicId, 74 | messages: Iterable, 75 | configure: PubsubMessage.Builder.() -> Unit = {}, 76 | ): List = 77 | publish(topicId, messages.map { PubsubMessage.newBuilder().setData(it).apply(configure).build() }) 78 | 79 | @JvmName("publishString") 80 | public suspend fun GcpPublisher.publish( 81 | topicId: TopicId, 82 | messages: Iterable, 83 | configure: PubsubMessage.Builder.() -> Unit = {}, 84 | ): List = 85 | publish( 86 | topicId, 87 | messages.map { 88 | PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8(it)).apply(configure).build() 89 | }, 90 | ) 91 | 92 | @JvmName("publishByteBuffer") 93 | public suspend fun GcpPublisher.publish( 94 | topicId: TopicId, 95 | messages: Iterable, 96 | configure: PubsubMessage.Builder.() -> Unit = {}, 97 | ): List = 98 | publish( 99 | topicId, 100 | messages.map { 101 | PubsubMessage.newBuilder().setData(ByteString.copyFrom(it)).apply(configure).build() 102 | }, 103 | ) 104 | 105 | @JvmName("publishByteArray") 106 | public suspend fun GcpPublisher.publish( 107 | topicId: TopicId, 108 | messages: Iterable, 109 | configure: PubsubMessage.Builder.() -> Unit = {}, 110 | ): List = 111 | publish( 112 | topicId, 113 | messages.map { 114 | PubsubMessage.newBuilder().setData(ByteString.copyFrom(it)).apply(configure).build() 115 | }, 116 | ) 117 | 118 | private class DefaultGcpPublisher( 119 | val projectId: ProjectId, 120 | val configure: Publisher.Builder.(topicId: TopicId) -> Unit, 121 | ) : GcpPublisher { 122 | val publisherCache = ConcurrentHashMap() 123 | 124 | override suspend fun publish(topicId: TopicId, message: PubsubMessage): String = 125 | getOrCreatePublisher(topicId).publish(message).await() 126 | 127 | override suspend fun publish(topicId: TopicId, messages: Iterable): List { 128 | val publisher = getOrCreatePublisher(topicId) 129 | return ApiFutures.allAsList(messages.map(publisher::publish)).await() 130 | } 131 | 132 | override fun close() { 133 | publisherCache.forEachValue(1, Publisher::shutdown) 134 | } 135 | 136 | fun getOrCreatePublisher(topicId: TopicId): Publisher = 137 | publisherCache[topicId] 138 | ?: publisherCache.computeIfAbsent(topicId) { 139 | Publisher.newBuilder(topicId.toProjectTopicName(projectId)) 140 | .apply { configure(topicId) } 141 | .build() 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /pubsub/src/main/kotlin/io/github/nomisrev/gcp/pubsub/GcpPull.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub 2 | 3 | import com.google.cloud.pubsub.v1.stub.SubscriberStub 4 | import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings 5 | import com.google.protobuf.Empty 6 | import com.google.pubsub.v1.AcknowledgeRequest 7 | import com.google.pubsub.v1.ModifyAckDeadlineRequest 8 | import com.google.pubsub.v1.ProjectSubscriptionName 9 | import com.google.pubsub.v1.PubsubMessage 10 | import com.google.pubsub.v1.PubsubMessageOrBuilder 11 | import com.google.pubsub.v1.PullRequest 12 | import com.google.pubsub.v1.ReceivedMessage 13 | import io.github.nomisrev.gcp.core.asDeferred 14 | import io.github.nomisrev.gcp.core.await 15 | import kotlinx.coroutines.Deferred 16 | import kotlinx.coroutines.channels.Channel 17 | import kotlinx.coroutines.flow.Flow 18 | import kotlinx.coroutines.flow.buffer 19 | import kotlinx.coroutines.flow.flow 20 | 21 | public fun GcpPull(projectId: ProjectId): GcpPull = DefaultGcpPull(projectId) 22 | 23 | public interface GcpPull { 24 | /** 25 | * Create an infinite stream [Flow] of [AckPubSubMessage] objects. 26 | * 27 | * The [Flow] respects backpressure by using of Pub/Sub Synchronous Pull to retrieve batches of up 28 | * to the requested number of messages until the full demand is fulfilled or subscription 29 | * terminated. 30 | * 31 | * Any exceptions that are thrown by the Pub/Sub client will be passed as an error to the stream. 32 | * The error handling operators, like [Flow.retry], can be used to recover and continue streaming 33 | * messages. 34 | * 35 | * Uses [Channel.RENDEZVOUS] such that Gcp PubSub is pulled whilst the downstream is processing, 36 | * use the [buffer] operator if you want to increase the amount of [SubscriberStub.pullCallable] 37 | * you want to be in memory. This can positively influence the latency, and throughput but 38 | * increase memory usage. 39 | * 40 | * @param subscription subscription from which to retrieve messages. 41 | * @param maxMessages max number of messages that may be pulled from the source subscription in 42 | * @return infinite [Flow] of [AckPubSubMessage] objects. 43 | */ 44 | public fun pull( 45 | subscription: SubscriptionId, 46 | maxMessages: Int = Int.MAX_VALUE, 47 | configure: SubscriberStubSettings.Builder.() -> Unit = {}, 48 | ): Flow> 49 | } 50 | 51 | private class DefaultGcpPull(val projectId: ProjectId) : GcpPull { 52 | override fun pull( 53 | subscription: SubscriptionId, 54 | maxMessages: Int, 55 | configure: SubscriberStubSettings.Builder.() -> Unit, 56 | ): Flow> { 57 | val pullRequest: PullRequest = 58 | PullRequest.newBuilder() 59 | .setSubscription(ProjectSubscriptionName.of(projectId.value, subscription.value).toString()) 60 | .setMaxMessages(maxMessages) 61 | .build() 62 | 63 | val subscriberStub = SubscriberStubSettings.newBuilder().apply(configure).build().createStub() 64 | 65 | return flow> { 66 | while (true) { 67 | subscriberStub.pullOnce(pullRequest) 68 | } 69 | } 70 | .buffer(Channel.RENDEZVOUS) 71 | } 72 | 73 | private suspend fun SubscriberStub.pullOnce(pullRequest: PullRequest): List = 74 | pullCallable() 75 | .futureCall(pullRequest) 76 | .await() 77 | .receivedMessagesList 78 | .toAckPubSubMessage(this, SubscriptionId(pullRequest.subscription)) 79 | 80 | private fun List.toAckPubSubMessage( 81 | subscriberStub: SubscriberStub, 82 | subscription: SubscriptionId, 83 | ): List = map { message -> 84 | DefaultAckPubSubMessage(projectId, subscription, message.message, message.ackId, subscriberStub) 85 | } 86 | 87 | private fun SubscriberStub.ack( 88 | subscriptionId: String, 89 | ackIds: Collection, 90 | ): Deferred { 91 | val acknowledgeRequest = 92 | AcknowledgeRequest.newBuilder().addAllAckIds(ackIds).setSubscription(subscriptionId).build() 93 | return acknowledgeCallable().futureCall(acknowledgeRequest).asDeferred() 94 | } 95 | 96 | private fun SubscriberStub.modifyAckDeadline( 97 | subscriptionId: String, 98 | ackIds: Collection, 99 | ackDeadlineSeconds: Int, 100 | ): Deferred { 101 | val modifyAckDeadlineRequest = 102 | ModifyAckDeadlineRequest.newBuilder() 103 | .setAckDeadlineSeconds(ackDeadlineSeconds) 104 | .addAllAckIds(ackIds) 105 | .setSubscription(subscriptionId) 106 | .build() 107 | return modifyAckDeadlineCallable().futureCall(modifyAckDeadlineRequest).asDeferred() 108 | } 109 | 110 | private inner class DefaultAckPubSubMessage( 111 | override val projectId: ProjectId, 112 | override val subscriptionId: SubscriptionId, 113 | override val pubSubMessage: PubsubMessage, 114 | override val ackId: String, 115 | private val subscriberStub: SubscriberStub, 116 | ) : AckPubSubMessage, PubsubMessageOrBuilder by pubSubMessage { 117 | override fun ack(): Deferred = subscriberStub.ack(subscriptionId.value, listOf(ackId)) 118 | 119 | override fun modifyAckDeadline(ackDeadlineSeconds: Int): Deferred = 120 | subscriberStub.modifyAckDeadline(subscriptionId.value, listOf(ackId), ackDeadlineSeconds) 121 | 122 | override fun toString(): String = 123 | """ 124 | AckPubSubMessage { 125 | projectId = $projectId, 126 | subscriptionId = $subscriptionId, 127 | message = $pubSubMessage, 128 | ackId = $ackId 129 | } 130 | """ 131 | .trimIndent() 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /pubsub/src/main/kotlin/io/github/nomisrev/gcp/pubsub/GcpSubscriber.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub 2 | 3 | import com.google.api.core.ApiService 4 | import com.google.api.gax.batching.FlowControlSettings 5 | import com.google.cloud.pubsub.v1.AckReplyConsumerWithResponse 6 | import com.google.cloud.pubsub.v1.Subscriber 7 | import com.google.common.util.concurrent.MoreExecutors 8 | import com.google.pubsub.v1.PubsubMessage 9 | import kotlinx.coroutines.channels.Channel 10 | import kotlinx.coroutines.channels.awaitClose 11 | import kotlinx.coroutines.channels.trySendBlocking 12 | import kotlinx.coroutines.flow.Flow 13 | import kotlinx.coroutines.flow.buffer 14 | import kotlinx.coroutines.flow.channelFlow 15 | import kotlinx.coroutines.flow.map 16 | 17 | public interface MessageDecoder { 18 | public suspend fun decode(message: PubsubMessage): A 19 | } 20 | 21 | public fun GcpSubscriber( 22 | projectId: ProjectId? = null, 23 | configure: Subscriber.Builder.(subscriptionId: SubscriptionId) -> Unit = {}, 24 | ): GcpSubscriber = DefaultGcpSubscriber(projectId, configure) 25 | 26 | public interface GcpSubscriber { 27 | 28 | /** 29 | * Basic implementation to subscribe to Google PubSub. 30 | * 31 | * `PubsubRecord` is offloaded into a [Channel] with [Channel.RENDEZVOUS], and you can use 32 | * [Flow.buffer] to increase the size of the [Channel], if you want to keep more message in memory 33 | * before back-pressuring the [Subscriber]. 34 | * 35 | * ```kotlin 36 | * val subscription = SubsriptionId("my-subscription") 37 | * val subscriber: GcpSubscriber = TODO("Create subscriber") 38 | * subscriber.subscribe(subscription) 39 | * .collect { record -> 40 | * println("Processing - ${record.message.data.toStringUtf8()}") 41 | * record.ack() 42 | * } 43 | * ``` 44 | * 45 | * @param subscriptionId the subscription you want to subscribe to 46 | * @param configure the [Subscriber] using [Subscriber.Builder]. To deal with backpressure you can 47 | * provide [FlowControlSettings], which allows limiting how many messages subscribers pull. You 48 | * can limit both the number of messages and the maximum size of messages held by the client at 49 | * one time, to not overburden a single client. See [Subscriber] for more details. 50 | * @return [Flow] with the received [PubsubRecord]. 51 | */ 52 | public fun subscribe( 53 | subscriptionId: SubscriptionId, 54 | configure: Subscriber.Builder.() -> Unit = {}, 55 | ): Flow 56 | 57 | /** 58 | * Utility variant that automatically decodes [PubsubRecord] into [A] using [MessageDecoder]. 59 | * 60 | * ```kotlin 61 | * data class Event(val key: String, val message: String) 62 | * 63 | * val eventDecoder = object : MessageDecoder { 64 | * override suspend fun decode(message: PubsubMessage): A = 65 | * Event(message.key, message.data.toStringUtf8()) 66 | * } 67 | * 68 | * val subscription = SubsriptionId("my-subscription") 69 | * val subscriber: GcpSubscriber = TODO("Create subscriber") 70 | * subscriber.subscribe(subscription, eventDecoder) 71 | * .collect { (event: Event, record: PubsubRecord) -> 72 | * println("event.key: ${event.key}, event.message: ${event.message}") 73 | * record.ack() 74 | * } 75 | * ``` 76 | * 77 | * @see subscribe for full documentation 78 | */ 79 | public fun subscribe( 80 | subscriptionId: SubscriptionId, 81 | decoder: MessageDecoder, 82 | configure: Subscriber.Builder.() -> Unit = {}, 83 | ): Flow> = 84 | subscribe(subscriptionId, configure).map { record -> 85 | AcknowledgeableValue(decoder.decode(record.message), record) 86 | } 87 | } 88 | 89 | /** Default implementation build on top of Gcloud library */ 90 | private class DefaultGcpSubscriber( 91 | val projectId: ProjectId?, 92 | val globalConfigure: Subscriber.Builder.(subscriptionId: SubscriptionId) -> Unit = {}, 93 | ) : GcpSubscriber { 94 | 95 | override fun subscribe( 96 | subscriptionId: SubscriptionId, 97 | configure: Subscriber.Builder.() -> Unit, 98 | ): Flow = 99 | channelFlow { 100 | val projectSubscriptionName = subscriptionId.toProjectSubscriptionName(projectId) 101 | 102 | // Create Subscriber for projectId & subscriptionId 103 | val subscriber = 104 | Subscriber.newBuilder(projectSubscriptionName) { 105 | message: PubsubMessage, 106 | consumer: AckReplyConsumerWithResponse -> 107 | // Block the upstream when Channel cannot keep up with messages 108 | trySendBlocking( 109 | PubsubRecord( 110 | message, 111 | consumer, 112 | ProjectId(projectSubscriptionName.project), 113 | subscriptionId, 114 | ) 115 | ) 116 | .getOrThrow() 117 | } 118 | .apply { 119 | globalConfigure(subscriptionId) 120 | configure() 121 | } 122 | .build() 123 | 124 | subscriber 125 | .apply { 126 | addListener( 127 | object : ApiService.Listener() { 128 | override fun failed(from: ApiService.State?, failure: Throwable?) { 129 | super.failed(from, failure) 130 | close(failure) 131 | } 132 | }, 133 | MoreExecutors.directExecutor(), 134 | ) 135 | } 136 | .startAsync() 137 | 138 | awaitClose { subscriber.stopAsync().awaitTerminated() } 139 | } 140 | .buffer(Channel.RENDEZVOUS) 141 | } 142 | -------------------------------------------------------------------------------- /pubsub-kotlinx-serialization-json/src/main/kotlin/io/github/nomisrev/gcp/pubsub/serialization/KotlinXSerialization.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub.serialization 2 | 3 | import com.google.cloud.pubsub.v1.Subscriber 4 | import com.google.protobuf.ByteString 5 | import com.google.pubsub.v1.PubsubMessage 6 | import io.github.nomisrev.gcp.pubsub.AcknowledgeableValue 7 | import io.github.nomisrev.gcp.pubsub.GcpPublisher 8 | import io.github.nomisrev.gcp.pubsub.GcpSubscriber 9 | import io.github.nomisrev.gcp.pubsub.MessageDecoder 10 | import io.github.nomisrev.gcp.pubsub.MessageEncoder 11 | import io.github.nomisrev.gcp.pubsub.PubsubRecord 12 | import io.github.nomisrev.gcp.pubsub.SubscriptionId 13 | import io.github.nomisrev.gcp.pubsub.TopicId 14 | import kotlinx.coroutines.flow.Flow 15 | import kotlinx.serialization.KSerializer 16 | import kotlinx.serialization.StringFormat 17 | import kotlinx.serialization.json.Json 18 | import kotlinx.serialization.serializer 19 | 20 | /** 21 | * KotlinX Serialization implementation of [MessageEncoder]. It's fixed to [StringFormat], so can be 22 | * re-used for other `string` based serializations. 23 | */ 24 | public class KotlinXJsonEncoder( 25 | private val stringFormat: StringFormat, 26 | private val serializer: KSerializer, 27 | private val configure: PubsubMessage.Builder.() -> Unit, 28 | ) : MessageEncoder { 29 | override suspend fun encode(value: A): PubsubMessage = 30 | PubsubMessage.newBuilder() 31 | .setData(ByteString.copyFromUtf8(stringFormat.encodeToString(serializer, value))) 32 | .apply(configure) 33 | .build() 34 | } 35 | 36 | /** 37 | * KotlinX Serialization implementation of [MessageDecoder]. It's fixed to [StringFormat], so can be 38 | * re-used for other `string` based serializations. 39 | */ 40 | public class KotlinXJsonDecoder( 41 | private val stringFormat: StringFormat, 42 | private val serializer: KSerializer, 43 | ) : MessageDecoder { 44 | override suspend fun decode(message: PubsubMessage): A = 45 | stringFormat.decodeFromString(serializer, message.data.toStringUtf8()) 46 | } 47 | 48 | /** 49 | * Allows publishing values of [A], using KotlinX Serialization for Json format. 50 | * 51 | * @param topicId which to publish the value 52 | * @param message that is going to be published 53 | * @param serializer that is going to be used to serialize [A] to [Json] 54 | * @param json the KotlinX Serialization [Json] instance to be used 55 | * @param configure lambda that allows additional configuration to [PubsubMessage.Builder], such as 56 | * [PubsubMessage.Builder.setOrderingKey]. 57 | */ 58 | public suspend inline fun GcpPublisher.publish( 59 | topicId: TopicId, 60 | message: A, 61 | json: Json = Json, 62 | serializer: KSerializer = serializer(), 63 | noinline configure: PubsubMessage.Builder.() -> Unit = {}, 64 | ): String = publish(topicId, message, KotlinXJsonEncoder(json, serializer, configure)) 65 | 66 | /** 67 | * Allows publishing values of [Iterable] of [A], using KotlinX Serialization for Json format. 68 | * 69 | * @param topicId which to publish the value 70 | * @param messages that are going to be published 71 | * @param serializer that is going to be used to serialize [A] to [Json] 72 | * @param json the KotlinX Serialization [Json] instance to be used 73 | * @param configure lambda that allows additional configuration to [PubsubMessage.Builder], such as 74 | * [PubsubMessage.Builder.setOrderingKey]. 75 | */ 76 | public suspend inline fun GcpPublisher.publish( 77 | topicId: TopicId, 78 | messages: Iterable, 79 | json: Json = Json, 80 | serializer: KSerializer = serializer(), 81 | noinline configure: PubsubMessage.Builder.() -> Unit = {}, 82 | ): List = publish(topicId, messages, KotlinXJsonEncoder(json, serializer, configure)) 83 | 84 | /** 85 | * Allows deserializing [PubsubMessage.getData] from [PubsubRecord] using KotlinX serialization. 86 | * 87 | * ```kotlin 88 | * @Serializable 89 | * data class Event(val key: String, val message: String) 90 | * 91 | * val subscription = SubsriptionId("my-subscription") 92 | * val subscriber: GcpSubscriber = TODO("Create subscriber") 93 | * subscriber.subscribe(subscription) 94 | * .collect { record: PubsubRecord -> 95 | * val event: Event = event.deserialized() 96 | * println("event.key: ${event.key}, event.message: ${event.message}") 97 | * record.ack() 98 | * } 99 | * ``` 100 | * 101 | * @param json the KotlinX Serialization [Json] instance to be used 102 | * @param serializer that is going to be used to serialize [A] to [Json] 103 | */ 104 | public inline fun PubsubRecord.deserialized( 105 | json: Json = Json, 106 | serializer: KSerializer = serializer(), 107 | ): A = json.decodeFromString(serializer, data.toStringUtf8()) 108 | 109 | /** 110 | * This signature looks the same as the regular [GcpSubscriber.subscribe], but instead takes a 111 | * generic argument. 112 | * 113 | * To use this method you need to explicitly pass the generic argument, otherwise use 114 | * [subscribeDeserialized]. Alternatively you can also use [deserialized]. 115 | * 116 | * ```kotlin 117 | * @Serializable 118 | * data class Event(val key: String, val message: String) 119 | * 120 | * val subscription = SubsriptionId("my-subscription") 121 | * val subscriber: GcpSubscriber = TODO("Create subscriber") 122 | * subscriber.subscribe(subscription) 123 | * .collect { (event: Event, record: PubsubRecord) -> 124 | * println("event.key: ${event.key}, event.message: ${event.message}") 125 | * record.ack() 126 | * } 127 | * ``` 128 | * 129 | * @see GcpSubscriber.subscribe for full documentation. 130 | */ 131 | public inline fun GcpSubscriber.subscribe( 132 | subscriptionId: SubscriptionId, 133 | json: Json = Json, 134 | serializer: KSerializer = serializer(), 135 | noinline configure: Subscriber.Builder.() -> Unit = {}, 136 | ): Flow> = 137 | subscribe(subscriptionId, KotlinXJsonDecoder(json, serializer), configure) 138 | 139 | /** Alias for subscribe(..) */ 140 | public inline fun GcpSubscriber.subscribeDeserialized( 141 | subscriptionId: SubscriptionId, 142 | json: Json = Json, 143 | serializer: KSerializer = serializer(), 144 | noinline configure: Subscriber.Builder.() -> Unit = {}, 145 | ): Flow> = subscribe(subscriptionId, json, serializer, configure) 146 | -------------------------------------------------------------------------------- /pubsub-test/src/main/kotlin/io/github/nomisrev/gcp/pubsub/test/PubSubEmulator.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub.test 2 | 3 | import com.google.api.gax.core.CredentialsProvider 4 | import com.google.api.gax.core.NoCredentialsProvider 5 | import com.google.api.gax.grpc.GrpcTransportChannel 6 | import com.google.api.gax.rpc.FixedTransportChannelProvider 7 | import com.google.api.gax.rpc.TransportChannelProvider 8 | import com.google.cloud.pubsub.v1.Publisher 9 | import com.google.cloud.pubsub.v1.Subscriber 10 | import com.google.cloud.pubsub.v1.SubscriptionAdminClient 11 | import com.google.cloud.pubsub.v1.SubscriptionAdminSettings 12 | import com.google.cloud.pubsub.v1.TopicAdminClient 13 | import com.google.cloud.pubsub.v1.TopicAdminSettings 14 | import io.github.nomisrev.gcp.pubsub.GcpPublisher 15 | import io.github.nomisrev.gcp.pubsub.GcpPubsSubAdmin 16 | import io.github.nomisrev.gcp.pubsub.GcpSubscriber 17 | import io.github.nomisrev.gcp.pubsub.ProjectId 18 | import io.github.nomisrev.gcp.pubsub.SubscriptionId 19 | import io.github.nomisrev.gcp.pubsub.TopicId 20 | import io.github.nomisrev.gcp.pubsub.ktor.GcpPubSub 21 | import io.grpc.ManagedChannel 22 | import io.grpc.ManagedChannelBuilder 23 | import io.ktor.server.application.Application 24 | import io.ktor.server.application.BaseApplicationPlugin 25 | import io.ktor.server.application.install 26 | import io.ktor.server.application.pluginOrNull 27 | import io.ktor.util.AttributeKey 28 | import java.time.Duration 29 | import java.util.UUID 30 | import java.util.concurrent.TimeUnit 31 | import org.junit.rules.ExternalResource 32 | import org.testcontainers.containers.PubSubEmulatorContainer 33 | import org.testcontainers.lifecycle.Startable 34 | import org.testcontainers.utility.DockerImageName 35 | 36 | public class PubSubEmulator( 37 | imageName: DockerImageName = 38 | DockerImageName.parse("gcr.io/google.com/cloudsdktool/cloud-sdk:316.0.0-emulators"), 39 | private val credentials: CredentialsProvider = NoCredentialsProvider.create(), 40 | ) : ExternalResource(), Startable, AutoCloseable, BaseApplicationPlugin { 41 | 42 | override val key: AttributeKey = AttributeKey("PubSubEmulatorExtension") 43 | 44 | @Suppress("UNUSED", "INVISIBLE_MEMBER", "INVISIBLE_REFERENCE") 45 | override fun install(pipeline: Application, configure: Unit.() -> Unit) { 46 | val plugin = (pipeline.pluginOrNull(GcpPubSub) ?: pipeline.install(GcpPubSub)) 47 | val configurePublisher = plugin.configurePublisher 48 | plugin.configurePublisher = { projectId, topicId -> 49 | configurePublisher(projectId, topicId) 50 | setChannelProvider(channel) 51 | setCredentialsProvider(credentials) 52 | } 53 | val configureSubscriber = plugin.configureSubscriber 54 | plugin.configureSubscriber = { 55 | configureSubscriber(it) 56 | setChannelProvider(channel) 57 | setCredentialsProvider(credentials) 58 | } 59 | val configureTopicAdmin = plugin.configureTopicAdmin 60 | plugin.configureTopicAdmin = { 61 | configureTopicAdmin?.invoke(this, it) 62 | setTransportChannelProvider(channel) 63 | setCredentialsProvider(credentials) 64 | } 65 | val configureSubscriptionAdmin = plugin.configureSubscriptionAdmin 66 | plugin.configureSubscriptionAdmin = { 67 | configureSubscriptionAdmin?.invoke(this, it) 68 | setTransportChannelProvider(channel) 69 | setCredentialsProvider(credentials) 70 | } 71 | } 72 | 73 | private val container = 74 | PubSubEmulatorContainer(imageName) 75 | .withStartupTimeout(Duration.ofMinutes(2)) 76 | .withStartupAttempts(2) 77 | 78 | private val managedChannel: ManagedChannel by lazy { 79 | ManagedChannelBuilder.forTarget(container.emulatorEndpoint).usePlaintext().build() 80 | } 81 | 82 | private val channel: FixedTransportChannelProvider by lazy { 83 | FixedTransportChannelProvider.create(GrpcTransportChannel.create(managedChannel)) 84 | } 85 | 86 | /** Create [TopicAdminSettings] that is by default linked to test channel. */ 87 | public fun topicAdminSettings( 88 | transportChannelProvider: TransportChannelProvider = channel, 89 | credentialsProvider: CredentialsProvider = credentials, 90 | ): TopicAdminSettings = 91 | TopicAdminSettings.newBuilder() 92 | .setTransportChannelProvider(transportChannelProvider) 93 | .setCredentialsProvider(credentialsProvider) 94 | .build() 95 | 96 | /** Create [SubscriptionAdminSettings] that is by default linked to test channel. */ 97 | public fun subscriptionAdminSettings( 98 | transportChannelProvider: TransportChannelProvider = channel, 99 | credentialsProvider: CredentialsProvider = credentials, 100 | ): SubscriptionAdminSettings = 101 | SubscriptionAdminSettings.newBuilder() 102 | .setTransportChannelProvider(transportChannelProvider) 103 | .setCredentialsProvider(credentialsProvider) 104 | .build() 105 | 106 | /** Generate a unique topic name */ 107 | public fun uniqueTopic(): TopicId = TopicId("topic-${UUID.randomUUID()}") 108 | 109 | /** Generate a unique subscription name */ 110 | public fun uniqueSubscription(): SubscriptionId = 111 | SubscriptionId("subscription-${UUID.randomUUID()}") 112 | 113 | public fun subscriber( 114 | projectId: ProjectId, 115 | configure: Subscriber.Builder.(subscriptionId: SubscriptionId) -> Unit = {}, 116 | ): GcpSubscriber = 117 | GcpSubscriber(projectId) { 118 | configure(it) 119 | setChannelProvider(channel) 120 | setCredentialsProvider(credentials) 121 | } 122 | 123 | public fun publisher( 124 | projectId: ProjectId, 125 | configure: Publisher.Builder.(topicId: TopicId) -> Unit = {}, 126 | ): GcpPublisher = 127 | GcpPublisher(projectId) { 128 | configure(it) 129 | setChannelProvider(channel) 130 | setCredentialsProvider(credentials) 131 | } 132 | 133 | public fun admin(projectId: ProjectId): GcpPubsSubAdmin = 134 | GcpPubsSubAdmin( 135 | projectId, 136 | TopicAdminClient.create( 137 | TopicAdminSettings.newBuilder() 138 | .setTransportChannelProvider(channel) 139 | .setCredentialsProvider(credentials) 140 | .build() 141 | ), 142 | SubscriptionAdminClient.create( 143 | SubscriptionAdminSettings.newBuilder() 144 | .setTransportChannelProvider(channel) 145 | .setCredentialsProvider(credentials) 146 | .build() 147 | ), 148 | ) 149 | 150 | override fun before() { 151 | super.before() 152 | container.start() 153 | } 154 | 155 | override fun after() { 156 | super.after() 157 | container.stop() 158 | } 159 | 160 | override fun start() { 161 | container.start() 162 | } 163 | 164 | override fun stop() { 165 | container.stop() 166 | } 167 | 168 | override fun close() { 169 | managedChannel.shutdown().awaitTermination(5, TimeUnit.SECONDS) 170 | container.stop() 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /pubsub/README.MD: -------------------------------------------------------------------------------- 1 | # Module gcp-pubsub 2 | 3 | KotlinX integration for `TopicAdminClient`, `SubscriptionAdminClient`, `Susbcriber` and `Publisher`. 4 | 5 | This modules exposes following types that correspond to their relevant Gcp PubSub types but expose `suspend` APIs, that 6 | await the operations in a suspending way. 7 | 8 | | **Name** | **Description** 9 | |-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 10 | | [GcpPublisher](#GcpPublisher) | Exposes `publish` methods, and automatically caches the `Publisher` under-the-hood for optimal performance. All `publish` methods suspend until all methods are published to the topic. 11 | | [GcpPubSubAdmin](#GcpPubSubAdmin) | Combines the API of `TopicAdminClient` and `SubscriptionAdminClient`, and awaits the succesfully result of their operations in a suspending way. 12 | | [GcpSubscriber](#GcpSubscriber) | Exposes a single `subscribe` method that subscribes to PubSub and exposes the stream of messages as a KotlinX Flow. 13 | | [GcpPull] | Exposes a single `pull` method that pulls messages from PubSub in a back-pressured way. 14 | 15 | ## Examples 16 | 17 | All types can be created using the `ProjectId`, and a configuration lambda using the original `Builder` types from 18 | Google's Java lib so the original documentation and techniques apply. It keeps the original default settings, see all 19 | the examples below. 20 | 21 | ### GcpPublisher 22 | 23 | Configuring the `GcpPublisher` can conveniently be done using the lambda using the Google `Publisher.Builder` API. 24 | The `GcpPublisher` type implements `AutoCloseable` so most conveniently you can consume it with `use` from the Kotlin 25 | Std. Alternatively you could also use `try/finally`, or similar strategies. 26 | 27 | ```kotlin 28 | GcpPublisher(ProjectId("my-project")).use { publisher -> 29 | 30 | } 31 | ``` 32 | 33 | Publishing messages from different types `String`, `ByteArray`, `ByteBuffer`, `ByteString` is support out-of-the-box, 34 | this can be done by providing a single value or an `Iterable` of any of these types. 35 | 36 | ```kotlin 37 | publisher.publish(TopicId("my-topic"), "msg1") 38 | publisher.publish(TopicId("my-topic"), listOf("msg2", "msg3")) 39 | publisher.publish(TopicId("my-topic"), "msg1".toByteArray()) 40 | ``` 41 | 42 | You can also publish custom types, but need to provide a `MessageEncoder`. 43 | See [pubsub-kotlinx-serialization-json](../pubsub-kotlinx-serialization-json/README.MD) on how to publish messages 44 | using `Json` with KotlinX Serialization. 45 | 46 | If you want to implement your own serializers, you can use the `MessageEncoder` interface. 47 | 48 | ```kotlin 49 | data class MyEvent(val key: String, val message: String) 50 | 51 | object MyEventEncoder : MessageEncoder { 52 | override suspend fun encode(value: MyEvent): PubsubMessage = 53 | PubsubMessage.newBuilder() 54 | .setData(ByteString.copyFromUtf8(value.message)) 55 | .setOrderingKey(value.key) 56 | .build() 57 | } 58 | 59 | publisher.publish(TopicId("my-topic"), MyEvent("key", "msg1"), MessageEncoder) 60 | publisher.publish(TopicId("my-topic"), listOf(MyEvent("key", "msg2"), MyEvent("key", "msg3")), MessageEncoder) 61 | ``` 62 | 63 | ### GcpPubSubAdmin 64 | 65 | Configuring the `GcpPubSubAdmin` can conveniently be done by any of the overloads, it wraps both `TopicAdminClient` 66 | and `SubscriptionAdminClient`. You can manually construct both, and pass them into the constructor, 67 | remember that you need to close both `TopicAdmintClient` and `SubscriptionAdminClient` so you can either use `use` or 68 | manually close `TopicAdminClient` and `SubscriptionAdminClient`. 69 | 70 | ```kotlin 71 | val topicAdminClient: TopicAdmintClient = TODO("Create TopicAdmintClient") 72 | val subscriptionAdminClient: SubscriptionAdminClient = TODO("Create SubscriptionAdminClient") 73 | GcpPubsSubAdmin( 74 | ProjectId("my-project"), 75 | topicAdminClient, 76 | subscriptionAdminClient 77 | ).use { admin -> 78 | 79 | } 80 | ``` 81 | 82 | Or, there is also the option to rely on the lambdas to configure the `TopicAdminSettings.Builder` 83 | and `SubscriptionAdminSettings.Builder`. 84 | 85 | ```kotlin 86 | val credentials = GoogleCredentials.fromStream(FileInputStream(PATH_TO_JSON_KEY)) 87 | 88 | GcpPubsSubAdmin( 89 | ProjectId("my-project"), 90 | configureSubscriptionAdmin = { setCredentialsProvider(credentials) }, 91 | configureTopicAdmin = { setCredentialsProvider(credentials) } 92 | ).use { admin -> 93 | 94 | } 95 | ``` 96 | 97 | Once the admin is created we can use all of its convenient `suspend` functions. 98 | 99 | ```kotlin 100 | GcpPubsSubAdmin(ProjectId("my-project")).use { admin -> 101 | admin.createTopic(TopicId("my-topic")) 102 | admin.createSubscription(Subsriptionid("my-subscription"), TopicId("my-topic")) 103 | // delete, get or list topics and subscriptions 104 | } 105 | ``` 106 | 107 | ### GcpSubscriber 108 | 109 | The `GcpSubscriber` can easily be created by just providing a `ProjectId`, since a new [Subscriber] will be created for 110 | every [subscribe] call that is made. Optionally, a common `configure` lambda can be provided for configuration that is 111 | common for all [Subscriber] instances. `GcpSubscriber` doesn't implement `AutoCloseable` as it internally takes care of 112 | the lifecycles of the created [Subscriber]. 113 | 114 | ```kotlin 115 | val subscriber = GcpSubscriber(ProjectId("my-project")) 116 | 117 | val subscriber2 = GcpSubscriber(ProjectId("my-project")) { 118 | setCredentialsProvider(GoogleCredentials.fromStream(FileInputStream(PATH_TO_JSON_KEY))) 119 | setParallelPullCount(3) 120 | setMaxDurationPerAckExtension(Duration.ofMinutes(60)) 121 | } 122 | ``` 123 | 124 | Then we can simply call the `subscribe` method, of which two simple variants exist to subscribe to a 125 | given [SubscriptionId]. 126 | 127 | ```kotlin 128 | val subscription = SubsriptionId("my-subscription") 129 | val subscriber: GcpSubscriber = TODO("Create subscriber") 130 | subscriber.subscribe(subscription) 131 | .collect { record -> 132 | println("Processing - ${record.message.data.toStringUtf8()}") 133 | record.ack() 134 | } 135 | ``` 136 | 137 | There is also a variant that takes a `MessageDecoder` in case you want to process decoded messages of ` A`. 138 | 139 | ```kotlin 140 | data class Event(val key: String, val message: String) 141 | 142 | val eventDecoder = object : MessageDecoder { 143 | override suspend fun decode(message: PubsubMessage): A = 144 | Event(message.key, message.data.toStringUtf8()) 145 | } 146 | 147 | val subscription = SubsriptionId("my-subscription") 148 | val subscriber: GcpSubscriber = TODO("Create subscriber") 149 | subscriber.subscribe(subscription, eventDecoder) 150 | .collect { (event: Event, record: PubsubRecord) -> 151 | println("event.key: ${event.key}, event.message: ${event.message}") 152 | record.ack() 153 | } 154 | ``` 155 | 156 | ## Using in your projects 157 | 158 | ### Gradle 159 | 160 | Add dependencies (you can also add other modules that you need): 161 | 162 | ```kotlin 163 | dependencies { 164 | implementation("io.github.nomisrev:gcp-pubsub:1.0.0") 165 | } 166 | ``` 167 | 168 | ### Maven 169 | 170 | Add dependencies (you can also add other modules that you need): 171 | 172 | ```xml 173 | 174 | 175 | io.github.nomisrev 176 | gcp-pubsub 177 | 1.0.0 178 | 179 | ``` 180 | -------------------------------------------------------------------------------- /pubsub/src/test/kotlin/io/github/nomisrev/gcp/pubsub/PubSubTest.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub 2 | 3 | import com.google.api.gax.rpc.AlreadyExistsException 4 | import com.google.api.gax.rpc.InvalidArgumentException 5 | import com.google.api.gax.rpc.NotFoundException 6 | import io.github.nomisrev.gcp.pubsub.test.PubSubEmulator 7 | import java.lang.AssertionError 8 | import kotlin.test.Test 9 | import kotlin.test.assertEquals 10 | import kotlin.time.Duration.Companion.seconds 11 | import kotlinx.coroutines.flow.map 12 | import kotlinx.coroutines.flow.single 13 | import kotlinx.coroutines.flow.take 14 | import kotlinx.coroutines.flow.toList 15 | import kotlinx.coroutines.flow.toSet 16 | import kotlinx.coroutines.runBlocking 17 | import kotlinx.coroutines.withTimeoutOrNull 18 | import org.junit.AfterClass 19 | import org.junit.ClassRule 20 | 21 | class PubSubTest { 22 | 23 | @Test 24 | fun `Create empty topic`() = runBlocking { 25 | val actual = assertThrows { admin.createTopic(TopicId("")) } 26 | assertEquals( 27 | actual.message, 28 | "io.grpc.StatusRuntimeException: INVALID_ARGUMENT: Invalid [topics] name: (name=projects/my-project-id/topics/)", 29 | ) 30 | } 31 | 32 | @Test 33 | fun `Create topic twice`() = runBlocking { 34 | val topicId = extension.uniqueTopic() 35 | admin.createTopic(topicId) 36 | val actual = assertThrows { admin.createTopic(topicId) } 37 | assertEquals( 38 | actual.message, 39 | "io.grpc.StatusRuntimeException: ALREADY_EXISTS: Topic already exists", 40 | ) 41 | } 42 | 43 | @Test 44 | fun `Delete empty topic`() = runBlocking { 45 | val actual = assertThrows { admin.deleteTopic(TopicId("")) } 46 | assertEquals( 47 | actual.message, 48 | "io.grpc.StatusRuntimeException: INVALID_ARGUMENT: Invalid [topics] name: (name=projects/my-project-id/topics/)", 49 | ) 50 | } 51 | 52 | @Test 53 | fun `Delete non-existing topic `() = runBlocking { 54 | val topicId = extension.uniqueTopic() 55 | val actual = assertThrows { admin.deleteTopic(topicId) } 56 | assertEquals(actual.message, "io.grpc.StatusRuntimeException: NOT_FOUND: Topic not found") 57 | } 58 | 59 | @Test 60 | fun `Create empty subscription`() = runBlocking { 61 | val topicId = extension.uniqueTopic() 62 | admin.createTopic(topicId) 63 | val actual = 64 | assertThrows { 65 | admin.createSubscription(SubscriptionId(""), topicId) 66 | } 67 | assertEquals( 68 | actual.message, 69 | "io.grpc.StatusRuntimeException: INVALID_ARGUMENT: Invalid [subscriptions] name: (name=projects/my-project-id/subscriptions/)", 70 | ) 71 | } 72 | 73 | @Test 74 | fun `Create subscription for non-existing topic`() = runBlocking { 75 | val topicId = extension.uniqueTopic() 76 | val subscriptionId = extension.uniqueSubscription() 77 | val actual = 78 | assertThrows { admin.createSubscription(subscriptionId, topicId) } 79 | assertEquals( 80 | actual.message, 81 | "io.grpc.StatusRuntimeException: NOT_FOUND: Subscription topic does not exist", 82 | ) 83 | } 84 | 85 | @Test 86 | fun `Create subscription twice`() = runBlocking { 87 | val topicId = extension.uniqueTopic() 88 | val subscriptionId = extension.uniqueSubscription() 89 | admin.createTopic(topicId) 90 | admin.createSubscription(subscriptionId, topicId) 91 | val actual = 92 | assertThrows { admin.createSubscription(subscriptionId, topicId) } 93 | assertEquals( 94 | actual.message, 95 | "io.grpc.StatusRuntimeException: ALREADY_EXISTS: Subscription already exists", 96 | ) 97 | } 98 | 99 | @Test 100 | fun `Delete empty subscription`() = runBlocking { 101 | val actual = 102 | assertThrows { admin.deleteSubscription(SubscriptionId("")) } 103 | assertEquals( 104 | actual.message, 105 | "io.grpc.StatusRuntimeException: INVALID_ARGUMENT: Invalid [subscriptions] name: (name=projects/my-project-id/subscriptions/)", 106 | ) 107 | } 108 | 109 | @Test 110 | fun `Delete non-existing subscription`() = runBlocking { 111 | val subscriptionId = extension.uniqueSubscription() 112 | val actual = assertThrows { admin.deleteSubscription(subscriptionId) } 113 | assertEquals( 114 | actual.message, 115 | "io.grpc.StatusRuntimeException: NOT_FOUND: Subscription does not exist", 116 | ) 117 | } 118 | 119 | @Test 120 | fun `publish to non-existing topic`() = runBlocking { 121 | val actual = assertThrows { publisher.publish(TopicId("non-existing"), "") } 122 | assertEquals(actual.message, "io.grpc.StatusRuntimeException: NOT_FOUND: Topic not found") 123 | } 124 | 125 | @Suppress("MaxLineLength") 126 | @Test 127 | fun `subscribe to non-existing subscription`() = runBlocking { 128 | val actual = 129 | assertThrows { 130 | subscriber.subscribe(SubscriptionId("non-existing")).single() 131 | } 132 | assertEquals( 133 | actual.message, 134 | "com.google.api.gax.rpc.NotFoundException: io.grpc.StatusRuntimeException: NOT_FOUND: Subscription does not exist (resource=non-existing)", 135 | ) 136 | } 137 | 138 | @Test 139 | fun `publish and subscribe multiple messages`() = runBlocking { 140 | val expected = listOf("first-message", "second-message", "third-message") 141 | val topicId = extension.uniqueTopic() 142 | val subscriptionId = extension.uniqueSubscription() 143 | admin.createTopic(topicId) 144 | admin.createSubscription(subscriptionId, topicId) 145 | 146 | publisher.publish(topicId, expected) 147 | 148 | val actual = 149 | subscriber 150 | .subscribe(subscriptionId) 151 | .map { msg -> 152 | val str = msg.data.toStringUtf8() 153 | msg.ack() 154 | str 155 | } 156 | .take(3) 157 | .toSet() 158 | 159 | assertEquals(expected.toSet(), actual) 160 | 161 | val shouldTimeout = 162 | withTimeoutOrNull(1.seconds) { subscriber.subscribe(subscriptionId).take(1).toList() } 163 | assert(shouldTimeout == null) { "Messages were not acknowledged and received twice" } 164 | } 165 | 166 | @Test 167 | fun `publish and subscribe ordered multiple messages`() = runBlocking { 168 | val expected = listOf("first-message", "second-message", "third-message") 169 | val topicId = extension.uniqueTopic() 170 | val subscriptionId = extension.uniqueSubscription() 171 | admin.createTopic(topicId) 172 | admin.createSubscription(subscriptionId, topicId) 173 | 174 | extension 175 | .publisher(projectId) { setEnableMessageOrdering(true) } 176 | .use { publisher -> publisher.publish(topicId, expected) { setOrderingKey("key") } } 177 | 178 | val actual = 179 | subscriber 180 | .subscribe(subscriptionId) 181 | .map { msg -> 182 | val str = msg.data.toStringUtf8() 183 | msg.ack() 184 | str 185 | } 186 | .take(3) 187 | .toList() 188 | 189 | assertEquals(expected, actual) 190 | 191 | val shouldTimeout = 192 | withTimeoutOrNull(1.seconds) { subscriber.subscribe(subscriptionId).take(1).toList() } == null 193 | assert(shouldTimeout) { "Messages were not acknowledged and received twice" } 194 | } 195 | 196 | companion object { 197 | val projectId = ProjectId("my-project-id") 198 | 199 | @JvmStatic @get:ClassRule val extension = PubSubEmulator() 200 | 201 | val subscriber by lazy { extension.subscriber(projectId) } 202 | val publisher by lazy { extension.publisher(projectId) } 203 | val admin by lazy { extension.admin(projectId) } 204 | 205 | @JvmStatic 206 | @AfterClass 207 | fun destroy() { 208 | admin.close() 209 | publisher.close() 210 | } 211 | } 212 | } 213 | 214 | inline fun assertThrows(executable: () -> Unit): T = 215 | when (val exception = runCatching { executable() }.exceptionOrNull()) { 216 | null -> 217 | throw AssertionError( 218 | "Expected ${T::class.simpleName} to be thrown, but no exception was thrown." 219 | ) 220 | is T -> exception 221 | else -> 222 | throw AssertionError( 223 | "Expected ${T::class.simpleName} to be thrown, but found ${exception::class.simpleName}", 224 | exception, 225 | ) 226 | } 227 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Stop when "xargs" is not available. 209 | if ! command -v xargs >/dev/null 2>&1 210 | then 211 | die "xargs is not available" 212 | fi 213 | 214 | # Use "xargs" to parse quoted args. 215 | # 216 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 217 | # 218 | # In Bash we could simply go: 219 | # 220 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 221 | # set -- "${ARGS[@]}" "$@" 222 | # 223 | # but POSIX shell has neither arrays nor command substitution, so instead we 224 | # post-process each arg (as a line of input to sed) to backslash-escape any 225 | # character that might be a shell metacharacter, then use eval to reverse 226 | # that process (while maintaining the separation between arguments), and wrap 227 | # the whole thing up as a single "set" statement. 228 | # 229 | # This will of course break if any of these variables contains a newline or 230 | # an unmatched quote. 231 | # 232 | 233 | eval "set -- $( 234 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 235 | xargs -n1 | 236 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 237 | tr '\n' ' ' 238 | )" '"$@"' 239 | 240 | exec "$JAVACMD" "$@" 241 | -------------------------------------------------------------------------------- /pubsub/src/main/kotlin/io/github/nomisrev/gcp/pubsub/GcpPubsSubAdmin.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub 2 | 3 | import com.google.api.gax.rpc.AlreadyExistsException 4 | import com.google.api.gax.rpc.ApiException 5 | import com.google.api.gax.rpc.InvalidArgumentException 6 | import com.google.api.gax.rpc.NotFoundException 7 | import com.google.api.gax.rpc.StatusCode 8 | import com.google.cloud.pubsub.v1.SubscriptionAdminClient 9 | import com.google.cloud.pubsub.v1.SubscriptionAdminSettings 10 | import com.google.cloud.pubsub.v1.TopicAdminClient 11 | import com.google.cloud.pubsub.v1.TopicAdminSettings 12 | import com.google.pubsub.v1.DeleteSubscriptionRequest 13 | import com.google.pubsub.v1.DeleteTopicRequest 14 | import com.google.pubsub.v1.GetSubscriptionRequest 15 | import com.google.pubsub.v1.GetTopicRequest 16 | import com.google.pubsub.v1.ListSubscriptionsRequest 17 | import com.google.pubsub.v1.ListTopicsRequest 18 | import com.google.pubsub.v1.Subscription 19 | import com.google.pubsub.v1.Topic 20 | import io.github.nomisrev.gcp.core.await 21 | 22 | public fun GcpPubsSubAdmin( 23 | projectId: ProjectId, 24 | configureSubscriptionAdmin: SubscriptionAdminSettings.Builder.() -> Unit, 25 | configureTopicAdmin: TopicAdminSettings.Builder.() -> Unit, 26 | ): GcpPubsSubAdmin { 27 | val topicAdminClient = 28 | TopicAdminClient.create(TopicAdminSettings.newBuilder().apply(configureTopicAdmin).build()) 29 | 30 | val subscriptionAdminClient = 31 | try { 32 | SubscriptionAdminClient.create( 33 | SubscriptionAdminSettings.newBuilder().apply(configureSubscriptionAdmin).build() 34 | ) 35 | } catch (ex: Exception) { 36 | topicAdminClient.close() 37 | throw ex 38 | } 39 | return DefaultPubSubAdmin(projectId, topicAdminClient, subscriptionAdminClient) 40 | } 41 | 42 | public fun GcpPubsSubAdmin( 43 | projectId: ProjectId, 44 | topicAdminClient: TopicAdminClient, 45 | subscriptionAdminClient: SubscriptionAdminClient, 46 | ): GcpPubsSubAdmin { 47 | require(projectId.value.isNotBlank()) { "The project ID can't be null or empty." } 48 | return DefaultPubSubAdmin(projectId, topicAdminClient, subscriptionAdminClient) 49 | } 50 | 51 | public interface GcpPubsSubAdmin : AutoCloseable { 52 | /** 53 | * Creates the given topic with the given name. See the 54 | * [resource name rules](https://cloud.google.com/pubsub/docs/admin#resource_names). 55 | * 56 | * @param topicId The name of the topic. `{topic}` must start with a letter, and contain only 57 | * letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`), underscores (`_`), periods (`.`), 58 | * tildes (`~`), plus (`+`) or percent signs (`%`). It must be between 3 and 255 characters in 59 | * length, and it must not start with `"goog"`. 60 | * @throws [InvalidArgumentException] when topic doesn't follow correct naming requirements 61 | * @throws [AlreadyExistsException] when topic already exists 62 | * @throws [ApiException] if the remote call fails 63 | * @see TopicAdminClient.createTopic 64 | */ 65 | public suspend fun createTopic(topicId: TopicId): Topic 66 | 67 | /** 68 | * Deletes the topic with the given name. After a topic is deleted, a new topic may be created 69 | * with the same name; this is an entirely new topic with none of the old configuration or 70 | * subscriptions. Existing subscriptions to this topic are not deleted, but their `topic` field is 71 | * set to `_deleted-topic_`. 72 | * 73 | * @param topicId the name of the topic to be deleted 74 | * @throws [InvalidArgumentException] when empty topic name is specified 75 | * @throws [NotFoundException] when topic not found 76 | */ 77 | public suspend fun deleteTopic(topicId: TopicId) 78 | 79 | public suspend fun getTopic(topicId: TopicId): Topic? 80 | 81 | public suspend fun listTopics(): List 82 | 83 | /** 84 | * Creates a subscription to a given topic. See the 85 | * [resource name rules](https://cloud.google.com/pubsub/docs/admin#resource_names) 86 | * 87 | * If the name is not provided in the request, the server will assign a random name for this 88 | * subscription on the same project as the topic, conforming to the [resource name format] 89 | * (https://cloud.google.com/pubsub/docs/admin#resource_names). The generated name is populated in 90 | * the returned Subscription object. Note that for REST API requests, you must specify a name in 91 | * the request. 92 | * 93 | * @param configure the [Subscription] with configuration such as `ackDeadline`, `pushConfig`, 94 | * `messageOrdering`, dead lettering, etc. 95 | * @throws [InvalidArgumentException] when invalid subscription name is specified 96 | * @throws [AlreadyExistsException] when subscription already exists 97 | * @throws [NotFoundException] if the [topicId] doesn't exist 98 | */ 99 | public suspend fun createSubscription( 100 | subscriptionId: SubscriptionId, 101 | topicId: TopicId, 102 | configure: Subscription.Builder.() -> Unit = {}, 103 | ): Subscription 104 | 105 | /** Throws [NotFoundException] when subscription not found */ 106 | public suspend fun deleteSubscription(subscriptionId: SubscriptionId) 107 | 108 | /** 109 | * Get the configuration of a Google Cloud Pub/Sub subscription. 110 | * 111 | * @param subscriptionId short subscription name, e.g., "subscriptionId", or the fully-qualified 112 | * subscription name in the `projects/{project_name}/subscriptions/{subscription_name}` format 113 | * @return subscription configuration or `null` if subscription doesn't exist 114 | */ 115 | public suspend fun getSubscription(subscriptionId: SubscriptionId): Subscription? 116 | 117 | public suspend fun listSubscriptions(): List 118 | } 119 | 120 | private class DefaultPubSubAdmin( 121 | val projectId: ProjectId, 122 | val topicAdminClient: TopicAdminClient, 123 | val subscriptionAdminClient: SubscriptionAdminClient, 124 | ) : GcpPubsSubAdmin { 125 | 126 | override suspend fun createTopic(topicId: TopicId): Topic { 127 | return topicAdminClient 128 | .createTopicCallable() 129 | .futureCall(Topic.newBuilder().setName(topicId.toTopicName(projectId).toString()).build()) 130 | .await() 131 | } 132 | 133 | override suspend fun deleteTopic(topicId: TopicId) { 134 | topicAdminClient 135 | .deleteTopicCallable() 136 | .futureCall( 137 | DeleteTopicRequest.newBuilder().setTopic(topicId.toTopicName(projectId).toString()).build() 138 | ) 139 | .await() 140 | } 141 | 142 | override suspend fun getTopic(topicId: TopicId): Topic? { 143 | require(topicId.value.isNotBlank()) { "No topic name was specified." } 144 | return try { 145 | topicAdminClient.topicCallable 146 | .futureCall( 147 | GetTopicRequest.newBuilder().setTopic(topicId.toTopicName(projectId).toString()).build() 148 | ) 149 | .await() 150 | } catch (aex: ApiException) { 151 | if (aex.statusCode.code == StatusCode.Code.NOT_FOUND) null else throw aex 152 | } 153 | } 154 | 155 | override suspend fun listTopics(): List = 156 | topicAdminClient 157 | .listTopicsCallable() 158 | .futureCall(ListTopicsRequest.newBuilder().setProject(projectId.value).build()) 159 | .await() 160 | .topicsList 161 | 162 | override suspend fun createSubscription( 163 | subscriptionId: SubscriptionId, 164 | topicId: TopicId, 165 | configure: Subscription.Builder.() -> Unit, 166 | ): Subscription = 167 | subscriptionAdminClient 168 | .createSubscriptionCallable() 169 | .futureCall( 170 | Subscription.newBuilder() 171 | .setTopic(topicId.toTopicName(projectId).toString()) 172 | .setName(subscriptionId.toSubscriptionName(projectId).toString()) 173 | .apply(configure) 174 | .build() 175 | ) 176 | .await() 177 | 178 | override suspend fun deleteSubscription(subscriptionId: SubscriptionId) { 179 | subscriptionAdminClient 180 | .deleteSubscriptionCallable() 181 | .futureCall( 182 | DeleteSubscriptionRequest.newBuilder() 183 | .setSubscription(subscriptionId.toSubscriptionName(projectId).toString()) 184 | .build() 185 | ) 186 | .await() 187 | } 188 | 189 | override suspend fun getSubscription(subscriptionId: SubscriptionId): Subscription? { 190 | require(subscriptionId.value.isNotEmpty()) { "No subscription name was specified" } 191 | return try { 192 | subscriptionAdminClient.subscriptionCallable 193 | .futureCall( 194 | GetSubscriptionRequest.newBuilder() 195 | .setSubscription(subscriptionId.toSubscriptionName(projectId).toString()) 196 | .build() 197 | ) 198 | .await() 199 | } catch (aex: ApiException) { 200 | if (aex.statusCode.code == StatusCode.Code.NOT_FOUND) null else throw aex 201 | } 202 | } 203 | 204 | override suspend fun listSubscriptions(): List = 205 | subscriptionAdminClient 206 | .listSubscriptionsCallable() 207 | .futureCall(ListSubscriptionsRequest.getDefaultInstance()) 208 | .await() 209 | .subscriptionsList 210 | 211 | override fun close() { 212 | topicAdminClient.close() 213 | subscriptionAdminClient.close() 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /pubsub-ktor/src/main/kotlin/io/github/nomisrev/gcp/pubsub/ktor/GcpPubSub.kt: -------------------------------------------------------------------------------- 1 | package io.github.nomisrev.gcp.pubsub.ktor 2 | 3 | import com.google.api.gax.core.CredentialsProvider 4 | import com.google.cloud.pubsub.v1.Publisher 5 | import com.google.cloud.pubsub.v1.Subscriber 6 | import com.google.cloud.pubsub.v1.SubscriptionAdminClient 7 | import com.google.cloud.pubsub.v1.SubscriptionAdminSettings 8 | import com.google.cloud.pubsub.v1.TopicAdminClient 9 | import com.google.cloud.pubsub.v1.TopicAdminSettings 10 | import io.github.nomisrev.gcp.pubsub.GcpPublisher 11 | import io.github.nomisrev.gcp.pubsub.GcpPubsSubAdmin 12 | import io.github.nomisrev.gcp.pubsub.GcpSubscriber 13 | import io.github.nomisrev.gcp.pubsub.ProjectId 14 | import io.github.nomisrev.gcp.pubsub.PubsubRecord 15 | import io.github.nomisrev.gcp.pubsub.SubscriptionId 16 | import io.github.nomisrev.gcp.pubsub.TopicId 17 | import io.ktor.server.application.Application 18 | import io.ktor.server.application.ApplicationStopped 19 | import io.ktor.server.application.BaseApplicationPlugin 20 | import io.ktor.server.application.application 21 | import io.ktor.server.application.install 22 | import io.ktor.server.application.pluginOrNull 23 | import io.ktor.server.routing.RoutingContext 24 | import io.ktor.util.AttributeKey 25 | import io.ktor.utils.io.KtorDsl 26 | import java.util.concurrent.ConcurrentHashMap 27 | import kotlin.coroutines.CoroutineContext 28 | import kotlinx.coroutines.CoroutineScope 29 | import kotlinx.coroutines.ExperimentalCoroutinesApi 30 | import kotlinx.coroutines.Job 31 | import kotlinx.coroutines.flow.flattenMerge 32 | import kotlinx.coroutines.flow.flow 33 | import kotlinx.coroutines.flow.launchIn 34 | import kotlinx.coroutines.flow.map 35 | import kotlinx.coroutines.launch 36 | import kotlinx.coroutines.plus 37 | 38 | public class GcpPubSub(internal val application: Application, configuration: Configuration) { 39 | 40 | internal var configureSubscriber = configuration.configureSubscriber 41 | internal var configureTopicAdmin: (TopicAdminSettings.Builder.(projectId: ProjectId) -> Unit)? = 42 | configuration.configureTopicAdmin 43 | 44 | internal var configureSubscriptionAdmin: 45 | (SubscriptionAdminSettings.Builder.(projectId: ProjectId) -> Unit)? = 46 | configuration.configureSubscriptionAdmin 47 | 48 | internal var configurePublisher: 49 | Publisher.Builder.(projectId: ProjectId, topicId: TopicId) -> Unit = 50 | configuration.configurePublisher 51 | 52 | private val credentialsProvider: CredentialsProvider? = configuration.credentialsProvider 53 | private val adminCache = ConcurrentHashMap() 54 | private val publisherCache = ConcurrentHashMap() 55 | 56 | internal fun getOrCreateAdmin(projectId: ProjectId): GcpPubsSubAdmin = 57 | adminCache[projectId] 58 | ?: adminCache.computeIfAbsent(projectId) { 59 | GcpPubsSubAdmin( 60 | projectId, 61 | TopicAdminClient.create( 62 | TopicAdminSettings.newBuilder() 63 | .apply { 64 | configureTopicAdmin?.invoke(this, projectId) 65 | this@GcpPubSub.credentialsProvider?.let { setCredentialsProvider(it) } 66 | } 67 | .build() 68 | ), 69 | SubscriptionAdminClient.create( 70 | SubscriptionAdminSettings.newBuilder() 71 | .apply { 72 | configureSubscriptionAdmin?.invoke(this, projectId) 73 | this@GcpPubSub.credentialsProvider?.let { setCredentialsProvider(it) } 74 | } 75 | .build() 76 | ), 77 | ) 78 | .also { admin -> 79 | application.environment.monitor.subscribe(ApplicationStopped) { admin.close() } 80 | } 81 | } 82 | 83 | internal fun getOrCreatePublisher(projectId: ProjectId): GcpPublisher = 84 | publisherCache[projectId] 85 | ?: publisherCache.computeIfAbsent(projectId) { 86 | GcpPublisher(projectId) { configurePublisher(projectId, it) } 87 | .also { publisher -> 88 | application.environment.monitor.subscribe(ApplicationStopped) { publisher.close() } 89 | } 90 | } 91 | 92 | public fun admin(projectId: ProjectId): GcpPubsSubAdmin = getOrCreateAdmin(projectId) 93 | 94 | public fun publisher(projectId: ProjectId): GcpPublisher = getOrCreatePublisher(projectId) 95 | 96 | @OptIn(ExperimentalCoroutinesApi::class) 97 | internal fun gcpPubSub(projectId: ProjectId, scope: CoroutineScope): GcpPubSubSyntax = 98 | object : 99 | GcpPubSubSyntax, 100 | GcpPubsSubAdmin by getOrCreateAdmin(projectId), 101 | GcpPublisher by getOrCreatePublisher(projectId), 102 | CoroutineScope by scope { 103 | override val subscriber = GcpSubscriber(projectId, configureSubscriber) 104 | 105 | override fun subscribe( 106 | subscriptionId: SubscriptionId, 107 | concurrency: Int, 108 | context: CoroutineContext, 109 | configure: Subscriber.Builder.() -> Unit, 110 | handler: suspend (PubsubRecord) -> Unit, 111 | ): Job = 112 | subscriber 113 | .subscribe(subscriptionId) { configure() } 114 | .map { record -> flow { emit(handler(record)) } } 115 | .flattenMerge(concurrency) 116 | .launchIn(scope + context) 117 | 118 | // These are managed by application.environment.monitor.subscribe(ApplicationStopped) 119 | override fun close() {} 120 | } 121 | 122 | @KtorDsl 123 | public class Configuration { 124 | public var credentialsProvider: CredentialsProvider? = null 125 | internal var configureSubscriber: Subscriber.Builder.(subscriptionId: SubscriptionId) -> Unit = 126 | {} 127 | internal var configurePublisher: 128 | Publisher.Builder.(projectId: ProjectId, topicId: TopicId) -> Unit = 129 | { _, _ -> 130 | } 131 | internal var configureTopicAdmin: (TopicAdminSettings.Builder.(projectId: ProjectId) -> Unit)? = 132 | null 133 | internal var configureSubscriptionAdmin: 134 | (SubscriptionAdminSettings.Builder.(projectId: ProjectId) -> Unit)? = 135 | null 136 | 137 | /** 138 | * Global [Subscriber.Builder] configuration that will be applied to all [Subscriber]'s created 139 | * by the [GcpPubSub] plugin. You can provide additional configuration on a per [subscriber] 140 | * basis. 141 | */ 142 | public fun subscriber(block: Subscriber.Builder.(subscriptionId: SubscriptionId) -> Unit) { 143 | configureSubscriber = block 144 | } 145 | 146 | /** 147 | * Global [Publisher.Builder] configuration that will be applied to all [Publisher]'s created by 148 | * the [GcpPubSub] plugin. It's not possible to provide additional configuration on a 149 | * per-publisher basis, since publishers are cached under the hood for performance reasons. 150 | * 151 | * If you need to add configuration for a specific [Publisher] do so by checking `projectId` and 152 | * `topicId` parameters of the lambda. 153 | */ 154 | public fun publisher( 155 | block: Publisher.Builder.(projectId: ProjectId, topicId: TopicId) -> Unit 156 | ) { 157 | configurePublisher = block 158 | } 159 | 160 | /** 161 | * Global [TopicAdminSettings.Builder] configuration that will be applied to all 162 | * [TopicAdminClient]'s created by the [GcpPubSub] plugin. It's not possible to provide 163 | * additional configuration on a per-publisher basis, since publishers are cached under the hood 164 | * for performance reasons. 165 | * 166 | * If you need to add configuration for a specific [TopicAdminClient] do so by checking 167 | * `projectId` parameters of the lambda. 168 | */ 169 | public fun topicAdmin(block: TopicAdminSettings.Builder.(projectId: ProjectId) -> Unit) { 170 | configureTopicAdmin = block 171 | } 172 | 173 | /** 174 | * Global [SubscriptionAdminSettings.Builder] configuration that will be applied to all 175 | * [SubscriptionAdminClient]'s created by the [GcpPubSub] plugin. It's not possible to provide 176 | * additional configuration on a per-publisher basis, since publishers are cached under the hood 177 | * for performance reasons. 178 | * 179 | * If you need to add configuration for a specific [SubscriptionAdminClient] do so by checking 180 | * `projectId` parameters of the lambda. 181 | */ 182 | public fun subscriptionAdmin( 183 | block: SubscriptionAdminSettings.Builder.(projectId: ProjectId) -> Unit 184 | ) { 185 | configureSubscriptionAdmin = block 186 | } 187 | } 188 | 189 | public companion object Plugin : BaseApplicationPlugin { 190 | override val key: AttributeKey = AttributeKey("GcpPubSubPlugin") 191 | 192 | public override fun install( 193 | pipeline: Application, 194 | configure: Configuration.() -> Unit, 195 | ): GcpPubSub = GcpPubSub(pipeline, Configuration().apply(configure)) 196 | } 197 | } 198 | 199 | /** 200 | * Retrieve the [GcpPubSub] plugin, and launch a coroutine that creates a Gcp PubSub based program 201 | * using [GcpPubSubSyntax]. It gives access to all functions from [GcpPubsSubAdmin] & 202 | * [GcpPublisher]. 203 | * 204 | * ```kotlin 205 | * fun Application.process(): Job = 206 | * pubSub(projectId) { 207 | * // Access all GcpPubSubAdmin functions 208 | * createTopic(TopicId("my-topic")) 209 | * createSubscription(SubscriptionId("my-subscription"), TopicId("my-topic")) 210 | * 211 | * // Access to all GcpPublisher functions 212 | * publish(TopicId("my-topic"), listOf("my-message1", "my-message2")) 213 | * 214 | * subscribe(SubscriptionId("my-subscription")) { record -> 215 | * println(record.message.data.toStringUtf8()) 216 | * record.ack(). 217 | * } 218 | * } 219 | * ``` 220 | */ 221 | public fun Application.pubSub( 222 | projectId: ProjectId, 223 | block: suspend GcpPubSubSyntax.() -> Unit, 224 | ): Job { 225 | val plugin = pubSub() 226 | return launch { block(plugin.gcpPubSub(projectId, this)) } 227 | } 228 | 229 | /** 230 | * Retrieve the GcpPubSub plugin, this allows access to the cached [GcpPubsSubAdmin] & 231 | * [GcpPublisher] instances 232 | * 233 | * ```kotlin 234 | * fun Application.route(): Routing = 235 | * routing { 236 | * post("/publish") { 237 | * pubSub().publisher(ProjectId("my-project")) 238 | * .publish(TopicId("my-topic"), "my-message") 239 | * call.respond(HttpStatusCode.Accepted) 240 | * } 241 | * 242 | * post("/delete/{topic}") { 243 | * val topic = 244 | * requireNotNull(call.parameters["topic"]) { "Missing parameter topic" } 245 | * pubSub().admin(projectId).deleteTopic(TopicId(topic)) 246 | * } 247 | * } 248 | * ``` 249 | */ 250 | public fun RoutingContext.pubSub(): GcpPubSub = call.application.pubSub() 251 | 252 | private fun Application.pubSub(): GcpPubSub = pluginOrNull(GcpPubSub) ?: install(GcpPubSub) 253 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /common-api/src/main/kotlin/io/github/nomisrev/gcp/core/ApiFuture.kt: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is inspired by 3 | * https://github.com/Kotlin/kotlinx.coroutines/blob/master/integration/kotlinx-coroutines-guava/src/ApiFuture.kt 4 | */ 5 | package io.github.nomisrev.gcp.core 6 | 7 | import com.google.api.core.ApiFuture 8 | import com.google.api.core.ApiFutureCallback 9 | import com.google.api.core.ApiFutures 10 | import com.google.common.util.concurrent.Futures 11 | import com.google.common.util.concurrent.MoreExecutors 12 | import com.google.common.util.concurrent.SettableFuture 13 | import com.google.common.util.concurrent.Uninterruptibles 14 | import com.google.common.util.concurrent.internal.InternalFutureFailureAccess 15 | import com.google.common.util.concurrent.internal.InternalFutures 16 | import java.util.concurrent.ExecutionException 17 | import java.util.concurrent.Executor 18 | import java.util.concurrent.TimeUnit 19 | import kotlin.coroutines.CoroutineContext 20 | import kotlin.coroutines.EmptyCoroutineContext 21 | import kotlin.coroutines.cancellation.CancellationException 22 | import kotlin.coroutines.resume 23 | import kotlin.coroutines.resumeWithException 24 | import kotlinx.coroutines.AbstractCoroutine 25 | import kotlinx.coroutines.CancellableContinuation 26 | import kotlinx.coroutines.CompletableDeferred 27 | import kotlinx.coroutines.Deferred 28 | import kotlinx.coroutines.InternalCoroutinesApi 29 | import kotlinx.coroutines.Job 30 | import kotlinx.coroutines.handleCoroutineException 31 | import kotlinx.coroutines.suspendCancellableCoroutine 32 | 33 | /** 34 | * Returns a [Deferred] that is completed or failed by `this` [ApiFuture]. 35 | * 36 | * Completion is non-atomic between the two promises. 37 | * 38 | * Cancellation is propagated bidirectionally. 39 | * 40 | * When `this` `ApiFuture` completes (either successfully or exceptionally) it will try to 41 | * complete the returned `Deferred` with the same value or exception. This will succeed, barring a 42 | * race with cancellation of the `Deferred`. 43 | * 44 | * When `this` `ApiFuture` is [successfully cancelled][java.util.concurrent.Future.cancel], 45 | * it will cancel the returned `Deferred`. 46 | * 47 | * When the returned `Deferred` is [cancelled][Deferred.cancel], it will try to propagate the 48 | * cancellation to `this` `ApiFuture`. Propagation will succeed, barring a race with the 49 | * `ApiFuture` completing normally. This is the only case in which the returned `Deferred` 50 | * will complete with a different outcome than `this` `ApiFuture`. 51 | */ 52 | public fun ApiFuture.asDeferred(): Deferred { 53 | /* This method creates very specific behaviour as it entangles the `Deferred` and 54 | * `ApiFuture`. This behaviour is the best discovered compromise between the possible 55 | * states and interface contracts of a `Future` and the states of a `Deferred`. The specific 56 | * behaviour is described here. 57 | * 58 | * When `this` `ApiFuture` is successfully cancelled - meaning 59 | * `ApiFuture.cancel()` returned `true` - it will synchronously cancel the returned 60 | * `Deferred`. This can only race with cancellation of the returned `Deferred`, so the 61 | * `Deferred` will always be put into its "cancelling" state and (barring uncooperative 62 | * cancellation) _eventually_ reach its "cancelled" state when either promise is successfully 63 | * cancelled. 64 | * 65 | * When the returned `Deferred` is cancelled, `ApiFuture.cancel()` will be synchronously 66 | * called on `this` `ApiFuture`. This will attempt to cancel the `Future`, though 67 | * cancellation may not succeed and the `ApiFuture` may complete in a non-cancelled 68 | * terminal state. 69 | * 70 | * The returned `Deferred` may receive and suppress the `true` return value from 71 | * `ApiFuture.cancel()` when the task is cancelled via the `Deferred` reference to it. 72 | * This is unavoidable, so make sure no idempotent cancellation work is performed by a 73 | * reference-holder of the `ApiFuture` task. The idempotent work won't get done if 74 | * cancellation was from the `Deferred` representation of the task. 75 | * 76 | * This is inherently a race. See `Future.cancel()` for a description of `Future` cancellation 77 | * semantics. See `Job` for a description of coroutine cancellation semantics. 78 | */ 79 | // First, try the fast-fast error path for Guava ApiFutures. This will save allocating an 80 | // Exception by using the same instance the Future created. 81 | if (this is InternalFutureFailureAccess) { 82 | val t: Throwable? = InternalFutures.tryInternalFastPathGetFailure(this) 83 | if (t != null) { 84 | return CompletableDeferred().also { it.completeExceptionally(t) } 85 | } 86 | } 87 | 88 | // Second, try the fast path for a completed Future. The Future is known to be done, so get() 89 | // will not block, and thus it won't be interrupted. Calling getUninterruptibly() instead of 90 | // getDone() in this known-non-interruptible case saves the volatile read that getDone() uses to 91 | // handle interruption. 92 | if (isDone) { 93 | return try { 94 | CompletableDeferred(Uninterruptibles.getUninterruptibly(this)) 95 | } catch (e: CancellationException) { 96 | CompletableDeferred().also { it.cancel(e) } 97 | } catch (e: ExecutionException) { 98 | // ExecutionException is the only kind of exception that can be thrown from a gotten 99 | // Future. Anything else showing up here indicates a very fundamental bug in a 100 | // Future implementation. 101 | CompletableDeferred().also { it.completeExceptionally(e.nonNullCause()) } 102 | } 103 | } 104 | 105 | // Finally, if this isn't done yet, attach a Listener that will complete the Deferred. 106 | val deferred = CompletableDeferred() 107 | @OptIn(InternalCoroutinesApi::class) 108 | ApiFutures.addCallback( 109 | this, 110 | object : ApiFutureCallback { 111 | override fun onSuccess(result: T) { 112 | runCatching { deferred.complete(result) } 113 | .onFailure { handleCoroutineException(EmptyCoroutineContext, it) } 114 | } 115 | 116 | override fun onFailure(t: Throwable) { 117 | runCatching { deferred.completeExceptionally(t) } 118 | .onFailure { handleCoroutineException(EmptyCoroutineContext, it) } 119 | } 120 | }, 121 | MoreExecutors.directExecutor() 122 | ) 123 | 124 | // ... And cancel the Future when the deferred completes. Since the return type of this method 125 | // is Deferred, the only interaction point from the caller is to cancel the Deferred. If this 126 | // completion handler runs before the Future is completed, the Deferred must have been 127 | // cancelled and should propagate its cancellation. If it runs after the Future is completed, 128 | // this is a no-op. 129 | deferred.invokeOnCompletion { cancel(false) } 130 | // Return hides the CompletableDeferred. This should prevent casting. 131 | return object : Deferred by deferred {} 132 | } 133 | 134 | /** 135 | * Awaits completion of `this` [ApiFuture] without blocking a thread. 136 | * 137 | * This suspend function is cancellable. 138 | * 139 | * If the [Job] of the current coroutine is cancelled or completed while this suspending function is 140 | * waiting, this function stops waiting for the future and immediately resumes with 141 | * [CancellationException][kotlinx.coroutines.CancellationException]. 142 | * 143 | * This method is intended to be used with one-shot Futures, so on coroutine cancellation, the 144 | * Future is cancelled as well. If cancelling the given future is undesired, use 145 | * [Futures.nonCancellationPropagating] or [kotlinx.coroutines.NonCancellable]. 146 | */ 147 | public suspend fun ApiFuture.await(): T { 148 | try { 149 | if (isDone) return Uninterruptibles.getUninterruptibly(this) 150 | } catch (e: ExecutionException) { 151 | // ExecutionException is the only kind of exception that can be thrown from a gotten 152 | // Future, other than CancellationException. Cancellation is propagated upward so that 153 | // the coroutine running this suspend function may process it. 154 | // Any other Exception showing up here indicates a very fundamental bug in a 155 | // Future implementation. 156 | throw e.nonNullCause() 157 | } 158 | 159 | return suspendCancellableCoroutine { cont: CancellableContinuation -> 160 | addListener(ToContinuation(this, cont), MoreExecutors.directExecutor()) 161 | cont.invokeOnCancellation { cancel(false) } 162 | } 163 | } 164 | 165 | /** 166 | * Propagates the outcome of [futureToObserve] to [continuation] on completion. 167 | * 168 | * Cancellation is propagated as cancelling the continuation. If [futureToObserve] completes and 169 | * fails, the cause of the Future will be propagated without a wrapping [ExecutionException] when 170 | * thrown. 171 | */ 172 | private class ToContinuation( 173 | val futureToObserve: ApiFuture, 174 | val continuation: CancellableContinuation 175 | ) : Runnable { 176 | override fun run() { 177 | if (futureToObserve.isCancelled) { 178 | continuation.cancel() 179 | } else { 180 | try { 181 | continuation.resume(Uninterruptibles.getUninterruptibly(futureToObserve)) 182 | } catch (e: ExecutionException) { 183 | // ExecutionException is the only kind of exception that can be thrown from a gotten 184 | // Future. Anything else showing up here indicates a very fundamental bug in a 185 | // Future implementation. 186 | continuation.resumeWithException(e.nonNullCause()) 187 | } 188 | } 189 | } 190 | } 191 | 192 | /** 193 | * Returns the cause from an [ExecutionException] thrown by a [Future.get] or similar. 194 | * 195 | * [ExecutionException] _always_ wraps a non-null cause when Future.get() throws. A Future cannot 196 | * fail without a non-null `cause`, because the only way a Future _can_ fail is an uncaught 197 | * [Exception]. 198 | * 199 | * If this !! throws [NullPointerException], a Future is breaking its interface contract and losing 200 | * state - a serious fundamental bug. 201 | */ 202 | private fun ExecutionException.nonNullCause(): Throwable { 203 | return this.cause!! 204 | } 205 | 206 | @InternalCoroutinesApi 207 | private class ApiFutureCoroutine(context: CoroutineContext) : 208 | AbstractCoroutine(context, initParentJob = true, active = true) { 209 | 210 | // JobApiFuture propagates external cancellation to `this` coroutine. See 211 | // JobApiFuture. 212 | @JvmField val future = JobApiFuture(this) 213 | 214 | override fun onCompleted(value: T) { 215 | future.complete(value) 216 | } 217 | 218 | override fun onCancelled(cause: Throwable, handled: Boolean) { 219 | // Note: if future was cancelled in a race with a cancellation of this 220 | // coroutine, and the future was successfully cancelled first, the cause of coroutine 221 | // cancellation is dropped in this promise. A Future can only be completed once. 222 | // 223 | // This is consistent with FutureTask behaviour. A race between a Future.cancel() and 224 | // a FutureTask.setException() for the same Future will similarly drop the 225 | // cause of a failure-after-cancellation. 226 | future.completeExceptionallyOrCancel(cause) 227 | } 228 | } 229 | 230 | /** 231 | * A [ApiFuture] that delegates to an internal [SettableFuture], collaborating with it. 232 | * 233 | * This setup allows the returned [ApiFuture] to maintain the following properties: 234 | * - Correct implementation of [Future]'s happens-after semantics documented for [get], [isDone] and 235 | * [isCancelled] methods 236 | * - Cancellation propagation both to and from [Deferred] 237 | * - Correct cancellation and completion semantics even when this [ApiFuture] is combined 238 | * with different concrete implementations of [ApiFuture] 239 | * - Fully correct cancellation and listener happens-after obeying [Future] and 240 | * [ApiFuture]'s documented and implicit contracts is surprisingly difficult to 241 | * achieve. The best way to be correct, especially given the fun corner cases from 242 | * [AbstractFuture.setFuture], is to just use an [AbstractFuture]. 243 | * - To maintain sanity, this class implements [ApiFuture] and uses an auxiliary 244 | * [SettableFuture] around coroutine's result as a state engine to establish 245 | * happens-after-completion. This could probably be compressed into one subclass of 246 | * [AbstractFuture] to save an allocation, at the cost of the implementation's readability. 247 | */ 248 | private class JobApiFuture(private val jobToCancel: Job) : ApiFuture { 249 | /** 250 | * Serves as a state machine for [Future] cancellation. 251 | * 252 | * [AbstractFuture] has a highly-correct atomic implementation of `Future`'s completion and 253 | * cancellation semantics. By using that type, the [JobApiFuture] can delegate its 254 | * semantics to `auxFuture.get()` the result in such a way that the `Deferred` is always complete 255 | * when returned. 256 | * 257 | * To preserve Coroutine's [CancellationException], this future points to either `T` or 258 | * [Cancelled]. 259 | */ 260 | private val auxFuture = SettableFuture.create() 261 | 262 | /** 263 | * `true` if [auxFuture.get][ApiFuture.get] throws [ExecutionException]. 264 | * 265 | * Note: this is eventually consistent with the state of [auxFuture]. 266 | * 267 | * Unfortunately, there's no API to figure out if [ApiFuture] throws [ExecutionException] 268 | * apart from calling [ApiFuture.get] on it. To avoid unnecessary [ExecutionException] 269 | * allocation we use this field as an optimization. 270 | */ 271 | private var auxFutureIsFailed: Boolean = false 272 | 273 | /** 274 | * When the attached coroutine [isCompleted][Job.isCompleted] successfully its outcome should be 275 | * passed to this method. 276 | * 277 | * This should succeed barring a race with external cancellation. 278 | */ 279 | fun complete(result: T): Boolean = auxFuture.set(result) 280 | 281 | /** 282 | * When the attached coroutine [isCompleted][Job.isCompleted] [exceptionally][Job.isCancelled] its 283 | * outcome should be passed to this method. 284 | * 285 | * This method will map coroutine's exception into corresponding Future's exception. 286 | * 287 | * This should succeed barring a race with external cancellation. 288 | */ 289 | // CancellationException is wrapped into `Cancelled` to preserve original cause and message. 290 | // All the other exceptions are delegated to SettableFuture.setException. 291 | fun completeExceptionallyOrCancel(t: Throwable): Boolean = 292 | if (t is CancellationException) auxFuture.set(Cancelled(t)) 293 | else auxFuture.setException(t).also { if (it) auxFutureIsFailed = true } 294 | 295 | /** 296 | * Returns cancellation _in the sense of [Future]_. This is _not_ equivalent to [Job.isCancelled]. 297 | * 298 | * When done, this Future is cancelled if its [auxFuture] is cancelled, or if [auxFuture] contains 299 | * [CancellationException]. 300 | * 301 | * See [cancel]. 302 | */ 303 | override fun isCancelled(): Boolean { 304 | // This expression ensures that isCancelled() will *never* return true when isDone() returns 305 | // false. 306 | // In the case that the deferred has completed with cancellation, completing `this`, its 307 | // reaching the "cancelled" state with a cause of CancellationException is treated as the 308 | // same thing as auxFuture getting cancelled. If the Job is in the "cancelling" state and 309 | // this Future hasn't itself been successfully cancelled, the Future will return 310 | // isCancelled() == false. This is the only discovered way to reconcile the two different 311 | // cancellation contracts. 312 | return auxFuture.isCancelled || 313 | isDone && 314 | !auxFutureIsFailed && 315 | try { 316 | Uninterruptibles.getUninterruptibly(auxFuture) is Cancelled 317 | } catch (e: CancellationException) { 318 | // `auxFuture` got cancelled right after `auxFuture.isCancelled` returned false. 319 | true 320 | } catch (e: ExecutionException) { 321 | // `auxFutureIsFailed` hasn't been updated yet. 322 | auxFutureIsFailed = true 323 | false 324 | } 325 | } 326 | 327 | /** 328 | * Waits for [auxFuture] to complete by blocking, then uses its `result` to get the `T` value 329 | * `this` [ApiFuture] is pointing to or throw a [CancellationException]. This establishes 330 | * happens-after ordering for completion of the entangled coroutine. 331 | * 332 | * [SettableFuture.get] can only throw [CancellationException] if it was cancelled externally. 333 | * Otherwise, it returns [Cancelled] that encapsulates outcome of the entangled coroutine. 334 | * 335 | * [auxFuture] _must be complete_ in order for the [isDone] and [isCancelled] happens-after 336 | * contract of [Future] to be correctly followed. 337 | */ 338 | override fun get(): T { 339 | return getInternal(auxFuture.get()) 340 | } 341 | 342 | /** See [get()]. */ 343 | override fun get(timeout: Long, unit: TimeUnit): T { 344 | return getInternal(auxFuture.get(timeout, unit)) 345 | } 346 | 347 | /** See [get()]. */ 348 | private fun getInternal(result: Any?): T = 349 | if (result is Cancelled) { 350 | throw CancellationException().initCause(result.exception) 351 | } else { 352 | // We know that `auxFuture` can contain either `T` or `Cancelled`. 353 | @Suppress("UNCHECKED_CAST") 354 | result as T 355 | } 356 | 357 | override fun addListener(listener: Runnable, executor: Executor) { 358 | auxFuture.addListener(listener, executor) 359 | } 360 | 361 | override fun isDone(): Boolean { 362 | return auxFuture.isDone 363 | } 364 | 365 | /** 366 | * Tries to cancel [jobToCancel] if `this` future was cancelled. This is fundamentally racy. 367 | * 368 | * The call to `cancel()` will try to cancel [auxFuture]: if and only if cancellation of 369 | * [auxFuture] succeeds, [jobToCancel] will have its [Job.cancel] called. 370 | * 371 | * This arrangement means that [jobToCancel] _might not successfully cancel_, if the race resolves 372 | * in a particular way. [jobToCancel] may also be in its "cancelling" state while this 373 | * ApiFuture is complete and cancelled. 374 | */ 375 | override fun cancel(mayInterruptIfRunning: Boolean): Boolean { 376 | // TODO: call jobToCancel.cancel() _before_ running the listeners. 377 | // `auxFuture.cancel()` will execute auxFuture's listeners. This delays cancellation of 378 | // `jobToCancel` until after auxFuture's listeners have already run. 379 | // Consider moving `jobToCancel.cancel()` into [AbstractFuture.afterDone] when the API is 380 | // finalized. 381 | return if (auxFuture.cancel(mayInterruptIfRunning)) { 382 | jobToCancel.cancel() 383 | true 384 | } else { 385 | false 386 | } 387 | } 388 | 389 | override fun toString(): String = buildString { 390 | append(super.toString()) 391 | append("[status=") 392 | if (isDone) { 393 | try { 394 | when (val result = Uninterruptibles.getUninterruptibly(auxFuture)) { 395 | is Cancelled -> append("CANCELLED, cause=[${result.exception}]") 396 | else -> append("SUCCESS, result=[$result]") 397 | } 398 | } catch (e: CancellationException) { 399 | // `this` future was cancelled by `Future.cancel`. In this case there's no cause or message. 400 | append("CANCELLED") 401 | } catch (e: ExecutionException) { 402 | append("FAILURE, cause=[${e.cause}]") 403 | } catch (t: Throwable) { 404 | // Violation of Future's contract, should never happen. 405 | append("UNKNOWN, cause=[${t.javaClass} thrown from get()]") 406 | } 407 | } else { 408 | append("PENDING, delegate=[$auxFuture]") 409 | } 410 | append(']') 411 | } 412 | } 413 | 414 | /** 415 | * A wrapper for `Coroutine`'s [CancellationException]. 416 | * 417 | * If the coroutine is _cancelled normally_, we want to show the reason of cancellation to the user. 418 | * Unfortunately, [SettableFuture] can't store the reason of cancellation. To mitigate this, we wrap 419 | * cancellation exception into this class and pass it into [SettableFuture.complete]. See 420 | * implementation of [JobApiFuture]. 421 | */ 422 | private class Cancelled(@JvmField val exception: CancellationException) 423 | --------------------------------------------------------------------------------