├── .dockerignore ├── .github └── workflows │ ├── build-image.yml │ └── codeql-analysis.yml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── RELEASES.md ├── SECURITY.md ├── backend ├── .gitignore ├── cmd │ └── api │ │ └── main.go ├── go.mod ├── go.sum └── pkg │ ├── api │ ├── api.go │ ├── config.go │ ├── handle_acls.go │ ├── handle_api_versions.go │ ├── handle_brokers.go │ ├── handle_cluster.go │ ├── handle_consumer_group.go │ ├── handle_frontend.go │ ├── handle_kafka_connect.go │ ├── handle_kowl.go │ ├── handle_operations.go │ ├── handle_probes.go │ ├── handle_quotas.go │ ├── handle_schema.go │ ├── handle_topic_create.go │ ├── handle_topic_documentation.go │ ├── handle_topic_messages.go │ ├── handle_topic_publish_records.go │ ├── handle_topics.go │ ├── hooks.go │ ├── middlewares.go │ ├── routes.go │ ├── validator.go │ ├── validator_test.go │ ├── version.go │ ├── websocket_client.go │ └── websocket_progress_reporter.go │ ├── connect │ ├── config.go │ ├── config_cluster.go │ ├── config_cluster_tls.go │ ├── create_connector.go │ ├── delete_connector.go │ ├── errors.go │ ├── get_cluster_info.go │ ├── get_connectors.go │ ├── pause_connector.go │ ├── put_connector_config.go │ ├── restart_connector.go │ ├── restart_task.go │ ├── resume_connector.go │ ├── service.go │ ├── util.go │ └── validate_connector.go │ ├── filesystem │ ├── config.go │ ├── file.go │ ├── service.go │ └── util.go │ ├── git │ ├── config.go │ ├── config_auth_basic.go │ ├── config_auth_ssh.go │ ├── config_repository.go │ ├── service.go │ ├── util.go │ └── util_test.go │ ├── interpreter │ └── find_function.go │ ├── kafka │ ├── api_version.go │ ├── client_hooks.go │ ├── config.go │ ├── config_helper.go │ ├── config_sasl.go │ ├── config_sasl_aws_iam.go │ ├── config_sasl_gssapi.go │ ├── config_sasl_oauth.go │ ├── config_tls.go │ ├── consumer.go │ ├── consumer_worker.go │ ├── create_topic.go │ ├── delete_records.go │ ├── delete_topics.go │ ├── describe_broker_config.go │ ├── describe_consumer_groups.go │ ├── describe_quotas.go │ ├── describe_topic_configs.go │ ├── deserializer.go │ ├── edit_consumer_group_offsets.go │ ├── get_metadata.go │ ├── health_check.go │ ├── incremental_alter_configs.go │ ├── list_acls.go │ ├── list_consumer_group_offsets.go │ ├── list_consumer_groups.go │ ├── log_dir.go │ ├── logger.go │ ├── partition_reassignments.go │ ├── produce_records.go │ ├── service.go │ ├── utils.go │ └── water_mark.go │ ├── msgpack │ ├── config.go │ ├── service.go │ └── utils.go │ ├── owl │ ├── api_versions.go │ ├── broker_config.go │ ├── cluster_info.go │ ├── common.go │ ├── config.go │ ├── config_topic_documentation.go │ ├── consumer_group_offsets.go │ ├── consumer_group_overview.go │ ├── create_topic.go │ ├── delete_consumer_group_offsets.go │ ├── delete_topic.go │ ├── describe_quotas.go │ ├── edit_consumer_group_offsets.go │ ├── endpoint_compatibility.go │ ├── errors.go │ ├── incremental_alter_configs.go │ ├── kafka_error.go │ ├── list_acls.go │ ├── list_messages.go │ ├── list_messages_test.go │ ├── list_offsets.go │ ├── log_dir_broker.go │ ├── log_dir_topic.go │ ├── owl.go │ ├── partition_reassignments.go │ ├── produce_records.go │ ├── schema_details.go │ ├── schema_overview.go │ ├── service.go │ ├── topic_config.go │ ├── topic_consumers.go │ ├── topic_documentation.go │ ├── topic_overview.go │ ├── topic_partitions.go │ └── util.go │ ├── proto │ ├── config.go │ ├── config_schema_registry.go │ ├── config_topic_mapping.go │ ├── service.go │ └── trigger_refresh.go │ └── schema │ ├── client.go │ ├── client_errors.go │ ├── client_test.go │ ├── config.go │ ├── config_tls.go │ └── service.go ├── docs ├── README.md ├── assets │ ├── identity-provider-setup │ │ ├── github │ │ │ ├── create-oauth-app-step1.png │ │ │ ├── create-oauth-app-step2.png │ │ │ ├── create-personal-access-token-1.png │ │ │ └── create-personal-access-token-2.png │ │ ├── google │ │ │ ├── create-google-project.png │ │ │ ├── oauth-consent-setup.png │ │ │ ├── oauth-credentials-setup.png │ │ │ ├── sa-google-groups.png │ │ │ ├── sa-google-groups2.png │ │ │ ├── sa-google-groups3.png │ │ │ ├── sa-google-groups4.png │ │ │ ├── sa-google-groups5.png │ │ │ ├── sa-google-groups6.jpg │ │ │ └── sa-google-groups7.jpg │ │ └── okta │ │ │ ├── add-oidc-application.png │ │ │ ├── create-api-token.png │ │ │ ├── get-client-credentials.png │ │ │ ├── setup-wizard-step1.png │ │ │ └── setup-wizard-step2.png │ ├── preview.gif │ ├── social-preview.png │ ├── sponsors │ │ └── rewe-digital-logo.png │ └── topic-documentation.png ├── authentication │ └── authentication.md ├── authorization │ ├── groups-sync.md │ ├── role-bindings.md │ └── roles.md ├── config │ ├── kowl-business-role-bindings.yaml │ ├── kowl-business-roles.yaml │ ├── kowl-business.yaml │ └── kowl.yaml ├── features │ ├── hosting.md │ ├── kafka-connect.md │ ├── protobuf.md │ └── topic-documentation.md ├── installation.md ├── local │ └── docker-compose.yaml ├── menu.md └── provider-setup │ ├── github.md │ ├── google.md │ └── okta.md └── frontend ├── .eslintrc.json ├── .gitignore ├── _.eslintrc.json ├── config-overrides.js ├── package-lock.json ├── package.json ├── public ├── favicon-32x32.png ├── favicon.ico ├── favicon.png ├── index.html ├── logo2.pdn ├── logo2.png ├── manifest.json └── robots.txt ├── src ├── assets │ ├── circle-stop.svg │ ├── connectors │ │ ├── amazon-s3.png │ │ ├── apache.svg │ │ ├── cassandra.png │ │ ├── confluent.png │ │ ├── db2.png │ │ ├── debezium.png │ │ ├── elastic.svg │ │ ├── google-bigquery.svg │ │ ├── google-pub-sub.svg │ │ ├── hdfs.png │ │ ├── ibm-mq.svg │ │ ├── jdbc.png │ │ ├── mongodb.png │ │ ├── mssql.png │ │ ├── mysql.svg │ │ ├── neo4j.svg │ │ ├── postgres.png │ │ ├── salesforce.png │ │ ├── servicenow.png │ │ ├── snowflake.png │ │ └── twitter.svg │ ├── elasticsearch.json │ ├── filter-example-1.png │ ├── filter-example-2.png │ ├── fonts │ │ ├── open-sans.css │ │ ├── open-sans │ │ │ ├── open-sans-v20-latin-ext_latin-600.eot │ │ │ ├── open-sans-v20-latin-ext_latin-600.svg │ │ │ ├── open-sans-v20-latin-ext_latin-600.ttf │ │ │ ├── open-sans-v20-latin-ext_latin-600.woff │ │ │ ├── open-sans-v20-latin-ext_latin-600.woff2 │ │ │ ├── open-sans-v20-latin-ext_latin-700.eot │ │ │ ├── open-sans-v20-latin-ext_latin-700.svg │ │ │ ├── open-sans-v20-latin-ext_latin-700.ttf │ │ │ ├── open-sans-v20-latin-ext_latin-700.woff │ │ │ ├── open-sans-v20-latin-ext_latin-700.woff2 │ │ │ ├── open-sans-v20-latin-ext_latin-regular.eot │ │ │ ├── open-sans-v20-latin-ext_latin-regular.svg │ │ │ ├── open-sans-v20-latin-ext_latin-regular.ttf │ │ │ ├── open-sans-v20-latin-ext_latin-regular.woff │ │ │ └── open-sans-v20-latin-ext_latin-regular.woff2 │ │ ├── poppins.css │ │ ├── poppins │ │ │ ├── poppins-v15-latin-ext_latin-300.eot │ │ │ ├── poppins-v15-latin-ext_latin-300.svg │ │ │ ├── poppins-v15-latin-ext_latin-300.ttf │ │ │ ├── poppins-v15-latin-ext_latin-300.woff │ │ │ ├── poppins-v15-latin-ext_latin-300.woff2 │ │ │ ├── poppins-v15-latin-ext_latin-500.eot │ │ │ ├── poppins-v15-latin-ext_latin-500.svg │ │ │ ├── poppins-v15-latin-ext_latin-500.ttf │ │ │ ├── poppins-v15-latin-ext_latin-500.woff │ │ │ ├── poppins-v15-latin-ext_latin-500.woff2 │ │ │ ├── poppins-v15-latin-ext_latin-600.eot │ │ │ ├── poppins-v15-latin-ext_latin-600.svg │ │ │ ├── poppins-v15-latin-ext_latin-600.ttf │ │ │ ├── poppins-v15-latin-ext_latin-600.woff │ │ │ ├── poppins-v15-latin-ext_latin-600.woff2 │ │ │ ├── poppins-v15-latin-ext_latin-700.eot │ │ │ ├── poppins-v15-latin-ext_latin-700.svg │ │ │ ├── poppins-v15-latin-ext_latin-700.ttf │ │ │ ├── poppins-v15-latin-ext_latin-700.woff │ │ │ ├── poppins-v15-latin-ext_latin-700.woff2 │ │ │ ├── poppins-v15-latin-ext_latin-regular.eot │ │ │ ├── poppins-v15-latin-ext_latin-regular.svg │ │ │ ├── poppins-v15-latin-ext_latin-regular.ttf │ │ │ ├── poppins-v15-latin-ext_latin-regular.woff │ │ │ └── poppins-v15-latin-ext_latin-regular.woff2 │ │ ├── quicksand.css │ │ └── quicksand │ │ │ ├── quicksand-v24-latin-ext_latin-500.eot │ │ │ ├── quicksand-v24-latin-ext_latin-500.svg │ │ │ ├── quicksand-v24-latin-ext_latin-500.ttf │ │ │ ├── quicksand-v24-latin-ext_latin-500.woff │ │ │ ├── quicksand-v24-latin-ext_latin-500.woff2 │ │ │ ├── quicksand-v24-latin-ext_latin-600.eot │ │ │ ├── quicksand-v24-latin-ext_latin-600.svg │ │ │ ├── quicksand-v24-latin-ext_latin-600.ttf │ │ │ ├── quicksand-v24-latin-ext_latin-600.woff │ │ │ ├── quicksand-v24-latin-ext_latin-600.woff2 │ │ │ ├── quicksand-v24-latin-ext_latin-700.eot │ │ │ ├── quicksand-v24-latin-ext_latin-700.svg │ │ │ ├── quicksand-v24-latin-ext_latin-700.ttf │ │ │ ├── quicksand-v24-latin-ext_latin-700.woff │ │ │ ├── quicksand-v24-latin-ext_latin-700.woff2 │ │ │ ├── quicksand-v24-latin-ext_latin-regular.eot │ │ │ ├── quicksand-v24-latin-ext_latin-regular.svg │ │ │ ├── quicksand-v24-latin-ext_latin-regular.ttf │ │ │ ├── quicksand-v24-latin-ext_latin-regular.woff │ │ │ └── quicksand-v24-latin-ext_latin-regular.woff2 │ ├── globExample.png │ ├── login_wave.svg │ ├── logo.svg │ ├── logo2.png │ ├── pattern3.png │ ├── postgres.json │ ├── topicConfigInfo.json │ └── twitter.json ├── components │ ├── App.tsx │ ├── misc │ │ ├── BoxCard.module.scss │ │ ├── BoxCard.tsx │ │ ├── Card.tsx │ │ ├── ConfigList.module.scss │ │ ├── ConfigList.tsx │ │ ├── ErrorBoundary.tsx │ │ ├── ErrorDisplay.tsx │ │ ├── ErrorModal.tsx │ │ ├── HiddenRadioList.module.scss │ │ ├── HiddenRadioList.tsx │ │ ├── HideStatisticsBarButton.tsx │ │ ├── KowlEditor.tsx │ │ ├── KowlJsonView.module.scss │ │ ├── KowlJsonView.tsx │ │ ├── KowlTable.module.scss │ │ ├── KowlTable.tsx │ │ ├── KowlTimePicker.tsx │ │ ├── NoClipboardPopover.tsx │ │ ├── SearchBar.tsx │ │ ├── ShortNum.tsx │ │ ├── UserButton.tsx │ │ ├── UserPreferences.tsx │ │ ├── Wizard.module.scss │ │ ├── Wizard.tsx │ │ ├── common.tsx │ │ ├── login-complete.tsx │ │ ├── login.tsx │ │ └── tabs │ │ │ ├── Tabs.module.scss │ │ │ ├── Tabs.test.tsx │ │ │ └── Tabs.tsx │ ├── pages │ │ ├── Page.ts │ │ ├── Settings.tsx │ │ ├── UrlTestPage.tsx │ │ ├── acls │ │ │ └── Acl.List.tsx │ │ ├── admin │ │ │ ├── Admin.RoleBindings.tsx │ │ │ ├── Admin.Roles.tsx │ │ │ ├── Admin.Users.tsx │ │ │ └── AdminPage.tsx │ │ ├── brokers │ │ │ └── Broker.List.tsx │ │ ├── connect │ │ │ ├── Cluster.Details.tsx │ │ │ ├── Connector.Details.tsx │ │ │ ├── ConnectorBoxCard.module.scss │ │ │ ├── ConnectorBoxCard.tsx │ │ │ ├── CreateConnector.module.scss │ │ │ ├── CreateConnector.tsx │ │ │ ├── Overview.tsx │ │ │ ├── dynamic-ui │ │ │ │ ├── DebugEditor.tsx │ │ │ │ ├── List.tsx │ │ │ │ ├── PropertyComponent.tsx │ │ │ │ ├── PropertyGroup.tsx │ │ │ │ └── components.tsx │ │ │ └── helper.tsx │ │ ├── consumers │ │ │ ├── Group.Details.tsx │ │ │ ├── Group.List.tsx │ │ │ └── Modals.tsx │ │ ├── quotas │ │ │ └── Quotas.List.tsx │ │ ├── reassign-partitions │ │ │ ├── ReassignPartitions.tsx │ │ │ ├── Step1.Partitions.tsx │ │ │ ├── Step2.Brokers.tsx │ │ │ ├── Step3.Review.tsx │ │ │ ├── components │ │ │ │ ├── ActiveReassignments.tsx │ │ │ │ ├── BandwidthSlider.tsx │ │ │ │ ├── BrokerList.tsx │ │ │ │ ├── IndeterminateCheckbox.tsx │ │ │ │ ├── StatisticsBars.tsx │ │ │ │ └── tsxUtils.tsx │ │ │ └── logic │ │ │ │ ├── reassignLogic.ts │ │ │ │ ├── reassignTests.ts │ │ │ │ ├── reassignmentTracker.ts │ │ │ │ └── utils.ts │ │ ├── schemas │ │ │ ├── Schema.Details.tsx │ │ │ ├── Schema.List.scss │ │ │ └── Schema.List.tsx │ │ └── topics │ │ │ ├── DeleteRecordsModal │ │ │ ├── DeleteRecordsModal.module.scss │ │ │ ├── DeleteRecordsModal.test.tsx │ │ │ └── DeleteRecordsModal.tsx │ │ │ ├── PublishMessagesModal │ │ │ ├── Headers.tsx │ │ │ ├── PublishMessagesModal.tsx │ │ │ └── headersEditor.scss │ │ │ ├── QuickInfo.tsx │ │ │ ├── Tab.Acl │ │ │ ├── AclList.test.tsx │ │ │ └── AclList.tsx │ │ │ ├── Tab.Config.tsx │ │ │ ├── Tab.Consumers.tsx │ │ │ ├── Tab.Docu.tsx │ │ │ ├── Tab.Messages │ │ │ ├── PreviewSettings.tsx │ │ │ ├── index.tsx │ │ │ └── styles.module.scss │ │ │ ├── Tab.Partitions.tsx │ │ │ ├── Topic.Details.tsx │ │ │ └── Topic.List.tsx │ └── routes.tsx ├── index.scss ├── index.tsx ├── react-app-env.d.ts ├── setupTests.js ├── state │ ├── appGlobal.ts │ ├── backendApi.ts │ ├── restInterfaces.ts │ ├── supportedFeatures.ts │ ├── typeExperiments.ts │ ├── ui.ts │ └── uiState.ts └── utils │ ├── LazyMap.ts │ ├── animationProps.tsx │ ├── arrayExtensions.ts │ ├── createAutoModal.tsx │ ├── env.ts │ ├── extensions.ts │ ├── featureDetection.ts │ ├── fetchWithTimeout.ts │ ├── filterHelper.ts │ ├── filterableDataSource.ts │ ├── formatters │ └── ConfigValueFormatter.ts │ ├── interpreter │ ├── findFunction.test.ts │ ├── findFunction.ts │ ├── global.ts │ ├── helpers.ts │ └── tsconfig.json │ ├── jsonUtils.ts │ ├── numberExtensions.ts │ ├── queryHelper.ts │ ├── svg │ └── OktaLogo.tsx │ ├── tsxUtils.tsx │ └── utils.ts └── tsconfig.json /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.dockerignore 2 | **/.git 3 | **/.github 4 | **/.gitignore 5 | **/.vscode 6 | **/.idea 7 | 8 | **/Dockerfile* 9 | **/docker-compose* 10 | 11 | **/node_modules 12 | **/npm-debug.log 13 | **/README.md 14 | **/LICENSE 15 | **/build 16 | 17 | backend/e2e 18 | docs 19 | 20 | -------------------------------------------------------------------------------- /.github/workflows/build-image.yml: -------------------------------------------------------------------------------- 1 | 2 | name: Build & Push Docker Image 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | release: 9 | types: [published] 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@master 17 | 18 | - name: Extract ref/branch name 19 | shell: bash 20 | run: echo "##[set-output name=ref;]$(echo ${GITHUB_REF##*/})" 21 | id: extract_ref 22 | 23 | - name: Set timestamp 24 | shell: bash 25 | run: echo "::set-output name=time::$(date +%s)" 26 | id: set_timestamp 27 | 28 | - name: Login to Quay 29 | uses: docker/login-action@v1 30 | with: 31 | registry: quay.io 32 | username: cloudhut+github_push 33 | password: ${{ secrets.QUAY_ENCRYPTED_PASS }} 34 | 35 | - name: Docker meta 36 | id: meta 37 | uses: docker/metadata-action@v3 38 | with: 39 | # list of Docker images to use as base name for tags 40 | images: | 41 | quay.io/cloudhut/kowl 42 | # generate Docker tags based on the following events/attributes 43 | tags: | 44 | type=raw,value=master 45 | type=sha,prefix=master-,format=short 46 | type=semver,pattern={{raw}},enable=${{ github.event.action == 'published' }} 47 | 48 | # Use Buildx with moby/buildkit to utilize registry caching and more 49 | # advanced building techniques (e.g. concurrent multistage builds) 50 | - name: Set up Docker Buildx 51 | uses: docker/setup-buildx-action@v1 52 | with: 53 | driver-opts: image=moby/buildkit:v0.9.0,network=host 54 | 55 | - name: Build and push Docker Image 56 | uses: docker/build-push-action@v2 57 | with: 58 | context: . 59 | push: true 60 | build-args: | 61 | KOWL_GIT_SHA=${{ github.sha }} 62 | KOWL_GIT_REF=${{ steps.extract_ref.outputs.ref }} 63 | KOWL_TIMESTAMP=${{ steps.set_timestamp.outputs.time }} 64 | BUILT_FROM_PUSH=${{ github.event.action != 'published' }} 65 | tags: ${{ steps.meta.outputs.tags }} 66 | cache-from: type=gha 67 | cache-to: type=gha,mode=max 68 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "Code scanning - action" 2 | 3 | on: 4 | push: 5 | pull_request: 6 | schedule: 7 | - cron: "0 13 * * 5" 8 | 9 | jobs: 10 | CodeQL-Build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout repository 15 | uses: actions/checkout@v2 16 | with: 17 | # We must fetch at least the immediate parents so that if this is 18 | # a pull request then we can checkout the head. 19 | fetch-depth: 2 20 | 21 | # Initializes the CodeQL tools for scanning. 22 | - name: Initialize CodeQL 23 | uses: github/codeql-action/init@v1 24 | # Override language selection by uncommenting this and choosing your languages 25 | with: 26 | languages: go, javascript 27 | 28 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 29 | # If this step fails, then you should remove it and run the build manually (see below) 30 | - name: Autobuild 31 | uses: github/codeql-action/autobuild@v1 32 | 33 | # ℹ️ Command-line programs to run using the OS shell. 34 | # 📚 https://git.io/JvXDl 35 | 36 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 37 | # and modify them (or add more) to build your code if your project 38 | # uses a compiled language 39 | 40 | #- run: | 41 | # make bootstrap 42 | # make release 43 | 44 | - name: Perform CodeQL Analysis 45 | uses: github/codeql-action/analyze@v1 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Docker Compose 2 | docker-compose/zk-single-kafka-single 3 | 4 | # Misc 5 | .DS_Store 6 | notes.md 7 | fixname.sh 8 | 9 | # IDEs 10 | **/.vscode 11 | **/.idea 12 | **/*.code-workspace 13 | 14 | # Helper Scripts 15 | run-docker.sh 16 | run-backend.sh 17 | requests.txt 18 | 19 | # Local Formatting 20 | .prettierrc 21 | 22 | docker-compose.yml -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Kowl Security and Disclosure Information 2 | 3 | As with any complex system, it is certain that bugs will be found, some of them security-relevant. 4 | If you find a _security bug_ please report it privately via email to info@cloudhut.dev. We will fix the issue as soon 5 | as possible and coordinate a release date with you. You will be able to choose if you want public acknowledgement of 6 | your effort and if you want to be mentioned by name. 7 | 8 | ## Public Disclosure Timing 9 | 10 | The public disclosure date is agreed between the CloudHut Team and the bug submitter. 11 | We prefer to fully disclose the bug as soon as possible, but only after a mitigation or fix is available. 12 | We will ask for delay if the bug or the fix is not yet fully understood or the solution is not tested to our standards 13 | yet. While there is no fixed time frame for fix & disclosure, we will try our best to be quick and do not expect to need 14 | the usual 90 days most companies ask or. For a vulnerability with a straightforward mitigation, we expect report date to 15 | disclosure date to be on the order of 7 days. 16 | -------------------------------------------------------------------------------- /backend/.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | __debug_bin 14 | 15 | # IDEs 16 | .idea 17 | .vscode 18 | 19 | # e2e data 20 | e2e/data 21 | 22 | # Compiled frontend 23 | build 24 | 25 | # Local configs 26 | config.yaml 27 | config 28 | config-*.yaml 29 | 30 | frontend/src/utils/interpreter/compiled 31 | 32 | *.pem -------------------------------------------------------------------------------- /backend/cmd/api/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | 6 | "github.com/cloudhut/kowl/backend/pkg/api" 7 | ) 8 | 9 | func main() { 10 | startupLogger := zap.NewExample() 11 | 12 | cfg, err := api.LoadConfig(startupLogger) 13 | if err != nil { 14 | startupLogger.Fatal("failed to load config", zap.Error(err)) 15 | } 16 | err = cfg.Validate() 17 | if err != nil { 18 | startupLogger.Fatal("failed to validate config", zap.Error(err)) 19 | } 20 | 21 | a := api.New(&cfg) 22 | a.Start() 23 | } 24 | -------------------------------------------------------------------------------- /backend/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/cloudhut/kowl/backend 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/ProtonMail/go-crypto v0.0.0-20210707164159-52430bf6b52c // indirect 7 | github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect 8 | github.com/basgys/goxml2json v1.1.0 9 | github.com/bitly/go-simplejson v0.5.0 // indirect 10 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 11 | github.com/cloudhut/common v0.6.0 12 | github.com/cloudhut/connect-client v0.0.0-20211109055846-c9f53449bdc5 13 | github.com/dop251/goja v0.0.0-20210406175830-1b11a6af686d 14 | github.com/gliderlabs/ssh v0.3.2 // indirect 15 | github.com/go-chi/chi v4.1.2+incompatible 16 | github.com/go-git/go-billy/v5 v5.3.1 17 | github.com/go-git/go-git/v5 v5.4.2 18 | github.com/go-resty/resty/v2 v2.7.0 19 | github.com/golang/snappy v0.0.4 // indirect 20 | github.com/google/uuid v1.3.0 // indirect 21 | github.com/gorilla/schema v1.2.0 22 | github.com/gorilla/websocket v1.4.2 23 | github.com/jarcoal/httpmock v1.0.8 24 | github.com/jcmturner/gokrb5/v8 v8.4.2 25 | github.com/jhump/protoreflect v1.8.2 26 | github.com/kevinburke/ssh_config v1.1.0 // indirect 27 | github.com/knadh/koanf v0.16.0 28 | github.com/linkedin/goavro/v2 v2.10.0 29 | github.com/mitchellh/copystructure v1.1.2 // indirect 30 | github.com/mitchellh/mapstructure v1.4.1 31 | github.com/pkg/errors v0.9.1 32 | github.com/prometheus/client_golang v1.10.0 33 | github.com/prometheus/common v0.20.0 // indirect 34 | github.com/sergi/go-diff v1.2.0 // indirect 35 | github.com/stretchr/testify v1.7.0 36 | github.com/twmb/franz-go v1.4.2 37 | github.com/twmb/franz-go/pkg/kmsg v1.0.0 38 | github.com/vmihailenco/msgpack/v5 v5.3.1 39 | github.com/xanzy/ssh-agent v0.3.1 // indirect 40 | go.uber.org/zap v1.16.0 41 | golang.org/x/mod v0.4.1 // indirect 42 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f 43 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 44 | google.golang.org/genproto v0.0.0-20210416161957-9910b6c460de // indirect 45 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 46 | honnef.co/go/tools v0.1.1 // indirect 47 | ) 48 | -------------------------------------------------------------------------------- /backend/pkg/api/handle_api_versions.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/cloudhut/common/rest" 5 | "github.com/cloudhut/kowl/backend/pkg/owl" 6 | "net/http" 7 | ) 8 | 9 | func (api *API) handleGetAPIVersions() http.HandlerFunc { 10 | type response struct { 11 | APIVersions []owl.APIVersion `json:"apiVersions"` 12 | } 13 | 14 | return func(w http.ResponseWriter, r *http.Request) { 15 | versions, err := api.OwlSvc.GetAPIVersions(r.Context()) 16 | if err != nil { 17 | restErr := &rest.Error{ 18 | Err: err, 19 | Status: http.StatusInternalServerError, 20 | Message: "Could not get Kafka API versions", 21 | IsSilent: false, 22 | } 23 | rest.SendRESTError(w, r, api.Logger, restErr) 24 | return 25 | } 26 | 27 | response := response{ 28 | APIVersions: versions, 29 | } 30 | rest.SendResponse(w, r, api.Logger, http.StatusOK, response) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/api/handle_brokers.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "github.com/cloudhut/common/rest" 6 | "github.com/cloudhut/kowl/backend/pkg/owl" 7 | "github.com/go-chi/chi" 8 | "net/http" 9 | "strconv" 10 | ) 11 | 12 | func (api *API) handleBrokerConfig() http.HandlerFunc { 13 | type response struct { 14 | BrokerConfigs []owl.BrokerConfigEntry `json:"brokerConfigs"` 15 | } 16 | 17 | return func(w http.ResponseWriter, r *http.Request) { 18 | // 1. Parse broker ID parameter and validate input 19 | brokerIDStr := chi.URLParam(r, "brokerID") 20 | if brokerIDStr == "" || len(brokerIDStr) > 10 { 21 | restErr := &rest.Error{ 22 | Err: fmt.Errorf("broker id in URL not set"), 23 | Status: http.StatusBadRequest, 24 | Message: "Broker ID must be set and no longer than 10 characters", 25 | IsSilent: true, 26 | } 27 | rest.SendRESTError(w, r, api.Logger, restErr) 28 | return 29 | } 30 | brokerID, err := strconv.ParseInt(brokerIDStr, 10, 32) 31 | if err != nil { 32 | restErr := &rest.Error{ 33 | Err: fmt.Errorf("broker id in URL not set"), 34 | Status: http.StatusBadRequest, 35 | Message: "Broker ID must be a valid int32", 36 | IsSilent: true, 37 | } 38 | rest.SendRESTError(w, r, api.Logger, restErr) 39 | return 40 | } 41 | 42 | cfgs, restErr := api.OwlSvc.GetBrokerConfig(r.Context(), int32(brokerID)) 43 | if restErr != nil { 44 | rest.SendRESTError(w, r, api.Logger, restErr) 45 | return 46 | } 47 | 48 | response := response{ 49 | BrokerConfigs: cfgs, 50 | } 51 | rest.SendResponse(w, r, api.Logger, http.StatusOK, response) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /backend/pkg/api/handle_cluster.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/cloudhut/common/rest" 5 | "github.com/cloudhut/kowl/backend/pkg/owl" 6 | "net/http" 7 | ) 8 | 9 | func (api *API) handleDescribeCluster() http.HandlerFunc { 10 | type response struct { 11 | ClusterInfo *owl.ClusterInfo `json:"clusterInfo"` 12 | } 13 | 14 | return func(w http.ResponseWriter, r *http.Request) { 15 | clusterInfo, err := api.OwlSvc.GetClusterInfo(r.Context()) 16 | if err != nil { 17 | restErr := &rest.Error{ 18 | Err: err, 19 | Status: http.StatusInternalServerError, 20 | Message: "Could not describe cluster", 21 | IsSilent: false, 22 | } 23 | rest.SendRESTError(w, r, api.Logger, restErr) 24 | return 25 | } 26 | 27 | response := response{ 28 | ClusterInfo: clusterInfo, 29 | } 30 | rest.SendResponse(w, r, api.Logger, http.StatusOK, response) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/api/handle_kowl.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/cloudhut/common/rest" 5 | "github.com/cloudhut/kowl/backend/pkg/owl" 6 | "net/http" 7 | ) 8 | 9 | func (api *API) handleGetEndpoints() http.HandlerFunc { 10 | type response struct { 11 | EndpointCompatibility owl.EndpointCompatibility `json:"endpointCompatibility"` 12 | } 13 | 14 | return func(w http.ResponseWriter, r *http.Request) { 15 | endpointCompatibility, err := api.OwlSvc.GetEndpointCompatibility(r.Context()) 16 | if err != nil { 17 | restErr := &rest.Error{ 18 | Err: err, 19 | Status: http.StatusInternalServerError, 20 | Message: "Could not get cluster config", 21 | IsSilent: false, 22 | } 23 | rest.SendRESTError(w, r, api.Logger, restErr) 24 | return 25 | } 26 | 27 | response := response{ 28 | EndpointCompatibility: endpointCompatibility, 29 | } 30 | rest.SendResponse(w, r, api.Logger, http.StatusOK, response) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/api/handle_probes.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/cloudhut/common/rest" 5 | "net/http" 6 | ) 7 | 8 | func (api *API) handleLivenessProbe() http.HandlerFunc { 9 | type response struct { 10 | IsHTTPOk bool `json:"isHttpOk"` 11 | } 12 | 13 | return func(w http.ResponseWriter, r *http.Request) { 14 | res := &response{ 15 | IsHTTPOk: true, 16 | } 17 | 18 | rest.SendResponse(w, r, api.Logger, http.StatusOK, res) 19 | } 20 | } 21 | 22 | func (api *API) handleStartupProbe() http.HandlerFunc { 23 | type response struct { 24 | IsHTTPOk bool `json:"isHttpOk"` 25 | IsKafkaOk bool `json:"isKafkaOk"` 26 | } 27 | 28 | return func(w http.ResponseWriter, r *http.Request) { 29 | // Check Kafka connectivity 30 | isKafkaOK := false 31 | err := api.KafkaSvc.IsHealthy(r.Context()) 32 | if err == nil { 33 | isKafkaOK = true 34 | } 35 | 36 | res := &response{ 37 | IsHTTPOk: true, 38 | IsKafkaOk: isKafkaOK, 39 | } 40 | 41 | rest.SendResponse(w, r, api.Logger, http.StatusOK, res) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /backend/pkg/api/handle_quotas.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "github.com/cloudhut/common/rest" 6 | "net/http" 7 | ) 8 | 9 | func (api *API) handleGetQuotas() http.HandlerFunc { 10 | return func(w http.ResponseWriter, r *http.Request) { 11 | quotas := api.OwlSvc.DescribeQuotas(r.Context()) 12 | 13 | // Check if logged in user is allowed to list Quotas 14 | isAllowed, restErr := api.Hooks.Owl.CanListQuotas(r.Context()) 15 | if restErr != nil { 16 | rest.SendRESTError(w, r, api.Logger, restErr) 17 | return 18 | } 19 | if !isAllowed { 20 | rest.SendRESTError(w, r, api.Logger, &rest.Error{ 21 | Err: fmt.Errorf("requester is not allowed to list Quotas"), 22 | Status: http.StatusForbidden, 23 | Message: "You are not allowed to list ACLs", 24 | IsSilent: true, 25 | }) 26 | return 27 | } 28 | 29 | rest.SendResponse(w, r, api.Logger, http.StatusOK, quotas) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /backend/pkg/api/handle_schema.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "github.com/cloudhut/common/rest" 6 | "github.com/cloudhut/kowl/backend/pkg/owl" 7 | "github.com/go-chi/chi" 8 | "net/http" 9 | ) 10 | 11 | func (api *API) handleGetSchemaOverview() http.HandlerFunc { 12 | type response struct { 13 | SchemaOverview *owl.SchemaOverview `json:"schemaOverview"` 14 | IsConfigured bool `json:"isConfigured"` 15 | } 16 | 17 | return func(w http.ResponseWriter, r *http.Request) { 18 | overview, err := api.OwlSvc.GetSchemaOverview(r.Context()) 19 | if err != nil { 20 | if err == owl.ErrSchemaRegistryNotConfigured { 21 | rest.SendResponse(w, r, api.Logger, http.StatusOK, &response{ 22 | SchemaOverview: nil, 23 | IsConfigured: false, 24 | }) 25 | return 26 | } 27 | 28 | rest.SendRESTError(w, r, api.Logger, &rest.Error{ 29 | Err: err, 30 | Status: http.StatusServiceUnavailable, 31 | Message: fmt.Sprintf("Schema overview request has failed: %v", err.Error()), 32 | IsSilent: false, 33 | }) 34 | return 35 | } 36 | 37 | rest.SendResponse(w, r, api.Logger, http.StatusOK, &response{ 38 | SchemaOverview: overview, 39 | IsConfigured: true, 40 | }) 41 | } 42 | } 43 | 44 | func (api *API) handleGetSchemaDetails() http.HandlerFunc { 45 | type response struct { 46 | SchemaDetails *owl.SchemaDetails `json:"schemaDetails"` 47 | IsConfigured bool `json:"isConfigured"` 48 | } 49 | 50 | return func(w http.ResponseWriter, r *http.Request) { 51 | subject := chi.URLParam(r, "subject") 52 | version := chi.URLParam(r, "version") 53 | schemaDetails, err := api.OwlSvc.GetSchemaDetails(r.Context(), subject, version) 54 | if err != nil { 55 | if err == owl.ErrSchemaRegistryNotConfigured { 56 | rest.SendResponse(w, r, api.Logger, http.StatusOK, &response{ 57 | SchemaDetails: nil, 58 | IsConfigured: false, 59 | }) 60 | return 61 | } 62 | 63 | rest.SendRESTError(w, r, api.Logger, &rest.Error{ 64 | Err: err, 65 | Status: http.StatusServiceUnavailable, 66 | Message: "Schema overview request has failed. Look into the server logs for more details.", 67 | IsSilent: false, 68 | }) 69 | return 70 | } 71 | 72 | rest.SendResponse(w, r, api.Logger, http.StatusOK, &response{ 73 | SchemaDetails: schemaDetails, 74 | IsConfigured: true, 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /backend/pkg/api/handle_topic_documentation.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/cloudhut/common/rest" 5 | "github.com/cloudhut/kowl/backend/pkg/owl" 6 | "github.com/go-chi/chi" 7 | "go.uber.org/zap" 8 | "net/http" 9 | ) 10 | 11 | // handleGetTopicDocumentation returns the respective topic documentation from the git repository 12 | func (api *API) handleGetTopicDocumentation() http.HandlerFunc { 13 | type response struct { 14 | TopicName string `json:"topicName"` 15 | Documentation *owl.TopicDocumentation `json:"documentation"` 16 | } 17 | 18 | return func(w http.ResponseWriter, r *http.Request) { 19 | topicName := chi.URLParam(r, "topicName") 20 | logger := api.Logger.With(zap.String("topic_name", topicName)) 21 | 22 | doc := api.OwlSvc.GetTopicDocumentation(topicName) 23 | 24 | rest.SendResponse(w, r, logger, http.StatusOK, &response{ 25 | TopicName: topicName, 26 | Documentation: doc, 27 | }) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /backend/pkg/api/validator.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import "regexp" 4 | 5 | var ( 6 | isAlpha = regexp.MustCompile(`^[A-Za-z]+$`).MatchString 7 | isValidKafkaTopicName = regexp.MustCompile(`^[\w.-]+$`).MatchString 8 | ) 9 | -------------------------------------------------------------------------------- /backend/pkg/api/validator_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestIsValidKafkaTopicname(t *testing.T) { 11 | tests := map[string]struct { 12 | input string 13 | want bool 14 | }{ 15 | // Valids 16 | "alphanumeric": {input: "abc123", want: true}, 17 | "alphanumeric-with-upper-case": {input: "ABC123", want: true}, 18 | "alphanumeric-with-mixed-case": {input: "abcABC123", want: true}, 19 | "alphanumeric-with-dash": {input: "abc-ABC-123", want: true}, 20 | "alphanumeric-with-dot": {input: "abc.ABC.123", want: true}, 21 | "alphanumeric-with-underscore": {input: "abc_ABC_123", want: true}, 22 | "alphanumeric-with-allowed-special-chars": {input: "abc.ABC-abc_should_work.123", want: true}, 23 | 24 | // Invalids 25 | "alphanumeric-with-colon": {input: "abc:ABC", want: false}, 26 | "alphanumeric-with-umlaut": {input: "abcÄBC", want: false}, 27 | "alphanumeric-with-special-chars": {input: "abc@ABC", want: false}, 28 | } 29 | 30 | for name, tc := range tests { 31 | got := isValidKafkaTopicName(tc.input) 32 | assert.Equal(t, tc.want, got, fmt.Sprintf("Name: %v, Input: '%v'", name, tc.input)) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /backend/pkg/api/version.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | "time" 7 | 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type versionInfo struct { 12 | isBusiness bool 13 | productName string 14 | 15 | timestamp time.Time 16 | timestampFriendly string 17 | 18 | gitSha string 19 | gitRef string 20 | 21 | gitShaBusiness string 22 | gitRefBusiness string 23 | } 24 | 25 | // loadVersionInfo loads various environment variables containing information 26 | // about the version/build (git shas, timestamp, branches) into a struct that can be worked with more easily 27 | func loadVersionInfo(logger *zap.Logger) versionInfo { 28 | version := versionInfo{ 29 | isBusiness: false, 30 | gitSha: os.Getenv("REACT_APP_KOWL_GIT_SHA"), 31 | gitRef: os.Getenv("REACT_APP_KOWL_GIT_REF"), 32 | gitShaBusiness: os.Getenv("REACT_APP_KOWL_BUSINESS_GIT_SHA"), 33 | gitRefBusiness: os.Getenv("REACT_APP_KOWL_BUSINESS_GIT_REF"), 34 | timestamp: time.Time{}, 35 | } 36 | 37 | timestamp := os.Getenv("REACT_APP_KOWL_TIMESTAMP") 38 | version.productName = "Kowl" 39 | if version.gitShaBusiness != "" { 40 | version.productName = "Kowl Business" 41 | version.isBusiness = true 42 | } 43 | 44 | // Early out: dev mode 45 | if version.gitSha == "" { 46 | version.gitSha = "dev" 47 | return version 48 | } 49 | 50 | // Parse timestamp 51 | t1, err := strconv.ParseInt(timestamp, 10, 64) 52 | if err != nil { 53 | logger.Warn("failed to parse timestamp as int64", zap.String("timestamp", timestamp), zap.Error(err)) 54 | version.timestampFriendly = "(parsing error)" 55 | } else { 56 | version.timestamp = time.Unix(t1, 0) 57 | version.timestampFriendly = version.timestamp.Format(time.RFC3339) 58 | } 59 | 60 | return version 61 | } 62 | -------------------------------------------------------------------------------- /backend/pkg/connect/config.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | ) 7 | 8 | type Config struct { 9 | Enabled bool `yaml:"enabled"` 10 | Clusters []ConfigCluster `yaml:"clusters"` 11 | } 12 | 13 | func (c *Config) SetDefaults() { 14 | for _, cluster := range c.Clusters { 15 | cluster.SetDefaults() 16 | } 17 | } 18 | 19 | // RegisterFlags registers all nested config flags. 20 | func (c *Config) RegisterFlags(f *flag.FlagSet) { 21 | for i, cluster := range c.Clusters { 22 | flagNamePrefix := fmt.Sprintf("connect.clusters.%d.", i) 23 | cluster.RegisterFlagsWithPrefix(f, flagNamePrefix) 24 | } 25 | } 26 | 27 | func (c *Config) Validate() error { 28 | for i, cluster := range c.Clusters { 29 | err := cluster.Validate() 30 | if err != nil { 31 | return fmt.Errorf("failed to validate cluster at index '%d' (name: '%v'): %w", i, cluster.Name, err) 32 | } 33 | } 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /backend/pkg/connect/config_cluster.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | ) 7 | 8 | type ConfigCluster struct { 9 | // Name will be shown in the Frontend to identify a connect cluster 10 | Name string `yaml:"name"` 11 | // URL is the HTTP address that will be set as base url for all requests 12 | URL string `yaml:"url"` 13 | 14 | // Authentication configuration 15 | // 16 | TLS ConfigClusterTLS `yaml:"tls"` 17 | Username string `yaml:"username"` 18 | Password string `yaml:"password"` 19 | Token string `yaml:"token"` 20 | } 21 | 22 | // RegisterFlagsWithPrefix registers all nested config flags. 23 | func (c *ConfigCluster) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { 24 | f.StringVar(&c.Password, prefix+"password", "", "Basic auth password for connect cluster authentication") 25 | f.StringVar(&c.Token, prefix+"token", "", "Bearer token for connect cluster authentication") 26 | } 27 | 28 | func (c *ConfigCluster) SetDefaults() { 29 | c.TLS.SetDefaults() 30 | } 31 | 32 | func (c *ConfigCluster) Validate() error { 33 | if c.Name == "" { 34 | return fmt.Errorf("a cluster name must be set to identify the connect cluster") 35 | } 36 | 37 | if c.URL == "" { 38 | return fmt.Errorf("url to access the Connect cluster API must be set") 39 | } 40 | 41 | err := c.TLS.Validate() 42 | if err != nil { 43 | return fmt.Errorf("failed to validate TLS config: %w", err) 44 | } 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /backend/pkg/connect/config_cluster_tls.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "encoding/pem" 7 | "fmt" 8 | "io/ioutil" 9 | ) 10 | 11 | // ConfigClusterTLS is the config if you want to connect to Kafka connect REST API via (mutual) TLS 12 | type ConfigClusterTLS struct { 13 | Enabled bool `yaml:"enabled"` 14 | CaFilepath string `yaml:"caFilepath"` 15 | CertFilepath string `yaml:"certFilepath"` 16 | KeyFilepath string `yaml:"keyFilepath"` 17 | InsecureSkipTLSVerify bool `yaml:"insecureSkipTlsVerify"` 18 | } 19 | 20 | func (c *ConfigClusterTLS) SetDefaults() { 21 | c.Enabled = false 22 | } 23 | 24 | func (c *ConfigClusterTLS) Validate() error { 25 | if !c.Enabled { 26 | return nil 27 | } 28 | 29 | return nil 30 | } 31 | 32 | func (c *ConfigClusterTLS) TLSConfig() (*tls.Config, error) { 33 | if !c.Enabled { 34 | return &tls.Config{}, nil 35 | } 36 | 37 | // 1. Create CA cert pool 38 | caCertPool := x509.NewCertPool() 39 | if c.CaFilepath != "" { 40 | ca, err := ioutil.ReadFile(c.CaFilepath) 41 | if err != nil { 42 | return nil, err 43 | } 44 | isSuccessful := caCertPool.AppendCertsFromPEM(ca) 45 | if !isSuccessful { 46 | return nil, fmt.Errorf("failed to append ca file to cert pool, is this a valid PEM format?") 47 | } 48 | } 49 | 50 | // 2. If configured load TLS cert & key - Mutual TLS 51 | var certificates []tls.Certificate 52 | if c.CertFilepath != "" && c.KeyFilepath != "" { 53 | cert, err := ioutil.ReadFile(c.CertFilepath) 54 | if err != nil { 55 | return nil, fmt.Errorf("failed to read cert file for schema registry client: %w", err) 56 | } 57 | 58 | privateKey, err := ioutil.ReadFile(c.KeyFilepath) 59 | if err != nil { 60 | return nil, fmt.Errorf("failed to read key file for schema registry client: %w", err) 61 | } 62 | 63 | pemBlock, _ := pem.Decode(privateKey) 64 | if pemBlock == nil { 65 | return nil, fmt.Errorf("no valid private key found") 66 | } 67 | 68 | tlsCert, err := tls.X509KeyPair(cert, privateKey) 69 | if err != nil { 70 | return nil, fmt.Errorf("failed to load certificate pair for schema registry client: %w", err) 71 | } 72 | certificates = []tls.Certificate{tlsCert} 73 | } 74 | 75 | return &tls.Config{ 76 | InsecureSkipVerify: c.InsecureSkipTLSVerify, 77 | Certificates: certificates, 78 | RootCAs: caCertPool, 79 | }, nil 80 | } 81 | -------------------------------------------------------------------------------- /backend/pkg/connect/create_connector.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/common/rest" 7 | con "github.com/cloudhut/connect-client" 8 | "go.uber.org/zap" 9 | "go.uber.org/zap/zapcore" 10 | "net/http" 11 | ) 12 | 13 | func (s *Service) CreateConnector(ctx context.Context, clusterName string, req con.CreateConnectorRequest) (con.ConnectorInfo, *rest.Error) { 14 | c, restErr := s.getConnectClusterByName(clusterName) 15 | if restErr != nil { 16 | return con.ConnectorInfo{}, restErr 17 | } 18 | 19 | cInfo, err := c.Client.CreateConnector(ctx, req) 20 | if err != nil { 21 | return con.ConnectorInfo{}, &rest.Error{ 22 | Err: fmt.Errorf("failed to create connector: %w", err), 23 | Status: http.StatusOK, 24 | Message: fmt.Sprintf("Failed to create Connector: %v", err.Error()), 25 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName)}, 26 | IsSilent: false, 27 | } 28 | } 29 | 30 | return cInfo, nil 31 | } 32 | -------------------------------------------------------------------------------- /backend/pkg/connect/delete_connector.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/common/rest" 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | "net/http" 10 | ) 11 | 12 | // DeleteConnector deletes a connector, halting all tasks and deleting its configuration. 13 | // Returns 409 (Conflict) if a rebalance is in process. 14 | func (s *Service) DeleteConnector(ctx context.Context, clusterName string, connector string) *rest.Error { 15 | c, restErr := s.getConnectClusterByName(clusterName) 16 | if restErr != nil { 17 | return restErr 18 | } 19 | 20 | err := c.Client.DeleteConnector(ctx, connector) 21 | if err != nil { 22 | return &rest.Error{ 23 | Err: err, 24 | Status: http.StatusServiceUnavailable, 25 | Message: fmt.Sprintf("Failed to delete connector: %v", err.Error()), 26 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName), zap.String("connector", connector)}, 27 | IsSilent: false, 28 | } 29 | } 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/connect/errors.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import "github.com/pkg/errors" 4 | 5 | var ( 6 | ErrKafkaConnectNotConfigured = errors.New("kafka connect not configured") 7 | ) 8 | -------------------------------------------------------------------------------- /backend/pkg/connect/get_cluster_info.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/common/rest" 7 | "github.com/cloudhut/connect-client" 8 | "go.uber.org/zap" 9 | "go.uber.org/zap/zapcore" 10 | "net/http" 11 | ) 12 | 13 | type ClusterInfo struct { 14 | Name string `json:"clusterName"` 15 | Host string `json:"host"` 16 | Version string `json:"clusterVersion"` 17 | Plugins []connect.ConnectorPluginInfo `json:"plugins"` 18 | } 19 | 20 | func (s *Service) GetClusterInfo(ctx context.Context, clusterName string) (ClusterInfo, *rest.Error) { 21 | c, restErr := s.getConnectClusterByName(clusterName) 22 | if restErr != nil { 23 | return ClusterInfo{}, restErr 24 | } 25 | 26 | rootInfo, err := c.Client.GetRoot(ctx) 27 | if err != nil { 28 | return ClusterInfo{}, &rest.Error{ 29 | Err: err, 30 | Status: http.StatusServiceUnavailable, 31 | Message: fmt.Sprintf("Failed to get cluster info: %v", err.Error()), 32 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName)}, 33 | IsSilent: false, 34 | } 35 | } 36 | 37 | plugins, err := c.Client.GetConnectorPlugins(ctx) 38 | if err != nil { 39 | return ClusterInfo{}, &rest.Error{ 40 | Err: err, 41 | Status: http.StatusServiceUnavailable, 42 | Message: fmt.Sprintf("Failed to get cluster plugins: %v", err.Error()), 43 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName)}, 44 | IsSilent: false, 45 | } 46 | } 47 | 48 | return ClusterInfo{ 49 | Name: c.Cfg.Name, 50 | Host: c.Cfg.URL, 51 | Version: rootInfo.Version, 52 | Plugins: plugins, 53 | }, nil 54 | } 55 | -------------------------------------------------------------------------------- /backend/pkg/connect/pause_connector.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/common/rest" 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | "net/http" 10 | ) 11 | 12 | // PauseConnector pauses the connector and its tasks, which stops message processing until the connector is resumed. 13 | // This call asynchronous and the tasks will not transition to PAUSED state at the same time. 14 | func (s *Service) PauseConnector(ctx context.Context, clusterName string, connector string) *rest.Error { 15 | c, restErr := s.getConnectClusterByName(clusterName) 16 | if restErr != nil { 17 | return restErr 18 | } 19 | 20 | err := c.Client.PauseConnector(ctx, connector) 21 | if err != nil { 22 | return &rest.Error{ 23 | Err: err, 24 | Status: http.StatusServiceUnavailable, 25 | Message: fmt.Sprintf("Failed to pause connector: %v", err.Error()), 26 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName), zap.String("connector", connector)}, 27 | IsSilent: false, 28 | } 29 | } 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/connect/put_connector_config.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/cloudhut/common/rest" 9 | con "github.com/cloudhut/connect-client" 10 | "go.uber.org/zap" 11 | "go.uber.org/zap/zapcore" 12 | ) 13 | 14 | func (s *Service) PutConnectorConfig(ctx context.Context, clusterName string, connectorName string, req con.PutConnectorConfigOptions) (con.ConnectorInfo, *rest.Error) { 15 | c, restErr := s.getConnectClusterByName(clusterName) 16 | if restErr != nil { 17 | return con.ConnectorInfo{}, restErr 18 | } 19 | 20 | cInfo, err := c.Client.PutConnectorConfig(ctx, connectorName, req) 21 | if err != nil { 22 | return con.ConnectorInfo{}, &rest.Error{ 23 | Err: fmt.Errorf("failed to patch connector config: %w", err), 24 | Status: http.StatusOK, 25 | Message: fmt.Sprintf("Failed to patch Connector config: %v", err.Error()), 26 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName), zap.String("connector_name", connectorName)}, 27 | IsSilent: false, 28 | } 29 | } 30 | 31 | return cInfo, nil 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/connect/restart_connector.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/common/rest" 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | "net/http" 10 | ) 11 | 12 | // RestartConnector restarts the connector. Return 409 (Conflict) if rebalance is in process. 13 | // No tasks are restarted as a result of a call to this endpoint. To restart tasks, see restart task. 14 | func (s *Service) RestartConnector(ctx context.Context, clusterName string, connector string) *rest.Error { 15 | c, restErr := s.getConnectClusterByName(clusterName) 16 | if restErr != nil { 17 | return restErr 18 | } 19 | 20 | err := c.Client.RestartConnector(ctx, connector) 21 | if err != nil { 22 | return &rest.Error{ 23 | Err: err, 24 | Status: http.StatusServiceUnavailable, 25 | Message: fmt.Sprintf("Failed to pause connector: %v", err.Error()), 26 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName), zap.String("connector", connector)}, 27 | IsSilent: false, 28 | } 29 | } 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/connect/restart_task.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/common/rest" 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | "net/http" 10 | ) 11 | 12 | // RestartConnectorTask restart an individual task. 13 | func (s *Service) RestartConnectorTask(ctx context.Context, clusterName string, connector string, taskID int) *rest.Error { 14 | c, restErr := s.getConnectClusterByName(clusterName) 15 | if restErr != nil { 16 | return restErr 17 | } 18 | 19 | err := c.Client.RestartConnectorTask(ctx, connector, taskID) 20 | if err != nil { 21 | return &rest.Error{ 22 | Err: err, 23 | Status: http.StatusServiceUnavailable, 24 | Message: fmt.Sprintf("Failed to restart connector task: %v", err.Error()), 25 | InternalLogs: []zapcore.Field{ 26 | zap.String("cluster_name", clusterName), 27 | zap.String("connector", connector), 28 | zap.Int("task_id", taskID)}, 29 | IsSilent: false, 30 | } 31 | } 32 | 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /backend/pkg/connect/resume_connector.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/common/rest" 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | "net/http" 10 | ) 11 | 12 | // ResumeConnector resumes a paused connector or do nothing if the connector is not paused. 13 | // This call asynchronous and the tasks will not transition to RUNNING state at the same time. 14 | func (s *Service) ResumeConnector(ctx context.Context, clusterName string, connector string) *rest.Error { 15 | c, restErr := s.getConnectClusterByName(clusterName) 16 | if restErr != nil { 17 | return restErr 18 | } 19 | 20 | err := c.Client.ResumeConnector(ctx, connector) 21 | if err != nil { 22 | return &rest.Error{ 23 | Err: err, 24 | Status: http.StatusServiceUnavailable, 25 | Message: fmt.Sprintf("Failed to pause connector: %v", err.Error()), 26 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName), zap.String("connector", connector)}, 27 | IsSilent: false, 28 | } 29 | } 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/connect/util.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "fmt" 5 | "github.com/cloudhut/common/rest" 6 | "go.uber.org/zap" 7 | "go.uber.org/zap/zapcore" 8 | "net/http" 9 | ) 10 | 11 | // getMapValueOrString returns the map entry for the given key. If this entry does not exist it will return the 12 | // passed fallback string. 13 | func getMapValueOrString(m map[string]string, key string, fallback string) string { 14 | if val, exists := m[key]; exists { 15 | return val 16 | } 17 | 18 | return fallback 19 | } 20 | 21 | func (s *Service) getConnectClusterByName(clusterName string) (*ClientWithConfig, *rest.Error) { 22 | c, exists := s.ClientsByCluster[clusterName] 23 | if !exists { 24 | return nil, &rest.Error{ 25 | Err: fmt.Errorf("a client for the given cluster name does not exist"), 26 | Status: http.StatusNotFound, 27 | Message: "There's no configured cluster with the given connect cluster name", 28 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName)}, 29 | IsSilent: false, 30 | } 31 | } 32 | 33 | return c, nil 34 | } 35 | -------------------------------------------------------------------------------- /backend/pkg/connect/validate_connector.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/cloudhut/common/rest" 9 | con "github.com/cloudhut/connect-client" 10 | "go.uber.org/zap" 11 | "go.uber.org/zap/zapcore" 12 | ) 13 | 14 | func (s *Service) ValidateConnectorConfig(ctx context.Context, clusterName string, pluginClassName string, options con.ValidateConnectorConfigOptions) (con.ConnectorValidationResult, *rest.Error) { 15 | c, restErr := s.getConnectClusterByName(clusterName) 16 | if restErr != nil { 17 | return con.ConnectorValidationResult{}, restErr 18 | } 19 | 20 | cValidationResult, err := c.Client.PutValidateConnectorConfig(ctx, pluginClassName, options) 21 | if err != nil { 22 | return con.ConnectorValidationResult{}, &rest.Error{ 23 | Err: fmt.Errorf("failed to validate connector config: %w", err), 24 | Status: http.StatusOK, 25 | Message: fmt.Sprintf("Failed to validate Connector config: %v", err.Error()), 26 | InternalLogs: []zapcore.Field{zap.String("cluster_name", clusterName), zap.String("plugin_class_name", pluginClassName)}, 27 | IsSilent: false, 28 | } 29 | } 30 | 31 | return cValidationResult, nil 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/filesystem/config.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // Config for Filesystem service 9 | type Config struct { 10 | Enabled bool `yaml:"enabled"` 11 | 12 | // AllowedFileExtensions specifies file extensions that shall be picked up. If at least one is specified all other 13 | // file extensions will be ignored. 14 | AllowedFileExtensions []string `yaml:"-"` 15 | 16 | // Max file size which will be considered. Files exceeding this size will be ignored and logged. 17 | MaxFileSize int64 `yaml:"-"` 18 | 19 | // Whether or not to use the filename or the full filepath as key in the map 20 | IndexByFullFilepath bool `yaml:"-"` 21 | 22 | // RefreshInterval specifies how often the repository shall be pulled to check for new changes. 23 | RefreshInterval time.Duration `yaml:"refreshInterval"` 24 | 25 | // Paths whose files shall be watched. Subdirectories and their files will be included. 26 | Paths []string `yaml:"paths"` 27 | } 28 | 29 | // Validate all root and child config structs 30 | func (c *Config) Validate() error { 31 | if !c.Enabled { 32 | return nil 33 | } 34 | if c.RefreshInterval == 0 { 35 | return fmt.Errorf("filesystem provider is enabled but refresh interval is set to 0") 36 | } 37 | 38 | return nil 39 | } 40 | 41 | // SetDefaults for all root and child config structs 42 | func (c *Config) SetDefaults() { 43 | c.MaxFileSize = 500 * 1000 // 500KB 44 | c.IndexByFullFilepath = false 45 | c.RefreshInterval = 5 * time.Minute 46 | } 47 | -------------------------------------------------------------------------------- /backend/pkg/filesystem/file.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | type File struct { 4 | Path string 5 | Filename string 6 | 7 | // TrimmedFilename is the filename without the recognized file extension 8 | TrimmedFilename string 9 | 10 | Payload []byte 11 | } 12 | -------------------------------------------------------------------------------- /backend/pkg/filesystem/util.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | import "strings" 4 | 5 | // isStringInSlice returns true if the given string exists in the string slice. 6 | func isStringInSlice(item string, arr []string) bool { 7 | for _, occurrence := range arr { 8 | if item == occurrence { 9 | return true 10 | } 11 | } 12 | return false 13 | } 14 | 15 | // isValidFileExtension returns: 16 | // 1. a bool which indicates whether the given filename has one of the allowed file extensions 17 | // 2. a string that is the filename with the trimmed extension suffix (e.g. "readme" instead of "readme.md") 18 | func (c *Service) isValidFileExtension(filename string) (bool, string) { 19 | i := strings.LastIndex(filename, ".") 20 | if i == -1 { 21 | // No file extension 22 | if c.Cfg.AllowedFileExtensions == nil { 23 | return true, filename 24 | } 25 | return false, filename 26 | } 27 | 28 | extension := filename[i+1:] 29 | trimmedFilename := strings.TrimSuffix(filename, "."+extension) 30 | if isStringInSlice(extension, c.Cfg.AllowedFileExtensions) { 31 | return true, trimmedFilename 32 | } 33 | return false, trimmedFilename 34 | } 35 | -------------------------------------------------------------------------------- /backend/pkg/git/config.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | // Config for Git Service 10 | type Config struct { 11 | Enabled bool `yaml:"enabled"` 12 | 13 | // AllowedFileExtensions specifies file extensions that shall be picked up. If at least one is specified all other 14 | // file extensions will be ignored. 15 | AllowedFileExtensions []string `yaml:"-"` 16 | 17 | // Max file size which will be considered. Files exceeding this size will be ignored and logged. 18 | MaxFileSize int64 `yaml:"-"` 19 | 20 | // Whether or not to use the filename or the full filepath as key in the map 21 | IndexByFullFilepath bool `yaml:"-"` 22 | 23 | // RefreshInterval specifies how often the repository shall be pulled to check for new changes. 24 | RefreshInterval time.Duration `yaml:"refreshInterval"` 25 | 26 | // Repository that contains markdown files that document a Kafka topic. 27 | Repository RepositoryConfig `yaml:"repository"` 28 | 29 | // Authentication Configs 30 | BasicAuth BasicAuthConfig `yaml:"basicAuth"` 31 | SSH SSHConfig `yaml:"ssh"` 32 | } 33 | 34 | // RegisterFlagsWithPrefix for all (sub)configs 35 | func (c *Config) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { 36 | c.BasicAuth.RegisterFlagsWithPrefix(f, prefix) 37 | c.SSH.RegisterFlagsWithPrefix(f, prefix) 38 | } 39 | 40 | // Validate all root and child config structs 41 | func (c *Config) Validate() error { 42 | if !c.Enabled { 43 | return nil 44 | } 45 | if c.RefreshInterval == 0 { 46 | return fmt.Errorf("git config is enabled but refresh interval is set to 0 (disabled)") 47 | } 48 | 49 | return c.Repository.Validate() 50 | } 51 | 52 | // SetDefaults for all root and child config structs 53 | func (c *Config) SetDefaults() { 54 | c.Repository.SetDefaults() 55 | 56 | c.RefreshInterval = time.Minute 57 | c.MaxFileSize = 500 * 1000 // 500KB 58 | c.IndexByFullFilepath = false 59 | } 60 | -------------------------------------------------------------------------------- /backend/pkg/git/config_auth_basic.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import "flag" 4 | 5 | type BasicAuthConfig struct { 6 | Enabled bool `yaml:"enabled"` 7 | Username string `yaml:"username"` 8 | Password string `yaml:"password"` 9 | } 10 | 11 | // RegisterFlagsWithPrefix for sensitive Basic Auth configs 12 | func (c *BasicAuthConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { 13 | f.StringVar(&c.Password, prefix+"git.basic-auth.password", "", "Basic Auth password") 14 | } 15 | -------------------------------------------------------------------------------- /backend/pkg/git/config_auth_ssh.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import "flag" 4 | 5 | type SSHConfig struct { 6 | Enabled bool `yaml:"enabled"` 7 | Username string `yaml:"username"` 8 | PrivateKey string `yaml:"privateKey"` // user can either pass the key directly or let Kowl load it from disk 9 | PrivateKeyFilePath string `yaml:"privateKeyFilepath"` 10 | Passphrase string `yaml:"passphrase"` 11 | } 12 | 13 | // RegisterFlagsWithPrefix for sensitive SSH configs 14 | func (c *SSHConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { 15 | f.StringVar(&c.PrivateKey, prefix+"git.ssh.private-key", "", "Private key for Git authentication") 16 | f.StringVar(&c.Passphrase, prefix+"git.ssh.passphrase", "", "Passphrase to decrypt private key") 17 | } 18 | -------------------------------------------------------------------------------- /backend/pkg/git/config_repository.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type RepositoryConfig struct { 8 | URL string `yaml:"url"` 9 | Branch string `yaml:"branch"` 10 | BaseDirectory string `yaml:"baseDirectory"` 11 | } 12 | 13 | // Validate given input for config properties 14 | func (c *RepositoryConfig) Validate() error { 15 | if c.URL == "" { 16 | return fmt.Errorf("you must set a repository url") 17 | } 18 | 19 | return nil 20 | } 21 | 22 | func (c *RepositoryConfig) SetDefaults() { 23 | c.BaseDirectory = "." 24 | } 25 | -------------------------------------------------------------------------------- /backend/pkg/git/util_test.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import ( 4 | "github.com/bmizerany/assert" 5 | "testing" 6 | ) 7 | 8 | func TestIsValidFileExtension(t *testing.T) { 9 | markdownSvc := Service{ 10 | Cfg: Config{ 11 | AllowedFileExtensions: []string{"md"}, 12 | }, 13 | } 14 | 15 | tests := []struct { 16 | input string 17 | wantIsValid bool 18 | wantTrimmedFilename string 19 | }{ 20 | {input: "test.md", wantIsValid: true, wantTrimmedFilename: "test"}, 21 | {input: ".md", wantIsValid: true, wantTrimmedFilename: ""}, 22 | {input: "test.MD", wantIsValid: false, wantTrimmedFilename: "test"}, 23 | {input: "test.bin", wantIsValid: false, wantTrimmedFilename: "test"}, 24 | {input: "weird-file.", wantIsValid: false, wantTrimmedFilename: "weird-file"}, 25 | } 26 | 27 | for _, tc := range tests { 28 | isValid, trimmedFilename := markdownSvc.isValidFileExtension(tc.input) 29 | assert.Equal(t, tc.wantIsValid, isValid) 30 | assert.Equal(t, tc.wantTrimmedFilename, trimmedFilename) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/kafka/api_version.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | ) 8 | 9 | // GetAPIVersions returns the supported Kafka API versions 10 | func (s *Service) GetAPIVersions(ctx context.Context) (*kmsg.ApiVersionsResponse, error) { 11 | req := kmsg.NewApiVersionsRequest() 12 | req.ClientSoftwareVersion = "NA" 13 | req.ClientSoftwareName = "Kowl" 14 | 15 | return req.RequestWith(ctx, s.KafkaClient) 16 | } 17 | -------------------------------------------------------------------------------- /backend/pkg/kafka/config_sasl.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | ) 7 | 8 | const ( 9 | SASLMechanismPlain = "PLAIN" 10 | SASLMechanismScramSHA256 = "SCRAM-SHA-256" 11 | SASLMechanismScramSHA512 = "SCRAM-SHA-512" 12 | SASLMechanismGSSAPI = "GSSAPI" 13 | SASLMechanismOAuthBearer = "OAUTHBEARER" 14 | SASLMechanismAWSManagedStreamingIAM = "AWS_MSK_IAM" 15 | ) 16 | 17 | // SASLConfig for Kafka client 18 | type SASLConfig struct { 19 | Enabled bool `yaml:"enabled"` 20 | Username string `yaml:"username"` 21 | Password string `yaml:"password"` 22 | Mechanism string `yaml:"mechanism"` 23 | OAUth SASLOAuthBearer `yaml:"oauth"` 24 | GSSAPIConfig SASLGSSAPIConfig `yaml:"gssapi"` 25 | AWSMskIam SASLAwsMskIam `yaml:"awsMskIam"` 26 | } 27 | 28 | // RegisterFlags for all sensitive Kafka SASL configs. 29 | func (c *SASLConfig) RegisterFlags(f *flag.FlagSet) { 30 | f.StringVar(&c.Password, "kafka.sasl.password", "", "SASL password") 31 | c.OAUth.RegisterFlags(f) 32 | c.GSSAPIConfig.RegisterFlags(f) 33 | c.AWSMskIam.RegisterFlags(f) 34 | } 35 | 36 | // SetDefaults for SASL Config 37 | func (c *SASLConfig) SetDefaults() { 38 | c.Mechanism = SASLMechanismPlain 39 | c.GSSAPIConfig.SetDefaults() 40 | } 41 | 42 | // Validate SASL config input 43 | func (c *SASLConfig) Validate() error { 44 | switch c.Mechanism { 45 | case SASLMechanismPlain, SASLMechanismScramSHA256, SASLMechanismScramSHA512: 46 | // Valid and supported 47 | case SASLMechanismGSSAPI: 48 | err := c.GSSAPIConfig.Validate() 49 | if err != nil { 50 | return fmt.Errorf("failed to validate gssapi config: %w", err) 51 | } 52 | case SASLMechanismOAuthBearer: 53 | err := c.OAUth.Validate() 54 | if err != nil { 55 | return fmt.Errorf("failed to validate OAuth Bearer config: %w", err) 56 | } 57 | case SASLMechanismAWSManagedStreamingIAM: 58 | err := c.AWSMskIam.Validate() 59 | if err != nil { 60 | return fmt.Errorf("failed to validate aws msk iam config: %w", err) 61 | } 62 | default: 63 | return fmt.Errorf("given sasl mechanism '%v' is invalid", c.Mechanism) 64 | } 65 | 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /backend/pkg/kafka/config_sasl_aws_iam.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "flag" 5 | ) 6 | 7 | // SASLAwsMskIam is the config for AWS IAM SASL mechanism, see: https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html 8 | type SASLAwsMskIam struct { 9 | AccessKey string `yaml:"accessKey"` 10 | SecretKey string `yaml:"secretKey"` 11 | 12 | // SessionToken, if non-empty, is a session / security token to use for authentication. 13 | // See: https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html 14 | SessionToken string `yaml:"sessionToken"` 15 | 16 | // UserAgent is the user agent to for the client to use when connecting 17 | // to Kafka, overriding the default "franz-go//". 18 | // 19 | // Setting a UserAgent allows authorizing based on the aws:UserAgent 20 | // condition key; see the following link for more details: 21 | // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_condition-keys.html#condition-keys-useragent 22 | UserAgent string `yaml:"userAgent"` 23 | } 24 | 25 | // RegisterFlags registers all sensitive Kerberos settings as flag 26 | func (c *SASLAwsMskIam) RegisterFlags(f *flag.FlagSet) { 27 | f.StringVar(&c.AccessKey, "kafka.sasl.aws-msk-iam.secret-key", "", "IAM Account secret key") 28 | f.StringVar(&c.SessionToken, "kafka.sasl.aws-msk-iam.session-token", "", "Optional session token for authentication purposes. Uses the AWS Security Token Service API") 29 | } 30 | 31 | func (c *SASLAwsMskIam) Validate() error { 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /backend/pkg/kafka/config_sasl_gssapi.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | ) 7 | 8 | // SASLGSSAPIConfig represents the Kafka Kerberos config 9 | type SASLGSSAPIConfig struct { 10 | AuthType string `yaml:"authType"` 11 | KeyTabPath string `yaml:"keyTabPath"` 12 | KerberosConfigPath string `yaml:"kerberosConfigPath"` 13 | ServiceName string `yaml:"serviceName"` 14 | Username string `yaml:"username"` 15 | Password string `yaml:"password"` 16 | Realm string `yaml:"realm"` 17 | 18 | // EnableFAST enables FAST, which is a pre-authentication framework for Kerberos. 19 | // It includes a mechanism for tunneling pre-authentication exchanges using armoured KDC messages. 20 | // FAST provides increased resistance to passive password guessing attacks. 21 | EnableFast bool `yaml:"enableFast"` 22 | } 23 | 24 | // RegisterFlags registers all sensitive Kerberos settings as flag 25 | func (c *SASLGSSAPIConfig) RegisterFlags(f *flag.FlagSet) { 26 | f.StringVar(&c.Password, "kafka.sasl.gssapi.password", "", "Kerberos password if auth type user auth is used") 27 | } 28 | 29 | func (c *SASLGSSAPIConfig) Validate() error { 30 | if c.AuthType != "USER_AUTH" && c.AuthType != "KEYTAB_AUTH" { 31 | return fmt.Errorf("auth type '%v' is invalid", c.AuthType) 32 | } 33 | 34 | return nil 35 | } 36 | 37 | func (s *SASLGSSAPIConfig) SetDefaults() { 38 | s.EnableFast = true 39 | } 40 | -------------------------------------------------------------------------------- /backend/pkg/kafka/config_sasl_oauth.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | ) 7 | 8 | // SASLOAuthBearer is the config struct for the SASL OAuthBearer mechanism 9 | type SASLOAuthBearer struct { 10 | Token string `yaml:"token"` 11 | } 12 | 13 | // RegisterFlags registers all sensitive Kerberos settings as flag 14 | func (c *SASLOAuthBearer) RegisterFlags(f *flag.FlagSet) { 15 | f.StringVar(&c.Token, "kafka.sasl.oauth.token", "", "OAuth Bearer Token") 16 | } 17 | 18 | func (c *SASLOAuthBearer) Validate() error { 19 | if c.Token == "" { 20 | return fmt.Errorf("OAuth Bearer token must be set") 21 | } 22 | 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /backend/pkg/kafka/config_tls.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import "flag" 4 | 5 | // TLSConfig to connect to Kafka via TLS 6 | type TLSConfig struct { 7 | Enabled bool `yaml:"enabled"` 8 | CaFilepath string `yaml:"caFilepath"` 9 | CertFilepath string `yaml:"certFilepath"` 10 | KeyFilepath string `yaml:"keyFilepath"` 11 | Passphrase string `yaml:"passphrase"` 12 | InsecureSkipTLSVerify bool `yaml:"insecureSkipTlsVerify"` 13 | } 14 | 15 | // RegisterFlags for all sensitive Kafka TLS configs 16 | func (c *TLSConfig) RegisterFlags(f *flag.FlagSet) { 17 | f.StringVar(&c.Passphrase, "kafka.tls.passphrase", "", "Passphrase to optionally decrypt the private key") 18 | } 19 | -------------------------------------------------------------------------------- /backend/pkg/kafka/create_topic.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/twmb/franz-go/pkg/kmsg" 8 | ) 9 | 10 | func (s *Service) CreateTopic(ctx context.Context, createTopicReq kmsg.CreateTopicsRequestTopic) (*kmsg.CreateTopicsResponseTopic, error) { 11 | req := kmsg.NewCreateTopicsRequest() 12 | req.Topics = []kmsg.CreateTopicsRequestTopic{createTopicReq} 13 | 14 | res, err := req.RequestWith(ctx, s.KafkaClient) 15 | if err != nil { 16 | return nil, fmt.Errorf("request has failed: %w", err) 17 | } 18 | if len(res.Topics) != 1 { 19 | return nil, fmt.Errorf("unexpected number of topic responses, expected exactly one but got '%v'", len(res.Topics)) 20 | } 21 | 22 | return &res.Topics[0], nil 23 | } 24 | -------------------------------------------------------------------------------- /backend/pkg/kafka/delete_records.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | ) 8 | 9 | func (s *Service) DeleteRecords(ctx context.Context, deleteReq kmsg.DeleteRecordsRequestTopic) (*kmsg.DeleteRecordsResponse, error) { 10 | req := kmsg.NewDeleteRecordsRequest() 11 | req.Topics = []kmsg.DeleteRecordsRequestTopic{deleteReq} 12 | 13 | res, err := req.RequestWith(ctx, s.KafkaClient) 14 | if err != nil { 15 | return nil, fmt.Errorf("failed to delete records: %w", err) 16 | } 17 | 18 | return res, nil 19 | } 20 | -------------------------------------------------------------------------------- /backend/pkg/kafka/delete_topics.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | ) 8 | 9 | func (s *Service) DeleteTopics(ctx context.Context, topicNames []string) (*kmsg.DeleteTopicsResponse, error) { 10 | req := kmsg.NewDeleteTopicsRequest() 11 | req.TopicNames = topicNames 12 | req.TimeoutMillis = 30 * 1000 // 30s 13 | 14 | res, err := req.RequestWith(ctx, s.KafkaClient) 15 | if err != nil { 16 | return nil, fmt.Errorf("failed to delete topics: %w", err) 17 | } 18 | 19 | return res, nil 20 | } 21 | -------------------------------------------------------------------------------- /backend/pkg/kafka/describe_broker_config.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "github.com/twmb/franz-go/pkg/kmsg" 6 | "strconv" 7 | ) 8 | 9 | // DescribeBrokerConfig fetches config entries which apply at the Broker Scope (e.g. offset.retention.minutes). 10 | // Use nil for configNames in order to get all config entries. 11 | func (s *Service) DescribeBrokerConfig(ctx context.Context, brokerID int32, configNames []string) (*kmsg.DescribeConfigsResponse, error) { 12 | resourceReq := kmsg.NewDescribeConfigsRequestResource() 13 | resourceReq.ResourceType = kmsg.ConfigResourceTypeBroker 14 | resourceReq.ResourceName = strconv.Itoa(int(brokerID)) // Empty string for all brokers (only works for dynamic broker configs) 15 | resourceReq.ConfigNames = configNames // Nil requests all 16 | 17 | req := kmsg.NewDescribeConfigsRequest() 18 | req.Resources = []kmsg.DescribeConfigsRequestResource{ 19 | resourceReq, 20 | } 21 | req.IncludeSynonyms = true 22 | req.IncludeDocumentation = true 23 | 24 | return req.RequestWith(ctx, s.KafkaClient) 25 | } 26 | -------------------------------------------------------------------------------- /backend/pkg/kafka/describe_quotas.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "github.com/twmb/franz-go/pkg/kmsg" 5 | "golang.org/x/net/context" 6 | ) 7 | 8 | func (s *Service) DescribeQuotas(ctx context.Context) (*kmsg.DescribeClientQuotasResponse, error) { 9 | r := kmsg.NewDescribeClientQuotasRequest() 10 | return r.RequestWith(ctx, s.KafkaClient) 11 | } 12 | -------------------------------------------------------------------------------- /backend/pkg/kafka/describe_topic_configs.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | "go.uber.org/zap" 8 | ) 9 | 10 | // DescribeTopicsConfigs fetches all topic config options for the given set of topic names and config names. 11 | // Use nil for configNames to fetch all configs. 12 | func (s *Service) DescribeTopicsConfigs(ctx context.Context, topicNames []string, configNames []string) (*kmsg.DescribeConfigsResponse, error) { 13 | resources := make([]kmsg.DescribeConfigsRequestResource, len(topicNames)) 14 | for i, topicName := range topicNames { 15 | r := kmsg.DescribeConfigsRequestResource{ 16 | ResourceType: kmsg.ConfigResourceTypeTopic, 17 | ResourceName: topicName, 18 | ConfigNames: configNames, 19 | } 20 | resources[i] = r 21 | } 22 | 23 | req := kmsg.NewDescribeConfigsRequest() 24 | req.Resources = resources 25 | req.IncludeDocumentation = true 26 | req.IncludeSynonyms = true 27 | 28 | res, err := req.RequestWith(ctx, s.KafkaClient) 29 | if err != nil { 30 | s.Logger.Error("could not describe topic configs", zap.Error(err)) 31 | return nil, fmt.Errorf("failed to request topic configs: %w", err) 32 | } 33 | 34 | return res, nil 35 | } 36 | -------------------------------------------------------------------------------- /backend/pkg/kafka/edit_consumer_group_offsets.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | ) 8 | 9 | // EditConsumerGroupOffsets edits the group offsets of an existing group. 10 | func (s *Service) EditConsumerGroupOffsets(ctx context.Context, groupID string, topics []kmsg.OffsetCommitRequestTopic) (*kmsg.OffsetCommitResponse, error) { 11 | req := kmsg.NewOffsetCommitRequest() 12 | req.Group = groupID 13 | req.Topics = topics 14 | 15 | res, err := req.RequestWith(ctx, s.KafkaClient) 16 | if err != nil { 17 | return nil, fmt.Errorf("failed to commit group offsets for group '%v': %w", groupID, err) 18 | } 19 | 20 | return res, nil 21 | } 22 | 23 | func (s *Service) DeleteConsumerGroupOffsets(ctx context.Context, groupID string, topics []kmsg.OffsetDeleteRequestTopic) (*kmsg.OffsetDeleteResponse, error) { 24 | req := kmsg.NewOffsetDeleteRequest() 25 | req.Group = groupID 26 | req.Topics = topics 27 | 28 | res, err := req.RequestWith(ctx, s.KafkaClient) 29 | if err != nil { 30 | return nil, fmt.Errorf("failed to commit group offset delete request for group '%v': %w", groupID, err) 31 | } 32 | 33 | return res, nil 34 | } 35 | -------------------------------------------------------------------------------- /backend/pkg/kafka/health_check.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | "go.uber.org/zap" 8 | ) 9 | 10 | // IsHealthy checks whether it can communicate with the Kafka cluster or not 11 | func (s *Service) IsHealthy(ctx context.Context) error { 12 | req := kmsg.MetadataRequest{ 13 | Topics: []kmsg.MetadataRequestTopic{}, 14 | AllowAutoTopicCreation: false, 15 | IncludeClusterAuthorizedOperations: true, 16 | IncludeTopicAuthorizedOperations: false, 17 | } 18 | kres, err := req.RequestWith(ctx, s.KafkaClient) 19 | if err != nil { 20 | s.Logger.Error("failed to request metadata in health check", zap.Error(err)) 21 | return fmt.Errorf("failed to request metadata: %w", err) 22 | } 23 | s.Logger.Debug("kafka cluster health check succeeded", 24 | zap.Int("broker_count", len(kres.Brokers)), 25 | ) 26 | 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /backend/pkg/kafka/incremental_alter_configs.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "github.com/twmb/franz-go/pkg/kmsg" 6 | ) 7 | 8 | func (s *Service) IncrementalAlterConfigs(ctx context.Context, alterConfigs []kmsg.IncrementalAlterConfigsRequestResource) (*kmsg.IncrementalAlterConfigsResponse, error) { 9 | req := kmsg.NewIncrementalAlterConfigsRequest() 10 | req.Resources = alterConfigs 11 | 12 | return req.RequestWith(ctx, s.KafkaClient) 13 | } 14 | -------------------------------------------------------------------------------- /backend/pkg/kafka/list_acls.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "github.com/twmb/franz-go/pkg/kmsg" 6 | ) 7 | 8 | // ListACLs sends a DescribeACL request for one or more specific filters 9 | // 10 | // Kafka Request documentation: 11 | // DescribeACLsRequest describes ACLs. Describing ACLs works on a filter basis: 12 | // anything that matches the filter is described. Note that there are two 13 | // "types" of filters in this request: the resource filter and the entry 14 | // filter, with entries corresponding to users. The first three fields form the 15 | // resource filter, the last four the entry filter. 16 | func (s *Service) ListACLs(ctx context.Context, req kmsg.DescribeACLsRequest) (*kmsg.DescribeACLsResponse, error) { 17 | return req.RequestWith(ctx, s.KafkaClient) 18 | } 19 | -------------------------------------------------------------------------------- /backend/pkg/kafka/list_consumer_group_offsets.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kerr" 7 | "github.com/twmb/franz-go/pkg/kmsg" 8 | "sync" 9 | 10 | "golang.org/x/sync/errgroup" 11 | ) 12 | 13 | // ListConsumerGroupOffsets returns the committed group offsets for a single group 14 | func (s *Service) ListConsumerGroupOffsets(ctx context.Context, group string) (*kmsg.OffsetFetchResponse, error) { 15 | req := kmsg.OffsetFetchRequest{ 16 | Group: group, 17 | Topics: nil, // Requests all topics for this consumer group 18 | } 19 | res, err := req.RequestWith(ctx, s.KafkaClient) 20 | if err != nil { 21 | return nil, fmt.Errorf("failed to request group offsets for group '%v': %w", group, err) 22 | } 23 | 24 | err = kerr.ErrorForCode(res.ErrorCode) 25 | if err != nil { 26 | return nil, fmt.Errorf("failed to request group offsets for group '%v'. Inner error: %w", group, err) 27 | } 28 | 29 | return res, nil 30 | } 31 | 32 | // ListConsumerGroupOffsetsBulk returns a map which has the Consumer group name as key 33 | func (s *Service) ListConsumerGroupOffsetsBulk(ctx context.Context, groups []string) (map[string]*kmsg.OffsetFetchResponse, error) { 34 | eg, _ := errgroup.WithContext(ctx) 35 | 36 | mutex := sync.Mutex{} 37 | res := make(map[string]*kmsg.OffsetFetchResponse) 38 | 39 | f := func(group string) func() error { 40 | return func() error { 41 | offsets, err := s.ListConsumerGroupOffsets(ctx, group) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | mutex.Lock() 47 | res[group] = offsets 48 | mutex.Unlock() 49 | return nil 50 | } 51 | } 52 | 53 | for _, group := range groups { 54 | eg.Go(f(group)) 55 | } 56 | 57 | if err := eg.Wait(); err != nil { 58 | return nil, err 59 | } 60 | 61 | return res, nil 62 | } 63 | -------------------------------------------------------------------------------- /backend/pkg/kafka/list_consumer_groups.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/twmb/franz-go/pkg/kgo" 8 | "github.com/twmb/franz-go/pkg/kmsg" 9 | ) 10 | 11 | type ListConsumerGroupsResponseSharded struct { 12 | Groups []ListConsumerGroupsResponse 13 | RequestsSent int 14 | RequestsFailed int 15 | } 16 | 17 | func (l *ListConsumerGroupsResponseSharded) GetGroupIDs() []string { 18 | groupIDs := make([]string, 0) 19 | for _, groupResp := range l.Groups { 20 | if groupResp.Error != nil || groupResp.Groups == nil { 21 | continue 22 | } 23 | for _, group := range groupResp.Groups.Groups { 24 | groupIDs = append(groupIDs, group.Group) 25 | } 26 | } 27 | return groupIDs 28 | } 29 | 30 | // LogDirResponse can have an error (if the broker failed to return data) or the actual LogDir response 31 | type ListConsumerGroupsResponse struct { 32 | BrokerMetadata kgo.BrokerMetadata 33 | Groups *kmsg.ListGroupsResponse 34 | Error error 35 | } 36 | 37 | // ListConsumerGroups returns an array of Consumer group ids. Failed broker requests will be returned in the response. 38 | // If all broker requests fail an error will be returned. 39 | func (s *Service) ListConsumerGroups(ctx context.Context) (*ListConsumerGroupsResponseSharded, error) { 40 | req := kmsg.ListGroupsRequest{} 41 | shardedResp := s.KafkaClient.RequestSharded(ctx, &req) 42 | 43 | result := &ListConsumerGroupsResponseSharded{ 44 | Groups: make([]ListConsumerGroupsResponse, len(shardedResp)), 45 | RequestsSent: 0, 46 | RequestsFailed: 0, 47 | } 48 | var lastErr error 49 | for _, kresp := range shardedResp { 50 | result.RequestsSent++ 51 | if kresp.Err != nil { 52 | result.RequestsFailed++ 53 | lastErr = kresp.Err 54 | } 55 | 56 | // Important: If we don't declare the second parameter, telling us if the cast succeeded, 57 | // we'll get a panic when the cast fails, instead of being able to continue. 58 | res, _ := kresp.Resp.(*kmsg.ListGroupsResponse) 59 | 60 | result.Groups = append(result.Groups, ListConsumerGroupsResponse{ 61 | BrokerMetadata: kresp.Meta, 62 | Groups: res, 63 | Error: kresp.Err, 64 | }) 65 | } 66 | 67 | if result.RequestsSent > 0 && result.RequestsSent == result.RequestsFailed { 68 | return result, fmt.Errorf("all '%v' requests have failed, last error: %w", len(shardedResp), lastErr) 69 | } 70 | 71 | return result, nil 72 | } 73 | -------------------------------------------------------------------------------- /backend/pkg/kafka/log_dir.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "github.com/twmb/franz-go/pkg/kgo" 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | ) 8 | 9 | type LogDirResponseSharded struct { 10 | LogDirResponses []LogDirResponse 11 | RequestsSent int 12 | RequestsFailed int 13 | } 14 | 15 | // LogDirResponse can have an error (if the broker failed to return data) or the actual LogDir response 16 | type LogDirResponse struct { 17 | BrokerMetadata kgo.BrokerMetadata 18 | LogDirs kmsg.DescribeLogDirsResponse 19 | Error error 20 | } 21 | 22 | // DescribeLogeDirs requests directory information for topic partitions. This request was added in KIP-113 and is 23 | // included in Kafka 1.1.0+ releases. 24 | // 25 | // Use nil for topicPartitions to describe all topics and partitions. 26 | func (s *Service) DescribeLogDirs(ctx context.Context, topicPartitions []kmsg.DescribeLogDirsRequestTopic) []LogDirResponse { 27 | req := kmsg.NewDescribeLogDirsRequest() 28 | req.Topics = topicPartitions 29 | shardedResp := s.KafkaClient.RequestSharded(ctx, &req) 30 | 31 | result := make([]LogDirResponse, len(shardedResp)) 32 | for i, kresp := range shardedResp { 33 | res, ok := kresp.Resp.(*kmsg.DescribeLogDirsResponse) 34 | if !ok { 35 | res = &kmsg.DescribeLogDirsResponse{} 36 | } 37 | result[i] = LogDirResponse{ 38 | BrokerMetadata: kresp.Meta, 39 | LogDirs: *res, 40 | Error: kresp.Err, 41 | } 42 | } 43 | 44 | return result 45 | } 46 | -------------------------------------------------------------------------------- /backend/pkg/kafka/logger.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "github.com/twmb/franz-go/pkg/kgo" 5 | "go.uber.org/zap" 6 | ) 7 | 8 | type KgoZapLogger struct { 9 | logger *zap.SugaredLogger 10 | } 11 | 12 | // Level Implements kgo.Logger interface. It returns the log level to log at. 13 | // We pin this to debug as the zap logger decides what to actually send to the output stream. 14 | func (k KgoZapLogger) Level() kgo.LogLevel { 15 | return kgo.LogLevelDebug 16 | } 17 | 18 | // Log implements kgo.Logger interface 19 | func (k KgoZapLogger) Log(level kgo.LogLevel, msg string, keyvals ...interface{}) { 20 | switch level { 21 | case kgo.LogLevelDebug: 22 | k.logger.Debugw(msg, keyvals...) 23 | case kgo.LogLevelInfo: 24 | k.logger.Infow(msg, keyvals...) 25 | case kgo.LogLevelWarn: 26 | k.logger.Warnw(msg, keyvals...) 27 | case kgo.LogLevelError: 28 | k.logger.Errorw(msg, keyvals...) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /backend/pkg/kafka/partition_reassignments.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "github.com/twmb/franz-go/pkg/kmsg" 6 | ) 7 | 8 | func (s *Service) ListPartitionReassignments(ctx context.Context) (*kmsg.ListPartitionReassignmentsResponse, error) { 9 | req := kmsg.NewListPartitionReassignmentsRequest() 10 | req.Topics = nil // List for all topics 11 | 12 | return req.RequestWith(ctx, s.KafkaClient) 13 | } 14 | 15 | func (s *Service) AlterPartitionAssignments(ctx context.Context, topics []kmsg.AlterPartitionAssignmentsRequestTopic) (*kmsg.AlterPartitionAssignmentsResponse, error) { 16 | req := kmsg.NewAlterPartitionAssignmentsRequest() 17 | req.Topics = topics 18 | 19 | return req.RequestWith(ctx, s.KafkaClient) 20 | } 21 | -------------------------------------------------------------------------------- /backend/pkg/kafka/utils.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "github.com/twmb/franz-go/pkg/kerr" 5 | "github.com/twmb/franz-go/pkg/kmsg" 6 | ) 7 | 8 | func (s *Service) PartitionsToPartitionIDs(partitions []kmsg.MetadataResponseTopicPartition) ([]int32, error) { 9 | var firstErr error 10 | 11 | partitionIDs := make([]int32, len(partitions)) 12 | for i, partition := range partitions { 13 | err := kerr.ErrorForCode(partition.ErrorCode) 14 | if err != nil && firstErr == nil { 15 | firstErr = err 16 | } else { 17 | partitionIDs[i] = partition.Partition 18 | } 19 | } 20 | 21 | return partitionIDs, firstErr 22 | } 23 | -------------------------------------------------------------------------------- /backend/pkg/msgpack/config.go: -------------------------------------------------------------------------------- 1 | package msgpack 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Config represents the message pack config. 8 | type Config struct { 9 | Enabled bool `yaml:"enabled"` 10 | 11 | // TopicNames is a list of topic names that shall be considered for messagepack decoding. 12 | // These names can be provided as regex string (e. g. "/.*/" or "/prefix-.*/") or as plain topic name 13 | // such as "frontend-activities". 14 | // This defaults to `/.*/` 15 | TopicNames []string `yaml:"topicNames"` 16 | } 17 | 18 | // Validate if provided TopicNames are valid. 19 | func (c *Config) Validate() error { 20 | if !c.Enabled { 21 | return nil 22 | } 23 | 24 | // Check whether each provided string is valid regex 25 | for _, topic := range c.TopicNames { 26 | _, err := compileRegex(topic) 27 | if err != nil { 28 | return fmt.Errorf("allowed topic string '%v' is not valid regex", topic) 29 | } 30 | } 31 | 32 | return nil 33 | } 34 | 35 | func (c *Config) SetDefaults() { 36 | c.TopicNames = []string{"/.*/"} 37 | } 38 | -------------------------------------------------------------------------------- /backend/pkg/msgpack/service.go: -------------------------------------------------------------------------------- 1 | package msgpack 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | // Service represents messagepack cfg, topic name regexes. 8 | type Service struct { 9 | cfg Config 10 | 11 | AllowedTopicsExpr []*regexp.Regexp 12 | } 13 | 14 | // NewService returns a new instance of Service with compiled regexes. 15 | func NewService(cfg Config) (*Service, error) { 16 | allowedTopicsExpr, err := compileRegexes(cfg.TopicNames) 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | return &Service{ 22 | cfg: cfg, 23 | AllowedTopicsExpr: allowedTopicsExpr, 24 | }, nil 25 | } 26 | 27 | // IsTopicAllowed validates if a topicName is permitted as per the config regexes. 28 | func (s *Service) IsTopicAllowed(topicName string) bool { 29 | isAllowed := false 30 | for _, regex := range s.AllowedTopicsExpr { 31 | if regex.MatchString(topicName) { 32 | isAllowed = true 33 | break 34 | } 35 | } 36 | 37 | return isAllowed 38 | } 39 | -------------------------------------------------------------------------------- /backend/pkg/msgpack/utils.go: -------------------------------------------------------------------------------- 1 | package msgpack 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | func compileRegex(expr string) (*regexp.Regexp, error) { 10 | if strings.HasPrefix(expr, "/") && strings.HasSuffix(expr, "/") { 11 | substr := expr[1 : len(expr)-1] 12 | regex, err := regexp.Compile(substr) 13 | if err != nil { 14 | return nil, err 15 | } 16 | 17 | return regex, nil 18 | } 19 | 20 | // If this is no regex input (which is marked by the slashes around it) then we escape it so that it's a literal 21 | regex, err := regexp.Compile("^" + expr + "$") 22 | if err != nil { 23 | return nil, err 24 | } 25 | return regex, nil 26 | } 27 | 28 | func compileRegexes(expr []string) ([]*regexp.Regexp, error) { 29 | compiledExpressions := make([]*regexp.Regexp, len(expr)) 30 | for i, exprStr := range expr { 31 | expr, err := compileRegex(exprStr) 32 | if err != nil { 33 | return nil, fmt.Errorf("failed to compile expression string '%v': %w", exprStr, err) 34 | } 35 | compiledExpressions[i] = expr 36 | } 37 | 38 | return compiledExpressions, nil 39 | } 40 | -------------------------------------------------------------------------------- /backend/pkg/owl/api_versions.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kerr" 7 | "github.com/twmb/franz-go/pkg/kmsg" 8 | ) 9 | 10 | type APIVersion struct { 11 | KeyID int16 `json:"keyId"` 12 | KeyName string `json:"keyName"` 13 | MaxVersion int16 `json:"maxVersion"` 14 | MinVersion int16 `json:"minVersion"` 15 | } 16 | 17 | // GetAPIVersions asks the brokers for the supported Kafka API requests and their supported 18 | // versions. This will be used by the frontend to figure out what functionality is available 19 | // or should be rendered as not available. 20 | func (s *Service) GetAPIVersions(ctx context.Context) ([]APIVersion, error) { 21 | versionsRes, err := s.kafkaSvc.GetAPIVersions(ctx) 22 | if err != nil { 23 | return nil, fmt.Errorf("failed to get kafka api version: %w", err) 24 | } 25 | 26 | err = kerr.ErrorForCode(versionsRes.ErrorCode) 27 | if err != nil { 28 | return nil, fmt.Errorf("failed to get kafka api version. Inner error: %w", err) 29 | } 30 | 31 | versions := make([]APIVersion, len(versionsRes.ApiKeys)) 32 | for i, version := range versionsRes.ApiKeys { 33 | versions[i] = APIVersion{ 34 | KeyID: version.ApiKey, 35 | KeyName: kmsg.NameForKey(version.ApiKey), 36 | MaxVersion: version.MaxVersion, 37 | MinVersion: version.MinVersion, 38 | } 39 | } 40 | 41 | return versions, nil 42 | } 43 | -------------------------------------------------------------------------------- /backend/pkg/owl/common.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import "github.com/twmb/franz-go/pkg/kgo" 4 | 5 | type BrokerRequestError struct { 6 | BrokerMeta kgo.BrokerMetadata `json:"brokerMetadata"` 7 | Error error `json:"error"` 8 | } 9 | -------------------------------------------------------------------------------- /backend/pkg/owl/config.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | ) 7 | 8 | type Config struct { 9 | TopicDocumentation ConfigTopicDocumentation `yaml:"topicDocumentation"` 10 | } 11 | 12 | func (c *Config) SetDefaults() { 13 | c.TopicDocumentation.SetDefaults() 14 | } 15 | 16 | func (c *Config) RegisterFlags(f *flag.FlagSet) { 17 | c.TopicDocumentation.RegisterFlags(f) 18 | } 19 | 20 | func (c *Config) Validate() error { 21 | err := c.TopicDocumentation.Validate() 22 | if err != nil { 23 | return fmt.Errorf("failed to validate topic documentation config: %w", err) 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /backend/pkg/owl/config_topic_documentation.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/cloudhut/kowl/backend/pkg/git" 7 | ) 8 | 9 | type ConfigTopicDocumentation struct { 10 | Enabled bool `yaml:"enabled"` 11 | Git git.Config `yaml:"git"` 12 | } 13 | 14 | func (c *ConfigTopicDocumentation) RegisterFlags(f *flag.FlagSet) { 15 | c.Git.RegisterFlagsWithPrefix(f, "owl.topic-documentation.") 16 | } 17 | 18 | func (c *ConfigTopicDocumentation) Validate() error { 19 | if !c.Enabled { 20 | return nil 21 | } 22 | if c.Enabled && !c.Git.Enabled { 23 | return fmt.Errorf("topic documentation is enabled, but git service is diabled. At least one source for topic documentations must be configured") 24 | } 25 | 26 | return c.Git.Validate() 27 | } 28 | 29 | func (c *ConfigTopicDocumentation) SetDefaults() { 30 | c.Git.SetDefaults() 31 | c.Git.AllowedFileExtensions = []string{".md"} 32 | } 33 | -------------------------------------------------------------------------------- /backend/pkg/owl/delete_consumer_group_offsets.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "context" 5 | "github.com/twmb/franz-go/pkg/kerr" 6 | "github.com/twmb/franz-go/pkg/kmsg" 7 | ) 8 | 9 | type DeleteConsumerGroupOffsetsResponseTopic struct { 10 | TopicName string `json:"topicName"` 11 | Partitions []DeleteConsumerGroupOffsetsResponseTopicPartition `json:"partitions"` 12 | } 13 | 14 | type DeleteConsumerGroupOffsetsResponseTopicPartition struct { 15 | ID int32 `json:"partitionID"` 16 | Error string `json:"error,omitempty"` 17 | } 18 | 19 | // 20 | func (s *Service) DeleteConsumerGroupOffsets(ctx context.Context, groupID string, topics []kmsg.OffsetDeleteRequestTopic) ([]DeleteConsumerGroupOffsetsResponseTopic, error) { 21 | commitResponse, err := s.kafkaSvc.DeleteConsumerGroupOffsets(ctx, groupID, topics) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | res := make([]DeleteConsumerGroupOffsetsResponseTopic, len(commitResponse.Topics)) 27 | for i, topic := range commitResponse.Topics { 28 | partitions := make([]DeleteConsumerGroupOffsetsResponseTopicPartition, len(topic.Partitions)) 29 | for j, partition := range topic.Partitions { 30 | err := kerr.ErrorForCode(partition.ErrorCode) 31 | var errMsg string 32 | if err != nil { 33 | errMsg = err.Error() 34 | } 35 | partitions[j] = DeleteConsumerGroupOffsetsResponseTopicPartition{ 36 | ID: partition.Partition, 37 | Error: errMsg, 38 | } 39 | } 40 | res[i] = DeleteConsumerGroupOffsetsResponseTopic{ 41 | TopicName: topic.Topic, 42 | Partitions: partitions, 43 | } 44 | } 45 | 46 | return res, nil 47 | } 48 | -------------------------------------------------------------------------------- /backend/pkg/owl/describe_quotas.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "fmt" 5 | "github.com/twmb/franz-go/pkg/kerr" 6 | "golang.org/x/net/context" 7 | ) 8 | 9 | type QuotaResponse struct { 10 | Error string `json:"error,omitempty"` 11 | Items []QuotaResponseItem `json:"items"` 12 | } 13 | 14 | type QuotaResponseItem struct { 15 | EntityType string `json:"entityType"` 16 | EntityName string `json:"entityName"` 17 | Settings []QuotaResponseSetting `json:"settings"` 18 | } 19 | 20 | type QuotaResponseSetting struct { 21 | Key string `json:"key"` 22 | Value float64 `json:"value"` 23 | } 24 | 25 | func (s *Service) DescribeQuotas(ctx context.Context) QuotaResponse { 26 | items := make([]QuotaResponseItem, 0) 27 | 28 | quotas, err := s.kafkaSvc.DescribeQuotas(ctx) 29 | if err != nil { 30 | return QuotaResponse{ 31 | Error: fmt.Errorf("kafka request has failed: %w", err).Error(), 32 | Items: nil, 33 | } 34 | } 35 | 36 | err = kerr.ErrorForCode(quotas.ErrorCode) 37 | if err != nil { 38 | return QuotaResponse{ 39 | Error: fmt.Errorf("inner kafka error: %w", err).Error(), 40 | Items: nil, 41 | } 42 | } 43 | 44 | // Flat map all quota settings from response into our items array 45 | for _, entry := range quotas.Entries { 46 | settings := make([]QuotaResponseSetting, len(entry.Values)) 47 | for i, setting := range entry.Values { 48 | settings[i] = QuotaResponseSetting{ 49 | Key: setting.Key, 50 | Value: setting.Value, 51 | } 52 | } 53 | 54 | for _, entity := range entry.Entity { 55 | // A nil value for entity.Name means that this quota is the default for the respective entity.Type 56 | entityName := "" 57 | if entity.Name != nil { 58 | entityName = *entity.Name 59 | } 60 | items = append(items, QuotaResponseItem{ 61 | EntityType: entity.Type, 62 | EntityName: entityName, 63 | Settings: settings, 64 | }) 65 | } 66 | } 67 | 68 | return QuotaResponse{ 69 | Error: "", 70 | Items: items, 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /backend/pkg/owl/errors.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrSchemaRegistryNotConfigured = errors.New("no schema registry configured") 7 | ) 8 | -------------------------------------------------------------------------------- /backend/pkg/owl/incremental_alter_configs.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/common/rest" 7 | "github.com/twmb/franz-go/pkg/kerr" 8 | "github.com/twmb/franz-go/pkg/kmsg" 9 | "net/http" 10 | ) 11 | 12 | type IncrementalAlterConfigsResourceResponse struct { 13 | Error string `json:"error,omitempty"` 14 | ResourceName string `json:"resourceName"` 15 | ResourceType int8 `json:"resourceType"` 16 | } 17 | 18 | func (s *Service) IncrementalAlterConfigs(ctx context.Context, 19 | alterConfigs []kmsg.IncrementalAlterConfigsRequestResource) ([]IncrementalAlterConfigsResourceResponse, *rest.Error) { 20 | configRes, err := s.kafkaSvc.IncrementalAlterConfigs(ctx, alterConfigs) 21 | if err != nil { 22 | return nil, &rest.Error{ 23 | Err: err, 24 | Status: http.StatusServiceUnavailable, 25 | Message: fmt.Sprintf("Incremental Alter Config request has failed: %v", err.Error()), 26 | IsSilent: false, 27 | } 28 | } 29 | 30 | patchedConfigs := make([]IncrementalAlterConfigsResourceResponse, len(configRes.Resources)) 31 | for i, res := range configRes.Resources { 32 | errMessage := "" 33 | err := kerr.ErrorForCode(res.ErrorCode) 34 | if err != nil { 35 | errMessage = err.Error() 36 | } 37 | patchedConfigs[i] = IncrementalAlterConfigsResourceResponse{ 38 | Error: errMessage, 39 | ResourceName: res.ResourceName, 40 | ResourceType: int8(res.ResourceType), 41 | } 42 | } 43 | 44 | return patchedConfigs, nil 45 | } 46 | -------------------------------------------------------------------------------- /backend/pkg/owl/kafka_error.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "fmt" 5 | "github.com/twmb/franz-go/pkg/kerr" 6 | ) 7 | 8 | type KafkaError struct { 9 | Code int16 `json:"code"` 10 | Message string `json:"message"` 11 | Description string `json:"description"` 12 | } 13 | 14 | func (e *KafkaError) Error() string { 15 | return fmt.Sprintf("%s: %s", e.Message, e.Description) 16 | } 17 | 18 | func newKafkaError(errCode int16) *KafkaError { 19 | typedError := kerr.TypedErrorForCode(errCode) 20 | if typedError == nil { 21 | return nil 22 | } 23 | 24 | return &KafkaError{ 25 | Code: typedError.Code, 26 | Message: typedError.Message, 27 | Description: typedError.Description, 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /backend/pkg/owl/list_acls.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/twmb/franz-go/pkg/kerr" 8 | "github.com/twmb/franz-go/pkg/kmsg" 9 | ) 10 | 11 | type AclOverview struct { 12 | AclResources []*AclResource `json:"aclResources"` 13 | IsAuthorizerEnabled bool `json:"isAuthorizerEnabled"` 14 | } 15 | 16 | // AclResource is all information we get when listing ACLs 17 | type AclResource struct { 18 | ResourceType string `json:"resourceType"` 19 | ResourceName string `json:"resourceName"` 20 | ResourcePatternType string `json:"resourcePatternType"` 21 | ACLs []*AclRule `json:"acls"` 22 | } 23 | 24 | type AclRule struct { 25 | Principal string `json:"principal"` 26 | Host string `json:"host"` 27 | Operation string `json:"operation"` 28 | PermissionType string `json:"permissionType"` 29 | } 30 | 31 | // ListAllACLs returns a list of all stored ACLs. 32 | func (s *Service) ListAllACLs(ctx context.Context, req kmsg.DescribeACLsRequest) (*AclOverview, error) { 33 | aclResponses, err := s.kafkaSvc.ListACLs(ctx, req) 34 | if err != nil { 35 | return nil, fmt.Errorf("failed to get ACLs from Kafka: %w", err) 36 | } 37 | 38 | kafkaErr := kerr.TypedErrorForCode(aclResponses.ErrorCode) 39 | if kafkaErr != nil { 40 | if kafkaErr == kerr.SecurityDisabled { 41 | return &AclOverview{ 42 | AclResources: nil, 43 | IsAuthorizerEnabled: false, 44 | }, nil 45 | } 46 | return nil, fmt.Errorf("failed to get ACLs from Kafka: %v", kafkaErr.Error()) 47 | } 48 | 49 | resources := make([]*AclResource, len(aclResponses.Resources)) 50 | for i, aclResponse := range aclResponses.Resources { 51 | overview := &AclResource{ 52 | ResourceType: aclResponse.ResourceType.String(), 53 | ResourceName: aclResponse.ResourceName, 54 | ResourcePatternType: aclResponse.ResourcePatternType.String(), 55 | ACLs: nil, 56 | } 57 | 58 | acls := make([]*AclRule, len(aclResponse.ACLs)) 59 | for j, acl := range aclResponse.ACLs { 60 | acls[j] = &AclRule{ 61 | Principal: acl.Principal, 62 | Host: acl.Host, 63 | Operation: acl.Operation.String(), 64 | PermissionType: acl.PermissionType.String(), 65 | } 66 | } 67 | overview.ACLs = acls 68 | resources[i] = overview 69 | } 70 | 71 | return &AclOverview{ 72 | AclResources: resources, 73 | IsAuthorizerEnabled: true, 74 | }, nil 75 | } 76 | -------------------------------------------------------------------------------- /backend/pkg/owl/list_offsets.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kerr" 7 | ) 8 | 9 | type TopicOffset struct { 10 | TopicName string `json:"topicName"` 11 | Partitions []PartitionOffset `json:"partitions"` 12 | } 13 | 14 | type PartitionOffset struct { 15 | Error string `json:"error,omitempty"` 16 | PartitionID int32 `json:"partitionId"` 17 | Offset int64 `json:"offset"` 18 | Timestamp int64 `json:"timestamp"` 19 | } 20 | 21 | func (s *Service) ListOffsets(ctx context.Context, topicNames []string, timestamp int64) ([]TopicOffset, error) { 22 | metadata, err := s.kafkaSvc.GetMetadata(ctx, topicNames) 23 | if err != nil { 24 | return nil, fmt.Errorf("failed to request partition info for topics") 25 | } 26 | 27 | topicPartitions := make(map[string][]int32, len(metadata.Topics)) 28 | for _, topic := range metadata.Topics { 29 | err := kerr.ErrorForCode(topic.ErrorCode) 30 | if err != nil { 31 | return nil, fmt.Errorf("failed to request partition info for topic '%v': %w", topic.Topic, err) 32 | } 33 | 34 | for _, partition := range topic.Partitions { 35 | err := kerr.ErrorForCode(partition.ErrorCode) 36 | if err != nil { 37 | return nil, fmt.Errorf("failed to request partition info for topic '%v', partition: '%v': %w", topic.Topic, partition.Partition, err) 38 | } 39 | topicName := *topic.Topic 40 | topicPartitions[topicName] = append(topicPartitions[topicName], partition.Partition) 41 | } 42 | } 43 | offsets := s.kafkaSvc.ListOffsets(ctx, topicPartitions, timestamp) 44 | 45 | offsetResponses := make([]TopicOffset, 0, len(offsets)) 46 | for topicName, partitions := range offsets { 47 | pOffsets := make([]PartitionOffset, len(partitions)) 48 | for pID, partition := range partitions { 49 | if err != nil { 50 | pOffsets[pID] = PartitionOffset{ 51 | Error: err.Error(), 52 | PartitionID: pID, 53 | Offset: partition.Offset, 54 | } 55 | } 56 | 57 | pOffsets[pID] = PartitionOffset{ 58 | PartitionID: pID, 59 | Offset: partition.Offset, 60 | Timestamp: partition.Timestamp, 61 | } 62 | } 63 | offsetResponses = append(offsetResponses, TopicOffset{ 64 | TopicName: topicName, 65 | Partitions: pOffsets, 66 | }) 67 | } 68 | 69 | return offsetResponses, nil 70 | } 71 | -------------------------------------------------------------------------------- /backend/pkg/owl/owl.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | // Q: What's the owl package's responsibility? 4 | // 5 | // A: The Owl package is in charge of constructing the responses for our REST API. 6 | // It's common that a single invocation requires multiple upstream requests against Kafka 7 | // so that we can merge the data and provide the most valuable information for the users. 8 | // While the kafka package is the abstraction for communicating with Kafka, this package 9 | // proccesses incoming requests from the REST API by: 10 | // 1. Sending upstream requests (concurrently) against Kafka 11 | // 2. Merge the responses as needed 12 | // 3. Convert the data so that it is handy to use in the frontend 13 | -------------------------------------------------------------------------------- /backend/pkg/owl/produce_records.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/twmb/franz-go/pkg/kgo" 7 | ) 8 | 9 | type ProduceRecordsResponse struct { 10 | Records []ProduceRecordResponse `json:"records"` 11 | 12 | // Error indicates that producing for all records have failed. E.g. because creating a transaction has failed 13 | // when transactions were enabled. Another option could be that the Kafka client creation has failed because 14 | // brokers are temporarily offline. 15 | Error string `json:"error,omitempty"` 16 | } 17 | 18 | type ProduceRecordResponse struct { 19 | TopicName string `json:"topicName"` 20 | PartitionID int32 `json:"partitionId"` 21 | Offset int64 `json:"offset"` 22 | Error string `json:"error,omitempty"` 23 | } 24 | 25 | // ProduceRecords produces one or more records. This might involve multiple topics or a just a single topic. 26 | // If multiple records shall be produced the user can opt in for using transactions so that either none or all 27 | // records will be produced successfully. 28 | func (s *Service) ProduceRecords(ctx context.Context, records []*kgo.Record, useTransactions bool, compressionType int8) ProduceRecordsResponse { 29 | recordResponses, err := s.kafkaSvc.ProduceRecords(ctx, records, useTransactions, compressionType) 30 | if err != nil { 31 | return ProduceRecordsResponse{ 32 | Records: nil, 33 | Error: fmt.Sprintf("Failed to produce records: %v", err.Error()), 34 | } 35 | } 36 | 37 | formattedResponses := make([]ProduceRecordResponse, len(recordResponses)) 38 | for i, record := range recordResponses { 39 | var errorStr string 40 | if record.Error != nil { 41 | errorStr = record.Error.Error() 42 | } 43 | formattedResponses[i] = ProduceRecordResponse{ 44 | TopicName: record.TopicName, 45 | PartitionID: record.PartitionID, 46 | Offset: record.Offset, 47 | Error: errorStr, 48 | } 49 | } 50 | 51 | return ProduceRecordsResponse{ 52 | Records: formattedResponses, 53 | Error: "", // Will be omitted 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /backend/pkg/owl/schema_details.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cloudhut/kowl/backend/pkg/schema" 7 | 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type SchemaDetails struct { 12 | Subject string `json:"string"` 13 | SchemaID int `json:"schemaId"` 14 | Version int `json:"version"` 15 | Compatibility string `json:"compatibility"` 16 | Schema string `json:"schema"` 17 | RegisteredVersions []int `json:"registeredVersions"` 18 | Type string `json:"type"` 19 | } 20 | 21 | func (s *Service) GetSchemaDetails(_ context.Context, subject string, version string) (*SchemaDetails, error) { 22 | if s.kafkaSvc.SchemaService == nil { 23 | return nil, ErrSchemaRegistryNotConfigured 24 | } 25 | 26 | versions, err := s.kafkaSvc.SchemaService.GetSubjectVersions(subject) 27 | if err != nil { 28 | return nil, fmt.Errorf("failed to get versions for given subject: %w", err) 29 | } 30 | 31 | versionedSchema, err := s.kafkaSvc.SchemaService.GetSchemaBySubject(subject, version) 32 | if err != nil { 33 | return nil, fmt.Errorf("failed to get versioned schema for given subject: %w", err) 34 | } 35 | 36 | cfgRes, err := s.kafkaSvc.SchemaService.GetSubjectConfig(subject) 37 | if err != nil { 38 | s.logger.Debug("failed to get compatibility for given subject", zap.Error(err)) 39 | cfgRes, err = s.kafkaSvc.SchemaService.GetConfig() 40 | if err != nil { 41 | s.logger.Warn("failed to get subject and global compatibility for given subject", 42 | zap.String("subject", subject), 43 | zap.Error(err)) 44 | cfgRes = &schema.ConfigResponse{Compatibility: "UNKNOWN"} 45 | } 46 | } 47 | 48 | return &SchemaDetails{ 49 | Subject: subject, 50 | SchemaID: versionedSchema.SchemaID, 51 | Version: versionedSchema.Version, 52 | Compatibility: cfgRes.Compatibility, 53 | RegisteredVersions: versions.Versions, 54 | Schema: versionedSchema.Schema, 55 | Type: versionedSchema.Type, 56 | }, nil 57 | } 58 | -------------------------------------------------------------------------------- /backend/pkg/owl/service.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "fmt" 5 | "github.com/cloudhut/kowl/backend/pkg/git" 6 | "github.com/cloudhut/kowl/backend/pkg/kafka" 7 | "go.uber.org/zap" 8 | ) 9 | 10 | // Service offers all methods to serve the responses for the REST API. This usually only involves fetching 11 | // several responses from Kafka concurrently and constructing them so, that they are 12 | type Service struct { 13 | kafkaSvc *kafka.Service 14 | gitSvc *git.Service // Git service can be nil if not configured 15 | logger *zap.Logger 16 | } 17 | 18 | // NewService for the Owl package 19 | func NewService(cfg Config, logger *zap.Logger, kafkaSvc *kafka.Service) (*Service, error) { 20 | var gitSvc *git.Service 21 | cfg.TopicDocumentation.Git.AllowedFileExtensions = []string{"md"} 22 | if cfg.TopicDocumentation.Enabled && cfg.TopicDocumentation.Git.Enabled { 23 | svc, err := git.NewService(cfg.TopicDocumentation.Git, logger, nil) 24 | if err != nil { 25 | return nil, fmt.Errorf("failed to create git service: %w", err) 26 | } 27 | gitSvc = svc 28 | } 29 | return &Service{ 30 | kafkaSvc: kafkaSvc, 31 | gitSvc: gitSvc, 32 | logger: logger, 33 | }, nil 34 | } 35 | 36 | // Start starts all the (background) tasks which are required for this service to work properly. If any of these 37 | // tasks can not be setup an error will be returned which will cause the application to exit. 38 | func (s *Service) Start() error { 39 | if s.gitSvc == nil { 40 | return nil 41 | } 42 | return s.gitSvc.Start() 43 | } 44 | -------------------------------------------------------------------------------- /backend/pkg/owl/topic_consumers.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | ) 7 | 8 | // TopicConsumerGroup is a group along with it's accumulated topic log for a given topic 9 | type TopicConsumerGroup struct { 10 | GroupID string `json:"groupId"` 11 | SummedLag int64 `json:"summedLag"` 12 | } 13 | 14 | // ListTopicConsumers returns all consumer group names along with their accumulated lag across all partitions which 15 | // have at least one active offset on the given topic. 16 | func (s *Service) ListTopicConsumers(ctx context.Context, topicName string) ([]*TopicConsumerGroup, error) { 17 | groups, err := s.kafkaSvc.ListConsumerGroups(ctx) 18 | if err != nil { 19 | return nil, fmt.Errorf("failed to list consumer groups: %w", err) 20 | } 21 | 22 | groupIDs := groups.GetGroupIDs() 23 | 24 | offsetsByGroup, err := s.getConsumerGroupOffsets(ctx, groupIDs) 25 | if err != nil { 26 | return nil, fmt.Errorf("failed to get consumer group offsetsByGroup: %w", err) 27 | } 28 | 29 | response := make([]*TopicConsumerGroup, 0, len(offsetsByGroup)) 30 | for groupID, grpTopicOffsets := range offsetsByGroup { 31 | for _, topicLag := range grpTopicOffsets { 32 | if topicLag.Topic != topicName { 33 | continue 34 | } 35 | 36 | cg := &TopicConsumerGroup{GroupID: groupID, SummedLag: topicLag.SummedLag} 37 | response = append(response, cg) 38 | } 39 | } 40 | 41 | return response, nil 42 | } 43 | -------------------------------------------------------------------------------- /backend/pkg/owl/topic_documentation.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | // TopicDocumentation holds the Markdown with potential metadata (e. g. editor, last edited at etc). 4 | type TopicDocumentation struct { 5 | IsEnabled bool `json:"isEnabled"` 6 | Markdown []byte `json:"markdown"` 7 | } 8 | 9 | // GetTopicDocumentation returns the documentation for the given topic if available. 10 | func (s *Service) GetTopicDocumentation(topicName string) *TopicDocumentation { 11 | if s.gitSvc == nil { 12 | return &TopicDocumentation{ 13 | IsEnabled: false, 14 | Markdown: nil, 15 | } 16 | } 17 | 18 | markdown := s.gitSvc.GetFileByFilename(topicName) 19 | 20 | return &TopicDocumentation{ 21 | IsEnabled: true, 22 | Markdown: markdown.Payload, 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /backend/pkg/owl/util.go: -------------------------------------------------------------------------------- 1 | package owl 2 | 3 | func derefString(s *string) string { 4 | if s != nil { 5 | return *s 6 | } 7 | 8 | return "" 9 | } 10 | 11 | // find takes a slice and looks for an element in it. If found it will 12 | // return it's key, otherwise it will return -1 and a bool of false. 13 | func find(slice []string, val string) (int, bool) { 14 | for i, item := range slice { 15 | if item == val { 16 | return i, true 17 | } 18 | } 19 | return -1, false 20 | } 21 | 22 | func errToString(err error) string { 23 | if err == nil { 24 | return "" 25 | } 26 | return err.Error() 27 | } 28 | -------------------------------------------------------------------------------- /backend/pkg/proto/config.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/cloudhut/kowl/backend/pkg/filesystem" 7 | "github.com/cloudhut/kowl/backend/pkg/git" 8 | ) 9 | 10 | type Config struct { 11 | Enabled bool `json:"enabled"` 12 | 13 | // The required proto definitions can be provided via SchemaRegistry, Git or Filesystem 14 | SchemaRegistry SchemaRegistryConfig `json:"schemaRegistry"` 15 | Git git.Config `json:"git"` 16 | FileSystem filesystem.Config `json:"fileSystem"` 17 | 18 | // Mappings define what proto types shall be used for each Kafka topic. If SchemaRegistry is used, no mappings are required. 19 | Mappings []ConfigTopicMapping `json:"mappings"` 20 | } 21 | 22 | // RegisterFlags registers all nested config flags. 23 | func (c *Config) RegisterFlags(f *flag.FlagSet) { 24 | c.Git.RegisterFlagsWithPrefix(f, "kafka.protobuf.") 25 | } 26 | 27 | func (c *Config) Validate() error { 28 | if !c.Enabled { 29 | return nil 30 | } 31 | 32 | if !c.Git.Enabled && !c.FileSystem.Enabled && !c.SchemaRegistry.Enabled { 33 | return fmt.Errorf("protobuf deserializer is enabled, at least one source provider for proto files must be configured") 34 | } 35 | 36 | if len(c.Mappings) == 0 && !c.SchemaRegistry.Enabled { 37 | return fmt.Errorf("protobuf deserializer is enabled, but no topic mappings have been configured") 38 | } 39 | 40 | return nil 41 | } 42 | 43 | func (c *Config) SetDefaults() { 44 | c.Git.SetDefaults() 45 | c.FileSystem.SetDefaults() 46 | c.SchemaRegistry.SetDefaults() 47 | 48 | // Index by full filepath so that we support .proto files with the same filename in different directories 49 | c.Git.IndexByFullFilepath = true 50 | c.Git.AllowedFileExtensions = []string{"proto"} 51 | c.FileSystem.IndexByFullFilepath = true 52 | c.FileSystem.AllowedFileExtensions = []string{"proto"} 53 | } 54 | -------------------------------------------------------------------------------- /backend/pkg/proto/config_schema_registry.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import "time" 4 | 5 | // SchemaRegistryConfig that shall be used to get Protobuf types and Mappings from. The schema registry configuration 6 | // is not part of this config as the schema registry client that is configured under kafka.schemaRegistry will be 7 | // reused here. It is it's own configuration struct to remain extensible in the future without requiring breaking changes. 8 | type SchemaRegistryConfig struct { 9 | Enabled bool `json:"enabled"` 10 | RefreshInterval time.Duration `json:"refreshInterval"` 11 | } 12 | 13 | func (s *SchemaRegistryConfig) SetDefaults() { 14 | s.RefreshInterval = 5 * time.Minute 15 | } 16 | -------------------------------------------------------------------------------- /backend/pkg/proto/config_topic_mapping.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | type ConfigTopicMapping struct { 4 | TopicName string `yaml:"topicName"` 5 | 6 | // KeyProtoType is the proto's fully qualified name that shall be used for a Kafka record's key 7 | KeyProtoType string `yaml:"keyProtoType"` 8 | 9 | // ValueProtoType is the proto's fully qualified name that shall be used for a Kafka record's value 10 | ValueProtoType string `yaml:"valueProtoType"` 11 | } 12 | -------------------------------------------------------------------------------- /backend/pkg/proto/trigger_refresh.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "syscall" 7 | "time" 8 | ) 9 | 10 | func triggerRefresh(dur time.Duration, callback func()) { 11 | // Stop sync when we receive a signal 12 | quit := make(chan os.Signal, 1) 13 | signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) 14 | 15 | ticker := time.NewTicker(dur) 16 | for { 17 | select { 18 | case <-quit: 19 | return 20 | case <-ticker.C: 21 | callback() 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /backend/pkg/schema/client_errors.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | const ( 4 | codeSubjectNotFound = 40401 5 | codeSchemaNotFound = 40403 6 | codeBackendDatastoreError = 50001 7 | ) 8 | 9 | func IsSchemaNotFound(err error) bool { 10 | if err == nil { 11 | return false 12 | } 13 | 14 | if restErr, ok := err.(RestError); ok { 15 | return restErr.ErrorCode == codeSchemaNotFound 16 | } 17 | 18 | return false 19 | } 20 | -------------------------------------------------------------------------------- /backend/pkg/schema/config.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | ) 7 | 8 | // Config for using a (Confluent) Schema Registry 9 | type Config struct { 10 | Enabled bool `yaml:"enabled"` 11 | URLs []string `yaml:"urls"` 12 | 13 | // Credentials 14 | Username string `yaml:"username"` 15 | Password string `yaml:"password"` 16 | BearerToken string `yaml:"bearerToken"` 17 | 18 | // TLS / Custom CA 19 | TLS TLSConfig `yaml:"tls"` 20 | } 21 | 22 | // RegisterFlags registers all nested config flags. 23 | func (c *Config) RegisterFlags(f *flag.FlagSet) { 24 | f.StringVar(&c.Password, "schema.registry.password", "", "Password for authenticating against the schema registry (optional)") 25 | f.StringVar(&c.BearerToken, "schema.registry.token", "", "Bearer token for authenticating against the schema registry (optional)") 26 | } 27 | 28 | func (c *Config) Validate() error { 29 | if c.Enabled == false { 30 | return nil 31 | } 32 | 33 | if len(c.URLs) == 0 { 34 | return fmt.Errorf("schema registry is enabled but no URL is configured") 35 | } 36 | 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /backend/pkg/schema/config_tls.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | // TLSConfig to connect to Schema via TLS 4 | type TLSConfig struct { 5 | Enabled bool `yaml:"enabled"` 6 | CaFilepath string `yaml:"caFilepath"` 7 | CertFilepath string `yaml:"certFilepath"` 8 | KeyFilepath string `yaml:"keyFilepath"` 9 | InsecureSkipTLSVerify bool `yaml:"insecureSkipTlsVerify"` 10 | } 11 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Documentation 3 | path: /docs 4 | --- 5 | 6 | # Documentation 7 | 8 | In this folder you'll find documentation about how to configure Kowl and Kowl Business. Kowl Business is a wrapper around Kowl and provides additional functionality (see [Features](https://github.com/cloudhut/kowl#features)). This means that all config options of Kowl also apply to Kowl Business but not vice versa. Documentation pages which only apply to Kowl Business are clearly marked at the top of the page. 9 | 10 | If there are still open questions after reading this documentation please don't hesitate to submit an issue. 11 | 12 | - Getting started 13 | - [Installation](./installation.md) 14 | - [Helm Chart](https://github.com/cloudhut/charts) 15 | - [Terraform Module](https://github.com/cloudhut/terraform-modules) 16 | - Features 17 | - [Hosting](./features/hosting.md) 18 | - [Topic Documentation](./features/topic-documentation.md) 19 | - Kowl Business 20 | - [Authentication](./authentication/authentication.md) 21 | - Authorization 22 | - [Groups Sync](./authorization/groups-sync.md) 23 | - [Roles](./authorization/roles.md) 24 | - [RoleBindings](./authorization/role-bindings.md) 25 | - Reference Configs 26 | - [kowl.yaml](https://github.com/cloudhut/kowl/blob/master/docs/config/kowl.yaml) 27 | - [kowl-business.yaml](https://github.com/cloudhut/kowl/blob/master/docs/config/kowl-business.yaml) 28 | - [kowl-business-role-bindings.yaml](https://github.com/cloudhut/kowl/blob/master/docs/config/kowl-business-role-bindings.yaml) 29 | - [kowl-business-roles.yaml](https://github.com/cloudhut/kowl/blob/master/docs/config/kowl-business-roles.yaml) 30 | -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/github/create-oauth-app-step1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/github/create-oauth-app-step1.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/github/create-oauth-app-step2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/github/create-oauth-app-step2.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/github/create-personal-access-token-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/github/create-personal-access-token-1.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/github/create-personal-access-token-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/github/create-personal-access-token-2.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/create-google-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/create-google-project.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/oauth-consent-setup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/oauth-consent-setup.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/oauth-credentials-setup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/oauth-credentials-setup.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/sa-google-groups.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/sa-google-groups.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/sa-google-groups2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/sa-google-groups2.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/sa-google-groups3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/sa-google-groups3.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/sa-google-groups4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/sa-google-groups4.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/sa-google-groups5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/sa-google-groups5.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/sa-google-groups6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/sa-google-groups6.jpg -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/google/sa-google-groups7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/google/sa-google-groups7.jpg -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/okta/add-oidc-application.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/okta/add-oidc-application.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/okta/create-api-token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/okta/create-api-token.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/okta/get-client-credentials.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/okta/get-client-credentials.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/okta/setup-wizard-step1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/okta/setup-wizard-step1.png -------------------------------------------------------------------------------- /docs/assets/identity-provider-setup/okta/setup-wizard-step2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/identity-provider-setup/okta/setup-wizard-step2.png -------------------------------------------------------------------------------- /docs/assets/preview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/preview.gif -------------------------------------------------------------------------------- /docs/assets/social-preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/social-preview.png -------------------------------------------------------------------------------- /docs/assets/sponsors/rewe-digital-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/sponsors/rewe-digital-logo.png -------------------------------------------------------------------------------- /docs/assets/topic-documentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/docs/assets/topic-documentation.png -------------------------------------------------------------------------------- /docs/config/kowl-business-role-bindings.yaml: -------------------------------------------------------------------------------- 1 | # Role Bindings are used to attach roles to single users or groups of users 2 | roleBindings: 3 | - metadata: 4 | # Metadata properties will be shown in the UI. You can omit it if you want to 5 | name: Developers 6 | subjects: 7 | # You can specify all groups or users from different providers here which shall be bound to the same role 8 | - kind: group 9 | provider: Google 10 | name: dev-team-cloudhut@yourcompany.com 11 | - kind: user 12 | provider: Google 13 | name: john.doe@yourcompany.com 14 | - kind: group 15 | provider: GitHub 16 | organization: cloudhut 17 | name: kafka-owl-devs # This resolves to the team within the org 18 | - kind: user 19 | provider: GitHub 20 | name: rikimaru0345 21 | - kind: group 22 | provider: Okta 23 | name: 00qri1afoAa12G9js04x6 # Okta Group ID 24 | - kind: user 25 | provider: Okta 26 | name: weeco91@gmail.com # Okta user login 27 | roleName: developer 28 | -------------------------------------------------------------------------------- /docs/config/kowl-business-roles.yaml: -------------------------------------------------------------------------------- 1 | roles: 2 | # developer role can: 3 | # - administrate all consumer groups 4 | # - all view permissions on all topics except battle-logs 5 | # - 3 atomic permissions are allowed on the battle-logs topic 6 | - name: developer 7 | permissions: 8 | - resource: consumerGroups 9 | includes: ["/.*/"] 10 | allowedActions: ["admin"] 11 | 12 | - resource: topics 13 | includes: ["/.*/"] 14 | excludes: ["battle-logs"] 15 | allowedActions: ["viewer"] 16 | 17 | # Resource cluster does not have includes or excludes as there are no permissions implemented for individual ACL rules 18 | # Other actions also possible, i.e.: "reassignPartitions", "patchConfigs" 19 | - resource: cluster 20 | allowedActions: ["viewAcl"] 21 | 22 | - resource: topics 23 | includes: ["battle-logs"] 24 | # Other actions also possible, i.e.: "viewMessages", "useSearchFilter", "viewConsumers" 25 | allowedActions: ["seeTopic", "viewPartitions", "viewConfig"] 26 | -------------------------------------------------------------------------------- /docs/features/kafka-connect.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Kafka Connect 3 | path: /docs/features/kafka-connect 4 | --- 5 | 6 | # Kafka Connect 7 | 8 | Kowl provides a user interface that enables you to manage multiple Kafka connect clusters 9 | via a user interface. You can inspect the configured connectors, configure them, restart/pause/resume 10 | connectors and also delete them if desired. If you have more than one cluster configured some requests 11 | (such as listing connectors) will query all configured connect clusters and Kowl will aggregate the results 12 | so that you can see them in one spot. 13 | 14 | ## Configuration 15 | 16 | Below sample configuration can be put at the root level. For each cluster you have to provide at 17 | least a unique name, the HTTP address of the cluster, and the authentication settings if required. 18 | All available configuration options can be found in the [reference config](/docs/config/kowl.yaml). 19 | 20 | ```yaml 21 | connect: 22 | enabled: true 23 | clusters: 24 | - name: datawarehouse # Required field, will be used as identifier in the frontend 25 | url: http://dwh-connect.mycompany.com:8083 26 | tls: 27 | enabled: false # Trusted certs are still allowed by default 28 | username: admin 29 | # password: # Set via flag --connect.clusters.0.password=secret 30 | - name: analytics # Required field, will be used as identifier in the frontend 31 | url: http://analytics.mycompany.com:8083 32 | # No auth configured on that cluster, hence no username/password set 33 | ``` 34 | -------------------------------------------------------------------------------- /docs/menu.md: -------------------------------------------------------------------------------- 1 | - **Getting Started** 2 | - [Overview](./README.md) 3 | - [Installation](./installation.md) 4 | - **Features** 5 | - [Hosting](./features/hosting.md) 6 | - [Kafka Connect](./features/kafka-connect.md) 7 | - [Topic Documentation](./features/topic-documentation.md) 8 | - [Protobuf](./features/protobuf.md) 9 | - **Kowl Business** 10 | - [Authentication](./authentication/authentication.md) 11 | - **Authorization** 12 | - [Groups Sync](./authorization/groups-sync.md) 13 | - [Roles](./authorization/roles.md) 14 | - [RoleBindings](./authorization/role-bindings.md) 15 | - **Provider Setup** 16 | - [GitHub](./provider-setup/github.md) 17 | - [Google](./provider-setup/google.md) 18 | - [Okta](./provider-setup/okta.md) 19 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | node_modules/ 5 | 6 | /.pnp 7 | .pnp.js 8 | 9 | # testing 10 | /coverage 11 | 12 | # production 13 | /build 14 | 15 | # misc 16 | .DS_Store 17 | .env.local 18 | .env.development.local 19 | .env.test.local 20 | .env.production.local 21 | 22 | npm-debug.log* 23 | yarn-debug.log* 24 | yarn-error.log* 25 | 26 | # IDEs 27 | .idea 28 | .vscode 29 | 30 | notes.md 31 | 32 | **/utils/interpreter/compiled 33 | 34 | .env -------------------------------------------------------------------------------- /frontend/config-overrides.js: -------------------------------------------------------------------------------- 1 | 2 | const { override, fixBabelImports, addBabelPreset, addBabelPresets, addBabelPlugin, addBabelPlugins, addDecoratorsLegacy } = require('customize-cra'); 3 | 4 | 5 | module.exports = override( 6 | addBabelPlugins( 7 | "@babel/plugin-proposal-nullish-coalescing-operator", 8 | "@babel/plugin-proposal-logical-assignment-operators" 9 | ), 10 | 11 | // fixBabelImports('import', { 12 | // libraryName: 'antd', 13 | // libraryDirectory: 'es', 14 | // style: true, 15 | // }), 16 | 17 | // todo: to really get the dark theme working correctly we'll need 18 | // to fix a lot of stuff related to styling... 19 | // move inline styles out, invert stuff like #fff, ... 20 | // addLessLoader({ 21 | // javascriptEnabled: true, 22 | // modifyVars: darkTheme.default 23 | // }), 24 | ); -------------------------------------------------------------------------------- /frontend/public/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/public/favicon-32x32.png -------------------------------------------------------------------------------- /frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/public/favicon.ico -------------------------------------------------------------------------------- /frontend/public/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/public/favicon.png -------------------------------------------------------------------------------- /frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Kowl 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 |
42 | 43 | 44 | -------------------------------------------------------------------------------- /frontend/public/logo2.pdn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/public/logo2.pdn -------------------------------------------------------------------------------- /frontend/public/logo2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/public/logo2.png -------------------------------------------------------------------------------- /frontend/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "Kowl", 3 | "name": "Kowl", 4 | "icons": [ 5 | { 6 | "src": "favicon.png", 7 | "type": "image/png", 8 | "sizes": "200x200" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } -------------------------------------------------------------------------------- /frontend/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: / -------------------------------------------------------------------------------- /frontend/src/assets/circle-stop.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /frontend/src/assets/connectors/amazon-s3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/amazon-s3.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/cassandra.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/cassandra.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/confluent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/confluent.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/db2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/db2.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/debezium.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/debezium.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/elastic.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/src/assets/connectors/google-bigquery.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/src/assets/connectors/google-pub-sub.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/src/assets/connectors/hdfs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/hdfs.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/ibm-mq.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/src/assets/connectors/jdbc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/jdbc.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/mongodb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/mongodb.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/mssql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/mssql.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/postgres.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/postgres.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/salesforce.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/salesforce.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/servicenow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/servicenow.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/snowflake.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/connectors/snowflake.png -------------------------------------------------------------------------------- /frontend/src/assets/connectors/twitter.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 30 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /frontend/src/assets/filter-example-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/filter-example-1.png -------------------------------------------------------------------------------- /frontend/src/assets/filter-example-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/filter-example-2.png -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-600.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-600.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-600.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-600.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-600.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-600.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-600.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-600.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-700.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-700.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-700.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-700.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-700.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-700.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-700.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-700.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-regular.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-regular.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-regular.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/open-sans/open-sans-v20-latin-ext_latin-regular.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-300.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-300.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-300.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-300.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-300.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-300.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-300.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-300.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-500.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-500.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-500.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-500.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-500.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-500.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-500.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-500.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-600.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-600.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-600.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-600.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-600.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-600.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-600.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-600.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-700.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-700.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-700.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-700.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-700.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-700.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-700.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-700.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-regular.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-regular.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-regular.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/poppins/poppins-v15-latin-ext_latin-regular.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-500.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-500.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-500.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-500.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-500.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-500.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-500.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-500.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-600.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-600.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-600.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-600.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-600.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-600.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-600.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-600.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-700.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-700.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-700.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-700.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-700.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-700.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-700.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-700.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-regular.eot -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-regular.ttf -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-regular.woff -------------------------------------------------------------------------------- /frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/fonts/quicksand/quicksand-v24-latin-ext_latin-regular.woff2 -------------------------------------------------------------------------------- /frontend/src/assets/globExample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/globExample.png -------------------------------------------------------------------------------- /frontend/src/assets/login_wave.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/assets/logo2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/logo2.png -------------------------------------------------------------------------------- /frontend/src/assets/pattern3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ignitz/kowl/83d2bc107b5b9938f0c87681d5d7d81f368ce087/frontend/src/assets/pattern3.png -------------------------------------------------------------------------------- /frontend/src/components/misc/BoxCard.module.scss: -------------------------------------------------------------------------------- 1 | .boxCard { 2 | border: solid 1px #DDDDDD; 3 | padding: 16px; 4 | border-radius: 8px; 5 | box-sizing: border-box; 6 | } 7 | 8 | .dashed { 9 | border-style: dashed; 10 | } 11 | 12 | .hoverable:hover { 13 | border-color: lighten(#418fd8, 25%); 14 | } 15 | 16 | .active, 17 | .active:hover { 18 | border-color: #418fd8; 19 | } 20 | 21 | .hoverable:hover, 22 | .active, 23 | .medium { 24 | border-width: 2px; 25 | padding: 15px; 26 | } 27 | -------------------------------------------------------------------------------- /frontend/src/components/misc/BoxCard.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import styles from './BoxCard.module.scss'; 3 | 4 | export interface BoxCardProps { 5 | id?: string; 6 | borderStyle?: 'solid' | 'dashed'; 7 | borderWidth?: 'thin' | 'medium'; 8 | hoverable?: boolean; 9 | active?: boolean; 10 | children?: React.ReactNode; 11 | } 12 | 13 | export default function BoxCard({ id = undefined, borderStyle = 'solid', borderWidth = 'thin', hoverable = true, active = false, children }: BoxCardProps) { 14 | const classes = [styles.boxCard]; 15 | 16 | borderStyle === 'dashed' && classes.push(styles.dashed); 17 | borderWidth === 'medium' && classes.push(styles.medium); 18 | hoverable && classes.push(styles.hoverable) 19 | active && classes.push(styles.active) 20 | 21 | return
22 | {children} 23 |
; 24 | } 25 | -------------------------------------------------------------------------------- /frontend/src/components/misc/Card.tsx: -------------------------------------------------------------------------------- 1 | import React, { ReactNode, Component, CSSProperties } from "react"; 2 | 3 | 4 | class Card extends Component<{ id?: string, style?: CSSProperties, className?: string }> { 5 | 6 | render() { 7 | return
8 | {this.props.children} 9 |
10 | } 11 | } 12 | 13 | export default Card; 14 | -------------------------------------------------------------------------------- /frontend/src/components/misc/ConfigList.module.scss: -------------------------------------------------------------------------------- 1 | .default { 2 | color: #bfbfbf; 3 | } 4 | 5 | .overidden { 6 | color: #696969; 7 | 8 | .type { 9 | font-weight: 600; 10 | } 11 | } 12 | 13 | 14 | .name { 15 | display: flex; 16 | align-items: center; 17 | gap: 8px; 18 | 19 | .nameText { 20 | display: inline-block; 21 | padding: 0px 4px; 22 | 23 | border: solid thin #d1d1d1; 24 | border-radius: 3px; 25 | background-color: #f5f5f5; 26 | 27 | font-family: monospace; 28 | 29 | max-width: 360px; 30 | text-overflow: ellipsis; 31 | overflow: hidden; 32 | } 33 | 34 | .configFlags { 35 | display: flex; 36 | align-items: center; 37 | gap: 6px; 38 | 39 | margin-left: auto; // flags go to the right, next to the value 40 | } 41 | } 42 | 43 | 44 | .value { 45 | display: flex; 46 | gap: 6px; 47 | 48 | white-space: pre-line; 49 | } 50 | 51 | .type { 52 | color: #cb398f; 53 | font-family: monospace; 54 | } 55 | 56 | .sourceHeader { 57 | display: flex; 58 | gap: 6px; 59 | } 60 | 61 | .source { 62 | text-transform: capitalize; 63 | } 64 | 65 | 66 | .nested { 67 | :global(.ant-table) { 68 | background-color: transparent; 69 | } 70 | } 71 | 72 | .configEntryTable { 73 | 74 | // popovers are ugly when perfectly rectangular 75 | :global(.ant-popover-content, .ant-popover-inner) { 76 | border-radius: 6px; 77 | 78 | } 79 | 80 | :global(.ant-popover-inner-content) { 81 | padding: 6px 12px; 82 | font-family: monospace; 83 | } 84 | 85 | // antd default margin makes no sense 86 | :global(.ant-table-cell button.ant-table-row-expand-icon) { 87 | margin-left: 0; 88 | } 89 | 90 | td { 91 | // no wrapping by default 92 | white-space: nowrap; 93 | } 94 | 95 | :global(.ant-table-row-level-1) { 96 | background: #f5f5f5; 97 | 98 | td { 99 | border-bottom-color: #e7e7e7; 100 | white-space: nowrap; 101 | } 102 | 103 | // .name { 104 | // background-color: #ececec; 105 | // } 106 | } 107 | } -------------------------------------------------------------------------------- /frontend/src/components/misc/ErrorDisplay.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { observer } from "mobx-react"; 3 | import { Result } from 'antd'; 4 | import { Button } from 'antd'; 5 | import { api } from '../../state/backendApi'; 6 | import { CloseCircleOutlined } from '@ant-design/icons' 7 | import { toJson } from "../../utils/jsonUtils"; 8 | import { makeObservable } from 'mobx'; 9 | 10 | 11 | @observer 12 | export class ErrorDisplay extends React.Component { 13 | 14 | render() { 15 | if (api.errors.length === 0) 16 | return this.props.children; 17 | 18 | return <> 19 | 20 |
21 | 22 | 23 |
24 | {api.errors.map((e, i) =>
25 | {formatError(e)} 26 |
)} 27 |
28 |
29 | ; 30 | } 31 | } 32 | 33 | function formatError(err: any): string { 34 | if (err instanceof Error && err.message) { 35 | return err.message; 36 | } 37 | return String(err); 38 | } 39 | 40 | function clearErrors() { 41 | api.errors = []; 42 | } 43 | -------------------------------------------------------------------------------- /frontend/src/components/misc/HiddenRadioList.module.scss: -------------------------------------------------------------------------------- 1 | .radioCardGroup { 2 | list-style: none; 3 | margin: 1rem 0; 4 | padding: 0; 5 | 6 | display: grid; 7 | grid-template-columns: repeat(auto-fill, minmax(400px, 1fr)); 8 | grid-gap: 1rem; 9 | 10 | & > li, 11 | & > li > label { 12 | display: contents; 13 | } 14 | 15 | & > li > label { 16 | cursor: pointer; 17 | } 18 | 19 | & > li > label > input { 20 | display: none; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /frontend/src/components/misc/HiddenRadioList.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import styles from './HiddenRadioList.module.scss'; 3 | 4 | export interface HiddenRadioOption { 5 | checked?: boolean; 6 | disabled?: boolean; 7 | value: ValueType; 8 | render: (option: HiddenRadioOption) => JSX.Element 9 | } 10 | 11 | export interface HiddenRadioListProps { 12 | options: Array>; 13 | name: string; 14 | onChange: (value: ValueType) => void; 15 | value?: ValueType; 16 | disabled?: boolean; 17 | } 18 | 19 | export function HiddenRadioList({options, name, onChange, value, ...rest}: HiddenRadioListProps) { 20 | const allDisabled = rest.disabled ?? false; 21 | 22 | return (
    23 | {options.map((option, i) => { 24 | const checked = (option.value === value || option.checked) ?? false; 25 | const disabled = (allDisabled || option.disabled) ?? false; 26 | return ( 27 |
  • 28 | 38 |
    • 39 | ); 40 | })} 41 |
); 42 | } 43 | -------------------------------------------------------------------------------- /frontend/src/components/misc/HideStatisticsBarButton.tsx: -------------------------------------------------------------------------------- 1 | import { Component } from "react" 2 | import React from "react" 3 | import { Tooltip, message } from "antd" 4 | import { findPopupContainer } from "../../utils/tsxUtils" 5 | import { EyeClosedIcon } from "@primer/octicons-react" 6 | 7 | 8 | 9 | export class HideStatisticsBarButton extends Component<{ onClick: () => void }> { 10 | 11 | handleClick = () => { 12 | this.props.onClick(); 13 | message.info('Statistics bar hidden! You can enable it again in the preferences.', 8); 14 | } 15 | 16 | render() { 17 | return Hide statistics bar} 19 | getPopupContainer={findPopupContainer} 20 | arrowPointAtCenter={true} 21 | placement='right' 22 | > 23 |
24 |
25 | 26 |
27 |
28 | 29 | } 30 | } -------------------------------------------------------------------------------- /frontend/src/components/misc/KowlEditor.tsx: -------------------------------------------------------------------------------- 1 | import React, { CSSProperties } from 'react'; 2 | import Editor, { EditorProps, Monaco } from '@monaco-editor/react'; 3 | import { editor } from 'monaco-editor'; 4 | import merge from 'deepmerge'; 5 | 6 | type IStandaloneCodeEditor = editor.IStandaloneCodeEditor; 7 | 8 | export type { IStandaloneCodeEditor, Monaco } 9 | 10 | export type KowlEditorProps = EditorProps & { 11 | 12 | }; 13 | 14 | const defaultOptions: editor.IStandaloneEditorConstructionOptions = { 15 | minimap: { 16 | enabled: false, 17 | }, 18 | roundedSelection: false, 19 | padding: { 20 | top: 0, 21 | }, 22 | showFoldingControls: 'always', 23 | glyphMargin: false, 24 | scrollBeyondLastLine: false, 25 | cursorBlinking: 'phase', 26 | lineNumbersMinChars: 4, 27 | lineDecorationsWidth: 0, 28 | overviewRulerBorder: false, 29 | scrollbar: { 30 | alwaysConsumeMouseWheel: false, 31 | }, 32 | fontSize: 12, 33 | occurrencesHighlight: false, 34 | foldingHighlight: false, 35 | selectionHighlight: false, 36 | renderLineHighlight: 'all', 37 | } as const; 38 | 39 | export default function KowlEditor(props: KowlEditorProps) { 40 | const { options: givenOptions, ...rest } = props 41 | const options = Object.assign({}, defaultOptions, givenOptions ?? {}); 42 | 43 | 44 | return } 46 | wrapperProps={{ className: 'kowlEditor' }} 47 | defaultValue={'\n'.repeat(2)} 48 | options={options} 49 | {...rest} 50 | /> 51 | } 52 | 53 | const LoadingPlaceholder = () =>
54 | Loading Editor... 55 |
-------------------------------------------------------------------------------- /frontend/src/components/misc/KowlJsonView.module.scss: -------------------------------------------------------------------------------- 1 | .copyHint { 2 | position: absolute; 3 | top: 1px; 4 | right: 1px; 5 | padding: .4em 1em; 6 | 7 | background: rgba(0, 0, 0, 0.66); 8 | border-radius: 3px; 9 | 10 | color: #fff; 11 | font-family: 'Open Sans'; 12 | font-size: 12px; 13 | 14 | cursor: default; 15 | user-select: none; 16 | } 17 | 18 | .tooltipTransformFix { 19 | transform-origin: center center !important; 20 | } -------------------------------------------------------------------------------- /frontend/src/components/misc/KowlTable.module.scss: -------------------------------------------------------------------------------- 1 | .kowlTable { 2 | 3 | :global(.ant-table-footer) { 4 | padding: 0 !important; 5 | background: unset !important; 6 | 7 | display: flex; 8 | gap: 1em; 9 | margin: 16px 1px 4px 1px; 10 | 11 | :global(.ant-pagination) { 12 | margin-left: auto; 13 | } 14 | 15 | :global(.ant-pagination-total-text) { 16 | // font-family: 'Open Sans'; 17 | // font-size: 12px; 18 | cursor: default; 19 | } 20 | } 21 | 22 | th:global(.ant-table-cell) { 23 | // cursor: default; 24 | 25 | // &:not(:last-of-type) { 26 | // border-right: solid 1px hsl(220deg, 12%, 93%); 27 | // } 28 | 29 | /* 30 | ::after { 31 | content: ''; 32 | position: absolute; 33 | top: 0; 34 | right: 0; 35 | width: 10px; 36 | height: 10px; 37 | 38 | background: linear-gradient(45deg, transparent 50%, hsl(205deg, 100%, 60%) 50%); 39 | } 40 | */ 41 | } 42 | } -------------------------------------------------------------------------------- /frontend/src/components/misc/KowlTimePicker.tsx: -------------------------------------------------------------------------------- 1 | import { Component } from 'react'; 2 | import React from 'react'; 3 | import { DatePicker, Radio } from 'antd'; 4 | import { observer } from 'mobx-react'; 5 | import { makeObservable, observable } from 'mobx'; 6 | import moment from 'moment'; 7 | 8 | @observer 9 | export class KowlTimePicker extends Component<{ 10 | valueUtcMs: number; 11 | onChange: (utcMs: number) => void; 12 | disabled?: boolean; 13 | }> { 14 | @observable isLocalTimeMode = false; 15 | @observable timestampUtcMs: number = new Date().valueOf(); 16 | 17 | constructor(p: any) { 18 | super(p); 19 | this.timestampUtcMs = this.props.valueUtcMs; 20 | makeObservable(this); 21 | } 22 | 23 | render() { 24 | let format = 'DD.MM.YYYY HH:mm:ss'; 25 | let current: moment.Moment = moment.utc(this.timestampUtcMs); 26 | 27 | if (this.isLocalTimeMode) { 28 | current = current?.local(); 29 | format += ' [(Local)]'; 30 | } else { 31 | format += ' [(UTC)]'; 32 | } 33 | 34 | return ( 35 | this.footer()} 39 | format={format} 40 | value={current} 41 | onChange={(e) => { 42 | this.timestampUtcMs = e?.valueOf() ?? -1; 43 | this.props.onChange(this.timestampUtcMs); 44 | }} 45 | onOk={(e) => { 46 | this.timestampUtcMs = e.valueOf(); 47 | this.props.onChange(this.timestampUtcMs); 48 | }} 49 | disabled={this.props.disabled} 50 | /> 51 | ); 52 | } 53 | 54 | footer() { 55 | return ( 56 | { 59 | // console.log("date mode changed", { newValue: e.target.value, isLocalMode: this.isLocalTimeMode }); 60 | this.isLocalTimeMode = e.target.value == 'local'; 61 | }} 62 | > 63 | Local 64 | UTC 65 | 66 | ); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /frontend/src/components/misc/NoClipboardPopover.tsx: -------------------------------------------------------------------------------- 1 | import { Popover } from 'antd'; 2 | import React, { FunctionComponent, ReactElement } from 'react'; 3 | import { isClipboardAvailable } from '../../utils/featureDetection'; 4 | 5 | const popoverContent = ( 6 | <> 7 |

Due to browser restrictions, the clipboard is not accessible on unsecure connections.

8 |

Please make sure to run Kowl with SSL enabled to use this feature.

9 | 10 | ); 11 | 12 | export const NoClipboardPopover: FunctionComponent<{ 13 | children: ReactElement; 14 | placement?: 'left'|'top' 15 | }> = ({ children, placement = 'top' }) => 16 | isClipboardAvailable ? ( 17 | <>{children} 18 | ) : ( 19 | 20 | {children} 21 | 22 | ); 23 | -------------------------------------------------------------------------------- /frontend/src/components/misc/ShortNum.tsx: -------------------------------------------------------------------------------- 1 | import React, { ReactNode, Component, CSSProperties } from "react"; 2 | import { numberToThousandsString } from "../../utils/tsxUtils"; 3 | 4 | export function ShortNum(p: { value: number, tooltip?: boolean, className?: string, style?: CSSProperties }) { 5 | let { value, tooltip, className } = p; 6 | const style = p.style 7 | if (value == null) return ""; 8 | 9 | if (tooltip == null) tooltip = false; 10 | const originalValue = value; 11 | 12 | const million = 1000 * 1000; 13 | 14 | 15 | let decimals: number; 16 | let unit = ""; 17 | if (value >= million) { 18 | // 1000000 19 | unit = "M"; 20 | value /= million; 21 | decimals = 2; 22 | } 23 | else if (value >= 1000) { 24 | // 1000 25 | unit = "k"; 26 | value /= 1000; 27 | decimals = 1; 28 | } 29 | else { 30 | // 0-999 31 | decimals = 1; 32 | } 33 | 34 | // If, after down-scaling the number, it is still >=1k we need to add thousands separators 35 | // const needsThoudandsSeparators = value >= 1000; 36 | 37 | 38 | // Convert to fixed decimal string for rounding, 39 | // then drop trailing zeroes by converting to number and back to string 40 | const valString = Number(value.toFixed(decimals)).toLocaleString(); 41 | const str = unit 42 | ? valString + unit 43 | : valString; 44 | 45 | if (tooltip) { 46 | if (!className) className = 'tooltip'; 47 | else className += " tooltip"; 48 | 49 | return
50 | {str} 51 | {numberToThousandsString(originalValue)} 52 |
53 | } 54 | else { 55 | return 56 | {str} 57 | 58 | } 59 | } 60 | 61 | -------------------------------------------------------------------------------- /frontend/src/components/misc/Wizard.module.scss: -------------------------------------------------------------------------------- 1 | //.wizard { 2 | // :global(.ant-steps-item-custom.ant-steps-item-process .ant-steps-item-icon) > :global(.ant-steps-icon), 3 | // :global(.ant-steps-item-finish .ant-steps-item-icon) > :global(.ant-steps-icon) { 4 | // color: #5850EC; 5 | // } 6 | // 7 | // :global(.ant-steps-item-finish) > :global(.ant-steps-item-container) > :global(.ant-steps-item-content) > :global(.ant-steps-item-title::after), 8 | // background-color: #5850EC; 9 | // } 10 | //} 11 | 12 | .steps { 13 | margin-bottom: 35px; 14 | } 15 | 16 | .step { 17 | cursor: default; 18 | } 19 | 20 | .content { 21 | margin-bottom: 35px; 22 | } 23 | 24 | .footer { 25 | margin: 0; 26 | display: flex; 27 | justify-content: space-between; 28 | } 29 | 30 | .nextButton { 31 | margin-left: auto; 32 | } 33 | 34 | .prevButton { 35 | margin-right: 15px; 36 | } 37 | 38 | .nextButton, 39 | .prevButton { 40 | display: inline-flex; 41 | align-items: center; 42 | } -------------------------------------------------------------------------------- /frontend/src/components/misc/Wizard.tsx: -------------------------------------------------------------------------------- 1 | import styles from './Wizard.module.scss'; 2 | import { Button, Steps } from 'antd'; 3 | import React from 'react'; 4 | import { ChevronLeftIcon, ChevronRightIcon } from '@primer/octicons-react'; 5 | 6 | const { Step } = Steps; 7 | 8 | export function Wizard({ state }: { state: State }) { 9 | const [currentStepKey, currentStep] = state.getCurrentStep(); 10 | return (
11 | 12 | {state.getSteps().map((step, i) => )} 20 | 21 |
{currentStep.content}
22 |
23 | {/* {!state.isFirst() 24 | ? 31 | : null} */} 32 | 42 |
43 |
); 44 | } 45 | 46 | interface WizardState { 47 | getCurrentStep(): [number, WizardStep]; 48 | 49 | getSteps(): Array; 50 | 51 | canContinue(): boolean; 52 | 53 | next(): Promise; 54 | 55 | previous(): void; 56 | 57 | isLast(): boolean; 58 | 59 | isFirst(): boolean; 60 | } 61 | 62 | export interface WizardStep { 63 | title: React.ReactNode; 64 | description?: React.ReactNode; 65 | icon?: React.ReactNode; 66 | content: React.ReactNode; 67 | prevButtonLabel?: React.ReactNode; 68 | nextButtonLabel?: React.ReactNode; 69 | 70 | postConditionMet(): boolean; 71 | transitionConditionMet?(): Promise<{ conditionMet: boolean }> 72 | } 73 | -------------------------------------------------------------------------------- /frontend/src/components/misc/tabs/Tabs.module.scss: -------------------------------------------------------------------------------- 1 | @use "sass:selector"; 2 | @use "sass:string"; 3 | 4 | .wrapper { 5 | display: flex; 6 | flex-direction: column; 7 | } 8 | 9 | .content { 10 | flex-grow: 1; 11 | position: relative; 12 | } 13 | 14 | ul.navigationList { 15 | list-style: none; 16 | display: flex; 17 | margin: 0 0 1rem; 18 | padding: 0; 19 | border-bottom: solid thin #f0f0f0; 20 | 21 | li { 22 | flex: 1; 23 | min-width: 100px; 24 | } 25 | 26 | li>div.tabHeaderButton { 27 | display: flex; 28 | align-items: center; 29 | justify-content: center; 30 | border-bottom: solid 2px transparent; 31 | transform: translateY(1px); 32 | padding: 9px; 33 | color: rgba(0, 0, 0, 0.8); 34 | cursor: pointer; 35 | 36 | 37 | transition: background-color 0.2s ease-out, 38 | color .2s ease-out; 39 | &.default:hover { 40 | color: rgb(24, 144, 255); 41 | background: hsla(0, 0%, 50%, 0.03); 42 | } 43 | 44 | &.active { 45 | color: rgb(24, 144, 255); 46 | background-color: rgba(0, 185, 255, 0.09); 47 | border-bottom-color: rgb(48, 152, 232); 48 | } 49 | 50 | &.disabled { 51 | opacity: 0.25; // color: rgba(0, 0, 0, 0.25); 52 | cursor: default; 53 | } 54 | 55 | &.extra { 56 | flex: 1 1; 57 | display: flex; 58 | align-items: center; 59 | justify-content: flex-end; 60 | 61 | background: red; 62 | 63 | &:empty, 64 | &:blank { 65 | display: none; 66 | } 67 | } 68 | } 69 | 70 | } -------------------------------------------------------------------------------- /frontend/src/components/pages/Page.ts: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { ParsedQuery } from "query-string"; 3 | import { uiState } from "../../state/uiState"; 4 | 5 | 6 | // 7 | // Page Types 8 | // 9 | export type PageProps> = TRouteParams & { matchedPath: string; query: ParsedQuery; } 10 | 11 | export class PageInitHelper { 12 | set title(title: string) { uiState.pageTitle = title; } 13 | addBreadcrumb(title: string, to: string) { uiState.pageBreadcrumbs.push({ title: title, linkTo: to }) } 14 | } 15 | export abstract class PageComponent> extends React.Component> { 16 | 17 | constructor(props: Readonly>) { 18 | super(props); 19 | 20 | uiState.pageBreadcrumbs = []; 21 | 22 | this.initPage(new PageInitHelper()); 23 | } 24 | 25 | abstract initPage(p: PageInitHelper): void; 26 | } 27 | export type PageComponentType> = (new (props: PageProps) => PageComponent>); 28 | 29 | 30 | -------------------------------------------------------------------------------- /frontend/src/components/pages/UrlTestPage.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { Section } from "../misc/common"; 3 | import { PageComponent, PageInitHelper } from "./Page"; 4 | 5 | import { motion, AnimatePresence } from "framer-motion" 6 | import { animProps, MotionDiv } from "../../utils/animationProps"; 7 | import { observer } from "mobx-react"; 8 | import { Checkbox } from "antd"; 9 | import { makeObservable, observable } from "mobx"; 10 | 11 | @observer 12 | export class UrlTestPage extends PageComponent { 13 | 14 | @observable test: boolean = true; 15 | 16 | constructor(p: any) { 17 | super(p); 18 | makeObservable(this); 19 | } 20 | 21 | initPage(p: PageInitHelper) { 22 | p.title = 'DEBUG PLACEHOLDER'; 23 | } 24 | 25 | render() { 26 | const p = this.props; 27 | return ( 28 | 29 |
30 |

Path:

31 |

{p.matchedPath}

32 |
33 | 34 |
35 |

Query:

36 | 
{JSON.stringify(p.query, null, 4)}
37 |
38 | 39 |
40 |

All Props:

41 | 
{JSON.stringify(p, null, 4)}
42 |
43 | 44 |
45 |

Test

46 |
this.test = e.target.checked}>Test Prop
47 | 48 | 49 | {this.test 50 | 51 | ? 52 |

The first test container

53 | 54 | 55 | : 56 |

Another one! (This is the second container)

57 | 58 | 59 | } 60 | 61 |
62 | 63 | 64 | 65 | ); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /frontend/src/components/pages/connect/ConnectorBoxCard.module.scss: -------------------------------------------------------------------------------- 1 | .radioCardContent { 2 | display: flex; 3 | align-items: center; 4 | } 5 | 6 | .radioCardLogo { 7 | width: 32px; 8 | height: 32px; 9 | margin-right: 16px; 10 | flex-grow: 0; 11 | flex-shrink: 0; 12 | } 13 | 14 | .radioCardInfo { 15 | flex-basis: 100%; 16 | 17 | & > strong { 18 | font-size: 14px; 19 | line-height: 22px; 20 | font-weight: normal; 21 | } 22 | } 23 | 24 | .pluginType, 25 | .pluginMeta { 26 | color: lighten(black, 55%); 27 | } 28 | 29 | .pluginMeta { 30 | margin-bottom: 0; 31 | } 32 | -------------------------------------------------------------------------------- /frontend/src/components/pages/connect/ConnectorBoxCard.tsx: -------------------------------------------------------------------------------- 1 | import { findConnectorMetadata, removeNamespace } from './helper'; 2 | import styles from './ConnectorBoxCard.module.scss'; 3 | import React from 'react'; 4 | import BoxCard, { BoxCardProps } from '../../misc/BoxCard'; 5 | import { HiddenRadioOption } from '../../misc/HiddenRadioList'; 6 | 7 | interface ConnectorBoxCardProps extends Omit, Omit, "render" | "value"> { 8 | connectorPlugin: ConnectorPlugin; 9 | id?: string; 10 | } 11 | 12 | export function ConnectorBoxCard(props: ConnectorBoxCardProps) { 13 | const { id, checked, connectorPlugin, hoverable, active, borderWidth, borderStyle } = props; 14 | return ( 15 | 16 | ); 17 | } 18 | 19 | export type ConnectorPlugin = { class: string; type?: string; version?: string }; 20 | 21 | function ConnectorRadioCardContent({ connectorPlugin }: { connectorPlugin: ConnectorPlugin }) { 22 | const { friendlyName, logo, author = 'unknown' } = findConnectorMetadata(connectorPlugin.class) ?? {}; 23 | const displayName = friendlyName ?? removeNamespace(connectorPlugin.class); 24 | const type = connectorPlugin.type ?? 'unknown' 25 | const version = connectorPlugin.version ?? 'unknown' 26 | 27 | return
28 | {logo} 29 |
30 | {displayName} {connectorPlugin.type != null 31 | ? ({type}) 32 | : null} 33 | {connectorPlugin.version != null 34 | ?

35 | Version: {version} | Author: {author} 36 |

37 | : null} 38 |
39 |
; 40 | } 41 | -------------------------------------------------------------------------------- /frontend/src/components/pages/connect/CreateConnector.module.scss: -------------------------------------------------------------------------------- 1 | .wizardView { 2 | padding-top: 40px; 3 | 4 | h2 { 5 | margin-top: 35px; 6 | } 7 | } 8 | 9 | .motionContainer { 10 | margin: 0 1rem; 11 | } 12 | 13 | .connectorBoxCard { 14 | margin-bottom: 35px; 15 | max-width: 744px; 16 | } 17 | -------------------------------------------------------------------------------- /frontend/src/components/pages/connect/dynamic-ui/DebugEditor.tsx: -------------------------------------------------------------------------------- 1 | import { observer } from 'mobx-react'; 2 | 3 | import KowlEditor from '../../../misc/KowlEditor'; 4 | 5 | 6 | export const DebugEditor = observer((p: { observable: { jsonText: string } }) => { 7 | const obs = p.observable; 8 | 9 | return
10 |

Debug Editor

11 | { 16 | if (v) { 17 | if (!obs.jsonText && !v) 18 | return; // dont replace undefiend with empty (which would trigger our 'autorun') 19 | obs.jsonText = v; 20 | } 21 | }} 22 | height="300px" 23 | /> 24 |
25 | 26 | }); 27 | -------------------------------------------------------------------------------- /frontend/src/components/pages/connect/dynamic-ui/PropertyGroup.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-useless-escape */ 2 | import { Collapse } from 'antd'; 3 | import { observer } from 'mobx-react'; 4 | import { PropertyGroup } from './components'; 5 | import { PropertyComponent } from './PropertyComponent'; 6 | 7 | export const PropertyGroupComponent = observer((props: { group: PropertyGroup, allGroups: PropertyGroup[] }) => { 8 | const g = props.group; 9 | 10 | if (g.groupName == "Transforms") { 11 | // Transforms + its sub groups 12 | 13 | const subGroups = props.allGroups 14 | .filter(g => g.groupName.startsWith("Transforms: ")) 15 | .sort((a, b) => props.allGroups.indexOf(a) - props.allGroups.indexOf(b)); 16 | 17 | return
18 | {g.properties.map(p => )} 19 | 20 |
21 | 22 | 23 | {subGroups.map(subGroup => 24 | 0) ? 'hasErrors' : ''} 26 | key={subGroup.groupName} 27 | header={
28 | {subGroup.groupName} 29 | {subGroup.propertiesWithErrors.length} issues 30 |
} 31 | > 32 | 33 | 34 | )} 35 | 36 | 37 |
38 |
39 | 40 | } 41 | else { 42 | // Normal group 43 | return
44 | {g.properties.map(p => )} 45 |
46 | } 47 | 48 | }); 49 | -------------------------------------------------------------------------------- /frontend/src/components/pages/reassign-partitions/components/IndeterminateCheckbox.tsx: -------------------------------------------------------------------------------- 1 | import React, { Component } from "react"; 2 | import { observer } from "mobx-react"; 3 | 4 | 5 | @observer 6 | export class IndeterminateCheckbox extends Component<{ originalCheckbox: React.ReactNode; getCheckState: () => { checked: boolean; indeterminate: boolean; }; }> { 7 | 8 | render() { 9 | const state = this.props.getCheckState(); 10 | 11 | // console.log(`checkbox${index} props: ${(originNode as any).props?.indeterminate}`) 12 | const clone = React.cloneElement(this.props.originalCheckbox as any, { 13 | checked: state.checked, 14 | indeterminate: state.indeterminate, 15 | }); 16 | return clone; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /frontend/src/components/pages/reassign-partitions/components/tsxUtils.tsx: -------------------------------------------------------------------------------- 1 | 2 | export { }; -------------------------------------------------------------------------------- /frontend/src/components/pages/schemas/Schema.List.scss: -------------------------------------------------------------------------------- 1 | .SchemaList__error-card:empty { 2 | display: none; 3 | } 4 | 5 | .SchemaList__alert + .SchemaList__alert { 6 | margin-top: 1rem; 7 | } -------------------------------------------------------------------------------- /frontend/src/components/pages/topics/DeleteRecordsModal/DeleteRecordsModal.module.scss: -------------------------------------------------------------------------------- 1 | .sliderContainer { 2 | display: flex; 3 | align-items: center; 4 | } 5 | 6 | .sliderValue { 7 | width: calc(124rem / 16); 8 | margin-left: 2rem; 9 | line-height: calc(22rem / 16); 10 | padding-top: 0; 11 | padding-bottom: 0; 12 | } 13 | 14 | .slider { 15 | flex-grow: 1; 16 | } 17 | 18 | .partitionSelect { 19 | margin-top: 1em; 20 | } 21 | 22 | .twoCol { 23 | display: grid; 24 | grid-template-columns: 66px 1fr; 25 | gap: 30px; 26 | margin-bottom: 25px; 27 | } -------------------------------------------------------------------------------- /frontend/src/components/pages/topics/DeleteRecordsModal/DeleteRecordsModal.test.tsx: -------------------------------------------------------------------------------- 1 | import { fireEvent, render } from '@testing-library/react'; 2 | import { Topic } from '../../../../state/restInterfaces'; 3 | import DeleteRecordsModal from './DeleteRecordsModal'; 4 | 5 | const testTopic: Topic = { 6 | allowedActions: ['all'], 7 | cleanupPolicy: 'compact', 8 | partitionCount: 3, 9 | replicationFactor: 3, 10 | isInternal: false, 11 | topicName: 'test_topic', 12 | logDirSummary: { 13 | totalSizeBytes: 1024, 14 | hint: null, 15 | replicaErrors: [], 16 | }, 17 | }; 18 | 19 | it('renders all expected elements in step 1', () => { 20 | const { getByLabelText, getByText } = render(); 21 | 22 | expect(getByText('Delete records in topic')).toBeInTheDocument(); 23 | expect(getByText('All Partitions')).toBeInTheDocument(); 24 | expect(getByText('Specific Partition')).toBeInTheDocument(); 25 | expect(getByText('Cancel')).toBeInTheDocument(); 26 | expect(getByText('Choose End Offset')).toBeInTheDocument(); 27 | 28 | expect(getByLabelText(/All Partitions/)).toBeChecked(); 29 | }); 30 | 31 | it('renders all expected elements in step 2', () => { 32 | const { getByLabelText, getByText } = render(); 33 | 34 | fireEvent.click(getByText('Choose End Offset')); 35 | 36 | expect(getByText('Manual Offset')).toBeInTheDocument(); 37 | expect(getByText('Timestamp')).toBeInTheDocument(); 38 | expect(getByText('Cancel')).toBeInTheDocument(); 39 | expect(getByText('Delete Records')).toBeInTheDocument(); 40 | 41 | expect(getByLabelText(/Manual Offset/)).toBeChecked(); 42 | }); 43 | -------------------------------------------------------------------------------- /frontend/src/components/pages/topics/Tab.Acl/AclList.test.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render } from '@testing-library/react'; 3 | import AclList from './AclList'; 4 | import { observable } from 'mobx'; 5 | import { ResourcePatternType } from '../../../../state/restInterfaces'; 6 | 7 | it('renders an empty table when no data is present', () => { 8 | const store = observable({ 9 | isAuthorizerEnabled: true, 10 | aclResources: [], 11 | }); 12 | 13 | const { getByText } = render(); 14 | expect(getByText('No Data')).toBeInTheDocument(); 15 | }); 16 | 17 | it('a table with one entry', () => { 18 | const store = observable({ 19 | isAuthorizerEnabled: true, 20 | aclResources: [ 21 | { 22 | resourceType: 'Topic', 23 | resourceName: 'Test Topic', 24 | resourcePatternType: ResourcePatternType.UNKNOWN, 25 | acls: [ 26 | { 27 | principal: 'test principal', 28 | host: 'test host', 29 | operation: 'test operation', 30 | permissionType: 'test permission type', 31 | }, 32 | ], 33 | }, 34 | ], 35 | }); 36 | 37 | const { getByText } = render(); 38 | 39 | expect(getByText('Topic')).toBeInTheDocument(); 40 | expect(getByText('Test Topic')).toBeInTheDocument(); 41 | expect(getByText('0')).toBeInTheDocument(); 42 | expect(getByText('test principal')).toBeInTheDocument(); 43 | expect(getByText('test host')).toBeInTheDocument(); 44 | expect(getByText('test operation')).toBeInTheDocument(); 45 | expect(getByText('test permission type')).toBeInTheDocument(); 46 | }); 47 | 48 | it('informs user about missing permission to view ACLs', () => { 49 | const { getByText } = render(); 50 | expect(getByText('You do not have the necessary permissions to view ACLs')).toBeInTheDocument(); 51 | }); 52 | 53 | it('informs user about missing authorizer config in Kafka cluster', () => { 54 | const store = observable({ 55 | isAuthorizerEnabled: false, 56 | aclResources: [], 57 | }); 58 | 59 | const { getByText } = render(); 60 | expect(getByText("There's no authorizer configured in your Kafka cluster")).toBeInTheDocument(); 61 | }); 62 | -------------------------------------------------------------------------------- /frontend/src/components/pages/topics/Tab.Messages/styles.module.scss: -------------------------------------------------------------------------------- 1 | .filterbar { 2 | display: flex; 3 | flex-wrap: wrap; 4 | } 5 | 6 | .filters { 7 | width: calc(100% - 200px); 8 | display: inline-flex; 9 | row-gap: 2px; 10 | flex-wrap: wrap; 11 | } 12 | 13 | .filterName { 14 | display: inline-block; 15 | } 16 | 17 | .addFilter { 18 | vertical-align: middle; 19 | margin-top: -3px; 20 | } 21 | 22 | .metaSection { 23 | display: flex; 24 | justify-content: flex-end; 25 | align-items: center; 26 | margin-left: auto; 27 | width: 200px; 28 | 29 | font-size: 12px; 30 | font-weight: 500; 31 | padding: 0 5px; 32 | 33 | .bytesIcon { 34 | color: #1890ff; 35 | } 36 | 37 | .time { 38 | margin-left: 20px; 39 | } 40 | 41 | .timeIcon { 42 | color: #1890ff; 43 | } 44 | } 45 | 46 | .topicActionsWrapper, 47 | .quickSearch { 48 | height: 100%; 49 | display: flex; 50 | align-items: center; 51 | justify-content: center; 52 | } 53 | 54 | .topicActionsWrapper { 55 | margin-left: auto; 56 | } 57 | 58 | .quickSearchWrapper { 59 | margin-left: 1rem; 60 | } 61 | 62 | .quickSearchInput { 63 | width: 200px; 64 | padding: 2px 8px; 65 | white-space: nowrap; 66 | height: 32px; 67 | } -------------------------------------------------------------------------------- /frontend/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import { 4 | BrowserRouter, 5 | withRouter, 6 | RouteComponentProps 7 | } from "react-router-dom"; 8 | import { configure, when } from "mobx"; 9 | 10 | import "antd/dist/antd.css"; 11 | import "./index.scss"; 12 | 13 | import App from "./components/App"; 14 | import { appGlobal } from "./state/appGlobal"; 15 | import { basePathS, IsBusiness } from "./utils/env"; 16 | import { api } from "./state/backendApi"; 17 | 18 | import './assets/fonts/open-sans.css'; 19 | import './assets/fonts/poppins.css'; 20 | import './assets/fonts/quicksand.css'; 21 | 22 | const HistorySetter = withRouter((p: RouteComponentProps) => { 23 | appGlobal.history = p.history; 24 | return <>; 25 | }); 26 | 27 | // Configure MobX 28 | configure({ 29 | enforceActions: 'never', 30 | safeDescriptors: true, 31 | }); 32 | 33 | // Get supported endpoints / kafka cluster version 34 | // In the business version, that endpoint (like any other api endpoint) is 35 | // protected, so we need to delay the call until the user is logged in. 36 | if (!IsBusiness) { 37 | api.refreshSupportedEndpoints(true); 38 | } else { 39 | when(() => Boolean(api.userData), () => { 40 | setImmediate(() => { 41 | api.refreshSupportedEndpoints(true); 42 | }); 43 | }); 44 | } 45 | 46 | ReactDOM.render( 47 | 48 | 49 | 50 | , 51 | document.getElementById("root") 52 | ); 53 | -------------------------------------------------------------------------------- /frontend/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /frontend/src/setupTests.js: -------------------------------------------------------------------------------- 1 | import '@testing-library/jest-dom/extend-expect'; 2 | 3 | Object.defineProperty(global, 'matchMedia', { 4 | writable: true, 5 | value: (query) => ({ 6 | matches: false, 7 | media: query, 8 | onchange: null, 9 | addListener: jest.fn(), // deprecated 10 | removeListener: jest.fn(), // deprecated 11 | addEventListener: jest.fn(), 12 | removeEventListener: jest.fn(), 13 | dispatchEvent: jest.fn(), 14 | }), 15 | }); 16 | -------------------------------------------------------------------------------- /frontend/src/state/appGlobal.ts: -------------------------------------------------------------------------------- 1 | 2 | import { History } from 'history'; 3 | import { api } from './backendApi'; 4 | import { uiState } from './uiState'; 5 | 6 | class AppGlobal { 7 | private _history = (null as unknown as History); 8 | get history() { return this._history } 9 | 10 | set history(h: History) { 11 | if (this._history === h || !h) return; 12 | if (this._history) throw new Error('_history should not be overwritten'); 13 | 14 | this._history = h; 15 | 16 | h.listen((location, action) => { 17 | api.errors = []; 18 | uiState.pathName = location.pathname; 19 | }); 20 | uiState.pathName = h.location.pathname; 21 | } 22 | 23 | onRefresh: (() => void) = () => { 24 | // intended for pages to set 25 | } 26 | } 27 | export const appGlobal = new AppGlobal(); -------------------------------------------------------------------------------- /frontend/src/utils/LazyMap.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | export class LazyMap extends Map { 4 | constructor(private defaultCreate: (key: K) => V) { 5 | super(); 6 | } 7 | 8 | /** 9 | * @description Returns the value corrosponding to key 10 | * @param key Key of the value 11 | * @param create An optional `create` method to use instead of `defaultCreate` to create missing values 12 | */ 13 | get(key: K, create?: (key: K) => V): V { 14 | let v = super.get(key); 15 | if (v !== undefined) { 16 | return v; 17 | } 18 | 19 | v = this.handleMiss(key, create); 20 | this.set(key, v); 21 | return v; 22 | } 23 | 24 | private handleMiss(key: K, create?: ((key: K) => V)): V { 25 | if (create) { 26 | return create(key); 27 | } 28 | return this.defaultCreate(key); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /frontend/src/utils/extensions.ts: -------------------------------------------------------------------------------- 1 | 2 | export { } 3 | 4 | declare global { 5 | interface String { 6 | removePrefix(this: string, prefix: string): string; 7 | removeSuffix(this: string, suffix: string): string; 8 | } 9 | } 10 | 11 | 12 | String.prototype.removePrefix = function (this: string, prefix: string) { 13 | if (prefix.length == 0) return this; 14 | 15 | if (this.toLowerCase().startsWith(prefix.toLowerCase())) 16 | return this.substr(prefix.length); 17 | return this; 18 | } 19 | 20 | String.prototype.removeSuffix = function (this: string, suffix: string) { 21 | if (suffix.length == 0) return this; 22 | 23 | if (this.toLowerCase().endsWith(suffix.toLowerCase())) 24 | return this.substr(0, this.length - suffix.length); 25 | return this; 26 | } 27 | 28 | -------------------------------------------------------------------------------- /frontend/src/utils/featureDetection.ts: -------------------------------------------------------------------------------- 1 | export const isClipboardAvailable = Boolean(navigator?.clipboard) -------------------------------------------------------------------------------- /frontend/src/utils/fetchWithTimeout.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | export class RestTimeoutError extends Error { 4 | constructor(m: string) { 5 | super(m); 6 | Object.setPrototypeOf(this, RestTimeoutError.prototype); 7 | } 8 | } 9 | 10 | export default function fetchWithTimeout(url: RequestInfo, timeoutMs: number, options?: RequestInit): Promise { 11 | 12 | const requestPromise = fetch(url, options); 13 | const timeoutPromise = new Promise((_, reject) => { 14 | setTimeout(() => { 15 | reject(new RestTimeoutError("Request timed out after " + (timeoutMs / 1000).toFixed(1) + " sec: " + url)); 16 | }, timeoutMs); 17 | }); 18 | 19 | return Promise.race([requestPromise, timeoutPromise]); 20 | } 21 | -------------------------------------------------------------------------------- /frontend/src/utils/filterHelper.ts: -------------------------------------------------------------------------------- 1 | 2 | export function wrapFilterFragment(filterFragment: string) { 3 | if (!filterFragment.includes('return ')) 4 | filterFragment = `return (${filterFragment})`; 5 | return filterFragment; 6 | } 7 | 8 | export function injectFindFunc(fullFilterCode: string) { 9 | const code = fullFilterCode; 10 | if (!code.includes('find')) 11 | return code; 12 | 13 | 14 | const regexFindPropGlobal = /findProp\('(.*?)'\)/g; 15 | const regexFindProp = /findProp\('(.*?)'\)/; 16 | 17 | // Find the group of occurrences 18 | const findPropOccurences = code.match(regexFindPropGlobal) || []; 19 | 20 | findPropOccurences.forEach(occurence => { 21 | // Extract the attributes or keys from the string 22 | const keysMatch = occurence.match(regexFindProp) || []; 23 | const keys = keysMatch[1]?.split('.') ?? []; 24 | 25 | // Map it from 'key.key' into ['key']['key'] 26 | const replacedKeys = keys.map(key => `['${key}']`); 27 | const replacedString = `value${replacedKeys.join('')}`; 28 | 29 | // Replace the code with the replaced string 30 | // From: findProp('key.key') 31 | // To: value['key']['key'] 32 | // code = code.replace(occurence, replacedString); 33 | }); 34 | } 35 | 36 | export function sanitizeString(input: string) { 37 | return input.split('') 38 | .map((char: string) => { 39 | const code = char.charCodeAt(0); 40 | if (code > 0 && code < 128) { 41 | return char; 42 | } else if (code >= 128 && code <= 255) { 43 | //Hex escape encoding 44 | return `/x${code.toString(16)}`.replace('/', '\\'); 45 | } 46 | return ''; 47 | }) 48 | .join(''); 49 | } -------------------------------------------------------------------------------- /frontend/src/utils/filterableDataSource.ts: -------------------------------------------------------------------------------- 1 | import { observable, autorun, IReactionDisposer, computed, transaction, action, makeObservable } from "mobx"; 2 | 3 | /* 4 | Intended use: 5 | create a FilterableDataSource and give it a function (a mobx @computed) that acts as the data source, 6 | as well as a filter function (that accepts or rejects elements of the data source based on the filterText). 7 | When either the filterText or the dataSource change, the filter will be applied to all elements of the source, 8 | and the result will be set to 'data' (which is observable as well of course) 9 | */ 10 | export class FilterableDataSource { 11 | private reactionDisposer?: IReactionDisposer; 12 | 13 | @observable filterText: string = ''; // set by the user (from an input field or so, can be read/write) 14 | 15 | @observable private _lastFilterText: string = ''; 16 | @computed get lastFilterText() { return this._lastFilterText; } 17 | @observable.ref private resultData: T[] = []; // set by this class (so only exposed through computed prop) 18 | @computed get data(): T[] { return this.resultData; } 19 | 20 | constructor( 21 | private dataSource: () => T[] | undefined, 22 | private filter: (filterText: string, item: T) => boolean, 23 | debounceMilliseconds?: number 24 | ) { 25 | if (!debounceMilliseconds) debounceMilliseconds = 100; 26 | this.reactionDisposer = autorun(this.update.bind(this), { delay: debounceMilliseconds, name: 'FilterableDataSource' }); 27 | 28 | makeObservable(this); 29 | } 30 | 31 | private update() { 32 | transaction(() => { 33 | const source = this.dataSource(); 34 | const filterText = this.filterText; 35 | if (source) { 36 | this.resultData = source.filter(x => this.filter(filterText, x)); 37 | //console.log('updating filterableDataSource: ...'); 38 | } else { 39 | this.resultData = []; 40 | //console.log('updating filterableDataSource: source == undefined|null'); 41 | } 42 | this._lastFilterText = this.filterText; 43 | }); 44 | } 45 | 46 | dispose() { 47 | if (this.reactionDisposer) { 48 | this.reactionDisposer(); 49 | this.reactionDisposer = undefined; 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /frontend/src/utils/interpreter/global.ts: -------------------------------------------------------------------------------- 1 | 2 | /*eslint @typescript-eslint/no-namespace: off */ 3 | 4 | declare namespace NodeJS { 5 | interface Global { 6 | value: any; 7 | } 8 | } 9 | 10 | global.value = {}; -------------------------------------------------------------------------------- /frontend/src/utils/interpreter/helpers.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | const toJson = JSON.stringify; 5 | export let successfulTests = 0; 6 | 7 | 8 | export function expectRefEq(test: { name: string, actual: any, expected: any }) { 9 | const { name, actual, expected } = test; 10 | 11 | expect(`${name} (type equality)`, () => typeof actual === typeof expected); 12 | successfulTests--; 13 | 14 | expect(`${name} (ref equality)`, () => actual === expected); 15 | successfulTests--; 16 | 17 | successfulTests++; 18 | } 19 | export function expectEq(test: { name: string, actual: any, expected: any }) { 20 | const jActual = toJson(test.actual); 21 | const jExpected = toJson(test.expected); 22 | if (jActual == jExpected) { 23 | successfulTests++; 24 | return; 25 | } 26 | 27 | throw new Error(` 28 | Test failed: ${test.name} 29 | 30 | Actual: 31 | ${jActual} 32 | 33 | Expected: 34 | ${jExpected} 35 | 36 | `); 37 | } 38 | 39 | export function expect(test: () => boolean): void; 40 | export function expect(name: string, test: () => boolean): void; 41 | export function expect(testOrName: string | (() => boolean), test?: () => boolean) { 42 | const testFunc = typeof testOrName === 'function' 43 | ? testOrName as (() => boolean) 44 | : test!; 45 | 46 | const name = typeof testOrName === 'string' 47 | ? testOrName as string 48 | : null; 49 | 50 | if (testFunc()) { 51 | successfulTests++; 52 | return; // Success 53 | } 54 | 55 | // Failed! 56 | throw new Error(` 57 | Test failed ${name ? (': ' + name) : ''} 58 | ${testFunc.toString()} 59 | 60 | `); 61 | } 62 | -------------------------------------------------------------------------------- /frontend/src/utils/interpreter/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES5", 4 | "lib": [], 5 | "moduleResolution": "node", 6 | "module": "commonjs", 7 | "allowJs": false, 8 | "skipLibCheck": true, 9 | "esModuleInterop": true, 10 | "allowSyntheticDefaultImports": true, 11 | "strict": false, 12 | "isolatedModules": false, 13 | "strictPropertyInitialization": false, 14 | "outDir": "compiled", 15 | "removeComments": true, 16 | "noEmitHelpers": true, 17 | "noEmitOnError": true, 18 | "noImplicitUseStrict": true, 19 | }, 20 | "include": [ 21 | "findFunction.ts" 22 | ], 23 | } -------------------------------------------------------------------------------- /frontend/src/utils/jsonUtils.ts: -------------------------------------------------------------------------------- 1 | const seen = new Set(); 2 | // Serialize object to json, handling reference loops gracefully 3 | export function toJson(obj: any, space?: string | number | undefined): string { 4 | seen.clear(); 5 | try { 6 | return JSON.stringify(obj, 7 | (key: string, value: any) => { 8 | if (typeof value === "object" && value !== null) { 9 | if (seen.has(value)) { 10 | return; 11 | } 12 | seen.add(value); 13 | } 14 | 15 | if (value instanceof Error) { 16 | return value.toString(); 17 | } 18 | 19 | return value; 20 | }, 21 | space 22 | ); 23 | } 24 | finally { 25 | seen.clear(); 26 | } 27 | } 28 | // Clone object using serialization 29 | 30 | export function clone(obj: T): T { 31 | if (!obj) 32 | return obj; 33 | return JSON.parse(toJson(obj)); 34 | } 35 | // Accesses all members of an object by serializing it 36 | 37 | 38 | export function touch(obj: any): void { 39 | JSON.stringify(obj, (k, v) => { 40 | if (typeof v === 'object') 41 | return v; 42 | return ''; 43 | }); 44 | } 45 | -------------------------------------------------------------------------------- /frontend/src/utils/numberExtensions.ts: -------------------------------------------------------------------------------- 1 | 2 | export { } 3 | 4 | declare global { 5 | interface Number { 6 | /** 7 | * linear interpolation to another number 8 | * @param to number to interpolate to 9 | * @param t factor between 0 and 1 10 | */ 11 | lerp(this: number, to: number, t: number): number; 12 | 13 | clamp(this: number, min: number | undefined, max: number | undefined): number; 14 | } 15 | } 16 | 17 | Number.prototype.lerp = function (this: number, to: number, t: number): number { 18 | const d = to - this; 19 | return this + d * t; 20 | }; 21 | 22 | Number.prototype.clamp = function (this: number, min: number | undefined, max: number | undefined): number { 23 | if (max !== undefined) 24 | if (this > max) return max; 25 | if (min !== undefined) 26 | if (this < min) return min; 27 | return this; 28 | }; 29 | -------------------------------------------------------------------------------- /frontend/src/utils/queryHelper.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | import queryString, { ParseOptions, StringifyOptions, ParsedQuery } from 'query-string'; 4 | 5 | import { appGlobal } from '../state/appGlobal'; 6 | 7 | const parseOptions: ParseOptions = { 8 | arrayFormat: 'comma', 9 | parseBooleans: true, 10 | parseNumbers: true, 11 | }; 12 | const stringifyOptions: StringifyOptions = { 13 | strict: false, 14 | encode: true, 15 | arrayFormat: 'comma', 16 | sort: false, 17 | }; 18 | 19 | export const queryToObj = (str: string) => queryString.parse(str, parseOptions); 20 | export const objToQuery = (obj: { [key: string]: any; }) => '?' + queryString.stringify(obj, stringifyOptions); 21 | 22 | 23 | // edit the current search query, 24 | // IFF you make any changes inside editFunction, it returns the stringified version of the search query 25 | export function editQuery(editFunction: (queryObject: ParsedQuery) => void) { 26 | const urlParams = queryString.parse(window.location.search); 27 | editFunction(urlParams); 28 | const query = queryString.stringify(urlParams); 29 | const search = '?' + query; 30 | 31 | if (window.location.search != search) { 32 | //console.log(`changing search: (${window.location.search}) -> (${search})`); 33 | appGlobal.history.location.search = search; 34 | appGlobal.history.replace(appGlobal.history.location); 35 | } 36 | } 37 | 38 | -------------------------------------------------------------------------------- /frontend/src/utils/svg/OktaLogo.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | 4 | export default 5 | 6 | 7 | 14 | 21 | 25 | 26 | -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "experimentalDecorators": true, 4 | "useDefineForClassFields": true, 5 | "target": "es2019", 6 | "lib": [ 7 | "dom", 8 | "dom.iterable", 9 | "esnext" 10 | ], 11 | "allowJs": true, 12 | "skipLibCheck": true, 13 | "esModuleInterop": true, 14 | "allowSyntheticDefaultImports": true, 15 | "strict": true, 16 | "forceConsistentCasingInFileNames": true, 17 | "module": "esnext", 18 | "moduleResolution": "node", 19 | "resolveJsonModule": true, 20 | "isolatedModules": true, 21 | "noEmit": true, 22 | "jsx": "react-jsx", 23 | "strictPropertyInitialization": false, 24 | "noFallthroughCasesInSwitch": true, 25 | }, 26 | "include": [ 27 | "src" 28 | ], 29 | "exclude": [ 30 | "src/utils/interpreter/**" 31 | ] 32 | } --------------------------------------------------------------------------------