├── .dockerignore ├── .docs ├── assets │ ├── bulker-summary.excalidraw.png │ └── stream-batch.excalidraw.png ├── db-feature-matrix.md ├── http-api.md ├── rfcs │ ├── data_types.md │ └── http.md └── server-config.md ├── .github └── workflows │ ├── tag-release.yml │ └── test-build-image.yml ├── .gitignore ├── CONTRIBUTING.md ├── GOWORK.md ├── LICENSE ├── README.md ├── TODO.md ├── admin ├── go.mod ├── go.sum ├── kafka.go └── main.go ├── all.Dockerfile ├── allbuild.sh ├── bulker.Dockerfile ├── bulkerapp ├── app │ ├── abstract_batch_consumer.go │ ├── abstract_consumer.go │ ├── app.go │ ├── app_config.go │ ├── batch_consumer.go │ ├── configuration_source.go │ ├── cron.go │ ├── fast_store.go │ ├── http_configuration_source.go │ ├── integration_test.go │ ├── load_test.go │ ├── metrics_server.go │ ├── multi_configuration_source.go │ ├── postgres_configuration_source.go │ ├── producer.go │ ├── redis_configuration_source.go │ ├── repository.go │ ├── retry_consumer.go │ ├── router.go │ ├── stream_consumer.go │ ├── test_data │ │ ├── badbatch.ndjson │ │ ├── bulker.env │ │ ├── config.yaml │ │ ├── goodbatch.ndjson │ │ └── single.ndjson │ ├── testcontainers │ │ └── kafka │ │ │ ├── docker-compose.yml │ │ │ └── kafka_container.go │ ├── topic_manager.go │ └── topic_manager_test.go ├── default.pgo ├── go.mod ├── go.sum ├── main.go └── metrics │ └── metrics.go ├── bulkerbuild.sh ├── bulkerlib ├── bulker.go ├── bulker_state_test.go ├── go.mod ├── go.sum ├── implementations │ ├── api_based │ │ ├── mixpanel.go │ │ ├── transactional_stream.go │ │ └── webhook.go │ ├── file.go │ ├── file_storage │ │ ├── abstract.go │ │ ├── bulker_test.go │ │ ├── gcs_bulker.go │ │ ├── replacepartition_stream.go │ │ ├── replacetable_stream.go │ │ ├── s3_bulker.go │ │ ├── test_data │ │ │ ├── empty.ndjson │ │ │ ├── no_repeated_ids.ndjson │ │ │ ├── repeated_ids.ndjson │ │ │ └── repeated_ids_discr.ndjson │ │ ├── testcontainers │ │ │ └── minio_container.go │ │ └── transactional_stream.go │ ├── flattener.go │ ├── google_cloud_storage.go │ ├── s3.go │ └── sql │ │ ├── abstract.go │ │ ├── abstract_transactional.go │ │ ├── autocommit_stream.go │ │ ├── autocommit_stream_test.go │ │ ├── batch_header.go │ │ ├── batch_header_test.go │ │ ├── bigdata_test.go │ │ ├── bigquery.go │ │ ├── bulker_test.go │ │ ├── clickhouse.go │ │ ├── datasource_config.go │ │ ├── dedup_with_discr_test.go │ │ ├── delete_condition.go │ │ ├── duckdb.go │ │ ├── existing_table_test.go │ │ ├── mergewindow_test.go │ │ ├── mysql.go │ │ ├── namespace_test.go │ │ ├── naming_test.go │ │ ├── options.go │ │ ├── postgres.go │ │ ├── processor.go │ │ ├── reconnect_test.go │ │ ├── redshift.go │ │ ├── redshift_driver │ │ ├── client.go │ │ ├── connection.go │ │ ├── connection_test.go │ │ ├── connector.go │ │ ├── driver.go │ │ ├── driver_test.go │ │ ├── dsn.go │ │ ├── dsn_test.go │ │ ├── errors.go │ │ ├── result.go │ │ ├── rows.go │ │ ├── statement.go │ │ ├── tx.go │ │ └── utils.go │ │ ├── redshift_iam.go │ │ ├── replacepartition_stream.go │ │ ├── replacepartition_stream_test.go │ │ ├── replacetable_stream.go │ │ ├── replacetable_stream_test.go │ │ ├── schema_freeze_test.go │ │ ├── snowflake.go │ │ ├── sql_adapter.go │ │ ├── sql_adapter_base.go │ │ ├── table.go │ │ ├── table_helper.go │ │ ├── test_data │ │ ├── columns_added.ndjson │ │ ├── columns_added2.ndjson │ │ ├── columns_added_partial.ndjson │ │ ├── columns_added_partial2.ndjson │ │ ├── data_types.ndjson │ │ ├── emoji.ndjson │ │ ├── empty.ndjson │ │ ├── existing_table1.ndjson │ │ ├── existing_table2.ndjson │ │ ├── existing_table_num.ndjson │ │ ├── existing_table_text.ndjson │ │ ├── identifiers.ndjson │ │ ├── merge_window1.ndjson │ │ ├── merge_window2.ndjson │ │ ├── merge_window3.ndjson │ │ ├── multiline.ndjson │ │ ├── nested.ndjson │ │ ├── non_utc_timestamp.ndjson │ │ ├── partition1.ndjson │ │ ├── partition1_1.ndjson │ │ ├── partition2.ndjson │ │ ├── repeated_ids.ndjson │ │ ├── repeated_ids2.ndjson │ │ ├── repeated_ids_discr.ndjson │ │ ├── repeated_ids_multi.ndjson │ │ ├── repeated_ids_multi2.ndjson │ │ ├── replace_table.ndjson │ │ ├── schema_option.ndjson │ │ ├── simple.ndjson │ │ ├── simple2.ndjson │ │ ├── type_hints.ndjson │ │ ├── type_hints_bq.ndjson │ │ ├── types.ndjson │ │ ├── types2.ndjson │ │ ├── types_coalesce.ndjson │ │ ├── types_collision.ndjson │ │ ├── types_collision2.ndjson │ │ ├── types_json.ndjson │ │ ├── types_json_noarr_part1.ndjson │ │ ├── types_json_noarr_part2.ndjson │ │ ├── types_json_part1.ndjson │ │ ├── types_json_part2.ndjson │ │ └── types_json_part3.ndjson │ │ ├── testcontainers │ │ ├── ch_container.go │ │ ├── clickhouse │ │ │ ├── ch_cluster_container.go │ │ │ ├── clickhouse01 │ │ │ │ ├── config.xml │ │ │ │ └── users.xml │ │ │ ├── clickhouse02 │ │ │ │ ├── config.xml │ │ │ │ └── users.xml │ │ │ ├── clickhouse03 │ │ │ │ ├── config.xml │ │ │ │ └── users.xml │ │ │ ├── clickhouse04 │ │ │ │ ├── config.xml │ │ │ │ └── users.xml │ │ │ └── docker-compose.yml │ │ ├── clickhouse_noshards │ │ │ ├── ch_cluster_container.go │ │ │ ├── clickhouse01 │ │ │ │ ├── config.xml │ │ │ │ └── users.xml │ │ │ ├── clickhouse02 │ │ │ │ ├── config.xml │ │ │ │ └── users.xml │ │ │ └── docker-compose.yml │ │ ├── mysql_container.go │ │ └── pg_container.go │ │ ├── transactional_stream.go │ │ ├── transactional_stream_test.go │ │ ├── tx_wrapper.go │ │ ├── type_resolver.go │ │ ├── types_test.go │ │ └── utils.go ├── loader.go ├── options.go └── types │ ├── avro.go │ ├── converter.go │ ├── datatype.go │ ├── datatype_test.go │ ├── error.go │ ├── marshaller.go │ ├── object.go │ ├── schema.go │ └── sqltype.go ├── config-keeper ├── app.go ├── config.go ├── go.mod ├── main.go └── router.go ├── connectors ├── airbytecdk │ ├── LICENCE │ ├── README.md │ ├── cmdparser.go │ ├── doc.go │ ├── go.mod │ ├── inferschema.go │ ├── protocol.go │ ├── safewriter.go │ ├── schema │ │ └── schema.go │ ├── source.go │ ├── sourceRunner.go │ └── trackers.go ├── firebase.Dockerfile ├── firebase │ ├── firebase.go │ ├── firebase_test.go │ ├── go.mod │ ├── go.sum │ └── main.go └── firebasebuild.sh ├── eventslog ├── ch_events_log.go ├── events_log.go ├── events_log_config.go ├── events_log_test.go ├── go.mod ├── go.sum ├── metrics.go ├── redis_events_log.go └── testcontainers │ └── redis.go ├── go.work ├── go.work.sum ├── ingest.Dockerfile ├── ingest ├── app.go ├── backup_logger.go ├── config.go ├── consumer_monitor.go ├── default.pgo ├── destination_types.go ├── filters.go ├── go.mod ├── go.sum ├── main.go ├── metrics.go ├── metrics_server.go ├── repository.go ├── router.go ├── router_batch_handler.go ├── router_classic_handler.go ├── router_ingest_handler.go ├── router_pixel_handler.go ├── router_script_handler.go ├── router_segment_settings_handler.go ├── script_repository.go └── types.go ├── ingestbuild.sh ├── ingress-manager ├── app.go ├── config.go ├── go.mod ├── go.sum ├── k8s.go ├── main.go ├── manager.go └── router.go ├── jitsubase ├── appbase │ ├── abstract_repository.go │ ├── app_base.go │ ├── http_repository.go │ ├── router_base.go │ ├── router_base_test.go │ └── service_base.go ├── coordination │ └── interface.go ├── errorj │ └── types.go ├── go.mod ├── go.sum ├── jsoniter │ ├── LICENSE │ ├── README.md │ ├── adapter.go │ ├── any.go │ ├── any_array.go │ ├── any_bool.go │ ├── any_float.go │ ├── any_int32.go │ ├── any_int64.go │ ├── any_invalid.go │ ├── any_nil.go │ ├── any_number.go │ ├── any_object.go │ ├── any_str.go │ ├── any_uint32.go │ ├── any_uint64.go │ ├── config.go │ ├── example_test.go │ ├── iter.go │ ├── iter_array.go │ ├── iter_float.go │ ├── iter_int.go │ ├── iter_object.go │ ├── iter_skip.go │ ├── iter_skip_sloppy.go │ ├── iter_skip_sloppy_test.go │ ├── iter_skip_strict.go │ ├── iter_str.go │ ├── jsoniter.go │ ├── pool.go │ ├── reflect.go │ ├── reflect_array.go │ ├── reflect_dynamic.go │ ├── reflect_extension.go │ ├── reflect_json_number.go │ ├── reflect_json_raw_message.go │ ├── reflect_map.go │ ├── reflect_marshaler.go │ ├── reflect_native.go │ ├── reflect_optional.go │ ├── reflect_slice.go │ ├── reflect_struct_decoder.go │ ├── reflect_struct_encoder.go │ ├── stream.go │ ├── stream_float.go │ ├── stream_int.go │ ├── stream_str.go │ └── stream_test.go ├── jsonorder │ ├── LICENSE │ ├── README.md │ ├── adapter.go │ ├── any.go │ ├── any_array.go │ ├── any_bool.go │ ├── any_float.go │ ├── any_int32.go │ ├── any_int64.go │ ├── any_invalid.go │ ├── any_nil.go │ ├── any_number.go │ ├── any_object.go │ ├── any_str.go │ ├── any_uint32.go │ ├── any_uint64.go │ ├── config.go │ ├── example_test.go │ ├── iter.go │ ├── iter_array.go │ ├── iter_float.go │ ├── iter_int.go │ ├── iter_object.go │ ├── iter_skip.go │ ├── iter_skip_sloppy.go │ ├── iter_skip_sloppy_test.go │ ├── iter_skip_strict.go │ ├── iter_str.go │ ├── json_test.go │ ├── jsoniter.go │ ├── pool.go │ ├── reflect.go │ ├── reflect_array.go │ ├── reflect_dynamic.go │ ├── reflect_extension.go │ ├── reflect_json_number.go │ ├── reflect_json_raw_message.go │ ├── reflect_map.go │ ├── reflect_marshaler.go │ ├── reflect_native.go │ ├── reflect_optional.go │ ├── reflect_orderedmap.go │ ├── reflect_slice.go │ ├── reflect_struct_decoder.go │ ├── reflect_struct_encoder.go │ ├── stream.go │ ├── stream_float.go │ ├── stream_int.go │ ├── stream_str.go │ └── stream_test.go ├── locks │ └── types.go ├── logging │ ├── dual.go │ ├── global_logger.go │ ├── level.go │ ├── object_logger.go │ ├── proxy.go │ ├── query_logger.go │ ├── rolling_writer.go │ ├── string_writer.go │ ├── task_logger.go │ ├── utils.go │ └── writer_mock.go ├── main.go ├── pg │ └── pgpool.go ├── safego │ ├── safego.go │ └── safego_test.go ├── timestamp │ ├── format.go │ ├── timestamp.go │ └── timestamp_test.go ├── types │ ├── LICENSE │ ├── README.md │ ├── json.go │ ├── list.go │ ├── orderedmap.go │ ├── orderedset.go │ └── set.go ├── utils │ ├── arrays.go │ ├── bool.go │ ├── bytes.go │ ├── cache.go │ ├── crypto.go │ ├── crypto_test.go │ ├── errors.go │ ├── hash.go │ ├── maps.go │ ├── net.go │ ├── numbers.go │ ├── objects.go │ ├── sets.go │ ├── strings.go │ └── time.go └── uuid │ └── uuid.go ├── kafkabase ├── go.mod ├── go.sum ├── kafka_config.go ├── metrics.go ├── producer.go └── utils.go ├── release.sh ├── sidecar.Dockerfile ├── sidecarbuild.sh ├── sync-controller ├── README.md ├── app.go ├── config.go ├── db_schema.go ├── examples │ ├── docker-compose │ │ ├── README.md │ │ ├── catalog.json │ │ ├── config.json │ │ ├── github_compose.yaml │ │ └── state.json │ └── k8s │ │ ├── README.md │ │ └── github_pod.yaml ├── go.mod ├── go.sum ├── job_runner.go ├── k8s.go ├── main.go ├── router.go ├── task.go └── task_manager.go ├── sync-sidecar ├── README.md ├── db │ └── db.go ├── go.mod ├── go.sum ├── main.go ├── read.go ├── spec_catalog.go └── types.go ├── syncctl.Dockerfile └── syncctlbuild.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | .gitignore -------------------------------------------------------------------------------- /.docs/assets/bulker-summary.excalidraw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jitsucom/bulker/b03b7dcfce71c43f8ee6edb9ae7f9c4fe57c071c/.docs/assets/bulker-summary.excalidraw.png -------------------------------------------------------------------------------- /.docs/assets/stream-batch.excalidraw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jitsucom/bulker/b03b7dcfce71c43f8ee6edb9ae7f9c4fe57c071c/.docs/assets/stream-batch.excalidraw.png -------------------------------------------------------------------------------- /.docs/http-api.md: -------------------------------------------------------------------------------- 1 | # 🚚 Bulker-server HTTP API 2 | 3 | > **See also** 4 | > [Configure Bulker server](./server-config.md) 5 | 6 | ## Authorization 7 | 8 | All request should have `Authorization` header with value `Bearer ${token}`. Token should be one of the tokens defined in `BULKER_AUTH_TOKENS` env variable, 9 | [see configuration manual](./server-config.md#bulker_auth_tokens). 10 | 11 | ## `POST /post/:destinationId?tableName=` 12 | 13 | This the main bulker endpoint. It accepts JSON object and sends it down to procssing pipeline. 14 | 15 | The body of the request is JSON object representing single event. The response is either `{"success": true}` or `{"success": false, "error": ""}` 16 | 17 | `tableName` indicates which table in destination should be used. This is mandatory parameter. 18 | 19 | ### `GET /ready` 20 | 21 | Returns `HTTP 200` if server is ready to accept requests. Otherwise, returns `HTTP 503`. Userfull 22 | for health checks. 23 | 24 | ### `GET /metrics` 25 | 26 | Returns metrics for prometheus export 27 | -------------------------------------------------------------------------------- /.docs/rfcs/data_types.md: -------------------------------------------------------------------------------- 1 | # Data types problem 2 | 3 | During batch processing we get type of column from the first message where column appears. 4 | If latter messages have incompatible type for the same column - we fail to process entire batch. 5 | 6 | 1. We need be more smart when selection column types for tmp tables. 7 | 2. For existing columns we need to know about types incompatibility in specific message before trying to load entire batch. 8 | That way we can skip or workaround incompatible messages and successfully load rest of a batch. 9 | 10 | # Solution 11 | 12 | ## Part1 – For new columns get the lowest common ancestor type 13 | 14 | During accumulation of batch we should keep track of all value types per each new column. 15 | 16 | We can detect the lowest common ancestor type for the column. 17 | For example, if we have `INTEGER` and `FLOAT` - we use `FLOAT` for the column. 18 | It is not hard to do using `DataType` abstraction 19 | 20 | ## Part2 – For existing columns check that data is castable to the column type 21 | 22 | When destination table already exists we can check for each messages in a batch that all values are castable to the corresponding column types. 23 | 24 | If data is not castable we can skip or workaround the message. 25 | 26 | If data is castable we can explicitly cast it to the column type or rely on database implicit cast. 27 | 28 | ### Castability check 29 | 30 | We can't check castability using `DataType` abstraction because when we get `Table` schema from DB we loose information about `DataType` of each column. 31 | We need to be able to restore `DataType` information for table from in `GetTableSchema` 32 | Alternatively we can use `reflect.Type` but also need to enrich `Table` columns information with `reflect.Type` and build castability tree for `reflect.Type` 33 | 34 | ## Part3 – Workaround for not castable data types 35 | 36 | We can still load events with not castable data types: 37 | * We need to remove problematic fields from such events. 38 | * We need to add `_unmaped_data` field of `JSON` type to such events that will contain object with problematic fields and its original values. 39 | That will allow user to map problematic fields to the destination table columns manually. 40 | 41 | -------------------------------------------------------------------------------- /.github/workflows/tag-release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - 'v[0-9]+.[0-9]+.[0-9]+' 6 | 7 | jobs: 8 | docker: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | with: 13 | fetch-depth: 0 14 | ref: ${{ github.ref }} 15 | # Make sure the value of GITHUB_TOKEN will not be persisted in repo's config 16 | persist-credentials: false 17 | - name: Set outputs 18 | id: vars 19 | run: | 20 | echo "tag=$(echo ${{ github.ref_name }} | sed 's/v//')" >> $GITHUB_OUTPUT 21 | echo "timestamp=$(date '+%F_%H:%M:%S')" >> $GITHUB_OUTPUT 22 | - uses: benjlevesque/short-sha@v2.2 23 | id: short-sha 24 | - name: Set up QEMU 25 | uses: docker/setup-qemu-action@v3 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3 28 | - name: Login to Docker Hub 29 | uses: docker/login-action@v3 30 | with: 31 | username: ${{ secrets.DOCKERHUB_USERNAME }} 32 | password: ${{ secrets.DOCKERHUB_PASSWORD }} 33 | - name: Build and push 34 | uses: docker/build-push-action@v5 35 | with: 36 | file: "bulker.Dockerfile" 37 | push: true 38 | build-args: | 39 | VERSION=${{ github.ref_name }}.${{ steps.short-sha.outputs.sha }} 40 | BUILD_TIMESTAMP=${{ steps.vars.outputs.timestamp }} 41 | tags: jitsucom/bulker:latest,jitsucom/bulker:${{ steps.vars.outputs.tag }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /bulkerapp/config.yaml 2 | *.env 3 | 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | # Test binary, built with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | .idea/* 17 | 18 | **/.logs 19 | 20 | #Compiled binaries 21 | bulkerapp/bulkerapp 22 | bulker 23 | ingest/ingest 24 | sync-controller/syncctl 25 | syncctl 26 | sync-sidecar/sidecar 27 | sidecar 28 | 29 | #Go workspace files 30 | #go.work 31 | #go.work.sum 32 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | 3 | - `go: 1.21` 4 | - `docker: >= 19.03.0` 5 | - `node: 18.x` 6 | - `npx` 7 | 8 | # Releasing Docker packages 9 | 10 | We use [build-scripts](https://github.com/jitsucom/jitsu/tree/newjitsu/cli/build-scripts) along with `all.Dockerfile` to publish releases to Docker. 11 | 12 | ## Packages 13 | 14 | - `jitsucom/bulker` (./bulkerapp) - Bulker is a tool for streaming and batching large amount of semi-structured data into data warehouses 15 | - `jitsucom/ingest` (./ingest) - Ingestion API for Jitsu 16 | - `jitsucom/syncctl` (./sync-controller) - Controller for Jitsu Connectors 17 | - `jitsucom/sidecar` (./sync-sidecar) - Sidecar component that runs in the same pod as connector and responsible for running syncs 18 | 19 | To avoid confusion, always release all packages together, even if only one of them has changes. 20 | 21 | ## Common steps 22 | 23 | - Make sure that you are logged to your docker account `docker login` 24 | - All changes should be committed (check with `git status`). It's ok to release canary from branches! 25 | 26 | ## Beta releases 27 | 28 | - `./release.sh --dryRun` - to **dry-run** publishing. 29 | - `./release.sh` - to actually **publish** beta. 30 | 31 | ## Stable releases 32 | 33 | - `./release.sh --release latest --dryRun` - to **dry-run** publishing. 34 | - `./release.sh --release latest ` - to actually **publish** latest image. 35 | 36 | ## Bumping versions 37 | 38 | For initial release or to bump major/minor version pass `--version` argument to `./release.sh` script. 39 | 40 | - `./release.sh --version 2.5.0` 41 | - `./release.sh --release latest --version 2.5.0` 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /GOWORK.md: -------------------------------------------------------------------------------- 1 | # Go Workspace 2 | 3 | To initialize a new Go workspace, run the following command: 4 | 5 | ```bash 6 | go work init ./jitsubase ./kafkabase ./eventslog ./bulkerlib ./bulkerapp # and other modules that can be added in future 7 | ``` 8 | 9 | Google is against of committing go.work file to the repository, so it's added to `.gitignore` file. 10 | [Read here](https://go.googlesource.com/proposal/+/master/design/45713-workspace.md#preventing-files-from-being-checked-in-to-repositories) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Jitsu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | This document contains a list of bulker problems. We are aware of this problems, and they need to be fixed before final release. Those 4 | problems are split by different destinations 5 | 6 | ## Regular Redshift 7 | 8 | * Add load test for Redshift in streaming (aka autocommit) mode and make sure it could handle a decent load. If it couldn't handle a decent load, streaming 9 | mode for Redshift should be disabled of discouraged 10 | * Redshift supports [SORTKEY](https://docs.aws.amazon.com/redshift/latest/dg/c_best-practices-sort-key.html). We need to introduce an option to defined 11 | sort key columnin the destination table. If the option is present, we should make sure that the either already created, or create it. Once we know sort key 12 | we should take advantage of it while doing deduplication by defining deduplication window which should be also a config param 13 | * 14 | 15 | ## Serverless Redshift 16 | 17 | We should test how bulker works with Serverless Redshift and make sure it won't cost a lot. As a result we should either discourage using Serverlress Redshift, 18 | or suggest the cost optimization strategy. Bulker as is could cost a lot. 19 | 20 | -------------------------------------------------------------------------------- /admin/kafka.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/confluentinc/confluent-kafka-go/v2/kafka" 7 | "github.com/hjson/hjson-go/v4" 8 | "os" 9 | ) 10 | 11 | // add partitions to the topic 12 | func kafka1() { 13 | bootstapServers := os.Getenv("KAFKA_BOOTSTRAP_SERVERS") 14 | securityProtocol := os.Getenv("KAFKA_SECURITY_PROTOCOL") 15 | kafkaSasl := os.Getenv("KAFKA_SASL") 16 | fmt.Println("Kafka bootstrap servers:", bootstapServers, "security protocol:", securityProtocol, "kafka sasl:", kafkaSasl) 17 | kafkaConfig := &kafka.ConfigMap{ 18 | "client.id": "bulkerapp_admin", 19 | "bootstrap.servers": bootstapServers, 20 | "reconnect.backoff.ms": 1000, 21 | "reconnect.backoff.max.ms": 10000, 22 | } 23 | if securityProtocol != "" { 24 | _ = kafkaConfig.SetKey("security.protocol", securityProtocol) 25 | _ = kafkaConfig.SetKey("enable.ssl.certificate.verification", false) 26 | } 27 | if kafkaSasl != "" { 28 | sasl := map[string]interface{}{} 29 | err := hjson.Unmarshal([]byte(kafkaSasl), &sasl) 30 | if err != nil { 31 | panic(fmt.Errorf("error parsing Kafka SASL config: %v", err)) 32 | } 33 | for k, v := range sasl { 34 | _ = kafkaConfig.SetKey("sasl."+k, v) 35 | } 36 | } 37 | admin, err := kafka.NewAdminClient(kafkaConfig) 38 | if err != nil { 39 | panic(fmt.Errorf("error creating Kafka admin client: %v", err)) 40 | } 41 | m, err := admin.GetMetadata(nil, true, 10000) 42 | if err != nil { 43 | panic(fmt.Errorf("error getting Kafka metadata: %v", err)) 44 | } 45 | fmt.Println(m.Brokers) 46 | fmt.Print("Enter topic name to increase partitions: ") 47 | var topic string 48 | _, err = fmt.Scanln(&topic) 49 | if err != nil { 50 | panic(fmt.Errorf("error reading topic name: %v", err)) 51 | } 52 | fmt.Printf("Enter new number of partitions for topic '%s': ", topic) 53 | var partitions int 54 | _, err = fmt.Scanln(&partitions) 55 | if err != nil { 56 | panic(fmt.Errorf("error reading partitions number: %v", err)) 57 | } 58 | res, err := admin.CreatePartitions(context.Background(), []kafka.PartitionsSpecification{ 59 | { 60 | Topic: topic, 61 | IncreaseTo: partitions, 62 | }, 63 | }) 64 | if err != nil { 65 | panic(fmt.Errorf("error creating partitions: %v", err)) 66 | } 67 | fmt.Println(res) 68 | } 69 | -------------------------------------------------------------------------------- /admin/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // add partitions to the topic 4 | func main() { 5 | kafka1() 6 | } 7 | -------------------------------------------------------------------------------- /allbuild.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | VERSION=$(git log -1 --pretty=%h) 4 | BUILD_TIMESTAMP=$( date '+%F_%H:%M:%S' ) 5 | 6 | docker buildx build --build-arg VERSION="$VERSION" --build-arg BUILD_TIMESTAMP="$BUILD_TIMESTAMP" --platform linux/amd64 -f all.Dockerfile --target bulker -t jitsucom/bulker:"$VERSION" -t jitsucom/bulker:beta --push . 7 | docker buildx build --build-arg VERSION="$VERSION" --build-arg BUILD_TIMESTAMP="$BUILD_TIMESTAMP" --platform linux/amd64 -f all.Dockerfile --target ingest -t jitsucom/ingest:"$VERSION" -t jitsucom/ingest:beta --push . 8 | docker buildx build --build-arg VERSION="$VERSION" --build-arg BUILD_TIMESTAMP="$BUILD_TIMESTAMP" --platform linux/amd64 -f all.Dockerfile --target sidecar -t jitsucom/sidecar:"$VERSION" -t jitsucom/sidecar:beta --push . 9 | docker buildx build --build-arg VERSION="$VERSION" --build-arg BUILD_TIMESTAMP="$BUILD_TIMESTAMP" --platform linux/amd64 -f all.Dockerfile --target syncctl -t jitsucom/syncctl:"$VERSION" -t jitsucom/syncctl:beta --push . -------------------------------------------------------------------------------- /bulker.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bullseye-slim as main 2 | 3 | RUN apt-get update -y 4 | RUN apt-get install -y ca-certificates curl 5 | 6 | ENV TZ=UTC 7 | 8 | FROM golang:1.24.2-bullseye as build 9 | 10 | ARG VERSION 11 | ENV VERSION $VERSION 12 | ARG BUILD_TIMESTAMP 13 | ENV BUILD_TIMESTAMP $BUILD_TIMESTAMP 14 | 15 | RUN apt-get install gcc libc6-dev 16 | 17 | #RUN wget -qO - https://packages.confluent.io/deb/7.2/archive.key | apt-key add - 18 | #RUN echo "deb https://packages.confluent.io/deb/7.2 stable main" > /etc/apt/sources.list.d/backports.list 19 | #RUN echo "deb https://packages.confluent.io/clients/deb buster main" > /etc/apt/sources.list.d/backports.list 20 | #RUN apt-get update 21 | #RUN apt-get install -y librdkafka1 librdkafka-dev 22 | 23 | RUN mkdir /app 24 | WORKDIR /app 25 | 26 | RUN mkdir jitsubase kafkabase eventslog bulkerlib bulkerapp 27 | 28 | COPY jitsubase/go.* ./jitsubase/ 29 | COPY kafkabase/go.* ./kafkabase/ 30 | COPY eventslog/go.* ./eventslog/ 31 | COPY bulkerlib/go.* ./bulkerlib/ 32 | COPY bulkerapp/go.* ./bulkerapp/ 33 | 34 | RUN go work init jitsubase kafkabase eventslog bulkerlib bulkerapp 35 | 36 | WORKDIR /app/bulkerapp 37 | 38 | RUN go mod download 39 | 40 | WORKDIR /app 41 | 42 | COPY . . 43 | 44 | # Build bulker 45 | RUN go build -ldflags="-X main.Commit=$VERSION -X main.Timestamp=$BUILD_TIMESTAMP" -o bulker ./bulkerapp 46 | 47 | ####################################### 48 | # FINAL STAGE 49 | FROM main as final 50 | 51 | RUN mkdir /app 52 | WORKDIR /app 53 | 54 | # Copy bulkerapp 55 | COPY --from=build /app/bulker ./ 56 | #COPY ./config.yaml ./ 57 | 58 | CMD ["/app/bulker"] 59 | -------------------------------------------------------------------------------- /bulkerapp/app/metrics_server.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/gin-gonic/gin" 7 | "github.com/jitsucom/bulker/jitsubase/appbase" 8 | "github.com/jitsucom/bulker/jitsubase/safego" 9 | "github.com/prometheus/client_golang/prometheus/promhttp" 10 | "net/http" 11 | "time" 12 | ) 13 | 14 | type MetricsServer struct { 15 | appbase.Service 16 | server *http.Server 17 | } 18 | 19 | func NewMetricsServer(appconfig *Config) *MetricsServer { 20 | base := appbase.NewServiceBase("metrics_server") 21 | engine := gin.New() 22 | engine.Use(gin.Recovery()) 23 | //expose prometheus metrics 24 | engine.GET("/metrics", gin.WrapH(promhttp.Handler())) 25 | 26 | server := &http.Server{ 27 | Addr: fmt.Sprintf("0.0.0.0:%d", appconfig.MetricsPort), 28 | Handler: engine, 29 | ReadTimeout: time.Second * 60, 30 | ReadHeaderTimeout: time.Second * 60, 31 | IdleTimeout: time.Second * 65, 32 | } 33 | m := &MetricsServer{Service: base, server: server} 34 | m.start() 35 | return m 36 | } 37 | 38 | func (s *MetricsServer) start() { 39 | safego.RunWithRestart(func() { 40 | s.Infof("Starting metrics server on %s", s.server.Addr) 41 | s.Infof("%v", s.server.ListenAndServe()) 42 | }) 43 | } 44 | 45 | func (s *MetricsServer) Stop() error { 46 | s.Infof("Stopping metrics server") 47 | return s.server.Shutdown(context.Background()) 48 | } 49 | -------------------------------------------------------------------------------- /bulkerapp/app/multi_configuration_source.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/safego" 5 | "reflect" 6 | ) 7 | 8 | type MultiConfigurationSource struct { 9 | configurationSources []ConfigurationSource 10 | changesChan chan bool 11 | closeChan chan struct{} 12 | } 13 | 14 | func NewMultiConfigurationSource(configurationSources []ConfigurationSource) *MultiConfigurationSource { 15 | mcs := MultiConfigurationSource{configurationSources: configurationSources, 16 | changesChan: make(chan bool, 1), 17 | closeChan: make(chan struct{})} 18 | // gather signals from all changes channels from configuration sources into one channel 19 | safego.RunWithRestart(func() { 20 | cases := make([]reflect.SelectCase, len(configurationSources)) 21 | for i, cs := range configurationSources { 22 | cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(cs.ChangesChannel())} 23 | } 24 | for { 25 | select { 26 | case <-mcs.closeChan: 27 | return 28 | default: 29 | // Use reflect.Select to select from the dynamic numbers of channels 30 | i, _, ok := reflect.Select(cases) 31 | if !ok { 32 | // The chosen channel has been closed, so zero out the channel to disable the case 33 | cases[i].Chan = reflect.ValueOf(nil) 34 | } else { 35 | mcs.changesChan <- true 36 | } 37 | } 38 | } 39 | }) 40 | return &mcs 41 | } 42 | 43 | func (mcs *MultiConfigurationSource) GetDestinationConfigs() []*DestinationConfig { 44 | results := make([]*DestinationConfig, 0) 45 | for _, cs := range mcs.configurationSources { 46 | results = append(results, cs.GetDestinationConfigs()...) 47 | } 48 | return results 49 | } 50 | 51 | func (mcs *MultiConfigurationSource) GetDestinationConfig(id string) *DestinationConfig { 52 | for _, cs := range mcs.configurationSources { 53 | cfg := cs.GetDestinationConfig(id) 54 | if cfg != nil { 55 | return cfg 56 | } 57 | } 58 | return nil 59 | } 60 | 61 | func (mcs *MultiConfigurationSource) ChangesChannel() <-chan bool { 62 | return mcs.changesChan 63 | } 64 | 65 | func (mcs *MultiConfigurationSource) Close() error { 66 | close(mcs.closeChan) 67 | for _, cs := range mcs.configurationSources { 68 | _ = cs.Close() 69 | } 70 | close(mcs.changesChan) 71 | return nil 72 | 73 | } 74 | -------------------------------------------------------------------------------- /bulkerapp/app/producer.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "github.com/confluentinc/confluent-kafka-go/v2/kafka" 5 | "github.com/jitsucom/bulker/kafkabase" 6 | ) 7 | 8 | type Producer struct { 9 | *kafkabase.Producer 10 | } 11 | 12 | // NewProducer creates new Producer 13 | func NewProducer(config *kafkabase.KafkaConfig, kafkaConfig *kafka.ConfigMap, reportQueueLength bool) (*Producer, error) { 14 | base, err := kafkabase.NewProducer(config, kafkaConfig, reportQueueLength, ProducerMessageLabels) 15 | if err != nil { 16 | return nil, err 17 | } 18 | return &Producer{ 19 | Producer: base, 20 | }, nil 21 | } 22 | 23 | func ProducerMessageLabels(topicId string, status, errText string) (topic, destinationId, mode, tableName, st string, err string) { 24 | destinationId, mode, tableName, topicErr := ParseTopicId(topicId) 25 | if topicErr != nil { 26 | return topicId, "", "", "", status, errText 27 | } else { 28 | return topicId, destinationId, mode, tableName, status, errText 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /bulkerapp/app/test_data/badbatch.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test2"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test3"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test4"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 5, "name": "test5"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 6, "name": "test6"} 7 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 7, "name": "test7"} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 8, "name": "test8"} 9 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 9, "name": "test9"} 10 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 10, "name": "test10"} 11 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 11, "name": "test11"} 12 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 12, "name": "test12"} 13 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 13, "name": "test13"} 14 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 14, "name": "test14"} 15 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 15, "name": "test15"} 16 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 16, "name": "test16"} 17 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 16, "name": "test17"} 18 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 18, "name": "test18"} 19 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 19, "name": "test19"} 20 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 20, "name": "test20"} 21 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 21, "name": "test21"} 22 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 22, "name": "test22"} 23 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 23, "name": "test23"} 24 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 24, "name": "test24"} 25 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 25, "name": "test25"} 26 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 26, "name": "test26"} 27 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 27, "name": "test27"} 28 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 28, "name": "test28"} 29 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 29, "name": "test29"} 30 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 30, "name": "test30"} 31 | -------------------------------------------------------------------------------- /bulkerapp/app/test_data/bulker.env: -------------------------------------------------------------------------------- 1 | CONFIG_SOURCE=[[CONFIG_SOURCE]] 2 | 3 | INSTANCE_ID=bulker_test 4 | 5 | KAFKA_BOOTSTRAP_SERVERS=127.0.0.1:19092 6 | KAFKA_ADMIN_METADATA_TIMEOUT_MS=10000 7 | 8 | AUTH_TOKENS=1b16b3f8-5f50-40e4-a928-4cf7f48b927f.pauqzVf89cWpIIpSqcA3fO/KyxGEFp0F9VJ5qd58R9Vhsa7VIn/KmSduw7WqiGfvd0W9Wzc1TVPlauupkE7PHA 9 | TOKEN_SECRET=dea42a58-acf4-45af-85bb-e77e94bd5025 10 | 11 | BATCH_RUNNER_WAIT_FOR_MESSAGES_SEC=3 -------------------------------------------------------------------------------- /bulkerapp/app/test_data/config.yaml: -------------------------------------------------------------------------------- 1 | destinations: 2 | batch_postgres: 3 | type: "postgres" 4 | options: 5 | mode: "batch" 6 | batchSize: 10 7 | primaryKey: "id" 8 | credentials: 9 | host: localhost 10 | port: [[POSTGRES_PORT]] 11 | database: test 12 | username: test 13 | password: test 14 | defaultSchema: bulker 15 | parameters: 16 | sslmode: disable 17 | batch_postgres_bytes: 18 | type: "postgres" 19 | options: 20 | mode: "batch" 21 | batchSize: 10000 22 | batchSizeBytes: 500 23 | primaryKey: "id" 24 | credentials: 25 | host: localhost 26 | port: [[POSTGRES_PORT]] 27 | database: test 28 | username: test 29 | password: test 30 | defaultSchema: bulker 31 | parameters: 32 | sslmode: disable 33 | stream_postgres: 34 | type: "postgres" 35 | options: 36 | mode: "stream" 37 | primaryKey: "id" 38 | credentials: 39 | host: localhost 40 | port: [[POSTGRES_PORT]] 41 | database: test 42 | username: test 43 | password: test 44 | defaultSchema: bulker 45 | parameters: 46 | sslmode: disable 47 | load_test_postgres: 48 | type: "postgres" 49 | options: 50 | mode: "batch" 51 | batchSize: 500000 52 | credentials: 53 | host: localhost 54 | port: [[POSTGRES_PORT]] 55 | database: test 56 | username: test 57 | password: test 58 | defaultSchema: bulker 59 | parameters: 60 | sslmode: disable 61 | -------------------------------------------------------------------------------- /bulkerapp/app/test_data/goodbatch.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test2"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test3"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test4"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 5, "name": "test5"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 6, "name": "test6"} 7 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 7, "name": "test7"} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 8, "name": "test8"} 9 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 9, "name": "test9"} 10 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 10, "name": "test10"} 11 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 11, "name": "test11"} 12 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 12, "name": "test12"} 13 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 13, "name": "test13"} 14 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 14, "name": "test14"} 15 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 15, "name": "test15"} 16 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 16, "name": "test16"} 17 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 17, "name": "test17"} 18 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 18, "name": "test18"} 19 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 19, "name": "test19"} 20 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 20, "name": "test20"} 21 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 21, "name": "test21"} 22 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 22, "name": "test22"} 23 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 23, "name": "test23"} 24 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 24, "name": "test24"} 25 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 25, "name": "test25"} 26 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 26, "name": "test26"} 27 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 27, "name": "test27"} 28 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 28, "name": "test28"} 29 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 29, "name": "test29"} 30 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 30, "name": "test30"} 31 | -------------------------------------------------------------------------------- /bulkerapp/app/test_data/single.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} -------------------------------------------------------------------------------- /bulkerapp/app/testcontainers/kafka/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | zookeeper: 4 | image: bitnami/zookeeper:3.9.2 5 | container_name: zoo 6 | expose: 7 | - 2181 8 | environment: 9 | ALLOW_ANONYMOUS_LOGIN: "yes" 10 | kafka: 11 | image: bitnami/kafka:3.7.1 12 | container_name: kafka 13 | depends_on: 14 | - zookeeper 15 | ports: 16 | - "19092:19092" 17 | healthcheck: 18 | test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server kafka:19092 --describe"] 19 | environment: 20 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19092 21 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:19092 22 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 23 | -------------------------------------------------------------------------------- /bulkerapp/app/testcontainers/kafka/kafka_container.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/jitsucom/bulker/jitsubase/logging" 7 | tc "github.com/testcontainers/testcontainers-go/modules/compose" 8 | ) 9 | 10 | type KafkaContainer struct { 11 | Identifier string 12 | Compose tc.ComposeStack 13 | Context context.Context 14 | } 15 | 16 | func NewKafkaContainer(ctx context.Context) (*KafkaContainer, error) { 17 | composeFilePaths := "testcontainers/kafka/docker-compose.yml" 18 | identifier := "bulker_kafka_compose" 19 | 20 | compose, err := tc.NewDockerComposeWith(tc.WithStackFiles(composeFilePaths), tc.StackIdentifier(identifier)) 21 | if err != nil { 22 | logging.Errorf("couldnt down docker compose: %s : %v", identifier, err) 23 | } 24 | err = compose.Down(ctx) 25 | if err != nil { 26 | logging.Errorf("couldnt down docker compose: %s : %v", identifier, err) 27 | } 28 | 29 | compose, err = tc.NewDockerComposeWith(tc.WithStackFiles(composeFilePaths), tc.StackIdentifier(identifier)) 30 | if err != nil { 31 | return nil, fmt.Errorf("could not run compose file: %v - %v", composeFilePaths, err) 32 | } 33 | err = compose.Up(ctx, tc.Wait(true)) 34 | if err != nil { 35 | return nil, fmt.Errorf("could not run compose file: %v - %v", composeFilePaths, err) 36 | } 37 | 38 | return &KafkaContainer{ 39 | Identifier: identifier, 40 | Compose: compose, 41 | Context: ctx, 42 | }, nil 43 | } 44 | 45 | // Close terminates underlying docker container 46 | func (ch *KafkaContainer) Close() error { 47 | if ch.Compose != nil { 48 | err := ch.Compose.Down(context.Background()) 49 | if err != nil { 50 | return fmt.Errorf("could down docker compose: %s", ch.Identifier) 51 | } 52 | } 53 | 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /bulkerapp/default.pgo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jitsucom/bulker/b03b7dcfce71c43f8ee6edb9ae7f9c4fe57c071c/bulkerapp/default.pgo -------------------------------------------------------------------------------- /bulkerapp/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/bulkerapp/app" 5 | _ "github.com/jitsucom/bulker/bulkerlib/implementations/api_based" 6 | _ "github.com/jitsucom/bulker/bulkerlib/implementations/file_storage" 7 | _ "github.com/jitsucom/bulker/bulkerlib/implementations/sql" 8 | "github.com/jitsucom/bulker/jitsubase/appbase" 9 | "github.com/jitsucom/bulker/jitsubase/logging" 10 | "os" 11 | ) 12 | 13 | var Commit string 14 | var Timestamp string 15 | 16 | func main() { 17 | logging.Infof("Starting bulker app. Version: %s Build timestamp: %s", Commit, Timestamp) 18 | 19 | settings := &appbase.AppSettings{ 20 | ConfigPath: os.Getenv("BULKER_CONFIG_PATH"), 21 | Name: "bulker", 22 | EnvPrefix: "BULKER", 23 | ConfigName: "bulker", 24 | ConfigType: "env", 25 | } 26 | application := appbase.NewApp[app.Config](&app.Context{}, settings) 27 | application.Run() 28 | } 29 | -------------------------------------------------------------------------------- /bulkerbuild.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | VERSION=$(git log -1 --pretty=%h) 4 | BUILD_TIMESTAMP=$( date '+%F_%H:%M:%S' ) 5 | 6 | docker buildx build --build-arg VERSION="$VERSION" --build-arg BUILD_TIMESTAMP="$BUILD_TIMESTAMP" --platform linux/amd64 -f bulker.Dockerfile -t jitsucom/bulker:"$VERSION" -t jitsucom/bulker:latest --push . -------------------------------------------------------------------------------- /bulkerlib/bulker_state_test.go: -------------------------------------------------------------------------------- 1 | package bulkerlib 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestBulkerState(t *testing.T) { 9 | st := State{} 10 | st.AddWarehouseState(WarehouseState{ 11 | Name: "first", 12 | TimeProcessedMs: 100, 13 | }) 14 | st.AddWarehouseState(WarehouseState{ 15 | Name: "second", 16 | TimeProcessedMs: 100, 17 | }) 18 | 19 | st2 := State{} 20 | st2.AddWarehouseState(WarehouseState{ 21 | Name: "first", 22 | TimeProcessedMs: 101, 23 | }) 24 | st2.AddWarehouseState(WarehouseState{ 25 | Name: "second", 26 | TimeProcessedMs: 120, 27 | }) 28 | st2.AddWarehouseState(WarehouseState{ 29 | Name: "third", 30 | TimeProcessedMs: 130, 31 | }) 32 | 33 | st.Merge(st2) 34 | 35 | fmt.Println(st.PrintWarehouseState()) 36 | } 37 | -------------------------------------------------------------------------------- /bulkerlib/implementations/file_storage/replacepartition_stream.go: -------------------------------------------------------------------------------- 1 | package file_storage 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | bulker "github.com/jitsucom/bulker/bulkerlib" 8 | "github.com/jitsucom/bulker/bulkerlib/implementations" 9 | ) 10 | 11 | type ReplacePartitionStream struct { 12 | AbstractFileStorageStream 13 | } 14 | 15 | func NewReplacePartitionStream(id string, p implementations.FileAdapter, tableName string, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error) { 16 | ps := ReplacePartitionStream{} 17 | so := bulker.StreamOptions{} 18 | for _, opt := range streamOptions { 19 | so.Add(opt) 20 | } 21 | partitionId := bulker.PartitionIdOption.Get(&so) 22 | if partitionId == "" { 23 | return nil, errors.New("WithPartition is required option for ReplacePartitionStream") 24 | } 25 | var err error 26 | filenameFunc := func(ctx context.Context) string { 27 | return fmt.Sprintf("%s/%s", tableName, partitionId) 28 | } 29 | ps.AbstractFileStorageStream, err = newAbstractFileStorageStream(id, p, filenameFunc, bulker.ReplacePartition, streamOptions...) 30 | if err != nil { 31 | return nil, err 32 | } 33 | return &ps, nil 34 | } 35 | 36 | func (ps *ReplacePartitionStream) Complete(ctx context.Context) (state bulker.State, err error) { 37 | if ps.state.Status != bulker.Active { 38 | return ps.state, errors.New("stream is not active") 39 | } 40 | defer func() { 41 | state, err = ps.postComplete(err) 42 | }() 43 | if ps.state.LastError == nil { 44 | //if at least one object was inserted 45 | if ps.state.SuccessfulRows > 0 { 46 | if ps.batchFile != nil { 47 | if err = ps.flushBatchFile(ctx); err != nil { 48 | return ps.state, err 49 | } 50 | } 51 | } else { 52 | //for ReplacePartitionStream we should replace existing file with empty one 53 | if err = ps.fileAdapter.UploadBytes(ps.filename(ctx), []byte{}); err != nil { 54 | return ps.state, err 55 | } 56 | } 57 | return 58 | } else { 59 | //if was any error - it will trigger transaction rollback in defer func 60 | err = ps.state.LastError 61 | return 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /bulkerlib/implementations/file_storage/replacetable_stream.go: -------------------------------------------------------------------------------- 1 | package file_storage 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | bulker "github.com/jitsucom/bulker/bulkerlib" 7 | "github.com/jitsucom/bulker/bulkerlib/implementations" 8 | ) 9 | 10 | type ReplaceTableStream struct { 11 | AbstractFileStorageStream 12 | } 13 | 14 | func NewReplaceTableStream(id string, p implementations.FileAdapter, tableName string, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error) { 15 | ps := ReplaceTableStream{} 16 | 17 | var err error 18 | ps.AbstractFileStorageStream, err = newAbstractFileStorageStream(id, p, func(ctx context.Context) string { 19 | return tableName 20 | }, bulker.ReplaceTable, streamOptions...) 21 | if err != nil { 22 | return nil, err 23 | } 24 | return &ps, nil 25 | } 26 | 27 | func (ps *ReplaceTableStream) Complete(ctx context.Context) (state bulker.State, err error) { 28 | if ps.state.Status != bulker.Active { 29 | return ps.state, errors.New("stream is not active") 30 | } 31 | defer func() { 32 | state, err = ps.postComplete(err) 33 | }() 34 | if ps.state.LastError == nil { 35 | //if at least one object was inserted 36 | if ps.state.SuccessfulRows > 0 { 37 | if ps.batchFile != nil { 38 | if err = ps.flushBatchFile(ctx); err != nil { 39 | return ps.state, err 40 | } 41 | } 42 | } else { 43 | //for ReplaceTable stream we should replace existing file with empty one 44 | if err = ps.fileAdapter.UploadBytes(ps.filename(ctx), []byte{}); err != nil { 45 | return ps.state, err 46 | } 47 | } 48 | return 49 | } else { 50 | //if was any error - it will trigger transaction rollback in defer func 51 | err = ps.state.LastError 52 | return 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /bulkerlib/implementations/file_storage/s3_bulker.go: -------------------------------------------------------------------------------- 1 | package file_storage 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | bulker "github.com/jitsucom/bulker/bulkerlib" 7 | "github.com/jitsucom/bulker/bulkerlib/implementations" 8 | "github.com/jitsucom/bulker/jitsubase/utils" 9 | ) 10 | 11 | const S3AutocommitUnsupported = "Stream mode is not supported for GCS. Please use 'batch' mode" 12 | 13 | func init() { 14 | bulker.RegisterBulker(implementations.S3BulkerTypeId, NewS3Bulker) 15 | } 16 | 17 | type S3Bulker struct { 18 | implementations.S3 19 | } 20 | 21 | func (s3 *S3Bulker) Type() string { 22 | return implementations.S3BulkerTypeId 23 | } 24 | 25 | func NewS3Bulker(bulkerConfig bulker.Config) (bulker.Bulker, error) { 26 | s3Config := &implementations.S3Config{} 27 | if err := utils.ParseObject(bulkerConfig.DestinationConfig, s3Config); err != nil { 28 | return nil, fmt.Errorf("failed to parse destination config: %v", err) 29 | } 30 | s3adapter, err := implementations.NewS3(s3Config) 31 | if err != nil { 32 | return nil, err 33 | } 34 | return &S3Bulker{*s3adapter}, nil 35 | } 36 | 37 | func (s3 *S3Bulker) CreateStream(id, tableName string, mode bulker.BulkMode, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error) { 38 | switch mode { 39 | case bulker.Stream: 40 | return nil, errors.New(S3AutocommitUnsupported) 41 | case bulker.Batch: 42 | return NewTransactionalStream(id, s3, tableName, streamOptions...) 43 | case bulker.ReplaceTable: 44 | return NewReplaceTableStream(id, s3, tableName, streamOptions...) 45 | case bulker.ReplacePartition: 46 | return NewReplacePartitionStream(id, s3, tableName, streamOptions...) 47 | } 48 | return nil, fmt.Errorf("unsupported bulk mode: %s", mode) 49 | } 50 | -------------------------------------------------------------------------------- /bulkerlib/implementations/file_storage/test_data/empty.ndjson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jitsucom/bulker/b03b7dcfce71c43f8ee6edb9ae7f9c4fe57c071c/bulkerlib/implementations/file_storage/test_data/empty.ndjson -------------------------------------------------------------------------------- /bulkerlib/implementations/file_storage/test_data/no_repeated_ids.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test1"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test2"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test3"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test4"} 5 | -------------------------------------------------------------------------------- /bulkerlib/implementations/file_storage/test_data/repeated_ids.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test1"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test2"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test3"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test4"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test5"} 7 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test6"} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test7"} -------------------------------------------------------------------------------- /bulkerlib/implementations/file_storage/test_data/repeated_ids_discr.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "C", "int1": 3, "nested": { "int1": 3}} 2 | {"_timestamp": "2022-08-15T14:17:22.375Z", "id": 1, "name": "B", "int1": 2, "nested": { "int1": 2}} 3 | {"_timestamp": "2022-08-15T14:17:22.375Z", "id": 2, "name": "B", "int1": 2, "nested": { "int1": 2}} 4 | {"_timestamp": "2022-08-10T14:17:22.375Z", "id": 3, "name": "A", "int1": 1, "nested": { "int1": 1}} 5 | {"_timestamp": "2022-08-15T14:17:22.375Z", "id": 3, "name": "B", "int1": 2, "nested": { "int1": 2}} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "C", "int1": 3, "nested": { "int1": 3}} 7 | {"_timestamp": "2022-08-10T14:17:22.375Z", "id": 1, "name": "A", "int1": 1, "nested": { "int1": 1}} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "C", "int1": 3, "nested": { "int1": 3}} 9 | {"_timestamp": "2022-08-10T14:17:22.375Z", "id": 2, "name": "A", "int1": 1, "nested": { "int1": 1}} 10 | -------------------------------------------------------------------------------- /bulkerlib/implementations/file_storage/transactional_stream.go: -------------------------------------------------------------------------------- 1 | package file_storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | bulker "github.com/jitsucom/bulker/bulkerlib" 7 | "github.com/jitsucom/bulker/bulkerlib/implementations" 8 | "github.com/jitsucom/bulker/jitsubase/timestamp" 9 | ) 10 | 11 | const FilenameDate = "2006_01_02T15_04_05" 12 | 13 | type TransactionalStream struct { 14 | AbstractFileStorageStream 15 | } 16 | 17 | func NewTransactionalStream(id string, p implementations.FileAdapter, tableName string, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error) { 18 | ps := TransactionalStream{} 19 | var err error 20 | streamStartDate := timestamp.Now() 21 | filenameFunc := func(ctx context.Context) string { 22 | batchNumStr := "" 23 | batchNum, ok := ctx.Value(bulker.BatchNumberCtxKey).(int) 24 | if ok { 25 | batchNumStr = fmt.Sprintf("_%d", batchNum) 26 | } 27 | return fmt.Sprintf("%s_%s%s", tableName, streamStartDate.Format(FilenameDate), batchNumStr) 28 | } 29 | ps.AbstractFileStorageStream, err = newAbstractFileStorageStream(id, p, filenameFunc, bulker.Batch, streamOptions...) 30 | if err != nil { 31 | return nil, err 32 | } 33 | return &ps, nil 34 | } 35 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/batch_header.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | types2 "github.com/jitsucom/bulker/bulkerlib/types" 5 | ) 6 | 7 | type Fields = []Field 8 | 9 | // TypesHeader is the schema result of parsing JSON objects 10 | type TypesHeader struct { 11 | TableName string 12 | Fields Fields 13 | Partition DatePartition 14 | } 15 | 16 | // Exists returns true if there is at least one field 17 | func (bh *TypesHeader) Exists() bool { 18 | return bh != nil && len(bh.Fields) > 0 19 | } 20 | 21 | // Field is a data type holder with sql type suggestion 22 | type Field struct { 23 | Name string 24 | dataType *types2.DataType 25 | suggestedType *types2.SQLColumn 26 | } 27 | 28 | // NewField returns Field instance 29 | func NewField(name string, t types2.DataType) Field { 30 | return Field{ 31 | Name: name, 32 | dataType: &t, 33 | } 34 | } 35 | 36 | // NewFieldWithSQLType returns Field instance with configured suggested sql types 37 | func NewFieldWithSQLType(name string, t types2.DataType, suggestedType *types2.SQLColumn) Field { 38 | return Field{ 39 | Name: name, 40 | dataType: &t, 41 | suggestedType: suggestedType, 42 | } 43 | } 44 | 45 | // GetSuggestedSQLType returns suggested SQL type if configured 46 | func (f Field) GetSuggestedSQLType() (types2.SQLColumn, bool) { 47 | if f.suggestedType != nil { 48 | return types2.SQLColumn{Type: f.suggestedType.Type, DdlType: f.suggestedType.DdlType, Override: true}, true 49 | } 50 | 51 | return types2.SQLColumn{}, false 52 | } 53 | 54 | // GetType get field type based on occurrence in one file 55 | // lazily get common ancestor type (typing.GetCommonAncestorType) 56 | func (f Field) GetType() types2.DataType { 57 | return *f.dataType 58 | } 59 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/datasource_config.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import "errors" 4 | 5 | // DataSourceConfig dto for deserialized datasource config (e.g. in Postgres or AwsRedshift destination) 6 | type DataSourceConfig struct { 7 | Host string `mapstructure:"host,omitempty" json:"host,omitempty" yaml:"host,omitempty"` 8 | Port int `mapstructure:"port,omitempty" json:"port,omitempty" yaml:"port,omitempty"` 9 | Db string `mapstructure:"database,omitempty" json:"database,omitempty" yaml:"database,omitempty"` 10 | Schema string `mapstructure:"defaultSchema,omitempty" json:"defaultSchema,omitempty" yaml:"defaultSchema,omitempty"` 11 | Username string `mapstructure:"username,omitempty" json:"username,omitempty" yaml:"username,omitempty"` 12 | Password string `mapstructure:"password,omitempty" json:"password,omitempty" yaml:"password,omitempty"` 13 | Parameters map[string]string `mapstructure:"parameters,omitempty" json:"parameters,omitempty" yaml:"parameters,omitempty"` 14 | } 15 | 16 | // Validate required fields in DataSourceConfig 17 | func (dsc *DataSourceConfig) Validate() error { 18 | if dsc == nil { 19 | return errors.New("Datasource config is required") 20 | } 21 | 22 | if dsc.Parameters == nil { 23 | dsc.Parameters = map[string]string{} 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/client.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/config" 7 | "github.com/aws/aws-sdk-go-v2/service/redshiftdata" 8 | ) 9 | 10 | type RedshiftClient interface { 11 | ExecuteStatement(ctx context.Context, params *redshiftdata.ExecuteStatementInput, optFns ...func(*redshiftdata.Options)) (*redshiftdata.ExecuteStatementOutput, error) 12 | DescribeStatement(ctx context.Context, params *redshiftdata.DescribeStatementInput, optFns ...func(*redshiftdata.Options)) (*redshiftdata.DescribeStatementOutput, error) 13 | CancelStatement(ctx context.Context, params *redshiftdata.CancelStatementInput, optFns ...func(*redshiftdata.Options)) (*redshiftdata.CancelStatementOutput, error) 14 | BatchExecuteStatement(ctx context.Context, params *redshiftdata.BatchExecuteStatementInput, optFns ...func(*redshiftdata.Options)) (*redshiftdata.BatchExecuteStatementOutput, error) 15 | redshiftdata.GetStatementResultAPIClient 16 | } 17 | 18 | func newRedshiftDataClient(ctx context.Context, cfg *RedshiftConfig, opts ...func(*config.LoadOptions) error) (RedshiftClient, error) { 19 | awsCfg, err := config.LoadDefaultConfig(ctx, opts...) 20 | if err != nil { 21 | return nil, err 22 | } 23 | client := redshiftdata.NewFromConfig(awsCfg, cfg.Opts()...) 24 | return client, nil 25 | } 26 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/connection_test.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestRewriteQuery(t *testing.T) { 10 | cases := []struct { 11 | casename string 12 | query string 13 | params bool 14 | expected string 15 | }{ 16 | { 17 | casename: "no params", 18 | query: `SELECT * FROM pg_user`, 19 | params: false, 20 | expected: `SELECT * FROM pg_user`, 21 | }, 22 | { 23 | casename: "no change", 24 | query: `SELECT * FROM pg_user WHERE usename = :name`, 25 | params: true, 26 | expected: `SELECT * FROM pg_user WHERE usename = :name`, 27 | }, 28 | { 29 | casename: "? rewrite", 30 | query: `SELECT 'hoge?' FROM pg_user WHERE usename = ? AND usesysid > ?`, 31 | params: true, 32 | expected: `SELECT 'hoge?' FROM pg_user WHERE usename = :1 AND usesysid > :2`, 33 | }, 34 | { 35 | casename: "$ rewrite", 36 | query: `SELECT '3$1$' FROM table WHERE "$column" = $1 AND column1 > $2 AND column2 < $1`, 37 | params: true, 38 | expected: `SELECT '3$1$' FROM table WHERE "$column" = :1 AND column1 > :2 AND column2 < :1`, 39 | }, 40 | } 41 | for _, c := range cases { 42 | t.Run(c.casename, func(t *testing.T) { 43 | actual := rewriteQuery(c.query, c.params) 44 | require.Equal(t, c.expected, actual) 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/connector.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "context" 5 | "database/sql/driver" 6 | ) 7 | 8 | func NewRedshiftConnector(cfg RedshiftConfig) driver.Connector { 9 | cfg.Sanitize() 10 | return &redshiftDataConnector{ 11 | d: &redshiftDataDriver{}, 12 | cfg: &cfg, 13 | } 14 | } 15 | 16 | type redshiftDataConnector struct { 17 | d *redshiftDataDriver 18 | cfg *RedshiftConfig 19 | } 20 | 21 | func (c *redshiftDataConnector) Connect(ctx context.Context) (driver.Conn, error) { 22 | loadOpts, err := c.cfg.LoadOpts(ctx) 23 | if err != nil { 24 | return nil, err 25 | } 26 | client, err := newRedshiftDataClient(ctx, c.cfg, loadOpts...) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return newConnection(client, c.cfg), nil 31 | } 32 | 33 | func (c *redshiftDataConnector) Driver() driver.Driver { 34 | return c.d 35 | } 36 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/driver.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "database/sql/driver" 7 | ) 8 | 9 | func init() { 10 | sql.Register("redshift-data", &redshiftDataDriver{}) 11 | } 12 | 13 | type redshiftDataDriver struct{} 14 | 15 | func (d *redshiftDataDriver) Open(dsn string) (driver.Conn, error) { 16 | connector, err := d.OpenConnector(dsn) 17 | if err != nil { 18 | return nil, err 19 | } 20 | return connector.Connect(context.Background()) 21 | } 22 | 23 | func (d *redshiftDataDriver) OpenConnector(dsn string) (driver.Connector, error) { 24 | cfg, err := ParseDSN(dsn) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return &redshiftDataConnector{ 29 | d: d, 30 | cfg: cfg, 31 | }, nil 32 | } 33 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/errors.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrNotSupported = errors.New("not supported") 7 | ErrDSNEmpty = errors.New("dsn is empty") 8 | ErrConnClosed = errors.New("connection closed") 9 | ErrBeforeCommit = errors.New("transaction is not committed") 10 | ErrNotInTx = errors.New("not in transaction") 11 | ErrInTx = errors.New("already in transaction") 12 | ) 13 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/result.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "database/sql/driver" 5 | "fmt" 6 | 7 | "github.com/aws/aws-sdk-go-v2/service/redshiftdata" 8 | "github.com/aws/aws-sdk-go-v2/service/redshiftdata/types" 9 | ) 10 | 11 | type redshiftResult struct { 12 | affectedRows int64 13 | } 14 | 15 | func newResult(output *redshiftdata.DescribeStatementOutput) *redshiftResult { 16 | return &redshiftResult{ 17 | affectedRows: output.ResultRows, 18 | } 19 | } 20 | 21 | func newResultWithSubStatementData(st types.SubStatementData) *redshiftResult { 22 | return &redshiftResult{ 23 | affectedRows: st.ResultRows, 24 | } 25 | } 26 | 27 | func (r *redshiftResult) LastInsertId() (int64, error) { 28 | return 0, fmt.Errorf("LastInsertId %w", ErrNotSupported) 29 | } 30 | 31 | func (r *redshiftResult) RowsAffected() (int64, error) { 32 | return r.affectedRows, nil 33 | } 34 | 35 | type redshiftDelayedResult struct { 36 | driver.Result 37 | } 38 | 39 | func (r *redshiftDelayedResult) LastInsertId() (int64, error) { 40 | if r.Result != nil { 41 | return r.Result.LastInsertId() 42 | } 43 | return 0, ErrBeforeCommit 44 | } 45 | 46 | func (r *redshiftDelayedResult) RowsAffected() (int64, error) { 47 | if r.Result != nil { 48 | return r.Result.RowsAffected() 49 | } 50 | return 0, ErrBeforeCommit 51 | } 52 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/statement.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "context" 5 | "database/sql/driver" 6 | ) 7 | 8 | type redshiftStatement struct { 9 | connection *redshiftConnection 10 | query string 11 | } 12 | 13 | func (_ *redshiftStatement) Close() error { 14 | return nil 15 | } 16 | 17 | func (_ *redshiftStatement) NumInput() int { 18 | return -1 19 | } 20 | 21 | func (s *redshiftStatement) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { 22 | return s.connection.ExecContext(ctx, s.query, args) 23 | } 24 | 25 | func (s *redshiftStatement) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { 26 | return s.connection.QueryContext(ctx, s.query, args) 27 | } 28 | 29 | func (_ *redshiftStatement) Exec(args []driver.Value) (driver.Result, error) { 30 | return nil, driver.ErrSkip 31 | } 32 | 33 | func (_ *redshiftStatement) Query(args []driver.Value) (driver.Rows, error) { 34 | return nil, driver.ErrSkip 35 | } 36 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/tx.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | type redshiftTx struct { 4 | onCommit func() error 5 | onRollback func() error 6 | } 7 | 8 | func (tx *redshiftTx) Commit() error { 9 | return tx.onCommit() 10 | } 11 | 12 | func (tx *redshiftTx) Rollback() error { 13 | return tx.onRollback() 14 | } 15 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/redshift_driver/utils.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | func nullStringIfEmpty(str string) *string { 4 | if str == "" { 5 | return nil 6 | } 7 | return &str 8 | } 9 | 10 | func coalesce[T any](strs ...*T) T { 11 | for _, str := range strs { 12 | if str != nil { 13 | return *str 14 | } 15 | } 16 | var zero T 17 | return zero 18 | } 19 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/columns_added.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test2", "column1": "data"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test3", "column1": "data", "column2": "data"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test2", "column1": "data"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 5, "name": "test"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 6, "name": "test4", "column1": "data", "column2": "data", "column3": "data"} 7 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/columns_added2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 7, "name": "test", "column4": "data"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 8, "name": "test2", "column5": "data"} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/columns_added_partial.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 9, "name": "test9"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 10, "name": "test10"} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/columns_added_partial2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 9, "name": "test9","column6": "data"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 10, "name": "test10", "column7": "data"} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/data_types.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "float_type": 2.13,"time_type": "2022-08-18T14:17:22.375Z", "bool_type": true, "string_type": "string"} 2 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/emoji.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test 😆"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test2"} 4 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/empty.ndjson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jitsucom/bulker/b03b7dcfce71c43f8ee6edb9ae7f9c4fe57c071c/bulkerlib/implementations/sql/test_data/empty.ndjson -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/existing_table1.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 22.2} 2 | {"id": "string_id2"} 3 | {"id": 1} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/existing_table2.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 2, "data": "string_id"} 2 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/existing_table_num.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "data": 1} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/existing_table_text.ndjson: -------------------------------------------------------------------------------- 1 | {"id": "string_id"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/identifiers.ndjson: -------------------------------------------------------------------------------- 1 | {"id":0, "name": "dummy", "_timestamp": "2022-08-18T14:17:22.375Z","?!": 1,"%#@": 2, "秒速５センチメートル": 3, "Université Français": 4, "Странное Имя": 5, "Test Name; DROP DATABASE public; SELECT 1 from DUAL;": 6, "Test Name": 7, "1test_name": 8, "2": 9, "": 10, "lorem_ipsum_dolor_sit_amet_consectetur_adipiscing_elit_sed_do_eiusmod_tempor_incididunt_ut_labore_et_dolore_magna_aliqua_ut_enim_ad_minim_veniam_quis_nostrud_exercitation_ullamco_laboris_nisi_ut_aliquip_ex_ea_commodo_consequat": 11, "camelCase": 12, "int": 13, "user": 14, "select": 15, "__ROOT__": 16, "hash": 17, "default": 18} 2 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/merge_window1.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2023-01-01T00:00:00.000Z", "id": 1, "name": "test1"} 2 | {"_timestamp": "2023-01-05T00:00:00.000Z", "id": 2, "name": "test2"} 3 | {"_timestamp": "2023-01-09T00:00:00.000Z", "id": 3, "name": "test3"} 4 | {"_timestamp": "2023-01-13T00:00:00.000Z", "id": 4, "name": "test4"} 5 | {"_timestamp": "2023-01-17T00:00:00.000Z", "id": 5, "name": "test5"} 6 | {"_timestamp": "2023-01-21T00:00:00.000Z", "id": 6, "name": "test6"} 7 | {"_timestamp": "2023-01-25T00:00:00.000Z", "id": 7, "name": "test7"} 8 | {"_timestamp": "2023-01-29T00:00:00.000Z", "id": 8, "name": "test8"} 9 | {"_timestamp": "2023-02-02T00:00:00.000Z", "id": 9, "name": "test9"} 10 | {"_timestamp": "2023-02-07T00:00:00.000Z", "id": 10, "name": "test10"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/merge_window2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2023-01-01T00:00:00.000Z", "id": 1, "name": "test1B"} 2 | {"_timestamp": "2023-01-05T00:00:00.000Z", "id": 2, "name": "test2B"} 3 | {"_timestamp": "2023-01-09T00:00:00.000Z", "id": 3, "name": "test3B"} 4 | {"_timestamp": "2023-01-13T00:00:00.000Z", "id": 4, "name": "test4B"} 5 | {"_timestamp": "2023-01-17T00:00:00.000Z", "id": 5, "name": "test5B"} 6 | {"_timestamp": "2023-01-21T00:00:00.000Z", "id": 6, "name": "test6B"} 7 | {"_timestamp": "2023-01-25T00:00:00.000Z", "id": 7, "name": "test7B"} 8 | {"_timestamp": "2023-01-29T00:00:00.000Z", "id": 8, "name": "test8B"} 9 | {"_timestamp": "2023-02-02T00:00:00.000Z", "id": 9, "name": "test9B"} 10 | {"_timestamp": "2023-02-07T00:00:00.000Z", "id": 10, "name": "test10B"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/merge_window3.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2023-01-25T00:00:00.000Z", "id": 7, "name": "test7C"} 2 | {"_timestamp": "2023-01-29T00:00:00.000Z", "id": 8, "name": "test8C"} 3 | {"_timestamp": "2023-02-02T00:00:00.000Z", "id": 9, "name": "test9C"} 4 | {"_timestamp": "2023-02-07T00:00:00.000Z", "id": 10, "name": "test10C"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/multiline.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "\n\ntest"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test\n\n"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "\ntest2\ntest3\n"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/nested.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "nested": {"id": 1, "name": "nested"}} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "nested": {"id": 1, "name": "nested", "extra": "extra"}} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/non_utc_timestamp.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2024-07-16T12:07:15+05:30", "id": 1} 2 | {"_timestamp": "2024-12-09T16:15:02+04:00", "id": 2} 3 | {"_timestamp": "2024-12-09T16:15:02Z", "id": 3} 4 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/partition1.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test2"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test3"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test4"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 5, "name": "test5"} 6 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/partition1_1.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 6, "name": "test6"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 7, "name": "test7"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 8, "name": "test8"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 9, "name": "test9"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 10, "name": "test10"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/partition2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 11, "name": "test11"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 12, "name": "test12"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 13, "name": "test13"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 14, "name": "test14"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 15, "name": "test15"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 16, "name": "test16"} 7 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 17, "name": "test17"} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 18, "name": "test18"} 9 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 19, "name": "test19"} 10 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 20, "name": "test20"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/repeated_ids.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test1"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test2"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test3"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test4"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test5"} 7 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test6"} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test7"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/repeated_ids2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test2"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test13"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 5, "name": "test15"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test14"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/repeated_ids_discr.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "C", "int1": 3, "float1": 3.3} 2 | {"_timestamp": "2022-08-15T14:17:22.375Z", "id": 1, "name": "B", "int1": 2, "float1": 2.2} 3 | {"_timestamp": "2022-08-15T14:17:22.375Z", "id": 2, "name": "B", "int1": 2, "float1": 2.2} 4 | {"_timestamp": "2022-08-10T14:17:22.375Z", "id": 3, "name": "A", "int1": 1, "float1": 1.1} 5 | {"_timestamp": "2022-08-15T14:17:22.375Z", "id": 3, "name": "B", "int1": 2, "float1": 2.2} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "C", "int1": 3, "float1": 3.3} 7 | {"_timestamp": "2022-08-10T14:17:22.375Z", "id": 1, "name": "A", "int1": 1, "float1": 1.1} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "C", "int1": 3, "float1": 3.3} 9 | {"_timestamp": "2022-08-10T14:17:22.375Z", "id": 2, "name": "A", "int1": 1, "float1": 1.1} 10 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/repeated_ids_multi.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "id2": "a", "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "id2": "b", "name": "test1"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "id2": "c", "name": "test2"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "id2": "c", "name": "test3"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "id2": "d", "name": "test4"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "id2": "d", "name": "test5"} 7 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "id2": "dd", "name": "test6"} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "id2": "c", "name": "test7"} 9 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "id2": "a", "name": "test8"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/repeated_ids_multi2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "id2": "a", "name": "test13"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 5, "id2": "d", "name": "test14"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "id2": "dd", "name": "test15"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "id2": "d", "name": "test16"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "id2": "a", "name": "test17"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/replace_table.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 11, "name": "test11", "name2": "a"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 12, "name": "test12", "name2": "a"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 13, "name": "test13", "name2": "a"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 14, "name": "test14", "name2": "a"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 15, "name": "test15", "name2": "a"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 16, "name": "test16", "name2": "a"} 7 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 17, "name": "test17", "name2": "a"} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 18, "name": "test18", "name2": "a"} 9 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 19, "name": "test19", "name2": "a"} 10 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 20, "name": "test20", "name2": "a"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/schema_option.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/simple.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test2", "extra": "extra"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/simple2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 1, "name": "test1"} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 2, "name": "test2"} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 3, "name": "test3"} 4 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 4, "name": "test4"} 5 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 5, "name": "test5"} 6 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 6, "name": "test6"} 7 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 7, "name": "test7"} 8 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 8, "name": "test8"} 9 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id": 9, "name": "test9"} 10 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/type_hints.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "time1": "2022-08-18T14:17:22.375Z", "name": "a", "int1": "27", "__sql_type_int1": "bigint","__sql_type_nested_json2": "json","__sql_type_nested_json3_nested_json_nested": "json", "nested_json1": {"a": 1, "__sql_type": "json" }, "nested_json2": {"a": "2", "__sql_type_a": "bigint"}, "nested_json3": {"a": 2, "nested_json_nested": {"a": 3}}, "nested_json4": {"a": "4"}} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/type_hints_bq.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "time1": "2022-08-18T14:17:22.375Z", "name": "a", "int1": "27", "__sql_type_int1": "INTEGER","__sql_type_nested_json2": "json","__sql_type_nested_json3_nested_json_nested": "json", "nested_json1": {"a": 1, "__sql_type": "json" }, "nested_json2": {"a": "2", "__sql_type_a": "bigint"}, "nested_json3": {"a": 2, "nested_json_nested": {"a": 3}}, "nested_json4": {"a": "4"}} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "time1": "2022-08-18T14:17:22.375Z","time2": "2022-08-18T14:17:22.375Z","date1": "2022-08-18", "int_1": 1, "roundfloat": 1.0, "float1": 1.2, "intstring": "1", "roundfloatstring": "1.0","floatstring": "1.1", "name": "test", "bool1": false, "bool2": true, "boolstring": "true", ",null1": null, "arr1": [], "arr2": [1,2,3], "arr3": ["a","b","c"]} 2 | {"id": 2, "time1": "2022-08-18T14:17:22.375Z","time2": "2022-08-18T14:17:22.375Z","date1": "2022-08-18", "int_1": 1.0, "roundfloat": 1.0, "float1": 1, "intstring": "1", "roundfloatstring": "1.0","floatstring": "1.0", "name": "test", "bool1": false, "bool2": true, "boolstring": "false", "null1": null, "arr1": [], "arr2": [1,2,3], "arr3": ["a","b","c"]} 3 | {"id": 3, "time1": "2022-08-18T14:17:22.375Z","time2": "2022-08-18T14:17:22.375Z","date1": "2022-08-18", "int_1": 1, "roundfloat": 1.0, "float1": 1.2, "intstring": "1", "roundfloatstring": "1.0","floatstring": "1.1", "name": "test", "bool1": false, "bool2": true, "boolstring": "true", ",null1": null, "arr1": [], "arr2": [1,2,3], "arr3": ["a","b","c"]} 4 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types2.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "time1": "2022-08-18T14:17:22.375Z","time2": "2022-08-18T14:17:22.375Z","date1": "2022-08-18", "int_1": 1.0, "roundfloat": 1.0, "float1": 1, "intstring": "1", "roundfloatstring": "1.0","floatstring": "1.0", "name": "test", "bool1": false, "bool2": true, "boolstring": "false", "null1": null} 2 | {"id": 2, "time1": "2022-08-18T14:17:22.375Z","time2": "2022-08-18T14:17:22.375Z","date1": "2022-08-18", "int_1": 1, "roundfloat": 1.0, "float1": 1.2, "intstring": "1", "roundfloatstring": "1.0","floatstring": "1.1", "name": "test", "bool1": false, "bool2": true, "boolstring": "true", ",null1": null} 3 | {"id": 3, "time1": "2022-08-18T14:17:22.375Z","time2": "2022-08-18T14:17:22.375Z","date1": "2022-08-18", "int_1": 1.0, "roundfloat": 1.0, "float1": 1, "intstring": "1", "roundfloatstring": "1.0","floatstring": "1.0", "name": "test", "bool1": false, "bool2": true, "boolstring": "false", "null1": null} 4 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_coalesce.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "str_1": 7, "float_1": 7, "int_1": 7, "bool_1": true, "str_2": "str"} 2 | {"id": 2, "str_1": 7, "float_1": 7, "int_1": 7, "bool_1": false, "str_2": "str"} 3 | {"id": 3, "str_1": 3.14, "float_1": 3.14, "int_1": 7, "bool_1": true, "str_2": "str"} 4 | {"id": 4, "str_1": "str", "float_1": 3.14, "int_1": 7, "bool_1": true, "str_2": "str"} 5 | {"id": 5, "str_1": "str", "float_1": true, "int_1": 9.0, "bool_1": 0, "str_2": "str"} 6 | {"id": 6, "str_1": "str", "float_1": false, "int_1": false, "bool_1": 1.0, "str_2": "str"} 7 | {"id": 7, "str_1": "str", "float_1": 7, "int_1": true, "bool_1": "true", "str_2": "str"} 8 | {"id": 8, "str_1": "str", "float_1": 7, "int_1": true, "bool_1": "false", "str_2": "str"} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_collision.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "time1": "2022-08-18T14:17:22.375Z","time2": "2022-08-18T14:17:22.375Z","time3": "2022-08-18", "int_1": 1, "roundfloat": 1.0, "float1": 1.2, "intstring": "1", "roundfloatstring": "1.0","floatstring": "1.1", "string1": "test", "bool1": false, "bool2": true, "null1": null} 2 | {"id": 2, "time1": "2022-08-18T14:17:22.375Z","time2": "2022-08-18T14:17:22.375Z","time3": "2022-08-18", "int_1": "a", "roundfloat": 1.0, "float1": 1, "intstring": "1.1", "roundfloatstring": "1.1","floatstring": "1.0", "string1": "test", "bool1": false, "bool2": true, "null1": null} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_collision2.ndjson: -------------------------------------------------------------------------------- 1 | {"id": 1, "time1": "2022-08-18T14:17:22.375Z", "string2": "test"} 2 | {"id": 2, "time1": "2022-08-18T14:17:22.375Z", "string2": "2022-08-18T14:17:22.375Z"} 3 | {"id": 3, "time1": "2022-08-18T14:17:22.375Z", "string2": null} 4 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_json.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":1, "name": "a", "json1": {"nested": 1}, "json2": {"nested": 1}, "array1": ["1","2","3"]} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":2, "name": "b", "json1": {"nested2": {"nested": 2}}, "json2": {"nested": {"nested": 2}}, "array1": [1,2,3]} 3 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":3, "name": "c", "json1": {"nested": 1}, "json2": {"nested": 1},"array1": [{"nested": 1},{"nested": 2},{"nested": 3}]} -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_json_noarr_part1.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":1, "name": "a", "json1": {"nested": 1}} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":2, "name": "b", "json1": {"nested": {"nested": 2}}} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_json_noarr_part2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":3, "name": "c", "json1": {"nested": 3}} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":4, "name": "d", "json1": {"nested": {"nested": 4}}} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_json_part1.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":1, "name": "a", "json1": {"nested": 1}, "array1": ["1","2","3"]} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":2, "name": "b", "json1": {"nested": {"nested": 2}}, "array1": [1,2,3]} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_json_part2.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":3, "name": "c", "json1": {"nested": 3}, "array1": ["1","2","3"]} 2 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":4, "name": "d", "json1": {"nested": {"nested": 4}}, "array1": [1,2,3]} 3 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/test_data/types_json_part3.ndjson: -------------------------------------------------------------------------------- 1 | {"_timestamp": "2022-08-18T14:17:22.375Z", "id":5, "name": "e", "json1": {"nested": 5}, "array1": ["1","2","3"]} 2 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/testcontainers/clickhouse/clickhouse01/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10000000000 6 | 0 7 | in_order 8 | 1 9 | 10 | 11 | 12 | 13 | 14 | 15 | default 16 | 17 | ::/0 18 | 19 | default 20 | 21 | 22 | 123 23 | default 24 | 25 | ::/0 26 | 27 | default 28 | 29 | 30 | 31 | 32 | 33 | 34 | 3600 35 | 0 36 | 0 37 | 0 38 | 0 39 | 0 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/testcontainers/clickhouse/clickhouse02/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10000000000 6 | 0 7 | in_order 8 | 1 9 | 10 | 11 | 12 | 13 | 14 | 15 | default 16 | 17 | ::/0 18 | 19 | default 20 | 21 | 22 | 123 23 | default 24 | 25 | ::/0 26 | 27 | default 28 | 29 | 30 | 31 | 32 | 33 | 34 | 3600 35 | 0 36 | 0 37 | 0 38 | 0 39 | 0 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/testcontainers/clickhouse/clickhouse03/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10000000000 6 | 0 7 | in_order 8 | 1 9 | 10 | 11 | 12 | 13 | 14 | 15 | default 16 | 17 | ::/0 18 | 19 | default 20 | 21 | 22 | 123 23 | default 24 | 25 | ::/0 26 | 27 | default 28 | 29 | 30 | 31 | 32 | 33 | 34 | 3600 35 | 0 36 | 0 37 | 0 38 | 0 39 | 0 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/testcontainers/clickhouse/clickhouse04/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10000000000 6 | 0 7 | in_order 8 | 1 9 | 10 | 11 | 12 | 13 | 14 | 15 | default 16 | 17 | ::/0 18 | 19 | default 20 | 21 | 22 | 123 23 | default 24 | 25 | ::/0 26 | 27 | default 28 | 29 | 30 | 31 | 32 | 33 | 34 | 3600 35 | 0 36 | 0 37 | 0 38 | 0 39 | 0 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/testcontainers/clickhouse_noshards/clickhouse01/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10000000000 6 | 0 7 | in_order 8 | 1 9 | 10 | 11 | 12 | 13 | 14 | 15 | default 16 | 17 | ::/0 18 | 19 | default 20 | 21 | 22 | 123 23 | default 24 | 25 | ::/0 26 | 27 | default 28 | 29 | 30 | 31 | 32 | 33 | 34 | 3600 35 | 0 36 | 0 37 | 0 38 | 0 39 | 0 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/testcontainers/clickhouse_noshards/clickhouse02/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10000000000 6 | 0 7 | in_order 8 | 1 9 | 10 | 11 | 12 | 13 | 14 | 15 | default 16 | 17 | ::/0 18 | 19 | default 20 | 21 | 22 | 123 23 | default 24 | 25 | ::/0 26 | 27 | default 28 | 29 | 30 | 31 | 32 | 33 | 34 | 3600 35 | 0 36 | 0 37 | 0 38 | 0 39 | 0 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /bulkerlib/implementations/sql/testcontainers/clickhouse_noshards/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | zookeeper2: 4 | image: zookeeper:3.9 5 | container_name: zookeeper2 6 | hostname: zookeeper2 7 | healthcheck: 8 | test: ["CMD-SHELL", "zkCli.sh -server zookeeper2:2181 ls /"] 9 | interval: 1s 10 | timeout: 3s 11 | retries: 30 12 | networks: 13 | clickhouse-network2: 14 | ipv4_address: 172.24.0.10 15 | clickhouse2_01: 16 | image: clickhouse/clickhouse-server:25.4-alpine 17 | container_name: clickhouse2_01 18 | hostname: clickhouse01 19 | networks: 20 | clickhouse-network2: 21 | ipv4_address: 172.24.0.11 22 | ports: 23 | - "8133:8123" 24 | - "9010:9000" 25 | volumes: 26 | - ./clickhouse01:/etc/clickhouse-server 27 | healthcheck: 28 | test: ["CMD-SHELL", "clickhouse-client --host clickhouse01 --query 'SELECT 1'"] 29 | interval: 1s 30 | timeout: 3s 31 | retries: 30 32 | environment: 33 | CLICKHOUSE_USER: default 34 | CLICKHOUSE_SKIP_USER_SETUP: 1 35 | depends_on: 36 | zookeeper2: 37 | condition: service_healthy 38 | clickhouse2_02: 39 | image: clickhouse/clickhouse-server:25.4-alpine 40 | container_name: clickhouse2_02 41 | hostname: clickhouse02 42 | networks: 43 | clickhouse-network2: 44 | ipv4_address: 172.24.0.12 45 | ports: 46 | - "8134:8123" 47 | - "9011:9000" 48 | volumes: 49 | - ./clickhouse02:/etc/clickhouse-server 50 | healthcheck: 51 | test: [ "CMD-SHELL", "clickhouse-client --host clickhouse02 --query 'SELECT 1'" ] 52 | interval: 1s 53 | timeout: 3s 54 | retries: 30 55 | environment: 56 | CLICKHOUSE_USER: default 57 | CLICKHOUSE_SKIP_USER_SETUP: 1 58 | depends_on: 59 | zookeeper2: 60 | condition: service_healthy 61 | 62 | networks: 63 | clickhouse-network2: 64 | name: clickhouse-network2 65 | ipam: 66 | config: 67 | - subnet: 172.24.0.0/24 68 | -------------------------------------------------------------------------------- /bulkerlib/types/avro.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | type AvroType struct { 4 | Type any `json:"type"` 5 | Name string `json:"name"` 6 | Default any `json:"default"` 7 | } 8 | 9 | type AvroSchema struct { 10 | Type string `json:"type"` 11 | Name string `json:"name"` 12 | Fields []AvroType `json:"fields"` 13 | DataTypes map[string]DataType `json:"-"` 14 | } 15 | -------------------------------------------------------------------------------- /bulkerlib/types/datatype_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func MustParseTime(layout, value string) time.Time { 9 | t, err := time.Parse(layout, value) 10 | if err != nil { 11 | panic(err) 12 | } 13 | return t 14 | } 15 | 16 | func TestReformatTimeValue(t *testing.T) { 17 | tests := []struct { 18 | name string 19 | input string 20 | expected time.Time 21 | }{ 22 | { 23 | "Shopify (Airbyte)", 24 | "2024-07-16T12:07:15+05:30", 25 | MustParseTime(time.RFC3339Nano, "2024-07-16T06:37:15Z"), 26 | }, 27 | } 28 | 29 | for _, tt := range tests { 30 | t.Run(tt.name, func(t *testing.T) { 31 | result, ok := ReformatTimeValue(tt.input, false) 32 | if !ok { 33 | t.Fatalf("Failed to parse datetime: %s", tt.input) 34 | } 35 | if result.UnixMicro() != tt.expected.UnixMicro() { 36 | t.Errorf("Expected %s, got %s", tt.expected, result) 37 | } 38 | }) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /bulkerlib/types/object.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/types" 5 | ) 6 | 7 | type Object = types.Json 8 | 9 | func NewObject(defaultCapacity int) Object { 10 | return types.NewJson(defaultCapacity) 11 | } 12 | 13 | func ObjectFromMap(mp map[string]any) Object { 14 | return types.JsonFromMap(mp) 15 | } 16 | 17 | func ObjectToMap(o Object) map[string]any { 18 | return types.JsonToMap(o) 19 | } 20 | -------------------------------------------------------------------------------- /bulkerlib/types/schema.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | type Schema struct { 4 | Name string `json:"name"` 5 | Fields []SchemaField `json:"fields"` 6 | } 7 | 8 | type SchemaField struct { 9 | Name string `json:"name"` 10 | Type DataType `json:"type"` 11 | } 12 | 13 | func (s Schema) IsEmpty() bool { 14 | return len(s.Fields) == 0 15 | } 16 | 17 | func (s Schema) ColumnsCount() int { 18 | return len(s.Fields) 19 | } 20 | -------------------------------------------------------------------------------- /bulkerlib/types/sqltype.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | type SQLTypes map[string]SQLColumn 4 | 5 | type SQLColumn struct { 6 | Type string `json:"type,omitempty"` 7 | DdlType string `json:"ddlType,omitempty"` 8 | Override bool `json:"override,omitempty"` 9 | // Important column is provided as part of bulkerlib.SchemaOption option. 10 | // It is not literally an Override, but we must give priority to this type 11 | Important bool 12 | DataType DataType 13 | // New column represents not commited part of a table schema 14 | New bool 15 | } 16 | 17 | func (c SQLColumn) GetDDLType() string { 18 | if c.DdlType != "" { 19 | return c.DdlType 20 | } 21 | return c.Type 22 | } 23 | 24 | func (s SQLTypes) With(name, sqlType string) SQLTypes { 25 | return s.WithDDL(name, sqlType, "") 26 | } 27 | 28 | func (s SQLTypes) WithDDL(name, sqlType, ddlType string) SQLTypes { 29 | if sqlType == "" { 30 | return s 31 | } else if ddlType == "" { 32 | s[name] = SQLColumn{Type: sqlType, DdlType: sqlType, Override: true} 33 | } else { 34 | s[name] = SQLColumn{Type: sqlType, DdlType: ddlType, Override: true} 35 | } 36 | return s 37 | } 38 | -------------------------------------------------------------------------------- /config-keeper/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/appbase" 5 | "github.com/jitsucom/bulker/jitsubase/utils" 6 | "github.com/spf13/viper" 7 | "os" 8 | ) 9 | 10 | type Config struct { 11 | appbase.Config `mapstructure:",squash"` 12 | 13 | //Cache dir for repositories data 14 | CacheDir string `mapstructure:"CACHE_DIR"` 15 | 16 | ScriptOrigin string `mapstructure:"SCRIPT_ORIGIN" default:"https://cdn.jsdelivr.net/npm/@jitsu/js@latest/dist/web/p.js.txt"` 17 | 18 | RepositoryBaseURL string `mapstructure:"REPOSITORY_BASE_URL"` 19 | RepositoryAuthToken string `mapstructure:"REPOSITORY_AUTH_TOKEN"` 20 | RepositoryRefreshPeriodSec int `mapstructure:"REPOSITORY_REFRESH_PERIOD_SEC" default:"5"` 21 | Repositories string `mapstructure:"REPOSITORIES" default:"streams-with-destinations,workspaces-with-profiles,functions,rotor-connections,bulker-connections"` 22 | } 23 | 24 | func init() { 25 | viper.SetDefault("HTTP_PORT", utils.NvlString(os.Getenv("PORT"), "3059")) 26 | } 27 | 28 | func (c *Config) PostInit(settings *appbase.AppSettings) error { 29 | return c.Config.PostInit(settings) 30 | } 31 | -------------------------------------------------------------------------------- /config-keeper/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/appbase" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | settings := &appbase.AppSettings{ 10 | ConfigPath: os.Getenv("CFGKPR_CONFIG_PATH"), 11 | Name: "cfgkpr", 12 | EnvPrefix: "CFGKPR", 13 | ConfigName: "cfgkpr", 14 | ConfigType: "env", 15 | } 16 | application := appbase.NewApp[Config](&Context{}, settings) 17 | application.Run() 18 | } 19 | -------------------------------------------------------------------------------- /connectors/airbytecdk/LICENCE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 bitstrapped 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /connectors/airbytecdk/cmdparser.go: -------------------------------------------------------------------------------- 1 | package airbyte 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | ) 8 | 9 | func getSourceConfigPath() (string, error) { 10 | if os.Args[2] != "--config" { 11 | return "", fmt.Errorf("expect --config") 12 | } 13 | return os.Args[3], nil 14 | } 15 | 16 | func getStatePath() (string, error) { 17 | if len(os.Args) <= 6 { 18 | return "", nil 19 | } 20 | if os.Args[6] != "--state" { 21 | return "", fmt.Errorf("expect --state") 22 | } 23 | return os.Args[7], nil 24 | } 25 | 26 | func getCatalogPath() (string, error) { 27 | if os.Args[4] != "--catalog" { 28 | return "", fmt.Errorf("expect --catalog") 29 | } 30 | return os.Args[5], nil 31 | } 32 | 33 | // UnmarshalFromPath is used to unmarshal json files into respective struct's 34 | // this is most commonly used to unmarshal your State between runs and also unmarshal SourceConfig's 35 | // 36 | // Example usage 37 | // 38 | // type CustomState struct { 39 | // Timestamp int `json:"timestamp"` 40 | // Foobar string `json:"foobar"` 41 | // } 42 | // 43 | // func (s *CustomSource) Read(stPath string, ...) error { 44 | // var cs CustomState 45 | // err = airbyte.UnmarshalFromPath(stPath, &cs) 46 | // if err != nil { 47 | // // handle error 48 | // } 49 | // // cs is populated 50 | // } 51 | func UnmarshalFromPath(path string, v interface{}) error { 52 | b, err := os.ReadFile(path) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | return json.Unmarshal(b, v) 58 | } 59 | -------------------------------------------------------------------------------- /connectors/airbytecdk/doc.go: -------------------------------------------------------------------------------- 1 | // Airbyte is the go-sdk/cdk to help build connectors quickly in go 2 | // This package abstracts away much of the "protocol" away from the user and lets them focus on biz logic 3 | // It focuses on developer efficiency and tries to be strongly typed as much as possible to help dev's move fast without mistakes 4 | package airbyte 5 | -------------------------------------------------------------------------------- /connectors/airbytecdk/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jitsucom/bulker/airbytecdk 2 | 3 | go 1.24 4 | -------------------------------------------------------------------------------- /connectors/airbytecdk/inferschema.go: -------------------------------------------------------------------------------- 1 | package airbyte 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/jitsucom/bulker/airbytecdk/schema" 7 | "reflect" 8 | ) 9 | 10 | // Infer schema translates golang structs to JSONSchema format 11 | func InferSchemaFromStruct(i interface{}, logTracker LogTracker) Properties { 12 | var prop Properties 13 | 14 | s, err := schema.Generate(reflect.TypeOf(i)) 15 | if err != nil { 16 | logTracker.Log(LogLevelError, fmt.Sprintf("generate schema error: %v", err)) 17 | return prop 18 | } 19 | 20 | b, err := json.Marshal(s) 21 | if err != nil { 22 | logTracker.Log(LogLevelError, fmt.Sprintf("json marshal schema error: %v", err)) 23 | return prop 24 | } 25 | 26 | err = json.Unmarshal(b, &prop) 27 | if err != nil { 28 | logTracker.Log(LogLevelError, fmt.Sprintf("unmarshal schema to propspec error: %v", err)) 29 | return prop 30 | } 31 | 32 | return prop 33 | } 34 | -------------------------------------------------------------------------------- /connectors/airbytecdk/safewriter.go: -------------------------------------------------------------------------------- 1 | package airbyte 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | ) 7 | 8 | type safeWriter struct { 9 | w io.Writer 10 | mu sync.Mutex 11 | } 12 | 13 | func newSafeWriter(w io.Writer) io.Writer { 14 | return &safeWriter{ 15 | w: w, 16 | } 17 | } 18 | 19 | func (sw *safeWriter) Write(p []byte) (int, error) { 20 | sw.mu.Lock() 21 | defer sw.mu.Unlock() 22 | return sw.w.Write(p) 23 | } 24 | -------------------------------------------------------------------------------- /connectors/airbytecdk/source.go: -------------------------------------------------------------------------------- 1 | package airbyte 2 | 3 | // Source is the only interface you need to define to create your source! 4 | type Source interface { 5 | // Spec returns the input "form" spec needed for your source 6 | Spec(logTracker LogTracker) (*ConnectorSpecification, error) 7 | // Check verifies the source - usually verify creds/connection etc. 8 | Check(srcCfgPath string, logTracker LogTracker) error 9 | // Discover returns the schema of the data you want to sync 10 | Discover(srcConfigPath string, logTracker LogTracker) (*Catalog, error) 11 | // Read will read the actual data from your source and use tracker.Record(), tracker.State() and tracker.Log() to sync data with airbyte/destinations 12 | // MessageTracker is thread-safe and so it is completely find to spin off goroutines to sync your data (just don't forget your waitgroups :)) 13 | // returning an error from this will cancel the sync and returning a nil from this will successfully end the sync 14 | Read(sourceCfgPath string, prevStatePath string, configuredCat *ConfiguredCatalog, 15 | tracker MessageTracker) error 16 | } 17 | -------------------------------------------------------------------------------- /connectors/airbytecdk/trackers.go: -------------------------------------------------------------------------------- 1 | package airbyte 2 | 3 | // MessageTracker is used to encap State tracking, Record tracking and Log tracking 4 | // It's thread safe 5 | type MessageTracker struct { 6 | // State will save an arbitrary JSON blob to airbyte state 7 | State StateWriter 8 | // Record will emit a record (data point) out to airbyte to sync with appropriate timestamps 9 | Record RecordWriter 10 | // Log logs out to airbyte 11 | Log LogWriter 12 | } 13 | 14 | // LogTracker is a single struct which holds a tracker which can be used for logs 15 | type LogTracker struct { 16 | Log LogWriter 17 | } 18 | -------------------------------------------------------------------------------- /connectors/firebase.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bullseye-slim as main 2 | 3 | RUN apt-get update -y 4 | RUN apt-get install -y ca-certificates curl 5 | 6 | ENV TZ=UTC 7 | 8 | FROM golang:1.24.2-bullseye as build 9 | 10 | RUN apt-get install gcc libc6-dev 11 | 12 | 13 | RUN mkdir /app 14 | WORKDIR /app 15 | 16 | RUN mkdir firebase airbytecdk 17 | 18 | 19 | COPY airbytecdk/go.* ./airbytecdk/ 20 | COPY firebase/go.* ./firebase/ 21 | 22 | RUN go work init airbytecdk firebase 23 | 24 | WORKDIR /app/firebase 25 | 26 | RUN go mod download 27 | 28 | WORKDIR /app 29 | 30 | COPY . . 31 | 32 | # Build bulker 33 | RUN go build -o firebase ./firebase 34 | 35 | ####################################### 36 | # FINAL STAGE 37 | FROM main as final 38 | 39 | RUN mkdir /app 40 | WORKDIR /app 41 | 42 | COPY --from=build /app/firebase ./ 43 | #COPY ./config.yaml ./ 44 | 45 | ENV AIRBYTE_ENTRYPOINT "/app/firebase" 46 | ENTRYPOINT ["/app/firebase"] 47 | -------------------------------------------------------------------------------- /connectors/firebase/firebase_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/jitsucom/bulker/airbytecdk" 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | ) 11 | 12 | func TestFirebase(t *testing.T) { 13 | os.Args = []string{os.Args[0], "spec"} 14 | hsrc := NewFirebaseSource() 15 | runner := airbyte.NewSourceRunner(hsrc, os.Stdout) 16 | err := runner.Start() 17 | if err != nil { 18 | t.Error(err) 19 | } 20 | 21 | //write config to tmp file 22 | config := FirebaseConfig{ 23 | ProjectID: "", 24 | ServiceAccountKey: `{ 25 | }`, 26 | } 27 | tmpfile, err := ioutil.TempFile("", "firebase") 28 | if err != nil { 29 | t.Error(err) 30 | } 31 | defer os.Remove(tmpfile.Name()) 32 | 33 | b, _ := json.Marshal(config) 34 | 35 | if _, err := tmpfile.Write(b); err != nil { 36 | t.Error(err) 37 | } 38 | fmt.Println("tmp file:" + tmpfile.Name()) 39 | 40 | os.Args = []string{os.Args[0], "check", "--config", tmpfile.Name()} 41 | 42 | err = runner.Start() 43 | if err != nil { 44 | t.Error(err) 45 | } 46 | 47 | os.Args = []string{os.Args[0], "discover", "--config", tmpfile.Name()} 48 | 49 | err = runner.Start() 50 | if err != nil { 51 | t.Error(err) 52 | } 53 | 54 | os.Args = []string{os.Args[0], "read", "--config", tmpfile.Name()} 55 | 56 | err = runner.Start() 57 | if err != nil { 58 | t.Error(err) 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /connectors/firebase/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | airbyte "github.com/jitsucom/bulker/airbytecdk" 5 | "log" 6 | "os" 7 | ) 8 | 9 | func main() { 10 | hsrc := NewFirebaseSource() 11 | runner := airbyte.NewSourceRunner(hsrc, os.Stdout) 12 | err := runner.Start() 13 | if err != nil { 14 | log.Fatal(err) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /connectors/firebasebuild.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker buildx build --platform linux/amd64,linux/arm64 -f firebase.Dockerfile -t jitsucom/source-firebase:0.0.3 --push . 4 | -------------------------------------------------------------------------------- /eventslog/events_log_config.go: -------------------------------------------------------------------------------- 1 | package eventslog 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/appbase" 5 | ) 6 | 7 | type EventsLogConfig struct { 8 | ClickhouseHost string `mapstructure:"CLICKHOUSE_HOST"` 9 | ClickhouseDatabase string `mapstructure:"CLICKHOUSE_DATABASE"` 10 | ClickhouseUsername string `mapstructure:"CLICKHOUSE_USERNAME"` 11 | ClickhousePassword string `mapstructure:"CLICKHOUSE_PASSWORD"` 12 | ClickhouseSSL bool `mapstructure:"CLICKHOUSE_SSL"` 13 | } 14 | 15 | func (e *EventsLogConfig) PostInit(settings *appbase.AppSettings) error { 16 | return nil 17 | } 18 | -------------------------------------------------------------------------------- /eventslog/metrics.go: -------------------------------------------------------------------------------- 1 | package eventslog 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var ( 9 | eventsLogError = promauto.NewCounterVec(prometheus.CounterOpts{ 10 | Namespace: "bulkerapp", 11 | Subsystem: "event_log", 12 | Name: "error", 13 | }, []string{"errorType"}) 14 | EventsLogError = func(errorType string) prometheus.Counter { 15 | return eventsLogError.WithLabelValues(errorType) 16 | } 17 | ) 18 | -------------------------------------------------------------------------------- /eventslog/testcontainers/redis.go: -------------------------------------------------------------------------------- 1 | package testcontainers 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/jitsucom/bulker/jitsubase/logging" 7 | "github.com/testcontainers/testcontainers-go" 8 | tcWait "github.com/testcontainers/testcontainers-go/wait" 9 | "time" 10 | ) 11 | 12 | const redisDefaultPort = "6379/tcp" 13 | 14 | // RedisContainer is a Redis testcontainer 15 | type RedisContainer struct { 16 | Container testcontainers.Container 17 | Host string 18 | Port int 19 | } 20 | 21 | func NewRedisContainer(ctx context.Context) (*RedisContainer, error) { 22 | exposedPort := redisDefaultPort 23 | 24 | container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ 25 | ContainerRequest: testcontainers.ContainerRequest{ 26 | Image: "redis:7-alpine", 27 | ExposedPorts: []string{exposedPort}, 28 | WaitingFor: tcWait.ForListeningPort(redisDefaultPort).WithStartupTimeout(60 * time.Second), 29 | }, 30 | Started: true, 31 | }) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | host, err := container.Host(ctx) 37 | if err != nil { 38 | _ = container.Terminate(ctx) 39 | return nil, err 40 | } 41 | port, err := container.MappedPort(ctx, "6379") 42 | if err != nil { 43 | _ = container.Terminate(ctx) 44 | return nil, err 45 | } 46 | return &RedisContainer{ 47 | Container: container, 48 | Host: host, 49 | Port: port.Int(), 50 | }, nil 51 | } 52 | 53 | func (rc *RedisContainer) Close() error { 54 | if rc.Container != nil { 55 | if err := rc.Container.Terminate(context.Background()); err != nil { 56 | logging.Errorf("Failed to stop redis container: %v", err) 57 | } 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // URL returns redis connection string 64 | func (rc *RedisContainer) URL() string { 65 | return fmt.Sprintf("redis://%s:%d", rc.Host, rc.Port) 66 | } 67 | -------------------------------------------------------------------------------- /go.work: -------------------------------------------------------------------------------- 1 | go 1.24 2 | 3 | use ( 4 | ./bulkerapp 5 | ./bulkerlib 6 | ./config-keeper 7 | ./connectors/airbytecdk 8 | ./connectors/firebase 9 | ./eventslog 10 | ./ingest 11 | ./ingress-manager 12 | ./jitsubase 13 | ./kafkabase 14 | ./sync-controller 15 | ./sync-sidecar 16 | admin 17 | ) 18 | -------------------------------------------------------------------------------- /ingest.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bullseye-slim as main 2 | 3 | RUN apt-get update -y 4 | RUN apt-get install -y ca-certificates curl 5 | 6 | ENV TZ=UTC 7 | 8 | FROM golang:1.24.2-bullseye as build 9 | 10 | ARG VERSION 11 | ENV VERSION $VERSION 12 | ARG BUILD_TIMESTAMP 13 | ENV BUILD_TIMESTAMP $BUILD_TIMESTAMP 14 | 15 | RUN apt-get install gcc libc6-dev 16 | 17 | #RUN wget -qO - https://packages.confluent.io/deb/7.2/archive.key | apt-key add - 18 | #RUN echo "deb https://packages.confluent.io/deb/7.2 stable main" > /etc/apt/sources.list.d/backports.list 19 | #RUN echo "deb https://packages.confluent.io/clients/deb buster main" > /etc/apt/sources.list.d/backports.list 20 | #RUN apt-get update 21 | #RUN apt-get install -y librdkafka1 librdkafka-dev 22 | 23 | RUN mkdir /app 24 | WORKDIR /app 25 | 26 | RUN mkdir jitsubase kafkabase eventslog ingest 27 | 28 | COPY jitsubase/go.* ./jitsubase/ 29 | COPY kafkabase/go.* ./kafkabase/ 30 | COPY eventslog/go.* ./eventslog/ 31 | COPY ingest/go.* ./ingest/ 32 | 33 | RUN go work init jitsubase kafkabase eventslog ingest 34 | 35 | WORKDIR /app/ingest 36 | 37 | RUN go mod download 38 | 39 | WORKDIR /app 40 | 41 | COPY . . 42 | 43 | # Build ingest 44 | RUN go build -ldflags="-X main.Commit=$VERSION -X main.Timestamp=$BUILD_TIMESTAMP" -o ingest ./ingest 45 | 46 | ####################################### 47 | # FINAL STAGE 48 | FROM main as final 49 | 50 | RUN mkdir /app 51 | WORKDIR /app 52 | 53 | # Copy ingest 54 | COPY --from=build /app/ingest ./ 55 | #COPY ./config.yaml ./ 56 | 57 | CMD ["/app/ingest"] 58 | -------------------------------------------------------------------------------- /ingest/default.pgo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jitsucom/bulker/b03b7dcfce71c43f8ee6edb9ae7f9c4fe57c071c/ingest/default.pgo -------------------------------------------------------------------------------- /ingest/destination_types.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | var DeviceOptions = map[string]map[string]any{ 4 | "logrocket": { 5 | "type": "internal-plugin", 6 | "name": "logrocket", 7 | }, 8 | "tag": { 9 | "type": "internal-plugin", 10 | "name": "tag", 11 | }, 12 | "ga4-tag": { 13 | "type": "internal-plugin", 14 | "name": "ga4-tag", 15 | }, 16 | "gtm": { 17 | "type": "internal-plugin", 18 | "name": "gtm", 19 | }, 20 | } 21 | -------------------------------------------------------------------------------- /ingest/filters.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/types" 5 | "github.com/jitsucom/bulker/jitsubase/utils" 6 | "strings" 7 | ) 8 | 9 | func SatisfyFilter(filter, subject string) bool { 10 | return filter == "*" || strings.TrimSpace(strings.ToLower(filter)) == strings.TrimSpace(strings.ToLower(subject)) 11 | } 12 | 13 | // SatisfyDomainFilter checks if subject satisfies filter. 14 | // if eager=true -> *.domain.com will match domain.com 15 | func SatisfyDomainFilter(filter, subject string, eager bool) bool { 16 | if filter == "*" { 17 | return true 18 | } 19 | if strings.HasPrefix(filter, "*.") { 20 | return strings.HasSuffix(subject, filter[1:]) || (eager && filter[2:] == subject) 21 | } else { 22 | return filter == subject 23 | } 24 | } 25 | 26 | func parseFilter(value any) []string { 27 | switch v := value.(type) { 28 | case string: 29 | return strings.Split(v, "\n") 30 | case []string: 31 | return v 32 | case nil: 33 | return []string{"*"} 34 | default: 35 | return []string{} 36 | } 37 | } 38 | 39 | func ApplyFilters(event types.Json, opts map[string]any) bool { 40 | eventsArray := parseFilter(opts["events"]) 41 | hostsArray := parseFilter(opts["hosts"]) 42 | 43 | return utils.ArrayContainsF(hostsArray, func(f string) bool { 44 | return SatisfyDomainFilter(f, event.GetPathS("context.page.host"), false) 45 | }) && (utils.ArrayContainsF(eventsArray, func(f string) bool { 46 | return SatisfyFilter(f, event.GetS("type")) 47 | }) || utils.ArrayContainsF(eventsArray, func(f string) bool { 48 | return SatisfyFilter(f, event.GetS("event")) 49 | })) 50 | } 51 | 52 | func ApplyAuthorizedJavaScriptDomainsFilter(domains string, origin string) bool { 53 | domainRules := strings.Split(domains, ",") 54 | return utils.ArrayContainsF(domainRules, func(rule string) bool { 55 | return SatisfyDomainFilter(sanitizeAuthorizedJavaScriptDomain(rule), origin, true) 56 | }) 57 | } 58 | 59 | func sanitizeAuthorizedJavaScriptDomain(domain string) string { 60 | domain = strings.TrimSpace(domain) 61 | domain, trimmed := strings.CutPrefix(domain, "https://") 62 | if !trimmed { 63 | domain = strings.TrimPrefix(domain, "http://") 64 | } 65 | domain = strings.TrimSuffix(domain, "/") 66 | return domain 67 | } 68 | -------------------------------------------------------------------------------- /ingest/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/appbase" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | settings := &appbase.AppSettings{ 10 | ConfigPath: os.Getenv("INGEST_CONFIG_PATH"), 11 | Name: "ingest", 12 | EnvPrefix: "INGEST", 13 | ConfigName: "ingest", 14 | ConfigType: "env", 15 | } 16 | application := appbase.NewApp[Config](&Context{}, settings) 17 | application.Run() 18 | } 19 | -------------------------------------------------------------------------------- /ingest/metrics.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var ( 9 | ingestHandlerRequests = promauto.NewCounterVec(prometheus.CounterOpts{ 10 | Namespace: "bulkerapp", 11 | Subsystem: "handler", 12 | Name: "ingest", 13 | Help: "Ingest handler errors by destination Id", 14 | }, []string{"slug", "status", "errorType"}) 15 | IngestHandlerRequests = func(slug, status, errorType string) prometheus.Counter { 16 | return ingestHandlerRequests.WithLabelValues(slug, status, errorType) 17 | } 18 | 19 | ingestedMessages = promauto.NewCounterVec(prometheus.CounterOpts{ 20 | Namespace: "bulkerapp", 21 | Subsystem: "ingest", 22 | Name: "messages", 23 | Help: "Messages ingested by destination Id", 24 | }, []string{"destinationId", "status", "errorType"}) 25 | IngestedMessages = func(destinationId, status, errorType string) prometheus.Counter { 26 | return ingestedMessages.WithLabelValues(destinationId, status, errorType) 27 | } 28 | 29 | deviceFunctions = promauto.NewCounterVec(prometheus.CounterOpts{ 30 | Namespace: "bulkerapp", 31 | Subsystem: "ingest", 32 | Name: "device_functions", 33 | Help: "Device Functions enrichment status by destination Id", 34 | }, []string{"destinationId", "status"}) 35 | DeviceFunctions = func(destinationId, status string) prometheus.Counter { 36 | return deviceFunctions.WithLabelValues(destinationId, status) 37 | } 38 | 39 | repositoryErrors = promauto.NewCounter(prometheus.CounterOpts{ 40 | Namespace: "ingest", 41 | Subsystem: "repository", 42 | Name: "error", 43 | }) 44 | RepositoryErrors = func() prometheus.Counter { 45 | return repositoryErrors 46 | } 47 | 48 | panics = promauto.NewCounter(prometheus.CounterOpts{ 49 | Namespace: "bulkerapp", 50 | Subsystem: "safego", 51 | Name: "panic", 52 | }) 53 | Panics = func() prometheus.Counter { 54 | return panics 55 | } 56 | ) 57 | -------------------------------------------------------------------------------- /ingest/metrics_server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/gin-gonic/gin" 7 | "github.com/jitsucom/bulker/jitsubase/appbase" 8 | "github.com/jitsucom/bulker/jitsubase/safego" 9 | "github.com/prometheus/client_golang/prometheus/promhttp" 10 | "net/http" 11 | "time" 12 | ) 13 | 14 | type MetricsServer struct { 15 | appbase.Service 16 | server *http.Server 17 | } 18 | 19 | func NewMetricsServer(appconfig *Config) *MetricsServer { 20 | base := appbase.NewServiceBase("metrics_server") 21 | engine := gin.New() 22 | engine.Use(gin.Recovery()) 23 | //expose prometheus metrics 24 | engine.GET("/metrics", gin.WrapH(promhttp.Handler())) 25 | 26 | server := &http.Server{ 27 | Addr: fmt.Sprintf("0.0.0.0:%d", appconfig.MetricsPort), 28 | Handler: engine, 29 | ReadTimeout: time.Second * 60, 30 | ReadHeaderTimeout: time.Second * 60, 31 | IdleTimeout: time.Second * 65, 32 | } 33 | m := &MetricsServer{Service: base, server: server} 34 | m.start() 35 | return m 36 | } 37 | 38 | func (s *MetricsServer) start() { 39 | safego.RunWithRestart(func() { 40 | s.Infof("Starting metrics server on %s", s.server.Addr) 41 | s.Infof("%v", s.server.ListenAndServe()) 42 | }) 43 | } 44 | 45 | func (s *MetricsServer) Stop() error { 46 | s.Infof("Stopping metrics server") 47 | return s.server.Shutdown(context.Background()) 48 | } 49 | -------------------------------------------------------------------------------- /ingest/router_script_handler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/gin-gonic/gin" 6 | "net/http" 7 | ) 8 | 9 | func (r *Router) ScriptHandler(c *gin.Context) { 10 | if c.Request.Method != "GET" && c.Request.Method != "HEAD" { 11 | c.AbortWithStatus(http.StatusMethodNotAllowed) 12 | return 13 | } 14 | if r.scriptRepository == nil { 15 | _ = c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("Script repository is absent")) 16 | return 17 | } 18 | script := r.scriptRepository.GetData() 19 | if script == nil { 20 | _ = c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("Script repository is not initialized")) 21 | return 22 | } 23 | 24 | ifNoneMatch := c.GetHeader("If-None-Match") 25 | etag := script.GetEtag() 26 | if etag != "" { 27 | c.Header("ETag", etag) 28 | if ifNoneMatch != "" && etag == ifNoneMatch { 29 | c.AbortWithStatus(http.StatusNotModified) 30 | return 31 | } 32 | } 33 | 34 | script.WriteScript(c, c.Request.Method == "HEAD", r.ShouldCompress(c.Request)) 35 | } 36 | -------------------------------------------------------------------------------- /ingest/router_segment_settings_handler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gin-gonic/gin" 5 | "net/http" 6 | ) 7 | 8 | const settingsHeader = "{\"integrations\":{\"Segment.io\":{\"apiKey\":\"" 9 | const settingsFooter = "\",\"versionSettings\":{\"version\":\"4.4.7\",\"componentTypes\":[\"browser\"]}}},\"plan\":{\"track\":{\"__default\":{\"enabled\":true}},\"identify\":{\"__default\":{\"enabled\":true}},\"group\":{\"__default\":{\"enabled\":true}}},\"analyticsNextEnabled\":true}" 10 | 11 | func (r *Router) SettingsHandler(c *gin.Context) { 12 | writeKey := c.Param("writeKey") 13 | writer := c.Writer 14 | writer.Header().Set("Content-Type", "application/json") 15 | writer.Header().Set("Cache-Control", "public, max-age=86400") 16 | writer.WriteHeader(200) 17 | _, err := writer.WriteString(settingsHeader) 18 | if err != nil { 19 | r.ResponseError(c, http.StatusBadRequest, "error writing response", false, err, true, true, false) 20 | return 21 | } 22 | _, err = writer.WriteString(writeKey) 23 | if err != nil { 24 | r.ResponseError(c, http.StatusBadRequest, "error writing response", false, err, true, true, false) 25 | return 26 | } 27 | _, err = writer.WriteString(settingsFooter) 28 | if err != nil { 29 | r.ResponseError(c, http.StatusBadRequest, "error writing response", false, err, true, true, false) 30 | return 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ingestbuild.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | VERSION=$(git log -1 --pretty=%h) 4 | TAG="$REPO$VERSION" 5 | LATEST="${REPO}latest" 6 | BUILD_TIMESTAMP=$( date '+%F_%H:%M:%S' ) 7 | 8 | docker buildx build --build-arg VERSION="$VERSION" --build-arg BUILD_TIMESTAMP="$BUILD_TIMESTAMP" --platform linux/amd64 -f ingest.Dockerfile -t jitsucom/ingest:"$VERSION" -t jitsucom/ingest:latest --push . -------------------------------------------------------------------------------- /ingress-manager/app.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | certificatemanager "cloud.google.com/go/certificatemanager/apiv1" 5 | "context" 6 | "fmt" 7 | "github.com/jitsucom/bulker/jitsubase/appbase" 8 | "google.golang.org/api/option" 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | type Context struct { 14 | config *Config 15 | server *http.Server 16 | certMgr *certificatemanager.Client 17 | manager *Manager 18 | } 19 | 20 | func (a *Context) InitContext(settings *appbase.AppSettings) error { 21 | var err error 22 | a.config = &Config{} 23 | err = appbase.InitAppConfig(a.config, settings) 24 | if err != nil { 25 | return err 26 | } 27 | ctx := context.Background() 28 | a.certMgr, err = certificatemanager.NewClient(ctx, option.WithCredentialsJSON([]byte(a.config.GoogleServiceAccountJson))) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | a.manager = NewManager(a) 34 | 35 | router := NewRouter(a) 36 | a.server = &http.Server{ 37 | Addr: fmt.Sprintf("0.0.0.0:%d", a.config.HTTPPort), 38 | Handler: router.Engine(), 39 | ReadTimeout: time.Second * 60, 40 | ReadHeaderTimeout: time.Second * 60, 41 | IdleTimeout: time.Second * 65, 42 | } 43 | return nil 44 | } 45 | 46 | func (a *Context) Cleanup() error { 47 | _ = a.certMgr.Close() 48 | return nil 49 | } 50 | 51 | func (a *Context) ShutdownSignal() error { 52 | _ = a.server.Shutdown(context.Background()) 53 | return nil 54 | } 55 | 56 | func (a *Context) Server() *http.Server { 57 | return a.server 58 | } 59 | 60 | func (a *Context) Config() *Config { 61 | return a.config 62 | } 63 | -------------------------------------------------------------------------------- /ingress-manager/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/jitsucom/bulker/jitsubase/appbase" 6 | "github.com/jitsucom/bulker/jitsubase/utils" 7 | "github.com/spf13/viper" 8 | "os" 9 | ) 10 | 11 | type Config struct { 12 | appbase.Config `mapstructure:",squash"` 13 | 14 | // KubernetesNamespace namespace of bulker app. Default: `default` 15 | KubernetesNamespace string `mapstructure:"KUBERNETES_NAMESPACE" default:"default"` 16 | KubernetesClientConfig string `mapstructure:"KUBERNETES_CLIENT_CONFIG" default:"local"` 17 | KubernetesContext string `mapstructure:"KUBERNETES_CONTEXT"` 18 | 19 | // InitialSetup if true, ingress-manager will create ingress on start 20 | InitialSetup bool `mapstructure:"INITIAL_SETUP" default:"false"` 21 | 22 | JitsuCnames string `mapstructure:"JITSU_CNAMES" default:"cname.jitsu.com,cname2.jitsu.com"` 23 | CertificateMapName string `mapstructure:"CERTIFICATE_MAP_NAME" default:"custom-domains"` 24 | GoogleServiceAccountJson string `mapstructure:"GOOGLE_SERVICE_ACCOUNT_JSON"` 25 | GoogleCloudProject string `mapstructure:"GOOGLE_CLOUD_PROJECT"` 26 | // CleanupCerts if true, ingress-manager will delete Certificates and CertificateMapEntry for domain names that no longer leads to a valid cnames 27 | CleanupCerts bool `mapstructure:"CLEANUP_CERTS" default:"false"` 28 | } 29 | 30 | func init() { 31 | viper.SetDefault("HTTP_PORT", utils.NvlString(os.Getenv("PORT"), "3051")) 32 | } 33 | 34 | func (c *Config) PostInit(settings *appbase.AppSettings) error { 35 | if c.KubernetesClientConfig == "" { 36 | return fmt.Errorf("%sKUBERNETES_CLIENT_CONFIG is required", settings.EnvPrefixWithUnderscore()) 37 | } 38 | return c.Config.PostInit(settings) 39 | } 40 | -------------------------------------------------------------------------------- /ingress-manager/k8s.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/jitsucom/bulker/jitsubase/utils" 6 | "k8s.io/client-go/kubernetes" 7 | "k8s.io/client-go/rest" 8 | "k8s.io/client-go/tools/clientcmd" 9 | "strings" 10 | ) 11 | 12 | func GetK8SClientSet(c *Config) (*kubernetes.Clientset, error) { 13 | config := c.KubernetesClientConfig 14 | if config == "" || config == "local" { 15 | // creates the in-cluster config 16 | cc, err := rest.InClusterConfig() 17 | if err != nil { 18 | return nil, fmt.Errorf("error getting in cluster config: %v", err) 19 | } 20 | clientset, err := kubernetes.NewForConfig(cc) 21 | if err != nil { 22 | return nil, fmt.Errorf("error creating kubernetes clientset: %v", err) 23 | } 24 | return clientset, nil 25 | } else if strings.ContainsRune(config, '\n') { 26 | // suppose yaml file 27 | clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(config)) 28 | if err != nil { 29 | return nil, fmt.Errorf("error parsing kubernetes client config: %v", err) 30 | } 31 | rawConfig, _ := clientconfig.RawConfig() 32 | clientconfig = clientcmd.NewNonInteractiveClientConfig(rawConfig, 33 | utils.NvlString(c.KubernetesContext, rawConfig.CurrentContext), 34 | &clientcmd.ConfigOverrides{}, 35 | &clientcmd.ClientConfigLoadingRules{}) 36 | cc, err := clientconfig.ClientConfig() 37 | if err != nil { 38 | return nil, fmt.Errorf("error creating kubernetes client config: %v", err) 39 | } 40 | clientset, err := kubernetes.NewForConfig(cc) 41 | if err != nil { 42 | return nil, fmt.Errorf("error creating kubernetes clientset: %v", err) 43 | } 44 | return clientset, nil 45 | } else { 46 | // suppose kubeconfig file path 47 | clientconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 48 | &clientcmd.ClientConfigLoadingRules{ExplicitPath: config}, 49 | &clientcmd.ConfigOverrides{ 50 | CurrentContext: c.KubernetesContext, 51 | }) 52 | cc, err := clientconfig.ClientConfig() 53 | if err != nil { 54 | return nil, fmt.Errorf("error creating kubernetes client config: %v", err) 55 | } 56 | clientset, err := kubernetes.NewForConfig(cc) 57 | if err != nil { 58 | return nil, fmt.Errorf("error creating kubernetes clientset: %v", err) 59 | } 60 | return clientset, nil 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /ingress-manager/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/appbase" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | settings := &appbase.AppSettings{ 10 | ConfigPath: os.Getenv("INGMGR_CONFIG_PATH"), 11 | Name: "ingmgr", 12 | EnvPrefix: "INGMGR", 13 | ConfigName: "ingmgr", 14 | ConfigType: "env", 15 | } 16 | application := appbase.NewApp[Config](&Context{}, settings) 17 | application.Run() 18 | } 19 | -------------------------------------------------------------------------------- /jitsubase/appbase/router_base_test.go: -------------------------------------------------------------------------------- 1 | package appbase 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/logging" 5 | "github.com/jitsucom/bulker/jitsubase/uuid" 6 | "testing" 7 | ) 8 | 9 | // test HashTokenHex function of router 10 | func TestHashToken(t *testing.T) { 11 | token := "21a2ae36-32994870a9fbf2f61ea6f6c8" 12 | salt := uuid.New() 13 | secret := "dea42a58-acf4-45af-85bb-e77e94bd5025" 14 | hashedToken := HashTokenHex(token, salt, secret) 15 | logging.Infof("hashedToken: %s.%s", salt, hashedToken) 16 | } 17 | -------------------------------------------------------------------------------- /jitsubase/appbase/service_base.go: -------------------------------------------------------------------------------- 1 | package appbase 2 | 3 | import ( 4 | "fmt" 5 | "github.com/jitsucom/bulker/jitsubase/logging" 6 | ) 7 | 8 | // Service base struct for typical service objects 9 | type Service struct { 10 | // ID is used as [ID] prefix in log and error messages 11 | ID string 12 | } 13 | 14 | func NewServiceBase(id string) Service { 15 | return Service{ 16 | ID: id, 17 | } 18 | } 19 | 20 | func (sb *Service) NewError(format string, a ...any) error { 21 | return fmt.Errorf("[%s] "+format, sb.args(a)...) 22 | } 23 | 24 | func (sb *Service) Infof(format string, a ...any) { 25 | logging.Infof("[%s] "+format, sb.args(a)...) 26 | } 27 | 28 | func (sb *Service) Errorf(format string, a ...any) { 29 | logging.Errorf("[%s] "+format, sb.args(a)...) 30 | } 31 | 32 | func (sb *Service) Warnf(format string, a ...any) { 33 | logging.Warnf("[%s] "+format, sb.args(a)...) 34 | } 35 | 36 | func (sb *Service) Debugf(format string, a ...any) { 37 | if logging.IsDebugEnabled() { 38 | logging.Debugf("[%s] "+format, sb.args(a)...) 39 | } 40 | } 41 | 42 | func (sb *Service) Fatalf(format string, a ...any) { 43 | logging.Fatalf("[%s] "+format, sb.args(a)...) 44 | } 45 | 46 | func (sb *Service) SystemErrorf(format string, a ...any) { 47 | logging.SystemErrorf("[%s] "+format, sb.args(a)...) 48 | } 49 | 50 | func (sb *Service) args(a []any) []any { 51 | b := make([]any, len(a)+1) 52 | copy(b[1:], a) 53 | b[0] = sb.ID 54 | return b 55 | } 56 | -------------------------------------------------------------------------------- /jitsubase/coordination/interface.go: -------------------------------------------------------------------------------- 1 | package coordination 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/locks" 5 | "io" 6 | ) 7 | 8 | // Service is a coordination service which is responsible for all distributed operations like: 9 | // - distributed locks 10 | // - obtain cluster information 11 | type Service interface { 12 | io.Closer 13 | //TODO: remove? 14 | GetJitsuInstancesInCluster() ([]string, error) 15 | CreateLock(name string) locks.Lock 16 | } 17 | 18 | type DummyCoordinationService struct { 19 | } 20 | 21 | func (s DummyCoordinationService) GetJitsuInstancesInCluster() ([]string, error) { 22 | return []string{}, nil 23 | } 24 | 25 | func (s DummyCoordinationService) CreateLock(name string) locks.Lock { 26 | return locks.DummyLock{} 27 | } 28 | 29 | func (s DummyCoordinationService) Close() error { 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 json-iterator 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/any_float.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type floatAny struct { 8 | baseAny 9 | val float64 10 | } 11 | 12 | func (any *floatAny) Parse() *Iterator { 13 | return nil 14 | } 15 | 16 | func (any *floatAny) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *floatAny) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *floatAny) LastError() error { 25 | return nil 26 | } 27 | 28 | func (any *floatAny) ToBool() bool { 29 | return any.ToFloat64() != 0 30 | } 31 | 32 | func (any *floatAny) ToInt() int { 33 | return int(any.val) 34 | } 35 | 36 | func (any *floatAny) ToInt32() int32 { 37 | return int32(any.val) 38 | } 39 | 40 | func (any *floatAny) ToInt64() int64 { 41 | return int64(any.val) 42 | } 43 | 44 | func (any *floatAny) ToUint() uint { 45 | if any.val > 0 { 46 | return uint(any.val) 47 | } 48 | return 0 49 | } 50 | 51 | func (any *floatAny) ToUint32() uint32 { 52 | if any.val > 0 { 53 | return uint32(any.val) 54 | } 55 | return 0 56 | } 57 | 58 | func (any *floatAny) ToUint64() uint64 { 59 | if any.val > 0 { 60 | return uint64(any.val) 61 | } 62 | return 0 63 | } 64 | 65 | func (any *floatAny) ToFloat32() float32 { 66 | return float32(any.val) 67 | } 68 | 69 | func (any *floatAny) ToFloat64() float64 { 70 | return any.val 71 | } 72 | 73 | func (any *floatAny) ToString() string { 74 | return strconv.FormatFloat(any.val, 'E', -1, 64) 75 | } 76 | 77 | func (any *floatAny) WriteTo(stream *Stream) { 78 | stream.WriteFloat64(any.val) 79 | } 80 | 81 | func (any *floatAny) GetInterface() interface{} { 82 | return any.val 83 | } 84 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/any_int32.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type int32Any struct { 8 | baseAny 9 | val int32 10 | } 11 | 12 | func (any *int32Any) LastError() error { 13 | return nil 14 | } 15 | 16 | func (any *int32Any) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *int32Any) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *int32Any) ToBool() bool { 25 | return any.val != 0 26 | } 27 | 28 | func (any *int32Any) ToInt() int { 29 | return int(any.val) 30 | } 31 | 32 | func (any *int32Any) ToInt32() int32 { 33 | return any.val 34 | } 35 | 36 | func (any *int32Any) ToInt64() int64 { 37 | return int64(any.val) 38 | } 39 | 40 | func (any *int32Any) ToUint() uint { 41 | return uint(any.val) 42 | } 43 | 44 | func (any *int32Any) ToUint32() uint32 { 45 | return uint32(any.val) 46 | } 47 | 48 | func (any *int32Any) ToUint64() uint64 { 49 | return uint64(any.val) 50 | } 51 | 52 | func (any *int32Any) ToFloat32() float32 { 53 | return float32(any.val) 54 | } 55 | 56 | func (any *int32Any) ToFloat64() float64 { 57 | return float64(any.val) 58 | } 59 | 60 | func (any *int32Any) ToString() string { 61 | return strconv.FormatInt(int64(any.val), 10) 62 | } 63 | 64 | func (any *int32Any) WriteTo(stream *Stream) { 65 | stream.WriteInt32(any.val) 66 | } 67 | 68 | func (any *int32Any) Parse() *Iterator { 69 | return nil 70 | } 71 | 72 | func (any *int32Any) GetInterface() interface{} { 73 | return any.val 74 | } 75 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/any_int64.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type int64Any struct { 8 | baseAny 9 | val int64 10 | } 11 | 12 | func (any *int64Any) LastError() error { 13 | return nil 14 | } 15 | 16 | func (any *int64Any) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *int64Any) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *int64Any) ToBool() bool { 25 | return any.val != 0 26 | } 27 | 28 | func (any *int64Any) ToInt() int { 29 | return int(any.val) 30 | } 31 | 32 | func (any *int64Any) ToInt32() int32 { 33 | return int32(any.val) 34 | } 35 | 36 | func (any *int64Any) ToInt64() int64 { 37 | return any.val 38 | } 39 | 40 | func (any *int64Any) ToUint() uint { 41 | return uint(any.val) 42 | } 43 | 44 | func (any *int64Any) ToUint32() uint32 { 45 | return uint32(any.val) 46 | } 47 | 48 | func (any *int64Any) ToUint64() uint64 { 49 | return uint64(any.val) 50 | } 51 | 52 | func (any *int64Any) ToFloat32() float32 { 53 | return float32(any.val) 54 | } 55 | 56 | func (any *int64Any) ToFloat64() float64 { 57 | return float64(any.val) 58 | } 59 | 60 | func (any *int64Any) ToString() string { 61 | return strconv.FormatInt(any.val, 10) 62 | } 63 | 64 | func (any *int64Any) WriteTo(stream *Stream) { 65 | stream.WriteInt64(any.val) 66 | } 67 | 68 | func (any *int64Any) Parse() *Iterator { 69 | return nil 70 | } 71 | 72 | func (any *int64Any) GetInterface() interface{} { 73 | return any.val 74 | } 75 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/any_invalid.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import "fmt" 4 | 5 | type invalidAny struct { 6 | baseAny 7 | err error 8 | } 9 | 10 | func newInvalidAny(path []interface{}) *invalidAny { 11 | return &invalidAny{baseAny{}, fmt.Errorf("%v not found", path)} 12 | } 13 | 14 | func (any *invalidAny) LastError() error { 15 | return any.err 16 | } 17 | 18 | func (any *invalidAny) ValueType() ValueType { 19 | return InvalidValue 20 | } 21 | 22 | func (any *invalidAny) MustBeValid() Any { 23 | panic(any.err) 24 | } 25 | 26 | func (any *invalidAny) ToBool() bool { 27 | return false 28 | } 29 | 30 | func (any *invalidAny) ToInt() int { 31 | return 0 32 | } 33 | 34 | func (any *invalidAny) ToInt32() int32 { 35 | return 0 36 | } 37 | 38 | func (any *invalidAny) ToInt64() int64 { 39 | return 0 40 | } 41 | 42 | func (any *invalidAny) ToUint() uint { 43 | return 0 44 | } 45 | 46 | func (any *invalidAny) ToUint32() uint32 { 47 | return 0 48 | } 49 | 50 | func (any *invalidAny) ToUint64() uint64 { 51 | return 0 52 | } 53 | 54 | func (any *invalidAny) ToFloat32() float32 { 55 | return 0 56 | } 57 | 58 | func (any *invalidAny) ToFloat64() float64 { 59 | return 0 60 | } 61 | 62 | func (any *invalidAny) ToString() string { 63 | return "" 64 | } 65 | 66 | func (any *invalidAny) WriteTo(stream *Stream) { 67 | } 68 | 69 | func (any *invalidAny) Get(path ...interface{}) Any { 70 | if any.err == nil { 71 | return &invalidAny{baseAny{}, fmt.Errorf("get %v from invalid", path)} 72 | } 73 | return &invalidAny{baseAny{}, fmt.Errorf("%v, get %v from invalid", any.err, path)} 74 | } 75 | 76 | func (any *invalidAny) Parse() *Iterator { 77 | return nil 78 | } 79 | 80 | func (any *invalidAny) GetInterface() interface{} { 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/any_nil.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | type nilAny struct { 4 | baseAny 5 | } 6 | 7 | func (any *nilAny) LastError() error { 8 | return nil 9 | } 10 | 11 | func (any *nilAny) ValueType() ValueType { 12 | return NilValue 13 | } 14 | 15 | func (any *nilAny) MustBeValid() Any { 16 | return any 17 | } 18 | 19 | func (any *nilAny) ToBool() bool { 20 | return false 21 | } 22 | 23 | func (any *nilAny) ToInt() int { 24 | return 0 25 | } 26 | 27 | func (any *nilAny) ToInt32() int32 { 28 | return 0 29 | } 30 | 31 | func (any *nilAny) ToInt64() int64 { 32 | return 0 33 | } 34 | 35 | func (any *nilAny) ToUint() uint { 36 | return 0 37 | } 38 | 39 | func (any *nilAny) ToUint32() uint32 { 40 | return 0 41 | } 42 | 43 | func (any *nilAny) ToUint64() uint64 { 44 | return 0 45 | } 46 | 47 | func (any *nilAny) ToFloat32() float32 { 48 | return 0 49 | } 50 | 51 | func (any *nilAny) ToFloat64() float64 { 52 | return 0 53 | } 54 | 55 | func (any *nilAny) ToString() string { 56 | return "" 57 | } 58 | 59 | func (any *nilAny) WriteTo(stream *Stream) { 60 | stream.WriteNil() 61 | } 62 | 63 | func (any *nilAny) Parse() *Iterator { 64 | return nil 65 | } 66 | 67 | func (any *nilAny) GetInterface() interface{} { 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/any_uint32.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type uint32Any struct { 8 | baseAny 9 | val uint32 10 | } 11 | 12 | func (any *uint32Any) LastError() error { 13 | return nil 14 | } 15 | 16 | func (any *uint32Any) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *uint32Any) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *uint32Any) ToBool() bool { 25 | return any.val != 0 26 | } 27 | 28 | func (any *uint32Any) ToInt() int { 29 | return int(any.val) 30 | } 31 | 32 | func (any *uint32Any) ToInt32() int32 { 33 | return int32(any.val) 34 | } 35 | 36 | func (any *uint32Any) ToInt64() int64 { 37 | return int64(any.val) 38 | } 39 | 40 | func (any *uint32Any) ToUint() uint { 41 | return uint(any.val) 42 | } 43 | 44 | func (any *uint32Any) ToUint32() uint32 { 45 | return any.val 46 | } 47 | 48 | func (any *uint32Any) ToUint64() uint64 { 49 | return uint64(any.val) 50 | } 51 | 52 | func (any *uint32Any) ToFloat32() float32 { 53 | return float32(any.val) 54 | } 55 | 56 | func (any *uint32Any) ToFloat64() float64 { 57 | return float64(any.val) 58 | } 59 | 60 | func (any *uint32Any) ToString() string { 61 | return strconv.FormatInt(int64(any.val), 10) 62 | } 63 | 64 | func (any *uint32Any) WriteTo(stream *Stream) { 65 | stream.WriteUint32(any.val) 66 | } 67 | 68 | func (any *uint32Any) Parse() *Iterator { 69 | return nil 70 | } 71 | 72 | func (any *uint32Any) GetInterface() interface{} { 73 | return any.val 74 | } 75 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/any_uint64.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type uint64Any struct { 8 | baseAny 9 | val uint64 10 | } 11 | 12 | func (any *uint64Any) LastError() error { 13 | return nil 14 | } 15 | 16 | func (any *uint64Any) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *uint64Any) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *uint64Any) ToBool() bool { 25 | return any.val != 0 26 | } 27 | 28 | func (any *uint64Any) ToInt() int { 29 | return int(any.val) 30 | } 31 | 32 | func (any *uint64Any) ToInt32() int32 { 33 | return int32(any.val) 34 | } 35 | 36 | func (any *uint64Any) ToInt64() int64 { 37 | return int64(any.val) 38 | } 39 | 40 | func (any *uint64Any) ToUint() uint { 41 | return uint(any.val) 42 | } 43 | 44 | func (any *uint64Any) ToUint32() uint32 { 45 | return uint32(any.val) 46 | } 47 | 48 | func (any *uint64Any) ToUint64() uint64 { 49 | return any.val 50 | } 51 | 52 | func (any *uint64Any) ToFloat32() float32 { 53 | return float32(any.val) 54 | } 55 | 56 | func (any *uint64Any) ToFloat64() float64 { 57 | return float64(any.val) 58 | } 59 | 60 | func (any *uint64Any) ToString() string { 61 | return strconv.FormatUint(any.val, 10) 62 | } 63 | 64 | func (any *uint64Any) WriteTo(stream *Stream) { 65 | stream.WriteUint64(any.val) 66 | } 67 | 68 | func (any *uint64Any) Parse() *Iterator { 69 | return nil 70 | } 71 | 72 | func (any *uint64Any) GetInterface() interface{} { 73 | return any.val 74 | } 75 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/iter_array.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | // ReadArray read array element, tells if the array has more element to read. 4 | func (iter *Iterator) ReadArray() (ret bool) { 5 | c := iter.nextToken() 6 | switch c { 7 | case 'n': 8 | iter.skipThreeBytes('u', 'l', 'l') 9 | return false // null 10 | case '[': 11 | c = iter.nextToken() 12 | if c != ']' { 13 | iter.unreadByte() 14 | return true 15 | } 16 | return false 17 | case ']': 18 | return false 19 | case ',': 20 | return true 21 | default: 22 | iter.ReportError("ReadArray", "expect [ or , or ] or n, but found "+string([]byte{c})) 23 | return 24 | } 25 | } 26 | 27 | // ReadArrayCB read array with callback 28 | func (iter *Iterator) ReadArrayCB(callback func(*Iterator) bool) (ret bool) { 29 | c := iter.nextToken() 30 | if c == '[' { 31 | if !iter.incrementDepth() { 32 | return false 33 | } 34 | c = iter.nextToken() 35 | if c != ']' { 36 | iter.unreadByte() 37 | if !callback(iter) { 38 | iter.decrementDepth() 39 | return false 40 | } 41 | c = iter.nextToken() 42 | for c == ',' { 43 | if !callback(iter) { 44 | iter.decrementDepth() 45 | return false 46 | } 47 | c = iter.nextToken() 48 | } 49 | if c != ']' { 50 | iter.ReportError("ReadArrayCB", "expect ] in the end, but found "+string([]byte{c})) 51 | iter.decrementDepth() 52 | return false 53 | } 54 | return iter.decrementDepth() 55 | } 56 | return iter.decrementDepth() 57 | } 58 | if c == 'n' { 59 | iter.skipThreeBytes('u', 'l', 'l') 60 | return true // null 61 | } 62 | iter.ReportError("ReadArrayCB", "expect [ or n, but found "+string([]byte{c})) 63 | return false 64 | } 65 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/jsoniter.go: -------------------------------------------------------------------------------- 1 | // Package jsoniter implements encoding and decoding of JSON as defined in 2 | // RFC 4627 and provides interfaces with identical syntax of standard lib encoding/json. 3 | // Converting from encoding/json to jsoniter is no more than replacing the package with jsoniter 4 | // and variable type declarations (if any). 5 | // jsoniter interfaces gives 100% compatibility with code using standard lib. 6 | // 7 | // "JSON and Go" 8 | // (https://golang.org/doc/articles/json_and_go.html) 9 | // gives a description of how Marshal/Unmarshal operate 10 | // between arbitrary or predefined json objects and bytes, 11 | // and it applies to jsoniter.Marshal/Unmarshal as well. 12 | // 13 | // Besides, jsoniter.Iterator provides a different set of interfaces 14 | // iterating given bytes/string/reader 15 | // and yielding parsed elements one by one. 16 | // This set of interfaces reads input as required and gives 17 | // better performance. 18 | package jsoniter 19 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/pool.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // IteratorPool a thread safe pool of iterators with same configuration 8 | type IteratorPool interface { 9 | BorrowIterator(data []byte) *Iterator 10 | ReturnIterator(iter *Iterator) 11 | } 12 | 13 | // StreamPool a thread safe pool of streams with same configuration 14 | type StreamPool interface { 15 | BorrowStream(writer io.Writer) *Stream 16 | ReturnStream(stream *Stream) 17 | } 18 | 19 | func (cfg *frozenConfig) BorrowStream(writer io.Writer) *Stream { 20 | stream := cfg.streamPool.Get().(*Stream) 21 | stream.Reset(writer) 22 | return stream 23 | } 24 | 25 | func (cfg *frozenConfig) ReturnStream(stream *Stream) { 26 | stream.out = nil 27 | stream.Error = nil 28 | stream.Attachment = nil 29 | cfg.streamPool.Put(stream) 30 | } 31 | 32 | func (cfg *frozenConfig) BorrowIterator(data []byte) *Iterator { 33 | iter := cfg.iteratorPool.Get().(*Iterator) 34 | iter.ResetBytes(data) 35 | return iter 36 | } 37 | 38 | func (cfg *frozenConfig) ReturnIterator(iter *Iterator) { 39 | iter.Error = nil 40 | iter.Attachment = nil 41 | cfg.iteratorPool.Put(iter) 42 | } 43 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/reflect_dynamic.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "github.com/modern-go/reflect2" 5 | "reflect" 6 | "unsafe" 7 | ) 8 | 9 | type dynamicEncoder struct { 10 | valType reflect2.Type 11 | } 12 | 13 | func (encoder *dynamicEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { 14 | obj := encoder.valType.UnsafeIndirect(ptr) 15 | stream.WriteVal(obj) 16 | } 17 | 18 | func (encoder *dynamicEncoder) IsEmpty(ptr unsafe.Pointer) bool { 19 | return encoder.valType.UnsafeIndirect(ptr) == nil 20 | } 21 | 22 | type efaceDecoder struct { 23 | } 24 | 25 | func (decoder *efaceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { 26 | pObj := (*interface{})(ptr) 27 | obj := *pObj 28 | if obj == nil { 29 | *pObj = iter.Read() 30 | return 31 | } 32 | typ := reflect2.TypeOf(obj) 33 | if typ.Kind() != reflect.Ptr { 34 | *pObj = iter.Read() 35 | return 36 | } 37 | ptrType := typ.(*reflect2.UnsafePtrType) 38 | ptrElemType := ptrType.Elem() 39 | if iter.WhatIsNext() == NilValue { 40 | if ptrElemType.Kind() != reflect.Ptr { 41 | iter.skipFourBytes('n', 'u', 'l', 'l') 42 | *pObj = nil 43 | return 44 | } 45 | } 46 | if reflect2.IsNil(obj) { 47 | obj := ptrElemType.New() 48 | iter.ReadVal(obj) 49 | *pObj = obj 50 | return 51 | } 52 | iter.ReadVal(obj) 53 | } 54 | 55 | type ifaceDecoder struct { 56 | valType *reflect2.UnsafeIFaceType 57 | } 58 | 59 | func (decoder *ifaceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { 60 | if iter.ReadNil() { 61 | decoder.valType.UnsafeSet(ptr, decoder.valType.UnsafeNew()) 62 | return 63 | } 64 | obj := decoder.valType.UnsafeIndirect(ptr) 65 | if reflect2.IsNil(obj) { 66 | iter.ReportError("decode non empty interface", "can not unmarshal into nil") 67 | return 68 | } 69 | iter.ReadVal(obj) 70 | } 71 | -------------------------------------------------------------------------------- /jitsubase/jsoniter/reflect_json_raw_message.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/modern-go/reflect2" 6 | "unsafe" 7 | ) 8 | 9 | var jsonRawMessageType = reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem() 10 | var jsoniterRawMessageType = reflect2.TypeOfPtr((*RawMessage)(nil)).Elem() 11 | 12 | func createEncoderOfJsonRawMessage(ctx *ctx, typ reflect2.Type) ValEncoder { 13 | if typ == jsonRawMessageType { 14 | return &jsonRawMessageCodec{} 15 | } 16 | if typ == jsoniterRawMessageType { 17 | return &jsoniterRawMessageCodec{} 18 | } 19 | return nil 20 | } 21 | 22 | func createDecoderOfJsonRawMessage(ctx *ctx, typ reflect2.Type) ValDecoder { 23 | if typ == jsonRawMessageType { 24 | return &jsonRawMessageCodec{} 25 | } 26 | if typ == jsoniterRawMessageType { 27 | return &jsoniterRawMessageCodec{} 28 | } 29 | return nil 30 | } 31 | 32 | type jsonRawMessageCodec struct { 33 | } 34 | 35 | func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { 36 | if iter.ReadNil() { 37 | *((*json.RawMessage)(ptr)) = nil 38 | } else { 39 | *((*json.RawMessage)(ptr)) = iter.SkipAndReturnBytes() 40 | } 41 | } 42 | 43 | func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { 44 | if *((*json.RawMessage)(ptr)) == nil { 45 | stream.WriteNil() 46 | } else { 47 | stream.WriteRaw(string(*((*json.RawMessage)(ptr)))) 48 | } 49 | } 50 | 51 | func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { 52 | return len(*((*json.RawMessage)(ptr))) == 0 53 | } 54 | 55 | type jsoniterRawMessageCodec struct { 56 | } 57 | 58 | func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { 59 | if iter.ReadNil() { 60 | *((*RawMessage)(ptr)) = nil 61 | } else { 62 | *((*RawMessage)(ptr)) = iter.SkipAndReturnBytes() 63 | } 64 | } 65 | 66 | func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { 67 | if *((*RawMessage)(ptr)) == nil { 68 | stream.WriteNil() 69 | } else { 70 | stream.WriteRaw(string(*((*RawMessage)(ptr)))) 71 | } 72 | } 73 | 74 | func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { 75 | return len(*((*RawMessage)(ptr))) == 0 76 | } 77 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 json-iterator 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/any_float.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type floatAny struct { 8 | baseAny 9 | val float64 10 | } 11 | 12 | func (any *floatAny) Parse() *Iterator { 13 | return nil 14 | } 15 | 16 | func (any *floatAny) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *floatAny) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *floatAny) LastError() error { 25 | return nil 26 | } 27 | 28 | func (any *floatAny) ToBool() bool { 29 | return any.ToFloat64() != 0 30 | } 31 | 32 | func (any *floatAny) ToInt() int { 33 | return int(any.val) 34 | } 35 | 36 | func (any *floatAny) ToInt32() int32 { 37 | return int32(any.val) 38 | } 39 | 40 | func (any *floatAny) ToInt64() int64 { 41 | return int64(any.val) 42 | } 43 | 44 | func (any *floatAny) ToUint() uint { 45 | if any.val > 0 { 46 | return uint(any.val) 47 | } 48 | return 0 49 | } 50 | 51 | func (any *floatAny) ToUint32() uint32 { 52 | if any.val > 0 { 53 | return uint32(any.val) 54 | } 55 | return 0 56 | } 57 | 58 | func (any *floatAny) ToUint64() uint64 { 59 | if any.val > 0 { 60 | return uint64(any.val) 61 | } 62 | return 0 63 | } 64 | 65 | func (any *floatAny) ToFloat32() float32 { 66 | return float32(any.val) 67 | } 68 | 69 | func (any *floatAny) ToFloat64() float64 { 70 | return any.val 71 | } 72 | 73 | func (any *floatAny) ToString() string { 74 | return strconv.FormatFloat(any.val, 'E', -1, 64) 75 | } 76 | 77 | func (any *floatAny) WriteTo(stream *Stream) { 78 | stream.WriteFloat64(any.val) 79 | } 80 | 81 | func (any *floatAny) GetInterface() interface{} { 82 | return any.val 83 | } 84 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/any_int32.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type int32Any struct { 8 | baseAny 9 | val int32 10 | } 11 | 12 | func (any *int32Any) LastError() error { 13 | return nil 14 | } 15 | 16 | func (any *int32Any) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *int32Any) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *int32Any) ToBool() bool { 25 | return any.val != 0 26 | } 27 | 28 | func (any *int32Any) ToInt() int { 29 | return int(any.val) 30 | } 31 | 32 | func (any *int32Any) ToInt32() int32 { 33 | return any.val 34 | } 35 | 36 | func (any *int32Any) ToInt64() int64 { 37 | return int64(any.val) 38 | } 39 | 40 | func (any *int32Any) ToUint() uint { 41 | return uint(any.val) 42 | } 43 | 44 | func (any *int32Any) ToUint32() uint32 { 45 | return uint32(any.val) 46 | } 47 | 48 | func (any *int32Any) ToUint64() uint64 { 49 | return uint64(any.val) 50 | } 51 | 52 | func (any *int32Any) ToFloat32() float32 { 53 | return float32(any.val) 54 | } 55 | 56 | func (any *int32Any) ToFloat64() float64 { 57 | return float64(any.val) 58 | } 59 | 60 | func (any *int32Any) ToString() string { 61 | return strconv.FormatInt(int64(any.val), 10) 62 | } 63 | 64 | func (any *int32Any) WriteTo(stream *Stream) { 65 | stream.WriteInt32(any.val) 66 | } 67 | 68 | func (any *int32Any) Parse() *Iterator { 69 | return nil 70 | } 71 | 72 | func (any *int32Any) GetInterface() interface{} { 73 | return any.val 74 | } 75 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/any_int64.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type int64Any struct { 8 | baseAny 9 | val int64 10 | } 11 | 12 | func (any *int64Any) LastError() error { 13 | return nil 14 | } 15 | 16 | func (any *int64Any) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *int64Any) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *int64Any) ToBool() bool { 25 | return any.val != 0 26 | } 27 | 28 | func (any *int64Any) ToInt() int { 29 | return int(any.val) 30 | } 31 | 32 | func (any *int64Any) ToInt32() int32 { 33 | return int32(any.val) 34 | } 35 | 36 | func (any *int64Any) ToInt64() int64 { 37 | return any.val 38 | } 39 | 40 | func (any *int64Any) ToUint() uint { 41 | return uint(any.val) 42 | } 43 | 44 | func (any *int64Any) ToUint32() uint32 { 45 | return uint32(any.val) 46 | } 47 | 48 | func (any *int64Any) ToUint64() uint64 { 49 | return uint64(any.val) 50 | } 51 | 52 | func (any *int64Any) ToFloat32() float32 { 53 | return float32(any.val) 54 | } 55 | 56 | func (any *int64Any) ToFloat64() float64 { 57 | return float64(any.val) 58 | } 59 | 60 | func (any *int64Any) ToString() string { 61 | return strconv.FormatInt(any.val, 10) 62 | } 63 | 64 | func (any *int64Any) WriteTo(stream *Stream) { 65 | stream.WriteInt64(any.val) 66 | } 67 | 68 | func (any *int64Any) Parse() *Iterator { 69 | return nil 70 | } 71 | 72 | func (any *int64Any) GetInterface() interface{} { 73 | return any.val 74 | } 75 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/any_invalid.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import "fmt" 4 | 5 | type invalidAny struct { 6 | baseAny 7 | err error 8 | } 9 | 10 | func newInvalidAny(path []interface{}) *invalidAny { 11 | return &invalidAny{baseAny{}, fmt.Errorf("%v not found", path)} 12 | } 13 | 14 | func (any *invalidAny) LastError() error { 15 | return any.err 16 | } 17 | 18 | func (any *invalidAny) ValueType() ValueType { 19 | return InvalidValue 20 | } 21 | 22 | func (any *invalidAny) MustBeValid() Any { 23 | panic(any.err) 24 | } 25 | 26 | func (any *invalidAny) ToBool() bool { 27 | return false 28 | } 29 | 30 | func (any *invalidAny) ToInt() int { 31 | return 0 32 | } 33 | 34 | func (any *invalidAny) ToInt32() int32 { 35 | return 0 36 | } 37 | 38 | func (any *invalidAny) ToInt64() int64 { 39 | return 0 40 | } 41 | 42 | func (any *invalidAny) ToUint() uint { 43 | return 0 44 | } 45 | 46 | func (any *invalidAny) ToUint32() uint32 { 47 | return 0 48 | } 49 | 50 | func (any *invalidAny) ToUint64() uint64 { 51 | return 0 52 | } 53 | 54 | func (any *invalidAny) ToFloat32() float32 { 55 | return 0 56 | } 57 | 58 | func (any *invalidAny) ToFloat64() float64 { 59 | return 0 60 | } 61 | 62 | func (any *invalidAny) ToString() string { 63 | return "" 64 | } 65 | 66 | func (any *invalidAny) WriteTo(stream *Stream) { 67 | } 68 | 69 | func (any *invalidAny) Get(path ...interface{}) Any { 70 | if any.err == nil { 71 | return &invalidAny{baseAny{}, fmt.Errorf("get %v from invalid", path)} 72 | } 73 | return &invalidAny{baseAny{}, fmt.Errorf("%v, get %v from invalid", any.err, path)} 74 | } 75 | 76 | func (any *invalidAny) Parse() *Iterator { 77 | return nil 78 | } 79 | 80 | func (any *invalidAny) GetInterface() interface{} { 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/any_nil.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | type nilAny struct { 4 | baseAny 5 | } 6 | 7 | func (any *nilAny) LastError() error { 8 | return nil 9 | } 10 | 11 | func (any *nilAny) ValueType() ValueType { 12 | return NilValue 13 | } 14 | 15 | func (any *nilAny) MustBeValid() Any { 16 | return any 17 | } 18 | 19 | func (any *nilAny) ToBool() bool { 20 | return false 21 | } 22 | 23 | func (any *nilAny) ToInt() int { 24 | return 0 25 | } 26 | 27 | func (any *nilAny) ToInt32() int32 { 28 | return 0 29 | } 30 | 31 | func (any *nilAny) ToInt64() int64 { 32 | return 0 33 | } 34 | 35 | func (any *nilAny) ToUint() uint { 36 | return 0 37 | } 38 | 39 | func (any *nilAny) ToUint32() uint32 { 40 | return 0 41 | } 42 | 43 | func (any *nilAny) ToUint64() uint64 { 44 | return 0 45 | } 46 | 47 | func (any *nilAny) ToFloat32() float32 { 48 | return 0 49 | } 50 | 51 | func (any *nilAny) ToFloat64() float64 { 52 | return 0 53 | } 54 | 55 | func (any *nilAny) ToString() string { 56 | return "" 57 | } 58 | 59 | func (any *nilAny) WriteTo(stream *Stream) { 60 | stream.WriteNil() 61 | } 62 | 63 | func (any *nilAny) Parse() *Iterator { 64 | return nil 65 | } 66 | 67 | func (any *nilAny) GetInterface() interface{} { 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/any_uint32.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type uint32Any struct { 8 | baseAny 9 | val uint32 10 | } 11 | 12 | func (any *uint32Any) LastError() error { 13 | return nil 14 | } 15 | 16 | func (any *uint32Any) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *uint32Any) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *uint32Any) ToBool() bool { 25 | return any.val != 0 26 | } 27 | 28 | func (any *uint32Any) ToInt() int { 29 | return int(any.val) 30 | } 31 | 32 | func (any *uint32Any) ToInt32() int32 { 33 | return int32(any.val) 34 | } 35 | 36 | func (any *uint32Any) ToInt64() int64 { 37 | return int64(any.val) 38 | } 39 | 40 | func (any *uint32Any) ToUint() uint { 41 | return uint(any.val) 42 | } 43 | 44 | func (any *uint32Any) ToUint32() uint32 { 45 | return any.val 46 | } 47 | 48 | func (any *uint32Any) ToUint64() uint64 { 49 | return uint64(any.val) 50 | } 51 | 52 | func (any *uint32Any) ToFloat32() float32 { 53 | return float32(any.val) 54 | } 55 | 56 | func (any *uint32Any) ToFloat64() float64 { 57 | return float64(any.val) 58 | } 59 | 60 | func (any *uint32Any) ToString() string { 61 | return strconv.FormatInt(int64(any.val), 10) 62 | } 63 | 64 | func (any *uint32Any) WriteTo(stream *Stream) { 65 | stream.WriteUint32(any.val) 66 | } 67 | 68 | func (any *uint32Any) Parse() *Iterator { 69 | return nil 70 | } 71 | 72 | func (any *uint32Any) GetInterface() interface{} { 73 | return any.val 74 | } 75 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/any_uint64.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type uint64Any struct { 8 | baseAny 9 | val uint64 10 | } 11 | 12 | func (any *uint64Any) LastError() error { 13 | return nil 14 | } 15 | 16 | func (any *uint64Any) ValueType() ValueType { 17 | return NumberValue 18 | } 19 | 20 | func (any *uint64Any) MustBeValid() Any { 21 | return any 22 | } 23 | 24 | func (any *uint64Any) ToBool() bool { 25 | return any.val != 0 26 | } 27 | 28 | func (any *uint64Any) ToInt() int { 29 | return int(any.val) 30 | } 31 | 32 | func (any *uint64Any) ToInt32() int32 { 33 | return int32(any.val) 34 | } 35 | 36 | func (any *uint64Any) ToInt64() int64 { 37 | return int64(any.val) 38 | } 39 | 40 | func (any *uint64Any) ToUint() uint { 41 | return uint(any.val) 42 | } 43 | 44 | func (any *uint64Any) ToUint32() uint32 { 45 | return uint32(any.val) 46 | } 47 | 48 | func (any *uint64Any) ToUint64() uint64 { 49 | return any.val 50 | } 51 | 52 | func (any *uint64Any) ToFloat32() float32 { 53 | return float32(any.val) 54 | } 55 | 56 | func (any *uint64Any) ToFloat64() float64 { 57 | return float64(any.val) 58 | } 59 | 60 | func (any *uint64Any) ToString() string { 61 | return strconv.FormatUint(any.val, 10) 62 | } 63 | 64 | func (any *uint64Any) WriteTo(stream *Stream) { 65 | stream.WriteUint64(any.val) 66 | } 67 | 68 | func (any *uint64Any) Parse() *Iterator { 69 | return nil 70 | } 71 | 72 | func (any *uint64Any) GetInterface() interface{} { 73 | return any.val 74 | } 75 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/iter_array.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | // ReadArray read array element, tells if the array has more element to read. 4 | func (iter *Iterator) ReadArray() (ret bool) { 5 | c := iter.nextToken() 6 | switch c { 7 | case 'n': 8 | iter.skipThreeBytes('u', 'l', 'l') 9 | return false // null 10 | case '[': 11 | c = iter.nextToken() 12 | if c != ']' { 13 | iter.unreadByte() 14 | return true 15 | } 16 | return false 17 | case ']': 18 | return false 19 | case ',': 20 | return true 21 | default: 22 | iter.ReportError("ReadArray", "expect [ or , or ] or n, but found "+string([]byte{c})) 23 | return 24 | } 25 | } 26 | 27 | // ReadArrayCB read array with callback 28 | func (iter *Iterator) ReadArrayCB(callback func(*Iterator) bool) (ret bool) { 29 | c := iter.nextToken() 30 | if c == '[' { 31 | if !iter.incrementDepth() { 32 | return false 33 | } 34 | c = iter.nextToken() 35 | if c != ']' { 36 | iter.unreadByte() 37 | if !callback(iter) { 38 | iter.decrementDepth() 39 | return false 40 | } 41 | c = iter.nextToken() 42 | for c == ',' { 43 | if !callback(iter) { 44 | iter.decrementDepth() 45 | return false 46 | } 47 | c = iter.nextToken() 48 | } 49 | if c != ']' { 50 | iter.ReportError("ReadArrayCB", "expect ] in the end, but found "+string([]byte{c})) 51 | iter.decrementDepth() 52 | return false 53 | } 54 | return iter.decrementDepth() 55 | } 56 | return iter.decrementDepth() 57 | } 58 | if c == 'n' { 59 | iter.skipThreeBytes('u', 'l', 'l') 60 | return true // null 61 | } 62 | iter.ReportError("ReadArrayCB", "expect [ or n, but found "+string([]byte{c})) 63 | return false 64 | } 65 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/jsoniter.go: -------------------------------------------------------------------------------- 1 | // Package jsoniter implements encoding and decoding of JSON as defined in 2 | // RFC 4627 and provides interfaces with identical syntax of standard lib encoding/json. 3 | // Converting from encoding/json to jsoniter is no more than replacing the package with jsoniter 4 | // and variable type declarations (if any). 5 | // jsoniter interfaces gives 100% compatibility with code using standard lib. 6 | // 7 | // "JSON and Go" 8 | // (https://golang.org/doc/articles/json_and_go.html) 9 | // gives a description of how Marshal/Unmarshal operate 10 | // between arbitrary or predefined json objects and bytes, 11 | // and it applies to jsoniter.Marshal/Unmarshal as well. 12 | // 13 | // Besides, jsoniter.Iterator provides a different set of interfaces 14 | // iterating given bytes/string/reader 15 | // and yielding parsed elements one by one. 16 | // This set of interfaces reads input as required and gives 17 | // better performance. 18 | package jsonorder 19 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/pool.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // IteratorPool a thread safe pool of iterators with same configuration 8 | type IteratorPool interface { 9 | BorrowIterator(data []byte) *Iterator 10 | ReturnIterator(iter *Iterator) 11 | } 12 | 13 | // StreamPool a thread safe pool of streams with same configuration 14 | type StreamPool interface { 15 | BorrowStream(writer io.Writer) *Stream 16 | ReturnStream(stream *Stream) 17 | } 18 | 19 | func (cfg *frozenConfig) BorrowStream(writer io.Writer) *Stream { 20 | stream := cfg.streamPool.Get().(*Stream) 21 | stream.Reset(writer) 22 | return stream 23 | } 24 | 25 | func (cfg *frozenConfig) ReturnStream(stream *Stream) { 26 | stream.out = nil 27 | stream.Error = nil 28 | stream.Attachment = nil 29 | cfg.streamPool.Put(stream) 30 | } 31 | 32 | func (cfg *frozenConfig) BorrowIterator(data []byte) *Iterator { 33 | iter := cfg.iteratorPool.Get().(*Iterator) 34 | iter.ResetBytes(data) 35 | return iter 36 | } 37 | 38 | func (cfg *frozenConfig) ReturnIterator(iter *Iterator) { 39 | iter.Error = nil 40 | iter.Attachment = nil 41 | cfg.iteratorPool.Put(iter) 42 | } 43 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/reflect_dynamic.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import ( 4 | "github.com/modern-go/reflect2" 5 | "reflect" 6 | "unsafe" 7 | ) 8 | 9 | type dynamicEncoder struct { 10 | valType reflect2.Type 11 | } 12 | 13 | func (encoder *dynamicEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { 14 | obj := encoder.valType.UnsafeIndirect(ptr) 15 | stream.WriteVal(obj) 16 | } 17 | 18 | func (encoder *dynamicEncoder) IsEmpty(ptr unsafe.Pointer) bool { 19 | return encoder.valType.UnsafeIndirect(ptr) == nil 20 | } 21 | 22 | type efaceDecoder struct { 23 | } 24 | 25 | func (decoder *efaceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { 26 | pObj := (*interface{})(ptr) 27 | obj := *pObj 28 | if obj == nil { 29 | *pObj = iter.Read() 30 | return 31 | } 32 | typ := reflect2.TypeOf(obj) 33 | if typ.Kind() != reflect.Ptr { 34 | *pObj = iter.Read() 35 | return 36 | } 37 | ptrType := typ.(*reflect2.UnsafePtrType) 38 | ptrElemType := ptrType.Elem() 39 | if iter.WhatIsNext() == NilValue { 40 | if ptrElemType.Kind() != reflect.Ptr { 41 | iter.skipFourBytes('n', 'u', 'l', 'l') 42 | *pObj = nil 43 | return 44 | } 45 | } 46 | if reflect2.IsNil(obj) { 47 | obj := ptrElemType.New() 48 | iter.ReadVal(obj) 49 | *pObj = obj 50 | return 51 | } 52 | iter.ReadVal(obj) 53 | } 54 | 55 | type ifaceDecoder struct { 56 | valType *reflect2.UnsafeIFaceType 57 | } 58 | 59 | func (decoder *ifaceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { 60 | if iter.ReadNil() { 61 | decoder.valType.UnsafeSet(ptr, decoder.valType.UnsafeNew()) 62 | return 63 | } 64 | obj := decoder.valType.UnsafeIndirect(ptr) 65 | if reflect2.IsNil(obj) { 66 | iter.ReportError("decode non empty interface", "can not unmarshal into nil") 67 | return 68 | } 69 | iter.ReadVal(obj) 70 | } 71 | -------------------------------------------------------------------------------- /jitsubase/jsonorder/reflect_json_raw_message.go: -------------------------------------------------------------------------------- 1 | package jsonorder 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/modern-go/reflect2" 6 | "unsafe" 7 | ) 8 | 9 | var jsonRawMessageType = reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem() 10 | var jsoniterRawMessageType = reflect2.TypeOfPtr((*RawMessage)(nil)).Elem() 11 | 12 | func createEncoderOfJsonRawMessage(ctx *ctx, typ reflect2.Type) ValEncoder { 13 | if typ == jsonRawMessageType { 14 | return &jsonRawMessageCodec{} 15 | } 16 | if typ == jsoniterRawMessageType { 17 | return &jsoniterRawMessageCodec{} 18 | } 19 | return nil 20 | } 21 | 22 | func createDecoderOfJsonRawMessage(ctx *ctx, typ reflect2.Type) ValDecoder { 23 | if typ == jsonRawMessageType { 24 | return &jsonRawMessageCodec{} 25 | } 26 | if typ == jsoniterRawMessageType { 27 | return &jsoniterRawMessageCodec{} 28 | } 29 | return nil 30 | } 31 | 32 | type jsonRawMessageCodec struct { 33 | } 34 | 35 | func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { 36 | if iter.ReadNil() { 37 | *((*json.RawMessage)(ptr)) = nil 38 | } else { 39 | *((*json.RawMessage)(ptr)) = iter.SkipAndReturnBytes() 40 | } 41 | } 42 | 43 | func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { 44 | if *((*json.RawMessage)(ptr)) == nil { 45 | stream.WriteNil() 46 | } else { 47 | stream.WriteRaw(string(*((*json.RawMessage)(ptr)))) 48 | } 49 | } 50 | 51 | func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { 52 | return len(*((*json.RawMessage)(ptr))) == 0 53 | } 54 | 55 | type jsoniterRawMessageCodec struct { 56 | } 57 | 58 | func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { 59 | if iter.ReadNil() { 60 | *((*RawMessage)(ptr)) = nil 61 | } else { 62 | *((*RawMessage)(ptr)) = iter.SkipAndReturnBytes() 63 | } 64 | } 65 | 66 | func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { 67 | if *((*RawMessage)(ptr)) == nil { 68 | stream.WriteNil() 69 | } else { 70 | stream.WriteRaw(string(*((*RawMessage)(ptr)))) 71 | } 72 | } 73 | 74 | func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { 75 | return len(*((*RawMessage)(ptr))) == 0 76 | } 77 | -------------------------------------------------------------------------------- /jitsubase/locks/types.go: -------------------------------------------------------------------------------- 1 | package locks 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // LockFactory creates lock and returns it (without locking) 8 | type LockFactory interface { 9 | CreateLock(name string) Lock 10 | } 11 | 12 | // Lock all operations with lock 13 | type Lock interface { 14 | // TryLock Attempts to acquire lock within given amount of time. If lock is not free by 15 | // that time, returns false. Otherwise, returns true 16 | TryLock(timeout time.Duration) (bool, error) 17 | Unlock() 18 | } 19 | 20 | type DummyLock struct { 21 | } 22 | 23 | func (l DummyLock) TryLock(timeout time.Duration) (bool, error) { 24 | return true, nil 25 | } 26 | func (l DummyLock) Unlock() { 27 | } 28 | -------------------------------------------------------------------------------- /jitsubase/logging/dual.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | //Dual write logs into fileWriter and into stdout as well 8 | type Dual struct { 9 | FileWriter io.Writer 10 | Stdout io.Writer 11 | } 12 | 13 | func (wp Dual) Write(bytes []byte) (int, error) { 14 | wp.Stdout.Write(bytes) 15 | return wp.FileWriter.Write(bytes) 16 | } 17 | -------------------------------------------------------------------------------- /jitsubase/logging/level.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import "strings" 4 | 5 | type Level int 6 | 7 | const ( 8 | UNKNOWN Level = iota 9 | DEBUG 10 | INFO 11 | WARN 12 | ERROR 13 | FATAL 14 | ) 15 | 16 | func (l Level) String() string { 17 | switch l { 18 | case UNKNOWN: 19 | return "unknown" 20 | case DEBUG: 21 | return "debug" 22 | case INFO: 23 | return "info" 24 | case WARN: 25 | return "warn" 26 | case ERROR: 27 | return "error" 28 | case FATAL: 29 | return "fatal" 30 | default: 31 | return "" 32 | } 33 | } 34 | 35 | func ToLevel(levelStr string) Level { 36 | switch strings.TrimSpace(strings.ToLower(levelStr)) { 37 | case "debug": 38 | return DEBUG 39 | case "info": 40 | return INFO 41 | case "warn": 42 | return WARN 43 | case "error": 44 | return ERROR 45 | case "fatal": 46 | return FATAL 47 | default: 48 | return UNKNOWN 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /jitsubase/logging/object_logger.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import "io" 4 | 5 | type ObjectLogger interface { 6 | io.Closer 7 | Consume(event map[string]any, tokenID string) 8 | ConsumeAny(obj any) 9 | } 10 | -------------------------------------------------------------------------------- /jitsubase/logging/proxy.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "github.com/jitsucom/bulker/jitsubase/timestamp" 5 | "io" 6 | ) 7 | 8 | type DateTimeWriterProxy struct { 9 | writer io.Writer 10 | } 11 | 12 | func (wp DateTimeWriterProxy) Write(bytes []byte) (int, error) { 13 | return wp.writer.Write([]byte(timestamp.Now().UTC().Format(timestamp.LogsLayout) + " " + string(bytes))) 14 | } 15 | 16 | type PrefixDateTimeProxy struct { 17 | prefix string 18 | writer io.Writer 19 | } 20 | 21 | func NewPrefixDateTimeProxy(prefix string, writer io.Writer) io.Writer { 22 | return &PrefixDateTimeProxy{prefix: prefix, writer: writer} 23 | } 24 | 25 | func (pwp PrefixDateTimeProxy) Write(bytes []byte) (int, error) { 26 | return pwp.writer.Write([]byte(timestamp.Now().UTC().Format(timestamp.LogsLayout) + " " + pwp.prefix + " " + string(bytes))) 27 | } 28 | -------------------------------------------------------------------------------- /jitsubase/logging/string_writer.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import "bytes" 4 | 5 | type StringWriter struct { 6 | buff *bytes.Buffer 7 | } 8 | 9 | func NewStringWriter() *StringWriter { 10 | return &StringWriter{ 11 | buff: bytes.NewBuffer([]byte{}), 12 | } 13 | } 14 | 15 | func (sw *StringWriter) String() string { 16 | return sw.buff.String() 17 | } 18 | 19 | func (sw *StringWriter) Bytes() []byte { 20 | return sw.buff.Bytes() 21 | } 22 | 23 | func (sw *StringWriter) Write(p []byte) (n int, err error) { 24 | return sw.buff.Write(p) 25 | } 26 | 27 | func (sw *StringWriter) Close() error { 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /jitsubase/logging/task_logger.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | type TaskLogger interface { 4 | INFO(format string, v ...any) 5 | ERROR(format string, v ...any) 6 | WARN(format string, v ...any) 7 | LOG(format, system string, level Level, v ...any) 8 | 9 | //Write is used by Singer 10 | Write(p []byte) (n int, err error) 11 | } 12 | -------------------------------------------------------------------------------- /jitsubase/logging/utils.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "os" 5 | "path" 6 | ) 7 | 8 | func IsDirWritable(dir string) bool { 9 | testFile := path.Join(dir, "tmp_test") 10 | err := os.WriteFile(testFile, []byte{}, 0644) 11 | if err != nil { 12 | return false 13 | } 14 | 15 | _ = os.Remove(testFile) 16 | return true 17 | } 18 | 19 | func EnsureDir(dir string) error { 20 | return os.MkdirAll(dir, 0766) 21 | } 22 | -------------------------------------------------------------------------------- /jitsubase/logging/writer_mock.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import "io" 4 | 5 | var InstanceMock *WriterMock 6 | 7 | type WriterMock struct { 8 | Data [][]byte 9 | } 10 | 11 | func InitInMemoryWriter() io.WriteCloser { 12 | InstanceMock = &WriterMock{ 13 | Data: [][]byte{}, 14 | } 15 | return InstanceMock 16 | } 17 | 18 | func (im *WriterMock) Write(dataToWrite []byte) (n int, err error) { 19 | im.Data = append(im.Data, dataToWrite) 20 | return len(dataToWrite), nil 21 | } 22 | 23 | func (im *WriterMock) Close() (err error) { 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /jitsubase/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/google/uuid" 6 | "github.com/jitsucom/bulker/jitsubase/appbase" 7 | ) 8 | 9 | func main() { 10 | fmt.Println("Enter token secret:") 11 | var tokenSecret string 12 | _, err := fmt.Scanln(&tokenSecret) 13 | if err != nil { 14 | panic(err) 15 | } 16 | fmt.Println("Enter auth token:") 17 | var authToken string 18 | _, err = fmt.Scanln(&authToken) 19 | if err != nil { 20 | panic(err) 21 | } 22 | salt := uuid.NewString() 23 | 24 | hashedToken := appbase.HashTokenHex(authToken, salt, tokenSecret) 25 | fmt.Printf("Hashed token: %s.%s", salt, hashedToken) 26 | } 27 | -------------------------------------------------------------------------------- /jitsubase/pg/pgpool.go: -------------------------------------------------------------------------------- 1 | package pg 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/jackc/pgx/v5" 7 | "github.com/jackc/pgx/v5/pgxpool" 8 | "regexp" 9 | ) 10 | 11 | var schemaRegex = regexp.MustCompile(`(?:search_path|schema)=([^$]+)`) 12 | 13 | func extractSchema(url string) string { 14 | parts := schemaRegex.FindStringSubmatch(url) 15 | if len(parts) == 2 { 16 | return parts[1] 17 | } else { 18 | return "" 19 | } 20 | } 21 | 22 | func NewPGPool(url string) (*pgxpool.Pool, error) { 23 | pgCfg, err := pgxpool.ParseConfig(url) 24 | if err != nil { 25 | return nil, fmt.Errorf("Unable to create postgres connection pool: %v\n", err) 26 | } 27 | schema := extractSchema(url) 28 | if schema != "" { 29 | pgCfg.ConnConfig.RuntimeParams["search_path"] = schema 30 | pgCfg.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { 31 | _, err := conn.Exec(ctx, fmt.Sprintf("SET search_path TO '%s'", schema)) 32 | return err 33 | } 34 | } 35 | dbpool, err := pgxpool.NewWithConfig(context.Background(), pgCfg) 36 | if err != nil { 37 | return nil, fmt.Errorf("Unable to create postgres connection pool: %v\n", err) 38 | } 39 | return dbpool, nil 40 | } 41 | -------------------------------------------------------------------------------- /jitsubase/safego/safego.go: -------------------------------------------------------------------------------- 1 | package safego 2 | 3 | import ( 4 | "fmt" 5 | "runtime/debug" 6 | "time" 7 | ) 8 | 9 | const defaultRestartTimeout = 2 * time.Second 10 | 11 | type RecoverHandler func(value any) 12 | 13 | var GlobalRecoverHandler RecoverHandler 14 | 15 | func init() { 16 | GlobalRecoverHandler = func(value interface{}) { 17 | fmt.Println("panic") 18 | fmt.Println(value) 19 | fmt.Println(string(debug.Stack())) 20 | } 21 | } 22 | 23 | type Execution struct { 24 | f func() 25 | recoverHandler RecoverHandler 26 | restartTimeout time.Duration 27 | } 28 | 29 | // Run runs a new goroutine and add panic handler (without restart) 30 | func Run(f func()) *Execution { 31 | exec := Execution{ 32 | f: f, 33 | recoverHandler: GlobalRecoverHandler, 34 | restartTimeout: 0, 35 | } 36 | return exec.run() 37 | } 38 | 39 | // RunWithRestart run a new goroutine and add panic handler: 40 | // write logs, wait 2 seconds and restart the goroutine 41 | func RunWithRestart(f func()) *Execution { 42 | exec := Execution{ 43 | f: f, 44 | recoverHandler: GlobalRecoverHandler, 45 | restartTimeout: defaultRestartTimeout, 46 | } 47 | return exec.run() 48 | } 49 | 50 | func (exec *Execution) run() *Execution { 51 | go func() { 52 | defer func() { 53 | if r := recover(); r != nil { 54 | exec.recoverHandler(r) 55 | 56 | if exec.restartTimeout > 0 { 57 | time.Sleep(exec.restartTimeout) 58 | exec.run() 59 | } 60 | } 61 | }() 62 | exec.f() 63 | }() 64 | return exec 65 | } 66 | 67 | func (exec *Execution) WithRestartTimeout(timeout time.Duration) *Execution { 68 | exec.restartTimeout = timeout 69 | return exec 70 | } 71 | -------------------------------------------------------------------------------- /jitsubase/safego/safego_test.go: -------------------------------------------------------------------------------- 1 | package safego 2 | 3 | import ( 4 | "github.com/stretchr/testify/require" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestHandlePanicAndRestart(t *testing.T) { 10 | defer func() { 11 | if r := recover(); r != nil { 12 | t.Fail() 13 | } 14 | }() 15 | 16 | GlobalRecoverHandler = func(value any) { 17 | } 18 | 19 | counter := 0 20 | 21 | RunWithRestart(func() { 22 | counter++ 23 | panic("panic") 24 | }).WithRestartTimeout(50 * time.Millisecond) 25 | 26 | time.Sleep(200 * time.Millisecond) 27 | require.True(t, counter > 1, "counter must be > 1") 28 | 29 | time.Sleep(100 * time.Millisecond) 30 | require.True(t, counter > 2, "counter must be > 2") 31 | 32 | if counter == 0 { 33 | t.Fail() 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /jitsubase/timestamp/format.go: -------------------------------------------------------------------------------- 1 | package timestamp 2 | 3 | import "time" 4 | 5 | // Key is a default key and format of event timestamp 6 | const Key = "_timestamp" 7 | 8 | // Layout is an ISO date time format. Note: for parsing use time.RFC3339Nano. 9 | const Layout = "2006-01-02T15:04:05.000000Z" 10 | 11 | // Layout is an ISO date time format. Note: for parsing use time.RFC3339Nano. 12 | const JsonISO = "2006-01-02T15:04:05.000Z" 13 | 14 | // DayLayout is a Day format of time.Time 15 | const DayLayout = "20060102" 16 | 17 | // MonthLayout is a Month format of time.Time 18 | const MonthLayout = "200601" 19 | 20 | // DashDayLayout is a Day format with dash delimiter of time.Time 21 | const DashDayLayout = "2006-01-02" 22 | 23 | // LogsLayout is a date time representation for log records prefixes 24 | const LogsLayout = "2006-01-02 15:04:05" 25 | 26 | // GolangLayout is a default golang layout that is returned on String() without formatting 27 | const GolangLayout = "2006-01-02T15:04:05-0700" 28 | 29 | // DBLayout is a time layout that usually comes from Airbyte database sources 30 | const DBLayout = "2006-01-02T15:04:05.999999999" 31 | 32 | // NowUTC returns ISO string representation of current UTC time 33 | func NowUTC() string { 34 | return Now().UTC().Format(Layout) 35 | } 36 | 37 | // ToISOFormat returns ISO string representation of input time.Time 38 | func ToISOFormat(t time.Time) string { 39 | return t.Format(Layout) 40 | } 41 | 42 | // ParseISOFormat returns time.Time from ISO time string representation 43 | func ParseISOFormat(t string) (time.Time, error) { 44 | return time.Parse(time.RFC3339Nano, t) 45 | } 46 | -------------------------------------------------------------------------------- /jitsubase/timestamp/timestamp.go: -------------------------------------------------------------------------------- 1 | package timestamp 2 | 3 | import ( 4 | "time" 5 | 6 | "go.uber.org/atomic" 7 | ) 8 | 9 | var ( 10 | // The value of frozen time that is used in all tests 11 | frozenTime = time.Date(2020, 06, 16, 23, 0, 0, 0, time.UTC) 12 | 13 | // The value for overwriting in tests 14 | currentFrozenTime = frozenTime 15 | 16 | // Indicator shows that time was frozen or was not frozen 17 | timeFrozen = atomic.NewBool(false) 18 | ) 19 | 20 | func Now() time.Time { 21 | if timeFrozen.Load() { 22 | return currentFrozenTime 23 | } 24 | return time.Now() 25 | } 26 | 27 | func FreezeTime() { 28 | timeFrozen.Store(true) 29 | } 30 | 31 | func SetFreezeTime(t time.Time) { 32 | currentFrozenTime = t 33 | } 34 | 35 | func UnfreezeTime() { 36 | timeFrozen.Store(false) 37 | currentFrozenTime = frozenTime 38 | } 39 | 40 | func MustParseTime(layout, value string) time.Time { 41 | t, err := time.Parse(layout, value) 42 | if err != nil { 43 | panic(err) 44 | } 45 | return t 46 | } 47 | -------------------------------------------------------------------------------- /jitsubase/timestamp/timestamp_test.go: -------------------------------------------------------------------------------- 1 | package timestamp 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestFreezing(t *testing.T) { 10 | 11 | require.NotEqual(t, frozenTime, Now(), "Now() should provide real current time") 12 | 13 | FreezeTime() 14 | 15 | require.Equal(t, frozenTime, Now(), "Now() should provide freezed time after freezing") 16 | 17 | UnfreezeTime() 18 | 19 | require.NotEqual(t, frozenTime, Now(), "Now() should provide real time after unfreezing") 20 | } 21 | -------------------------------------------------------------------------------- /jitsubase/types/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Elliot Chance 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /jitsubase/types/README.md: -------------------------------------------------------------------------------- 1 | # 🔃 github.com/elliotchance/orderedmap/v2 [![GoDoc](https://godoc.org/github.com/elliotchance/orderedmap/v2?status.svg)](https://godoc.org/github.com/elliotchance/orderedmap/v2) 2 | 3 | ## Basic Usage 4 | 5 | An `*OrderedMap` is a high performance ordered map that maintains amortized O(1) 6 | for `Set`, `Get`, `Delete` and `Len`: 7 | 8 | ```go 9 | import "github.com/elliotchance/orderedmap/v2" 10 | 11 | func main() { 12 | m := orderedmap.NewOrderedMap[string, any]() 13 | 14 | m.Set("foo", "bar") 15 | m.Set("qux", 1.23) 16 | m.Set("123", true) 17 | 18 | m.Delete("qux") 19 | } 20 | ``` 21 | 22 | *Note: v2 requires Go v1.18 for generics.* If you need to support Go 1.17 or 23 | below, you can use v1. 24 | 25 | Internally an `*OrderedMap` uses the composite type 26 | [map](https://go.dev/blog/maps) combined with a 27 | trimmed down linked list to maintain the order. 28 | 29 | ## Iterating 30 | 31 | Be careful using `Keys()` as it will create a copy of all of the keys so it's 32 | only suitable for a small number of items: 33 | 34 | ```go 35 | for _, key := range m.Keys() { 36 | value, _:= m.Get(key) 37 | fmt.Println(key, value) 38 | } 39 | ``` 40 | 41 | For larger maps you should use `Front()` or `Back()` to iterate per element: 42 | 43 | ```go 44 | // Iterate through all elements from oldest to newest: 45 | for el := m.Front(); el != nil; el = el.Next() { 46 | fmt.Println(el.Key, el.Value) 47 | } 48 | 49 | // You can also use Back and Prev to iterate in reverse: 50 | for el := m.Back(); el != nil; el = el.Prev() { 51 | fmt.Println(el.Key, el.Value) 52 | } 53 | ``` 54 | 55 | The iterator is safe to use bidirectionally, and will return `nil` once it goes 56 | beyond the first or last item. 57 | 58 | If the map is changing while the iteration is in-flight it may produce 59 | unexpected behavior. 60 | -------------------------------------------------------------------------------- /jitsubase/types/json.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "strings" 4 | 5 | const SqlTypePrefix = "__sql_type" 6 | 7 | type Json = *OrderedMap[string, any] 8 | 9 | func NewJson(defaultCapacity int) Json { 10 | return NewOrderedMap[string, any](defaultCapacity) 11 | } 12 | 13 | func JsonFromMap(mp map[string]any) Json { 14 | om := NewJson(len(mp)) 15 | for k, v := range mp { 16 | nested, ok := v.(map[string]any) 17 | if ok { 18 | om.Set(k, JsonFromMap(nested)) 19 | } else { 20 | om.Set(k, v) 21 | } 22 | } 23 | return om 24 | } 25 | 26 | func JsonToMap(j Json) map[string]any { 27 | mp := make(map[string]any, j.Len()) 28 | for el := j.Front(); el != nil; el = el.Next() { 29 | key := el.Key 30 | value := el.Value 31 | js, ok := value.(Json) 32 | if ok { 33 | mp[key] = JsonToMap(js) 34 | } else { 35 | mp[key] = value 36 | } 37 | } 38 | return mp 39 | } 40 | 41 | func FilterEvent(event Json) { 42 | _ = event.Delete("JITSU_TABLE_NAME") 43 | _ = event.Delete("JITSU_PROFILE_ID") 44 | filterEvent(event) 45 | } 46 | 47 | func filterEvent(event any) { 48 | switch v := event.(type) { 49 | case Json: 50 | for el := v.Front(); el != nil; { 51 | curEl := el 52 | // move to the next element before deleting the current one. otherwise iteration will be broken 53 | el = el.Next() 54 | if strings.HasPrefix(curEl.Key, SqlTypePrefix) { 55 | v.DeleteElement(curEl) 56 | } else { 57 | switch v2 := curEl.Value.(type) { 58 | case Json, []any: 59 | filterEvent(v2) 60 | } 61 | } 62 | } 63 | case []any: 64 | for _, a := range v { 65 | switch v2 := a.(type) { 66 | case Json, []any: 67 | filterEvent(v2) 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /jitsubase/types/set.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "cmp" 5 | "github.com/jitsucom/bulker/jitsubase/utils" 6 | "sort" 7 | "strconv" 8 | ) 9 | 10 | type Set[K cmp.Ordered] map[K]struct{} 11 | 12 | func NewSet[K cmp.Ordered](values ...K) Set[K] { 13 | s := make(Set[K]) 14 | s.PutAll(values) 15 | return s 16 | } 17 | 18 | func (s Set[K]) Contains(key K) bool { 19 | _, ok := s[key] 20 | return ok 21 | } 22 | 23 | func (s Set[K]) Put(key K) { 24 | s[key] = struct{}{} 25 | } 26 | 27 | func (s Set[K]) PutAll(keys []K) { 28 | for _, key := range keys { 29 | s.Put(key) 30 | } 31 | } 32 | 33 | func (s Set[K]) PutAllOrderedKeys(m *OrderedMap[K, any]) { 34 | for el := m.Front(); el != nil; el = el.Next() { 35 | s.Put(el.Key) 36 | } 37 | } 38 | 39 | func (s Set[K]) PutAllKeys(m map[K]any) { 40 | for key := range m { 41 | s.Put(key) 42 | } 43 | } 44 | 45 | func (s Set[K]) PutSet(keys Set[K]) { 46 | for key := range keys { 47 | s.Put(key) 48 | } 49 | } 50 | 51 | func (s Set[K]) Remove(key K) { 52 | delete(s, key) 53 | } 54 | 55 | func (s Set[K]) Clear() { 56 | for key := range s { 57 | delete(s, key) 58 | } 59 | } 60 | 61 | func (s Set[K]) Clone() Set[K] { 62 | newSet := make(Set[K]) 63 | for k := range s { 64 | newSet.Put(k) 65 | } 66 | return newSet 67 | } 68 | 69 | func (s Set[K]) Size() int { 70 | return len(s) 71 | } 72 | 73 | func (s Set[K]) ToSlice() []K { 74 | if len(s) == 0 { 75 | return []K{} 76 | } 77 | slice := make([]K, 0, len(s)) 78 | for k := range s { 79 | slice = append(slice, k) 80 | } 81 | sort.Slice(slice, func(i, j int) bool { 82 | return slice[i] < slice[j] 83 | }) 84 | return slice 85 | } 86 | 87 | func (s Set[K]) Equals(other Set[K]) bool { 88 | if len(s) != len(other) { 89 | return false 90 | } 91 | 92 | for k := range s { 93 | if !other.Contains(k) { 94 | return false 95 | } 96 | } 97 | return true 98 | } 99 | 100 | func (s Set[K]) Hash() string { 101 | h, _ := utils.HashAny(s) 102 | return strconv.FormatUint(h, 10) 103 | } 104 | -------------------------------------------------------------------------------- /jitsubase/utils/arrays.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func ArrayContains[T comparable](arr []T, value T) bool { 4 | for _, a := range arr { 5 | if a == value { 6 | return true 7 | } 8 | } 9 | return false 10 | } 11 | 12 | func ArrayExcluding[T comparable](arr []T, valueToExclude ...T) []T { 13 | res := make([]T, 0, len(arr)) 14 | for _, a := range arr { 15 | exclude := false 16 | for _, v := range valueToExclude { 17 | if a == v { 18 | exclude = true 19 | break 20 | } 21 | } 22 | if !exclude { 23 | res = append(res, a) 24 | } 25 | } 26 | return res 27 | } 28 | 29 | func ArrayIntersection[T comparable](firstArray []T, secondArray []T) []T { 30 | if len(secondArray) < len(firstArray) { 31 | firstArray, secondArray = secondArray, firstArray 32 | } 33 | res := make([]T, 0, len(firstArray)) 34 | for _, a := range firstArray { 35 | if ArrayContains(secondArray, a) { 36 | res = append(res, a) 37 | } 38 | } 39 | return res 40 | } 41 | 42 | func ArrayMap[V any, R any](arr []V, mappingFunc func(V) R) []R { 43 | result := make([]R, len(arr)) 44 | for i, v := range arr { 45 | result[i] = mappingFunc(v) 46 | } 47 | return result 48 | } 49 | 50 | func ArrayFilter[V any](arr []V, filterFunc func(V) bool) []V { 51 | result := make([]V, 0, len(arr)) 52 | for i, v := range arr { 53 | if filterFunc(v) { 54 | result = append(result, arr[i]) 55 | } 56 | } 57 | return result 58 | } 59 | 60 | func ArrayFilterMap[V any, R any](arr []V, filterFunc func(V) bool, mappingFunc func(V) R) []R { 61 | result := make([]R, 0, len(arr)) 62 | for i, v := range arr { 63 | if filterFunc(v) { 64 | result = append(result, mappingFunc(arr[i])) 65 | } 66 | } 67 | return result 68 | } 69 | 70 | func ArrayIndexOf[V any](arr []V, filterFunc func(V) bool) int { 71 | for i, v := range arr { 72 | if filterFunc(v) { 73 | return i 74 | } 75 | } 76 | return -1 77 | } 78 | 79 | func ArrayContainsF[V any](arr []V, filterFunc func(V) bool) bool { 80 | return ArrayIndexOf(arr, filterFunc) >= 0 81 | } 82 | -------------------------------------------------------------------------------- /jitsubase/utils/bool.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | // ParseBool parses value of string, int or bool into a bool. 10 | func ParseBool(value any) (bool, error) { 11 | switch v := value.(type) { 12 | case string: 13 | return strconv.ParseBool(v) 14 | case int: 15 | return v != 0, nil 16 | case bool: 17 | return v, nil 18 | default: 19 | return false, fmt.Errorf("ParseBool: invalid value type %T", value) 20 | } 21 | } 22 | 23 | func Ternary[T any](condition bool, yes T, no T) T { 24 | if condition { 25 | return yes 26 | } else { 27 | return no 28 | } 29 | } 30 | 31 | func IsTruish(value any) bool { 32 | switch v := value.(type) { 33 | case string: 34 | return strings.ToLower(v) == "true" || v == "1" 35 | case int: 36 | return v != 0 37 | case bool: 38 | return v 39 | default: 40 | return false 41 | } 42 | } 43 | 44 | func BoolPointer(b bool) *bool { 45 | return &b 46 | } 47 | -------------------------------------------------------------------------------- /jitsubase/utils/bytes.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func TruncateBytes(data []byte, maxSize int) []byte { 4 | if len(data) <= maxSize { 5 | return data 6 | } 7 | return data[:maxSize] 8 | } 9 | -------------------------------------------------------------------------------- /jitsubase/utils/cache.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | "time" 7 | ) 8 | 9 | type CacheEntry[T any] struct { 10 | addedAt int64 11 | value T 12 | } 13 | 14 | type Cache[T any] struct { 15 | sync.RWMutex 16 | ttlSeconds int64 17 | entries map[string]*CacheEntry[T] 18 | } 19 | 20 | func NewCache[T any](ttlSeconds int64) *Cache[T] { 21 | return &Cache[T]{ 22 | ttlSeconds: ttlSeconds, 23 | entries: make(map[string]*CacheEntry[T]), 24 | } 25 | } 26 | 27 | func (c *Cache[T]) Get(key string) (T, bool) { 28 | c.RLock() 29 | defer c.RUnlock() 30 | var dflt T 31 | entry, ok := c.entries[key] 32 | if !ok { 33 | return dflt, false 34 | } 35 | if entry.addedAt+c.ttlSeconds < time.Now().Unix() { 36 | return dflt, false 37 | } 38 | return entry.value, true 39 | } 40 | 41 | func (c *Cache[T]) Set(key string, value T) { 42 | c.Lock() 43 | defer c.Unlock() 44 | c.entries[key] = &CacheEntry[T]{addedAt: time.Now().Unix(), value: value} 45 | } 46 | 47 | type SyncMapCache[T any] struct { 48 | maxSize int64 49 | entries sync.Map 50 | length atomic.Int64 51 | } 52 | 53 | func NewSyncMapCache[T any](maxSize int) *SyncMapCache[T] { 54 | return &SyncMapCache[T]{ 55 | maxSize: int64(maxSize), 56 | entries: sync.Map{}, 57 | length: atomic.Int64{}, 58 | } 59 | } 60 | 61 | func (s *SyncMapCache[T]) Get(key string) (*T, bool) { 62 | entry, ok := s.entries.Load(key) 63 | if !ok { 64 | return nil, false 65 | } 66 | return entry.(*T), true 67 | } 68 | 69 | func (s *SyncMapCache[T]) Set(key string, value *T) { 70 | if s.length.Load() >= s.maxSize { 71 | s.entries.Clear() 72 | } 73 | _, ok := s.entries.Swap(key, value) 74 | if !ok { 75 | s.length.Add(1) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /jitsubase/utils/crypto.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/pem" 5 | "fmt" 6 | "github.com/youmark/pkcs8" 7 | ) 8 | 9 | func ParsePrivateKey(privateKey []byte, passphrase string) (pk interface{}, err error) { 10 | block, _ := pem.Decode(privateKey) 11 | if block == nil { 12 | return nil, fmt.Errorf("Failed to decode PEM block") 13 | } 14 | pk, _, err = pkcs8.ParsePrivateKey(block.Bytes, []byte(passphrase)) 15 | return pk, err 16 | } 17 | -------------------------------------------------------------------------------- /jitsubase/utils/crypto_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/rsa" 5 | "github.com/stretchr/testify/require" 6 | "testing" 7 | ) 8 | 9 | const pkWithPassphrase = `` 10 | 11 | const pkWithoutPassphrase = `` 12 | 13 | func TestParsePrivateKey(t *testing.T) { 14 | t.Skip() 15 | _, err := ParsePrivateKey([]byte(pkWithPassphrase), "test2") 16 | require.ErrorContains(t, err, "incorrect password") 17 | pk, err := ParsePrivateKey([]byte(pkWithPassphrase), "test") 18 | require.NoError(t, err) 19 | require.IsType(t, &rsa.PrivateKey{}, pk) 20 | 21 | _, err = ParsePrivateKey([]byte(pkWithoutPassphrase), "test2") 22 | require.ErrorContains(t, err, "only PKCS #5 v2.0 supported") 23 | pk, err = ParsePrivateKey([]byte(pkWithoutPassphrase), "") 24 | require.NoError(t, err) 25 | require.IsType(t, &rsa.PrivateKey{}, pk) 26 | } 27 | -------------------------------------------------------------------------------- /jitsubase/utils/errors.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | type RichError struct { 4 | error string 5 | payload any 6 | } 7 | 8 | func NewRichError(error string, payload any) *RichError { 9 | return &RichError{error: error, payload: payload} 10 | } 11 | 12 | func (r *RichError) Error() string { 13 | return r.error 14 | } 15 | 16 | func (r *RichError) Payload() any { 17 | return r.payload 18 | } 19 | -------------------------------------------------------------------------------- /jitsubase/utils/hash.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | "github.com/mitchellh/hashstructure/v2" 7 | "hash/fnv" 8 | ) 9 | 10 | var hashOptions = &hashstructure.HashOptions{SlicesAsSets: true} 11 | 12 | func HashString(value string) [16]byte { 13 | return md5.Sum([]byte(value)) 14 | } 15 | 16 | func HashStringS(value string) string { 17 | sum := md5.Sum([]byte(value)) 18 | return fmt.Sprintf("%x", sum) 19 | } 20 | 21 | func HashStringInt(value string) uint32 { 22 | h := fnv.New32a() 23 | _, _ = h.Write([]byte(value)) 24 | return h.Sum32() 25 | } 26 | 27 | func HashBytes(payload []byte) [16]byte { 28 | return md5.Sum(payload) 29 | } 30 | 31 | func HashAny(value interface{}) (uint64, error) { 32 | hash, err := hashstructure.Hash(value, hashstructure.FormatV2, hashOptions) 33 | if err != nil { 34 | return 0, err 35 | } 36 | 37 | return hash, nil 38 | } 39 | -------------------------------------------------------------------------------- /jitsubase/utils/net.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "net" 4 | 5 | // GetFreePort asks the kernel for a free open port that is ready to use. 6 | func GetFreePort() (int, error) { 7 | addr, err := net.ResolveTCPAddr("tcp", "localhost:0") 8 | if err != nil { 9 | return 0, err 10 | } 11 | 12 | l, err := net.ListenTCP("tcp", addr) 13 | if err != nil { 14 | return 0, err 15 | } 16 | defer l.Close() 17 | return l.Addr().(*net.TCPAddr).Port, nil 18 | } 19 | 20 | // GetPort is deprecated, use GetFreePort instead 21 | // Ask the kernel for a free open port that is ready to use 22 | func GetPort() int { 23 | port, err := GetFreePort() 24 | if err != nil { 25 | panic(err) 26 | } 27 | return port 28 | } 29 | 30 | // GetFreePort asks the kernel for free open ports that are ready to use. 31 | func GetFreePorts(count int) ([]int, error) { 32 | var ports []int 33 | for i := 0; i < count; i++ { 34 | addr, err := net.ResolveTCPAddr("tcp", "localhost:0") 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | l, err := net.ListenTCP("tcp", addr) 40 | if err != nil { 41 | return nil, err 42 | } 43 | defer l.Close() 44 | ports = append(ports, l.Addr().(*net.TCPAddr).Port) 45 | } 46 | return ports, nil 47 | } 48 | -------------------------------------------------------------------------------- /jitsubase/utils/numbers.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | // ParseInt parses value of string, int, integer float into int. 11 | func ParseInt(value any) (int, error) { 12 | switch v := value.(type) { 13 | case string: 14 | i, err := strconv.ParseInt(v, 10, 64) 15 | return int(i), err 16 | case int: 17 | return v, nil 18 | case int64: 19 | return int(v), nil 20 | case float64: 21 | if v == math.Trunc(v) { 22 | return int(v), nil 23 | } else { 24 | return 0, fmt.Errorf("can't parse float %f as int", v) 25 | } 26 | case nil: 27 | return 0, nil 28 | default: 29 | return 0, fmt.Errorf("ParseInt: invalid value type %T", value) 30 | } 31 | } 32 | 33 | // ParseFloat parses value of string, int, integer float into float64. 34 | func ParseFloat(value any) (float64, error) { 35 | switch v := value.(type) { 36 | case string: 37 | return strconv.ParseFloat(v, 64) 38 | case int: 39 | return float64(v), nil 40 | case int64: 41 | return float64(v), nil 42 | case float64: 43 | return v, nil 44 | case nil: 45 | return 0, nil 46 | default: 47 | return 0, fmt.Errorf("ParseFloat: invalid value type %T", value) 48 | } 49 | } 50 | 51 | func MaxInt(a, b int) int { 52 | if a > b { 53 | return a 54 | } 55 | return b 56 | } 57 | 58 | func MinInt64(a, b int64) int64 { 59 | if a < b { 60 | return a 61 | } 62 | return b 63 | } 64 | 65 | func MaxInt64(a, b int64) int64 { 66 | if a > b { 67 | return a 68 | } 69 | return b 70 | } 71 | 72 | func MaxDuration(a, b time.Duration) time.Duration { 73 | if a > b { 74 | return a 75 | } 76 | return b 77 | } 78 | -------------------------------------------------------------------------------- /jitsubase/utils/sets.go: -------------------------------------------------------------------------------- 1 | package utils 2 | -------------------------------------------------------------------------------- /jitsubase/utils/time.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "time" 4 | 5 | type Ticker struct { 6 | ticker *time.Ticker 7 | d time.Duration 8 | C chan time.Time 9 | Closed chan struct{} 10 | } 11 | 12 | func NewTicker(d time.Duration, startDelay time.Duration) *Ticker { 13 | c := make(chan time.Time, 1) 14 | closed := make(chan struct{}) 15 | t := &Ticker{ 16 | d: d, 17 | C: c, 18 | Closed: closed, 19 | } 20 | t.start(startDelay) 21 | return t 22 | } 23 | 24 | // start 25 | func (t *Ticker) start(startDelay time.Duration) { 26 | go func() { 27 | if startDelay != t.d { 28 | select { 29 | case tm := <-time.After(startDelay): 30 | t.C <- tm 31 | case <-t.Closed: 32 | return 33 | } 34 | } 35 | t.ticker = time.NewTicker(t.d) 36 | for { 37 | select { 38 | case tm := <-t.ticker.C: 39 | t.C <- tm 40 | case <-t.Closed: 41 | return 42 | } 43 | } 44 | }() 45 | } 46 | 47 | func (t *Ticker) Period() time.Duration { 48 | return t.d 49 | } 50 | 51 | // Stop turns off a ticker. After Stop, no more ticks will be sent. 52 | func (t *Ticker) Stop() { 53 | t.ticker.Stop() 54 | close(t.Closed) 55 | } 56 | -------------------------------------------------------------------------------- /jitsubase/uuid/uuid.go: -------------------------------------------------------------------------------- 1 | package uuid 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | googleuuid "github.com/google/uuid" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | var mock bool 12 | 13 | // InitMock initializes mock flag => New() func will return mock value everytime 14 | func InitMock() { 15 | mock = true 16 | } 17 | 18 | func init() { 19 | googleuuid.EnableRandPool() 20 | } 21 | 22 | // New returns uuid v4 string or the mocked value 23 | func New() string { 24 | if mock { 25 | return "mockeduuid" 26 | } 27 | 28 | return googleuuid.New().String() 29 | } 30 | 31 | // NewLettersNumbers returns uuid without "-" 32 | func NewLettersNumbers() string { 33 | if mock { 34 | return "mockeduuid" 35 | } 36 | 37 | uuidValue := googleuuid.New().String() 38 | return strings.ReplaceAll(uuidValue, "-", "") 39 | } 40 | 41 | // GetHash returns GetKeysHash result with keys from m 42 | func GetHash(m map[string]any) string { 43 | keys := make([]string, 0, len(m)) 44 | for k := range m { 45 | keys = append(keys, k) 46 | } 47 | 48 | return GetKeysHash(m, keys) 49 | } 50 | 51 | // GetKeysHash returns md5 hashsum of concatenated map values (sort keys before) 52 | func GetKeysHash(m map[string]any, keys []string) string { 53 | sort.Strings(keys) 54 | 55 | var str strings.Builder 56 | for _, k := range keys { 57 | str.WriteString(fmt.Sprint(m[k])) 58 | str.WriteRune('|') 59 | } 60 | 61 | return fmt.Sprintf("%x", md5.Sum([]byte(str.String()))) 62 | } 63 | 64 | // GetKeysUnhashed returns keys values joined by '_' 65 | func GetKeysUnhashed(m map[string]any, keys []string) string { 66 | sort.Strings(keys) 67 | 68 | var str strings.Builder 69 | for i, k := range keys { 70 | if i > 0 { 71 | str.WriteRune('_') 72 | } 73 | str.WriteString(fmt.Sprint(m[k])) 74 | } 75 | 76 | return str.String() 77 | } 78 | -------------------------------------------------------------------------------- /kafkabase/metrics.go: -------------------------------------------------------------------------------- 1 | package kafkabase 2 | 3 | import ( 4 | "fmt" 5 | "github.com/confluentinc/confluent-kafka-go/v2/kafka" 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/prometheus/client_golang/prometheus/promauto" 8 | ) 9 | 10 | var ( 11 | producerMessages = promauto.NewCounterVec(prometheus.CounterOpts{ 12 | Namespace: "bulkerapp", 13 | Subsystem: "producer", 14 | Name: "messages", 15 | }, []string{"topicId", "destinationId", "mode", "tableName", "status", "errorType"}) 16 | ProducerMessages = func(topicId, destinationId, mode, tableName, status, errorType string) prometheus.Counter { 17 | return producerMessages.WithLabelValues(topicId, destinationId, mode, tableName, status, errorType) 18 | } 19 | 20 | ProducerQueueLength = promauto.NewGauge(prometheus.GaugeOpts{ 21 | Namespace: "bulkerapp", 22 | Subsystem: "producer", 23 | Name: "queue_length", 24 | }) 25 | ) 26 | 27 | func KafkaErrorCode(err error) string { 28 | if err == nil { 29 | return "" 30 | } 31 | 32 | if kafkaError, ok := err.(kafka.Error); ok { 33 | return fmt.Sprintf("kafka error: %s", kafkaError.Code().String()) 34 | } 35 | 36 | return "kafka_error" 37 | } 38 | -------------------------------------------------------------------------------- /kafkabase/utils.go: -------------------------------------------------------------------------------- 1 | package kafkabase 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/confluentinc/confluent-kafka-go/v2/kafka" 7 | "github.com/jitsucom/bulker/jitsubase/timestamp" 8 | "strconv" 9 | "time" 10 | ) 11 | 12 | func GetKafkaHeader(message *kafka.Message, key string) string { 13 | for _, h := range message.Headers { 14 | if h.Key == key { 15 | return string(h.Value) 16 | } 17 | } 18 | return "" 19 | } 20 | 21 | func PutKafkaHeader(headers *[]kafka.Header, key string, value string) { 22 | for i, h := range *headers { 23 | if h.Key == key { 24 | (*headers)[i] = kafka.Header{Key: key, Value: []byte(value)} 25 | return 26 | } 27 | } 28 | *headers = append(*headers, kafka.Header{ 29 | Key: key, 30 | Value: []byte(value), 31 | }) 32 | } 33 | 34 | func GetKafkaIntHeader(message *kafka.Message, name string) (int, error) { 35 | v := GetKafkaHeader(message, name) 36 | if len(v) > 0 { 37 | return strconv.Atoi(v) 38 | } else { 39 | return 0, nil 40 | } 41 | } 42 | 43 | func GetKafkaTimeHeader(message *kafka.Message, name string) (time.Time, error) { 44 | v := GetKafkaHeader(message, name) 45 | if len(v) > 0 { 46 | return timestamp.ParseISOFormat(v) 47 | } else { 48 | return time.Time{}, nil 49 | } 50 | } 51 | 52 | func GetKafkaObjectHeader(message *kafka.Message, name string) (map[string]any, error) { 53 | v := GetKafkaHeader(message, name) 54 | if len(v) > 0 { 55 | var obj map[string]any 56 | err := json.Unmarshal([]byte(v), &obj) 57 | if err != nil { 58 | return nil, fmt.Errorf("Error unmarshalling object header %s=%s: %v", name, v, err) 59 | } 60 | return obj, nil 61 | } else { 62 | return nil, nil 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | npx jitsu-build-scripts docker -t bulker,ingest,syncctl,sidecar,ingmgr,cfgkpr --platform linux/amd64,linux/arm64 --push $@ 4 | 5 | -------------------------------------------------------------------------------- /sidecar.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24.2-bookworm as build 2 | 3 | RUN apt-get install gcc libc6-dev 4 | 5 | RUN mkdir /app 6 | WORKDIR /app 7 | 8 | RUN mkdir jitsubase sync-sidecar 9 | 10 | COPY jitsubase/go.* ./jitsubase/ 11 | COPY bulkerlib/go.* ./bulkerlib/ 12 | COPY sync-sidecar/go.* ./sync-sidecar/ 13 | 14 | RUN go work init jitsubase bulkerlib sync-sidecar 15 | 16 | WORKDIR /app/sync-sidecar 17 | 18 | RUN go mod download 19 | 20 | WORKDIR /app 21 | 22 | COPY . . 23 | 24 | # Build bulker 25 | RUN go build -o sidecar ./sync-sidecar 26 | 27 | ####################################### 28 | # FINAL STAGE 29 | FROM debian:bookworm-slim as final 30 | 31 | RUN apt-get update -y 32 | RUN apt-get install -y ca-certificates curl 33 | 34 | ENV TZ=UTC 35 | 36 | RUN mkdir /app 37 | WORKDIR /app 38 | 39 | # Copy bulkerapp 40 | COPY --from=build /app/sidecar ./ 41 | 42 | CMD ["/app/sidecar"] 43 | -------------------------------------------------------------------------------- /sidecarbuild.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker buildx build --platform linux/arm64 -f sidecar.Dockerfile -t jitsucom/sidecar:dev2 --push . -------------------------------------------------------------------------------- /sync-controller/app.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/jackc/pgx/v5/pgxpool" 7 | "github.com/jitsucom/bulker/jitsubase/appbase" 8 | "github.com/jitsucom/bulker/jitsubase/pg" 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | type Context struct { 14 | config *Config 15 | dbpool *pgxpool.Pool 16 | jobRunner *JobRunner 17 | taskManager *TaskManager 18 | server *http.Server 19 | } 20 | 21 | func (a *Context) InitContext(settings *appbase.AppSettings) error { 22 | var err error 23 | a.config = &Config{} 24 | err = appbase.InitAppConfig(a.config, settings) 25 | if err != nil { 26 | return err 27 | } 28 | a.dbpool, err = pg.NewPGPool(a.config.DatabaseURL) 29 | if err != nil { 30 | return fmt.Errorf("Unable to create postgres connection pool: %v\n", err) 31 | } 32 | //err = InitDBSchema(a.dbpool) 33 | //if err != nil { 34 | // return err 35 | //} 36 | a.jobRunner, err = NewJobRunner(a) 37 | if err != nil { 38 | return err 39 | } 40 | a.taskManager, err = NewTaskManager(a) 41 | 42 | router := NewRouter(a) 43 | a.server = &http.Server{ 44 | Addr: fmt.Sprintf("0.0.0.0:%d", a.config.HTTPPort), 45 | Handler: router.Engine(), 46 | ReadTimeout: time.Second * 60, 47 | ReadHeaderTimeout: time.Second * 60, 48 | IdleTimeout: time.Second * 65, 49 | } 50 | return nil 51 | } 52 | 53 | func (a *Context) Cleanup() error { 54 | a.taskManager.Close() 55 | a.jobRunner.Close() 56 | a.dbpool.Close() 57 | return nil 58 | } 59 | 60 | func (a *Context) ShutdownSignal() error { 61 | _ = a.server.Shutdown(context.Background()) 62 | return nil 63 | } 64 | 65 | func (a *Context) Server() *http.Server { 66 | return a.server 67 | } 68 | 69 | func (a *Context) Config() *Config { 70 | return a.config 71 | } 72 | -------------------------------------------------------------------------------- /sync-controller/examples/docker-compose/README.md: -------------------------------------------------------------------------------- 1 | # Proof of concept 2 | 3 | This is proof of concept. Docker-compose runtime is not supported by Sync Controller -------------------------------------------------------------------------------- /sync-controller/examples/docker-compose/catalog.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /sync-controller/examples/docker-compose/config.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /sync-controller/examples/docker-compose/github_compose.yaml: -------------------------------------------------------------------------------- 1 | version: "2.3" 2 | 3 | services: 4 | init: 5 | container_name: init 6 | image: busybox:1.36.0 7 | volumes: 8 | - pipes:/pipes 9 | entrypoint: ["sh", "-c", "rm -f /pipes/*; mkfifo /pipes/stdout; mkfifo /pipes/stderr"] 10 | restart: "no" 11 | jitsu: 12 | container_name: github 13 | image: airbyte/source-github:0.4.3 14 | entrypoint: [ 'sh', '-c', 'eval "$$AIRBYTE_ENTRYPOINT read --config /config/config.json --catalog /config/catalog.json --state /config/state.json" 2> /pipes/stderr > /pipes/stdout' ] 15 | depends_on: 16 | init: 17 | condition: service_completed_successfully 18 | volumes: 19 | - ./config.json:/config/config.json 20 | - ./catalog.json:/config/catalog.json 21 | - ./state.json:/config/state.json 22 | - pipes:/pipes 23 | restart: "no" 24 | sidecar: 25 | container_name: sidecar 26 | image: jitsucom/sidecar:latest 27 | volumes: 28 | - pipes:/pipes 29 | restart: "no" 30 | environment: 31 | SOURCE_ID: github 32 | TASK_ID: "1" 33 | BULKER_URL: http://localhost:3042 34 | BULKER_AUTH_TOKEN: 123 35 | CONNECTION_ID: ccc 36 | TASKS_CONNECTION_ID: tasks 37 | STATE_CONNECTION_ID: tasks_state 38 | STDOUT_PIPE_FILE: /pipes/stdout 39 | STDERR_PIPE_FILE: /pipes/stderr 40 | depends_on: 41 | init: 42 | condition: service_completed_successfully 43 | 44 | volumes: 45 | pipes: 46 | name: pipes 47 | -------------------------------------------------------------------------------- /sync-controller/examples/docker-compose/state.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /sync-controller/examples/k8s/README.md: -------------------------------------------------------------------------------- 1 | # Example 2 | 3 | That is example of entities Sync Controller deploys to k8s cluster to run sync task. 4 | Maybe be outdated and in some details may not reflect current state of entities. 5 | But general idea should be the same. -------------------------------------------------------------------------------- /sync-controller/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/jitsucom/bulker/jitsubase/appbase" 6 | "os" 7 | ) 8 | 9 | func main() { 10 | if os.Getenv("SYNCCTL_SYNCS_ENABLED") == "false" || os.Getenv("SYNCCTL_SYNCS_ENABLED") == "0" { 11 | fmt.Println("[syncctl] Syncs are disabled. Exiting...") 12 | os.Exit(0) 13 | } 14 | settings := &appbase.AppSettings{ 15 | ConfigPath: os.Getenv("SYNCCTL_CONFIG_PATH"), 16 | Name: "syncctl", 17 | EnvPrefix: "SYNCCTL", 18 | ConfigName: "syncctl", 19 | ConfigType: "env", 20 | } 21 | application := appbase.NewApp[Config](&Context{}, settings) 22 | application.Run() 23 | } 24 | -------------------------------------------------------------------------------- /sync-controller/router.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gin-gonic/gin" 5 | "github.com/jitsucom/bulker/jitsubase/appbase" 6 | "net/http" 7 | "net/http/pprof" 8 | ) 9 | 10 | type Router struct { 11 | *appbase.Router 12 | } 13 | 14 | func NewRouter(appContext *Context) *Router { 15 | base := appbase.NewRouterBase(appContext.config.Config, []string{"/health"}) 16 | 17 | router := &Router{ 18 | Router: base, 19 | } 20 | engine := router.Engine() 21 | engine.GET("/spec", appContext.taskManager.SpecHandler) 22 | engine.POST("/check", appContext.taskManager.CheckHandler) 23 | engine.POST("/discover", appContext.taskManager.DiscoverHandler) 24 | engine.POST("/read", appContext.taskManager.ReadHandler) 25 | engine.GET("/cancel", appContext.taskManager.CancelHandler) 26 | 27 | engine.GET("/health", func(c *gin.Context) { 28 | if appContext.jobRunner.Inited() { 29 | c.JSON(http.StatusOK, gin.H{"status": "pass"}) 30 | } else { 31 | c.JSON(http.StatusServiceUnavailable, gin.H{"status": "fail"}) 32 | } 33 | }) 34 | 35 | engine.GET("/debug/pprof/profile", gin.WrapF(pprof.Profile)) 36 | engine.GET("/debug/pprof/heap", gin.WrapF(pprof.Handler("heap").ServeHTTP)) 37 | engine.GET("/debug/pprof/goroutine", gin.WrapF(pprof.Handler("goroutine").ServeHTTP)) 38 | engine.GET("/debug/pprof/block", gin.WrapF(pprof.Handler("block").ServeHTTP)) 39 | engine.GET("/debug/pprof/threadcreate", gin.WrapF(pprof.Handler("threadcreate").ServeHTTP)) 40 | engine.GET("/debug/pprof/cmdline", gin.WrapF(pprof.Handler("cmdline").ServeHTTP)) 41 | engine.GET("/debug/pprof/symbol", gin.WrapF(pprof.Handler("symbol").ServeHTTP)) 42 | engine.GET("/debug/pprof/trace", gin.WrapF(pprof.Handler("trace").ServeHTTP)) 43 | engine.GET("/debug/pprof/mutex", gin.WrapF(pprof.Handler("mutex").ServeHTTP)) 44 | engine.GET("/debug/pprof", gin.WrapF(pprof.Index)) 45 | 46 | return router 47 | } 48 | -------------------------------------------------------------------------------- /sync-sidecar/README.md: -------------------------------------------------------------------------------- 1 | # 🏍️ Sync Sidecar 2 | 3 | Sync Sidecar is meant to be used as sidecar container to Airbyte protocol compatible Source connectors in Kubernetes Pod or Docker Compose. 4 | 5 | Sync Sidecar captures data rows, logs, state and results of spec, discover and check command from Source connector: 6 | 7 | - Data rows are sent to the target destination in Bulker instance, 8 | - Logs are sent to preconfigured Bulkers destinations. 9 | - `spec`, `discover` and `check` results goes to Postgres database tables. 10 | 11 | ## Named Pipes 12 | 13 | Bulker-Sidecar uses named pipes to communicate with Source connector. 14 | In k8s environment all containers from the same Pod are running on the same machine, and it is possible to use named pipes. 15 | Source connectors entrypoint must be changed to direct their stderr and stdout output to named pipes. 16 | 17 | Volume with named pipes should be mounted to both Bulker-Sidecar and Source connector containers. 18 | InitContainer can be used to create named pipes in advance. 19 | 20 | ## Configuration 21 | 22 | - `STDOUT_PIPE_FILE` - path of named pipe for stdout of Source connector 23 | - `STDERR_PIPE_FILE` - path of named pipe for stderr of Source connector 24 | - `COMMAND` - Command that is used to run Source connector. Should be one of `spec`, `discover`, `check` or `read` 25 | - `STARTED_AT` - Timestamp when task was triggered 26 | - `DATABASE_URL` - URL of Postgres database where spec, discover, check results and read task statuses should be stored 27 | - `PACKAGE` - Name of Source connector package 28 | - `PACKAGE_VERSION` - Version of Source connector package 29 | - `STORAGE_KEY` - key to store results of `check` and `discover` commands in a database. 30 | - `SYNC_ID` - id of sync entity (bulker destination id) where pulled events should be sent. For `read` command. 31 | - `TASK_ID` - id of current running task 32 | -------------------------------------------------------------------------------- /sync-sidecar/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jitsucom/bulker/sync-sidecar 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/google/uuid v1.6.0 7 | github.com/jackc/pgx/v5 v5.7.2 8 | ) 9 | 10 | require ( 11 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 12 | github.com/jackc/pgpassfile v1.0.0 // indirect 13 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 14 | github.com/jackc/puddle/v2 v2.2.2 // indirect 15 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 16 | github.com/stretchr/testify v1.10.0 // indirect 17 | golang.org/x/crypto v0.33.0 // indirect 18 | golang.org/x/sync v0.11.0 // indirect 19 | golang.org/x/text v0.22.0 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /syncctl.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bullseye-slim as main 2 | 3 | RUN apt-get update -y 4 | RUN apt-get install -y ca-certificates curl 5 | 6 | ENV TZ=UTC 7 | 8 | FROM golang:1.24.2-bullseye as build 9 | 10 | RUN apt-get install gcc libc6-dev 11 | 12 | #RUN wget -qO - https://packages.confluent.io/deb/7.2/archive.key | apt-key add - 13 | #RUN echo "deb https://packages.confluent.io/deb/7.2 stable main" > /etc/apt/sources.list.d/backports.list 14 | #RUN echo "deb https://packages.confluent.io/clients/deb buster main" > /etc/apt/sources.list.d/backports.list 15 | #RUN apt-get update 16 | #RUN apt-get install -y librdkafka1 librdkafka-dev 17 | 18 | RUN mkdir /app 19 | WORKDIR /app 20 | 21 | RUN mkdir jitsubase bulkerlib bulkerapp sync-controller sync-sidecar 22 | 23 | 24 | COPY jitsubase/go.* ./jitsubase/ 25 | COPY bulkerlib/go.* ./bulkerlib/ 26 | COPY bulkerapp/go.* ./bulkerapp/ 27 | COPY sync-controller/go.* ./sync-controller/ 28 | COPY sync-sidecar/go.* ./sync-sidecar/ 29 | 30 | 31 | RUN go work init jitsubase bulkerlib bulkerapp sync-controller sync-sidecar 32 | 33 | WORKDIR /app/sync-controller 34 | 35 | RUN go mod download 36 | 37 | WORKDIR /app 38 | 39 | COPY . . 40 | 41 | # Build bulker 42 | RUN go build -o syncctl ./sync-controller 43 | 44 | ####################################### 45 | # FINAL STAGE 46 | FROM main as final 47 | 48 | RUN mkdir /app 49 | WORKDIR /app 50 | 51 | COPY --from=build /app/syncctl ./ 52 | #COPY ./config.yaml ./ 53 | 54 | CMD ["/app/syncctl"] 55 | -------------------------------------------------------------------------------- /syncctlbuild.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker buildx build --platform linux/amd64 -f syncctl.Dockerfile -t jitsucom/syncctl:latest --push . --------------------------------------------------------------------------------