├── .codecov ├── scripts ├── parallel_read │ ├── .gitignore │ ├── ydb_20251006.csv │ ├── yq_preprod_20250422.csv │ ├── yq_20250303.csv │ └── yq_dev_20250810.csv ├── debug │ ├── dqrun │ │ ├── ydb.cloud.yql │ │ ├── ydb.local.yql │ │ ├── ch.cloud.yql │ │ ├── mysql.local.yql │ │ ├── pg.single.yql │ │ ├── pg.cloud.yql │ │ ├── debug.yql │ │ ├── ch.single.yql │ │ ├── pg.local.yql │ │ ├── join.local.benchmark_1g.yql │ │ ├── join.cloud.different.yql │ │ ├── join.cloud.example_1.yql │ │ ├── join.cloud.example_2.yql │ │ ├── ch.local.yql │ │ └── join.local.different.yql │ ├── kqprun │ │ ├── script.example.yql │ │ ├── script.mssql.local.txt │ │ ├── script.mysql.local.txt │ │ ├── script.mongodb.local.txt │ │ ├── script.redis.local.txt │ │ ├── script.benchmark_1g.txt │ │ ├── scheme.YQ-4416.txt │ │ ├── scheme.ydb.local.txt │ │ ├── script.join.yql │ │ ├── scheme.mysql.local.txt │ │ ├── scheme.mssql.local.txt │ │ ├── scheme.ydb.cloud.txt │ │ ├── scheme.clickhouse.local.txt │ │ ├── scheme.redis.local.txt │ │ ├── script.postgresql.local.txt │ │ ├── scheme.mongodb.local.txt │ │ ├── script.YQ-4416.txt │ │ ├── scheme.postgresql.local.txt │ │ ├── script.clickhouse.local.txt │ │ ├── script.ydb.local.txt │ │ └── scheme.join.txt │ ├── tls │ │ ├── grpc │ │ │ ├── README.md │ │ │ ├── localhost.crt │ │ │ ├── localhost.pem │ │ │ ├── root.crt │ │ │ └── root.pem │ │ └── oracle │ │ │ ├── extfile.cnf │ │ │ ├── generate_cert.sh │ │ │ └── client.csr │ ├── config │ │ ├── observation │ │ │ ├── kubernetes.yaml │ │ │ └── static.yaml │ │ ├── server │ │ │ ├── server.tls.yaml │ │ │ └── server.no_tls.txt │ │ └── client │ │ │ ├── prometheus.local.txt │ │ │ ├── ydb.local.txt │ │ │ ├── ydb_no_cred.local.txt │ │ │ ├── mysql.local.txt │ │ │ ├── s3.local.txt │ │ │ ├── redis.local.txt │ │ │ ├── ch.local.txt │ │ │ ├── opensearch.local.txt │ │ │ ├── ms_sql_server.local.txt │ │ │ ├── gp.local.txt │ │ │ ├── pg.local.txt │ │ │ ├── ydb_iam.local.txt │ │ │ └── mongodb.local.txt │ ├── postgresql │ │ └── maintain │ │ │ ├── list_schemas.sh │ │ │ └── show_table_size.sh │ └── clickhouse │ │ ├── config │ │ └── z_log_disable.xml │ │ └── maintain │ │ └── show_table_size.sh ├── timezone │ └── trino │ │ ├── catalog │ │ ├── postgresql.properties │ │ └── clickhouse.properties │ │ ├── clickhouse │ │ └── init │ │ │ └── init_db.sh │ │ ├── postgresql │ │ └── init │ │ │ └── init_db.sh │ │ └── docker-compose.yaml └── bench │ ├── ydb_sls_local.txt │ ├── ydb_sls_cloud.txt │ ├── redis_columns.txt │ ├── postgresql_datetime.txt │ ├── postgresql_queue.txt │ └── docker-compose.yaml ├── app ├── server │ ├── datasource │ │ ├── nosql │ │ │ ├── doc.go │ │ │ ├── mongodb │ │ │ │ └── doc.go │ │ │ ├── opensearch │ │ │ │ └── doc.go │ │ │ └── redis │ │ │ │ ├── trino │ │ │ │ ├── etc │ │ │ │ │ ├── log.properties │ │ │ │ │ ├── node.properties │ │ │ │ │ ├── config.properties │ │ │ │ │ ├── catalog │ │ │ │ │ │ └── redis.properties │ │ │ │ │ ├── jvm.config │ │ │ │ │ └── table-descriptions │ │ │ │ │ │ └── example_table.json │ │ │ │ └── docker-compose.yaml │ │ │ │ ├── consts.go │ │ │ │ └── demo │ │ │ │ └── docker-compose.yml │ │ ├── rdbms │ │ │ ├── mysql │ │ │ │ ├── doc.go │ │ │ │ ├── util.go │ │ │ │ ├── type_names.go │ │ │ │ └── table_metadata_query.go │ │ │ ├── ydb │ │ │ │ ├── doc.go │ │ │ │ ├── retry.go │ │ │ │ ├── split.proto │ │ │ │ └── table_metadata_cache │ │ │ │ │ ├── factory.go │ │ │ │ │ ├── value.proto │ │ │ │ │ ├── noop.go │ │ │ │ │ └── interface.go │ │ │ ├── oracle │ │ │ │ ├── doc.go │ │ │ │ ├── retry.go │ │ │ │ └── table_metadata_query.go │ │ │ ├── clickhouse │ │ │ │ ├── doc.go │ │ │ │ ├── type_names.go │ │ │ │ ├── common.go │ │ │ │ └── table_metadata_query.go │ │ │ ├── postgresql │ │ │ │ ├── doc.go │ │ │ │ └── table_metadata_query.go │ │ │ ├── doc.go │ │ │ ├── ms_sql_server │ │ │ │ ├── doc.go │ │ │ │ ├── table_metadata_query.go │ │ │ │ └── connection.go │ │ │ ├── utils │ │ │ │ ├── doc.go │ │ │ │ ├── split_provider.go │ │ │ │ ├── queryphase_string.go │ │ │ │ ├── select_query_args.go │ │ │ │ └── unit_test_helpers_test.go │ │ │ └── logging │ │ │ │ ├── consts.go │ │ │ │ └── split.proto │ │ ├── prometheus │ │ │ ├── analysis │ │ │ │ ├── Makefile │ │ │ │ └── trino │ │ │ │ │ ├── catalog │ │ │ │ │ └── prometheus.properties │ │ │ │ │ ├── info.trino │ │ │ │ │ ├── prometheus.yml │ │ │ │ │ ├── env.dockerfile │ │ │ │ │ ├── Makefile │ │ │ │ │ └── docker-compose.yml │ │ │ ├── errors.go │ │ │ ├── doc.go │ │ │ └── type_mapping.go │ │ ├── doc.go │ │ └── mock.go │ ├── utils │ │ ├── doc.go │ │ ├── service.go │ │ ├── counter.go │ │ ├── counter_test.go │ │ ├── retry │ │ │ └── error_checker.go │ │ └── decimal │ │ │ ├── deserialize.go │ │ │ └── serialize.go │ ├── paging │ │ ├── doc.go │ │ └── sink_string.go │ ├── streaming │ │ └── doc.go │ ├── doc.go │ ├── conversion │ │ ├── factory.go │ │ ├── strftime.go │ │ └── interface.go │ ├── config │ │ ├── config.prod.txt │ │ ├── config.debug.txt │ │ ├── config_test.go │ │ └── config.prod.yaml │ └── observation │ │ └── storage.go ├── client │ ├── doc.go │ ├── utils │ │ ├── flags.go │ │ └── preset.go │ ├── metrics │ │ ├── cmd.go │ │ └── client.go │ ├── cmd.go │ └── ydb │ │ └── cmd.go ├── bench │ ├── cpu_utilization_monitor.go │ ├── cpu_utilization_monitor_noop.go │ ├── cpu_utilization_monitor_linux.go │ └── cpu_utilization_monitor_darwin.go ├── observation │ ├── discovery │ │ ├── interface.go │ │ ├── factory.go │ │ └── static.go │ └── cmd.go ├── main.go ├── config │ ├── client.proto │ └── observation.proto └── version │ └── version.go ├── tools ├── ydb │ ├── dump_tablet_id_data │ │ └── .gitignore │ └── query_service_negative │ │ ├── init │ │ ├── init_ydb │ │ └── 01_basic.sh │ │ └── docker-compose.yml └── docker_compose_update │ └── main.go ├── .github ├── CODEOWNERS └── workflows │ └── copilot-reminder.yml ├── library └── go │ ├── yson │ ├── yson2json │ │ ├── gotest │ │ │ └── ya.make │ │ └── ya.make │ ├── testdata │ │ ├── line.yson │ │ └── line_simple.yson │ ├── gotest │ │ └── ya.make │ ├── decoder.go │ ├── compat.go │ ├── infer_example_test.go │ ├── reader_test.go │ ├── error.go │ ├── ya.make │ ├── struct_tag.go │ ├── struct_tag_test.go │ ├── generic_test.go │ ├── time.go │ └── infer.go │ ├── httputil │ └── headers │ │ ├── cookie.go │ │ ├── user_agent.go │ │ ├── tvm.go │ │ ├── authorization.go │ │ └── authorization_test.go │ ├── core │ ├── xerrors │ │ ├── doc.go │ │ ├── benchxerrors │ │ │ └── benchxerrors.go │ │ ├── errorf_multiple_errors_test.go │ │ ├── internal │ │ │ └── modes │ │ │ │ ├── stack_frames_count.go │ │ │ │ └── stack_trace_mode.go │ │ └── new.go │ ├── metrics │ │ ├── collect │ │ │ ├── collect.go │ │ │ └── policy │ │ │ │ └── inflight │ │ │ │ └── inflight_opts.go │ │ ├── prometheus │ │ │ ├── timer.go │ │ │ ├── histogram.go │ │ │ ├── timer_test.go │ │ │ ├── gauge.go │ │ │ ├── int_gauge.go │ │ │ ├── counter.go │ │ │ ├── stream.go │ │ │ ├── gauge_test.go │ │ │ └── counter_test.go │ │ ├── mock │ │ │ ├── timer.go │ │ │ ├── gauge.go │ │ │ ├── int_gauge.go │ │ │ ├── counter.go │ │ │ ├── histogram.go │ │ │ └── registry_opts.go │ │ ├── nop │ │ │ ├── timer.go │ │ │ ├── gauge.go │ │ │ ├── counter.go │ │ │ ├── int_gauge.go │ │ │ └── histogram.go │ │ ├── solomon │ │ │ ├── metrics_opts.go │ │ │ ├── spack_compression_test.go │ │ │ ├── timer_test.go │ │ │ ├── func_gauge_test.go │ │ │ └── func_int_gauge_test.go │ │ └── internal │ │ │ └── pkg │ │ │ ├── metricsutil │ │ │ └── buckets.go │ │ │ └── registryutil │ │ │ └── registryutil_test.go │ └── buildinfo │ │ └── test │ │ └── main.go │ ├── test │ ├── testhelpers │ │ ├── recurse.go │ │ └── remove_lines.go │ ├── yatest │ │ ├── arcadia.go │ │ ├── go.go │ │ └── env_test.go │ └── assertpb │ │ └── assert.go │ └── x │ ├── xreflect │ └── assign.go │ ├── xruntime │ ├── stacktrace_benchmark_test.go │ └── stacktrace.go │ └── xsync │ └── singleinflight.go ├── docs ├── type_mapping.jpg ├── type_mapping.png ├── sequence_diagram.png ├── providers_vs_connector.jpg ├── providers_vs_connector.png ├── append_to_arrow_builders.jpg ├── append_to_arrow_builders.png └── generate.py ├── .gitattributes ├── codecov.yml ├── tests ├── utils │ ├── constraints.go │ ├── id_array_builders.go │ └── context.go └── infra │ └── datasource │ ├── mysql │ ├── custom.cnf │ └── datasource.go │ ├── opensearch │ ├── init │ │ ├── opensearch.yml │ │ ├── entrypoint.sh │ │ └── Dockerfile │ └── datasource.go │ ├── ms_sql_server │ ├── init │ │ ├── entrypoint.sh │ │ ├── Dockerfile │ │ └── configure-db.sh │ └── datasource.go │ ├── ydb │ ├── init │ │ └── init_ydb │ └── datasource.go │ ├── datasource.go │ ├── greenplum │ ├── init │ │ └── entrypoint.sh │ └── datasource.go │ ├── redis │ └── datasource.go │ ├── postgresql │ └── datasource.go │ └── oracle │ └── datasource.go ├── common ├── consts.go ├── stacktrace.go ├── duration.go ├── call_stack.go ├── credentials.go ├── protobuf.go ├── endpoint.go ├── time.go ├── config.go └── client_basic.go ├── Dockerfile.base ├── .gitignore ├── examples ├── systemd │ └── fq-connector-go.service └── docker_compose │ ├── docker-compose.yaml │ └── README.md └── Dockerfile.release /.codecov: -------------------------------------------------------------------------------- 1 | comment: false 2 | -------------------------------------------------------------------------------- /scripts/parallel_read/.gitignore: -------------------------------------------------------------------------------- 1 | *.png 2 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/doc.go: -------------------------------------------------------------------------------- 1 | package nosql 2 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/mysql/doc.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/mongodb/doc.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/opensearch/doc.go: -------------------------------------------------------------------------------- 1 | package opensearch 2 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/ydb.cloud.yql: -------------------------------------------------------------------------------- 1 | SELECT * FROM ydb_dev.simple; 2 | -------------------------------------------------------------------------------- /tools/ydb/dump_tablet_id_data/.gitignore: -------------------------------------------------------------------------------- 1 | dump_tablet_id_data 2 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/trino/etc/log.properties: -------------------------------------------------------------------------------- 1 | io.trino=INFO -------------------------------------------------------------------------------- /scripts/debug/dqrun/ydb.local.yql: -------------------------------------------------------------------------------- 1 | SELECT * FROM ydb_local.simple; 2 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/ch.cloud.yql: -------------------------------------------------------------------------------- 1 | SELECT * FROM clickhouse_dev.example_1; 2 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.example.yql: -------------------------------------------------------------------------------- 1 | SELECT * FROM ch_local.primitives; 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @vitalyisaev2 @uzhastik @APozdniakov @GrigoriyPA @yumkam @buhtr 2 | -------------------------------------------------------------------------------- /library/go/yson/yson2json/gotest/ya.make: -------------------------------------------------------------------------------- 1 | GO_TEST_FOR(yt/go/yson/yson2json) 2 | 3 | END() 4 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/mysql.local.yql: -------------------------------------------------------------------------------- 1 | --SELECT * FROM mysql_local.simple; 2 | SELECT 1 * 2; 3 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.mssql.local.txt: -------------------------------------------------------------------------------- 1 | SELECT * FROM mssql_external_datasource.users; 2 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.mysql.local.txt: -------------------------------------------------------------------------------- 1 | SELECT * FROM mysql_external_datasource.datetimes; 2 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/pg.single.yql: -------------------------------------------------------------------------------- 1 | SELECT COL1 FROM rtmr_dev00_postgresql.column_projection; 2 | 3 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.mongodb.local.txt: -------------------------------------------------------------------------------- 1 | SELECT * FROM mongodb_external_datasource.primitives; 2 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.redis.local.txt: -------------------------------------------------------------------------------- 1 | SELECT * FROM external_data_source.`sample_session:*`; 2 | -------------------------------------------------------------------------------- /app/server/datasource/prometheus/analysis/Makefile: -------------------------------------------------------------------------------- 1 | bench: 2 | go run bench/client.go 3 | 4 | .PHONY: bench -------------------------------------------------------------------------------- /docs/type_mapping.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/docs/type_mapping.jpg -------------------------------------------------------------------------------- /docs/type_mapping.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/docs/type_mapping.png -------------------------------------------------------------------------------- /library/go/httputil/headers/cookie.go: -------------------------------------------------------------------------------- 1 | package headers 2 | 3 | const ( 4 | CookieKey = "Cookie" 5 | ) 6 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/pg.cloud.yql: -------------------------------------------------------------------------------- 1 | pragma UseBlocks; 2 | SELECT * FROM postgresql_streaming.example_1; 3 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ydb/doc.go: -------------------------------------------------------------------------------- 1 | // Package ydb contains code specific for YDB database. 2 | package ydb 3 | -------------------------------------------------------------------------------- /docs/sequence_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/docs/sequence_diagram.png -------------------------------------------------------------------------------- /library/go/httputil/headers/user_agent.go: -------------------------------------------------------------------------------- 1 | package headers 2 | 3 | const ( 4 | UserAgentKey = "User-Agent" 5 | ) 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.pb.go linguist-generated=true 2 | *.pb.go -diff 3 | *.pb.go linguist-vendored=true 4 | *.pb.go binary 5 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/debug.yql: -------------------------------------------------------------------------------- 1 | SELECT col_14_date, col_16_datetime , col_17_date64 FROM rtmr_dev00_clickhouse.primitives; -------------------------------------------------------------------------------- /app/server/datasource/rdbms/oracle/doc.go: -------------------------------------------------------------------------------- 1 | // Package oracle contains code specific for Oracle database. 2 | package oracle 3 | -------------------------------------------------------------------------------- /app/server/utils/doc.go: -------------------------------------------------------------------------------- 1 | // Package utils contains various helpers and utility functions 2 | package utils //nolint:revive 3 | -------------------------------------------------------------------------------- /docs/providers_vs_connector.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/docs/providers_vs_connector.jpg -------------------------------------------------------------------------------- /docs/providers_vs_connector.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/docs/providers_vs_connector.png -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "api" 3 | - "app/config" 4 | - "library" 5 | - "*_mock.go" 6 | - "*_test.go" 7 | - "tests" 8 | -------------------------------------------------------------------------------- /docs/append_to_arrow_builders.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/docs/append_to_arrow_builders.jpg -------------------------------------------------------------------------------- /docs/append_to_arrow_builders.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/docs/append_to_arrow_builders.png -------------------------------------------------------------------------------- /library/go/yson/testdata/line.yson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/library/go/yson/testdata/line.yson -------------------------------------------------------------------------------- /scripts/debug/dqrun/ch.single.yql: -------------------------------------------------------------------------------- 1 | pragma UseBlocks; 2 | 3 | SELECT id, id * id AS id_square FROM rtmr_dev00_clickhouse.example_1; 4 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/clickhouse/doc.go: -------------------------------------------------------------------------------- 1 | // Package clickhouse contains code specific for ClickHouse database. 2 | package clickhouse 3 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/postgresql/doc.go: -------------------------------------------------------------------------------- 1 | // Package postgresql contains code specific for PostgreSQL database. 2 | package postgresql 3 | -------------------------------------------------------------------------------- /tests/utils/constraints.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | type TableIDTypes interface { 4 | int32 | int64 | []byte | string 5 | } 6 | -------------------------------------------------------------------------------- /app/client/doc.go: -------------------------------------------------------------------------------- 1 | // Package client contains code of a simple CLI application 2 | // that can be used for server debugging. 3 | package client 4 | -------------------------------------------------------------------------------- /app/server/datasource/prometheus/analysis/trino/catalog/prometheus.properties: -------------------------------------------------------------------------------- 1 | connector.name=prometheus 2 | prometheus.uri=http://prometheus:9090 -------------------------------------------------------------------------------- /common/consts.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | const ( 4 | ForbidRetries = "forbid_retries" 5 | TestName = "test_name" 6 | ) 7 | -------------------------------------------------------------------------------- /library/go/yson/gotest/ya.make: -------------------------------------------------------------------------------- 1 | GO_TEST_FOR(yt/go/yson) 2 | 3 | DATA(arcadia/yt/go/yson/testdata) 4 | 5 | TEST_CWD(yt/go/yson) 6 | 7 | END() 8 | -------------------------------------------------------------------------------- /library/go/yson/testdata/line_simple.yson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/fq-connector-go/HEAD/library/go/yson/testdata/line_simple.yson -------------------------------------------------------------------------------- /scripts/parallel_read/ydb_20251006.csv: -------------------------------------------------------------------------------- 1 | MaxTasksPerStage,duration 2 | 4,14m 20s 3 | 8,7m 8s 4 | 16,4m 47s 5 | 32,3m 22s 6 | 64,3m 9s 7 | 128,3m 9s -------------------------------------------------------------------------------- /app/server/paging/doc.go: -------------------------------------------------------------------------------- 1 | // Package paging contains logic of splitting incoming data stream 2 | // into the separate pages or blocks. 3 | package paging 4 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/doc.go: -------------------------------------------------------------------------------- 1 | // Package rdbms contains generalized abstractions suitable for use with 2 | // any relational database. 3 | package rdbms 4 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ms_sql_server/doc.go: -------------------------------------------------------------------------------- 1 | // Package ms_sql_server contains code specific for Microsoft SQL Server database. 2 | package ms_sql_server 3 | -------------------------------------------------------------------------------- /tests/infra/datasource/mysql/custom.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | character-set-server = utf8 3 | collation-server = utf8_unicode_ci 4 | skip-character-set-client-handshake 5 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/trino/etc/node.properties: -------------------------------------------------------------------------------- 1 | node.environment=production 2 | node.id=ffffffff-ffff-ffff-ffff-ffffffffffff 3 | node.data-dir=/var/trino/data -------------------------------------------------------------------------------- /app/server/datasource/prometheus/errors.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrEmptyTimeSeries = errors.New("empty time series") 7 | ) 8 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/pg.local.yql: -------------------------------------------------------------------------------- 1 | -- SELECT col_19_bytea FROM rtmr_dev00_postgresql.primitives; 2 | SELECT MAX(id + LENGTH(col)) FROM rtmr_dev00_postgresql.benchmark_1g; 3 | -------------------------------------------------------------------------------- /tests/infra/datasource/opensearch/init/opensearch.yml: -------------------------------------------------------------------------------- 1 | cluster.name: opensearch-cluster 2 | node.name: opensearch-node 3 | network.host: 0.0.0.0 4 | discovery.type: single-node -------------------------------------------------------------------------------- /app/server/datasource/rdbms/utils/doc.go: -------------------------------------------------------------------------------- 1 | // Package utils contains helper types and functions that can be used by any 2 | // relational data source. 3 | package utils //nolint:revive 4 | -------------------------------------------------------------------------------- /library/go/core/xerrors/doc.go: -------------------------------------------------------------------------------- 1 | // package xerrors is a drop in replacement for errors and golang.org/x/xerrors packages and functionally for github.com/pkg/errors. 2 | package xerrors 3 | -------------------------------------------------------------------------------- /scripts/parallel_read/yq_preprod_20250422.csv: -------------------------------------------------------------------------------- 1 | threads,1 instance,3 instances 2 | 1,1m 9s,1m 37s 3 | 2,47s,42s 4 | 4,37s,45s 5 | 8,33s,25s 6 | 16,26s,20s 7 | 32,26s,13s 8 | 64,NaN,14s 9 | -------------------------------------------------------------------------------- /app/server/datasource/doc.go: -------------------------------------------------------------------------------- 1 | // Package datasource describes the interface that must be satisfied by any external data source 2 | // accessible through the connector. 3 | package datasource 4 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/trino/etc/config.properties: -------------------------------------------------------------------------------- 1 | coordinator=true 2 | node-scheduler.include-coordinator=true 3 | http-server.http.port=8080 4 | discovery.uri=http://localhost:8080 -------------------------------------------------------------------------------- /app/server/datasource/prometheus/analysis/trino/info.trino: -------------------------------------------------------------------------------- 1 | use prometheus.default; 2 | describe up; 3 | show tables like 'go_mem%'; 4 | select count(*) from up; 5 | select * from up limit 10; -------------------------------------------------------------------------------- /app/server/datasource/prometheus/doc.go: -------------------------------------------------------------------------------- 1 | // Package prometheus contains the implementation of Prometheus (monitoring system and time series database) based data source 2 | package prometheus 3 | -------------------------------------------------------------------------------- /scripts/timezone/trino/catalog/postgresql.properties: -------------------------------------------------------------------------------- 1 | connector.name=postgresql 2 | connection-url=jdbc:postgresql://postgres:5432/db 3 | connection-user=user 4 | connection-password=password -------------------------------------------------------------------------------- /scripts/debug/tls/grpc/README.md: -------------------------------------------------------------------------------- 1 | In order to avoid certificate issueing problems, the keypair and root CA was taken from https://github.com/grpc/grpc/tree/master/examples/python/auth/credentials 2 | -------------------------------------------------------------------------------- /scripts/debug/tls/oracle/extfile.cnf: -------------------------------------------------------------------------------- 1 | [v3_ca] 2 | basicConstraints = CA:FALSE 3 | keyUsage = digitalSignature, keyEncipherment 4 | subjectAltName = DNS:localhost, DNS:oracle, IP:127.0.0.1, IP:172.17.0.1 -------------------------------------------------------------------------------- /Dockerfile.base: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | LABEL org.opencontainers.image.source=https://github.com/ydb-platform/fq-connector-go 4 | 5 | RUN apk add libc6-compat busybox-extras bind-tools && apk cache clean 6 | -------------------------------------------------------------------------------- /app/server/datasource/prometheus/analysis/trino/prometheus.yml: -------------------------------------------------------------------------------- 1 | scrape_configs: 2 | 3 | - job_name: 'test-job' 4 | scrape_interval: 1s 5 | static_configs: 6 | - targets: [ 'test-server:8081' ] -------------------------------------------------------------------------------- /app/server/streaming/doc.go: -------------------------------------------------------------------------------- 1 | // Package streaming contains the logic of converting the data stream 2 | // coming from the data source into the data stream sent over the network to the client. 3 | package streaming 4 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.benchmark_1g.txt: -------------------------------------------------------------------------------- 1 | SELECT MAX(length(ch.col) + length(pg.col)) 2 | FROM clickhouse_cloud.benchmark_1g as ch 3 | JOIN postgresql_cloud.benchmark_1g as pg 4 | ON ch.id = pg.id; 5 | 6 | -------------------------------------------------------------------------------- /tests/infra/datasource/ms_sql_server/init/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start the script to create the DB and user 4 | /usr/config/configure-db.sh & 5 | 6 | # Start SQL Server 7 | /opt/mssql/bin/sqlservr 8 | -------------------------------------------------------------------------------- /tools/ydb/query_service_negative/init/init_ydb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | /ydb config profile create tests-ydb-client --endpoint grpc://localhost:2136 --database /local 6 | 7 | /bin/bash ./01_basic.sh 8 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/join.local.benchmark_1g.yql: -------------------------------------------------------------------------------- 1 | SELECT MAX(LENGTH(ch.col) - LENGTH(pg.col)) 2 | FROM rtmr_dev00_clickhouse.benchmark_1g as ch 3 | JOIN rtmr_dev00_postgresql.benchmark_1g as pg 4 | ON ch.id = pg.id; 5 | -------------------------------------------------------------------------------- /library/go/yson/yson2json/ya.make: -------------------------------------------------------------------------------- 1 | GO_LIBRARY() 2 | 3 | INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) 4 | 5 | SRCS(raw_message.go) 6 | 7 | GO_TEST_SRCS(raw_message_test.go) 8 | 9 | END() 10 | 11 | RECURSE(gotest) 12 | -------------------------------------------------------------------------------- /app/server/doc.go: -------------------------------------------------------------------------------- 1 | // Package server contains the code base of Connector GRPC service - 2 | // the component of YDB's Federated Query system providing the unified interface 3 | // to the external data sources. 4 | package server 5 | -------------------------------------------------------------------------------- /tests/infra/datasource/ydb/init/init_ydb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | /ydb config profile create tests-ydb-client --endpoint grpc://localhost:2136 --database /local 6 | 7 | /bin/bash ./01_basic.sh 8 | /bin/bash ./02_large.sh 9 | -------------------------------------------------------------------------------- /scripts/timezone/trino/catalog/clickhouse.properties: -------------------------------------------------------------------------------- 1 | connector.name=clickhouse 2 | connection-url=jdbc:clickhouse://clickhouse:8123/db 3 | connection-user=user 4 | connection-password=password 5 | unsupported-type-handling=CONVERT_TO_VARCHAR -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/trino/etc/catalog/redis.properties: -------------------------------------------------------------------------------- 1 | connector.name=redis 2 | redis.table-names=example_table 3 | redis.nodes=valkey:6379 4 | redis.default-schema=default 5 | redis.table-description-dir=/etc/trino/table-descriptions -------------------------------------------------------------------------------- /app/server/utils/service.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | // Service is an abstract interface representing some internal service 4 | // running in a distinct thread. 5 | type Service interface { 6 | Start() error 7 | Stop() 8 | } 9 | -------------------------------------------------------------------------------- /scripts/debug/config/observation/kubernetes.yaml: -------------------------------------------------------------------------------- 1 | endpoint: 2 | hostname: 0.0.0.0 3 | port: 8082 4 | 5 | discovery: 6 | kubernetes: 7 | label_selector: "kubernetes.io/service-name=yq-connector" 8 | 9 | polling_interval: "5s" 10 | -------------------------------------------------------------------------------- /tests/infra/datasource/datasource.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 5 | ) 6 | 7 | type DataSource struct { 8 | Instances []*api_common.TGenericDataSourceInstance 9 | } 10 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/mysql/util.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import "golang.org/x/exp/constraints" 4 | 5 | type number interface { 6 | constraints.Integer | constraints.Float 7 | } 8 | 9 | type stringLike interface { 10 | []byte | string 11 | } 12 | -------------------------------------------------------------------------------- /app/client/utils/flags.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | const ( 4 | ConfigFlag = "config" 5 | TableFlag = "table" 6 | DateTimeFormatFlag = "date-time-format" 7 | UserIDFlag = "user-id" 8 | SessionIDFlag = "session" 9 | ) 10 | -------------------------------------------------------------------------------- /library/go/core/metrics/collect/collect.go: -------------------------------------------------------------------------------- 1 | package collect 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 7 | ) 8 | 9 | type Func func(ctx context.Context, r metrics.Registry, c metrics.CollectPolicy) 10 | -------------------------------------------------------------------------------- /app/server/datasource/prometheus/analysis/trino/env.dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20.0 2 | 3 | WORKDIR /app 4 | 5 | COPY . . 6 | 7 | RUN export GOPATH="/app" 8 | 9 | RUN go version 10 | 11 | RUN go mod init promtrino && go mod edit -require github.com/slok/go-http-metrics@v0.12.0 && go mod tidy -------------------------------------------------------------------------------- /scripts/debug/dqrun/join.cloud.different.yql: -------------------------------------------------------------------------------- 1 | pragma UseBlocks; 2 | 3 | SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2 4 | FROM clickhouse_streaming.example_1 as ch 5 | JOIN postgresql_streaming.example_2 as pg 6 | ON ch.id = pg.id 7 | ORDER BY id; 8 | 9 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/join.cloud.example_1.yql: -------------------------------------------------------------------------------- 1 | pragma UseBlocks; 2 | 3 | SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2 4 | FROM rtmr_dev00_clickhouse.example_1 as ch 5 | JOIN rtmr_dev00_postgresql.example_1 as pg 6 | ON ch.id = pg.id 7 | ORDER BY id; 8 | 9 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/join.cloud.example_2.yql: -------------------------------------------------------------------------------- 1 | pragma UseBlocks; 2 | 3 | SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2 4 | FROM clickhouse_streaming.example_2 as ch 5 | JOIN postgresql_streaming.example_2 as pg 6 | ON ch.id = pg.id 7 | ORDER BY id; 8 | 9 | -------------------------------------------------------------------------------- /library/go/core/metrics/collect/policy/inflight/inflight_opts.go: -------------------------------------------------------------------------------- 1 | package inflight 2 | 3 | import "time" 4 | 5 | type Option func(*inflightPolicy) 6 | 7 | func WithMinCollectInterval(interval time.Duration) Option { 8 | return func(c *inflightPolicy) { 9 | c.minUpdateInterval = interval 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/ch.local.yql: -------------------------------------------------------------------------------- 1 | -- SELECT col_15_date32 FROM rtmr_dev00_clickhouse_native.primitives; 2 | -- SELECT MIN(LEN(col)) FROM rtmr_dev00_clickhouse_native.benchmark_1g; 3 | -- SELECT MIN(LEN(col1)+ LEN(col2)) FROM rtmr_dev00_clickhouse_native.benchmark_2g; 4 | SELECT * FROM rtmr_dev00_clickhouse_native.example_1; 5 | -------------------------------------------------------------------------------- /library/go/test/testhelpers/recurse.go: -------------------------------------------------------------------------------- 1 | package testhelpers 2 | 3 | // Recurse calls itself 'depth' times then executes 'f'. Useful for testing things where stack size matters. 4 | func Recurse(depth int, f func()) { 5 | if depth > 0 { 6 | depth-- 7 | Recurse(depth, f) 8 | return 9 | } 10 | 11 | f() 12 | } 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | fq-connector-go 2 | fq-connector-go-static 3 | fq-connector-go-tests 4 | ydbinit 5 | olap_inconsistency 6 | 7 | coverage* 8 | scripts/bench/postgresql 9 | scripts/bench/clickhouse 10 | scripts/bench/ydb 11 | __pycache__ 12 | *.test 13 | .vscode 14 | .idea 15 | .tmp 16 | .DS_Store 17 | 18 | .ipynb_checkpoints 19 | -------------------------------------------------------------------------------- /examples/systemd/fq-connector-go.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=YDB FQ Connector Go 3 | After=network.target 4 | 5 | [Service] 6 | Type=simple 7 | Restart=always 8 | RestartSec=10 9 | ExecStart=/opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /app/bench/cpu_utilization_monitor.go: -------------------------------------------------------------------------------- 1 | package bench 2 | 3 | // cpuUtilizationMonitor computes CPU utilization for a current process 4 | // in a platform-dependent way. 5 | type cpuUtilizationMonitor interface { 6 | // getPercentage returns utilization value in percents (like in tools similar to `top`) 7 | getPercentage() float64 8 | } 9 | -------------------------------------------------------------------------------- /app/server/conversion/factory.go: -------------------------------------------------------------------------------- 1 | package conversion 2 | 3 | import ( 4 | "github.com/ydb-platform/fq-connector-go/app/config" 5 | ) 6 | 7 | func NewCollection(cfg *config.TConversionConfig) Collection { 8 | if cfg.UseUnsafeConverters { 9 | return collectionUnsafe{} 10 | } 11 | 12 | return collectionDefault{} 13 | } 14 | -------------------------------------------------------------------------------- /library/go/httputil/headers/tvm.go: -------------------------------------------------------------------------------- 1 | package headers 2 | 3 | const ( 4 | // XYaServiceTicket is http header that should be used for service ticket transfer. 5 | XYaServiceTicketKey = "X-Ya-Service-Ticket" 6 | // XYaUserTicket is http header that should be used for user ticket transfer. 7 | XYaUserTicketKey = "X-Ya-User-Ticket" 8 | ) 9 | -------------------------------------------------------------------------------- /scripts/debug/config/observation/static.yaml: -------------------------------------------------------------------------------- 1 | endpoint: 2 | host: 0.0.0.0 3 | port: 8082 4 | 5 | discovery: 6 | static: 7 | endpoints: 8 | - host: 0.0.0.0 9 | port: 50053 10 | - host: 0.0.0.0 11 | port: 50054 12 | - host: 0.0.0.0 13 | port: 50055 14 | 15 | polling_interval: "5s" 16 | -------------------------------------------------------------------------------- /scripts/parallel_read/yq_20250303.csv: -------------------------------------------------------------------------------- 1 | threads,64_shards,256_shards,512_shards 2 | 1,7m 45s,8m 43s,10m 12s 3 | 2,3m 55s,4m 23s,5m 13s 4 | 4,2m 15s,2m 27s,2m 49s 5 | 8,1m 16s,1m 23s,1m 30s 6 | 16,1m 11s,52s,1m 11s 7 | 32,1m 1s,37s,54s 8 | 64,44s,45s,52s 9 | 128,42s,1m 7s,41s 10 | 256,53s,59s,38s 11 | 512,42s,39s,57s 12 | 1024,41s,37s,57s 13 | -------------------------------------------------------------------------------- /app/observation/discovery/interface.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | ) 8 | 9 | type Discovery interface { 10 | // Returns the list of Observation API endpoints 11 | GetEndpoints(*zap.Logger) ([]*api_common.TGenericEndpoint, error) 12 | } 13 | -------------------------------------------------------------------------------- /scripts/debug/dqrun/join.local.different.yql: -------------------------------------------------------------------------------- 1 | pragma UseBlocks; 2 | 3 | SELECT pg.col1 AS data_pg, ch.col1 AS data_ch, ydb.col1 AS data_ydb, gp.col1 AS data_gp 4 | FROM postgresql.simple AS pg 5 | JOIN clickhouse_native.simple AS ch 6 | ON pg.id = ch.id 7 | JOIN ydb.simple AS ydb 8 | ON pg.id = ydb.id 9 | JOIN greenplum.simple AS gp 10 | ON pg.id = gp.id; 11 | -------------------------------------------------------------------------------- /app/bench/cpu_utilization_monitor_noop.go: -------------------------------------------------------------------------------- 1 | //go:build !(cgo && (linux || darwin)) 2 | 3 | package bench 4 | 5 | type cpuUtilizationMonitorNoop struct { 6 | } 7 | 8 | func (mon cpuUtilizationMonitorNoop) getPercentage() float64 { 9 | return 0 10 | } 11 | 12 | func NewCPUUtilizationMonitor() cpuUtilizationMonitor { 13 | return cpuUtilizationMonitorNoop{} 14 | } 15 | -------------------------------------------------------------------------------- /library/go/core/buildinfo/test/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/ydb-platform/fq-connector-go/library/go/core/buildinfo" 7 | ) 8 | 9 | func main() { 10 | if buildinfo.Info.ProgramVersion != "" { 11 | fmt.Print(buildinfo.Info.ProgramVersion) 12 | } else { 13 | fmt.Printf("ProgramVersion is not available\n") 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /library/go/test/yatest/arcadia.go: -------------------------------------------------------------------------------- 1 | //go:build arcadia 2 | // +build arcadia 3 | 4 | package yatest 5 | 6 | import ( 7 | "os" 8 | ) 9 | 10 | func doInit() { 11 | initTestContext() 12 | } 13 | 14 | func init() { 15 | if val := os.Getenv("YA_TEST_CONTEXT_FILE"); val != "" { 16 | if _, err := os.Stat(val); err == nil { 17 | lazyInit() 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ydb/retry.go: -------------------------------------------------------------------------------- 1 | package ydb 2 | 3 | import ( 4 | grpc_codes "google.golang.org/grpc/codes" 5 | 6 | "github.com/ydb-platform/ydb-go-sdk/v3" 7 | ) 8 | 9 | func ErrorCheckerQuery(err error) bool { 10 | switch { 11 | case ydb.IsTransportError(err, grpc_codes.ResourceExhausted): 12 | return true 13 | default: 14 | return false 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.YQ-4416.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT ydb_local_password (TYPE SECRET) WITH (value = password); 2 | 3 | CREATE EXTERNAL DATA SOURCE external_datasource WITH ( 4 | SOURCE_TYPE="Ydb", 5 | LOCATION="localhost:2136", 6 | AUTH_METHOD="BASIC", 7 | LOGIN="admin", 8 | DATABASE_NAME="/Root", 9 | PASSWORD_SECRET_NAME="ydb_local_password" 10 | ); 11 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.ydb.local.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT ydb_local_password (TYPE SECRET) WITH (value = password); 2 | 3 | CREATE EXTERNAL DATA SOURCE external_datasource WITH ( 4 | SOURCE_TYPE="Ydb", 5 | LOCATION="localhost:2136", 6 | AUTH_METHOD="BASIC", 7 | LOGIN="admin", 8 | DATABASE_NAME="local", 9 | PASSWORD_SECRET_NAME="ydb_local_password" 10 | ); 11 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.join.yql: -------------------------------------------------------------------------------- 1 | SELECT pg.col1 AS data_pg, ch.col1 AS data_ch, ydb.col1 AS data_ydb, gp.col1 AS data_gp 2 | FROM postgresql_external_datasource.simple AS pg 3 | JOIN clickhouse_external_datasource.simple AS ch 4 | ON pg.id = ch.id 5 | JOIN ydb_external_datasource.simple AS ydb 6 | ON pg.id = ydb.id 7 | JOIN greenplum_external_datasource.simple AS gp 8 | ON pg.id = gp.id; 9 | -------------------------------------------------------------------------------- /scripts/debug/config/server/server.tls.yaml: -------------------------------------------------------------------------------- 1 | connector_server: 2 | endpoint: 3 | host: "0.0.0.0" 4 | port: 2130 5 | 6 | tls: 7 | ca: "./scripts/debug/tls/root.crt" 8 | key: "./scripts/debug/tls/localhost.key" 9 | cert: "./scripts/debug/tls/localhost.crt" 10 | 11 | logger: 12 | log_level: TRACE 13 | enable_sql_query_logging: true 14 | -------------------------------------------------------------------------------- /scripts/debug/config/client/prometheus.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: PROMETHEUS 13 | endpoint { 14 | host: "localhost" 15 | port: 9090 16 | } 17 | protocol: HTTP 18 | } 19 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.mysql.local.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT mysql_local_password (TYPE SECRET) WITH (value = "password"); 2 | 3 | CREATE EXTERNAL DATA SOURCE mysql_external_datasource WITH ( 4 | SOURCE_TYPE="MySQL", 5 | LOCATION="localhost:3306", 6 | AUTH_METHOD="BASIC", 7 | LOGIN="root", 8 | DATABASE_NAME="fq", 9 | PASSWORD_SECRET_NAME="mysql_local_password" 10 | ); 11 | -------------------------------------------------------------------------------- /app/server/datasource/prometheus/analysis/trino/Makefile: -------------------------------------------------------------------------------- 1 | build-env: 2 | docker build -t go_env:latest . -f env.dockerfile 3 | 4 | up: 5 | docker compose up -d 6 | 7 | trino-result: 8 | docker exec -it trino sh -c 'cd bin && ./trino -f /etc/trino/info.trino' 9 | 10 | down: 11 | @docker compose down 12 | 13 | rm: down 14 | @docker image rm go_env:latest 15 | 16 | .PHONY: up-all trino-result down rm -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.mssql.local.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT mssql_local_password (TYPE SECRET) WITH (value = "Qwerty12345!"); 2 | 3 | CREATE EXTERNAL DATA SOURCE mssql_external_datasource WITH ( 4 | SOURCE_TYPE="MsSQLServer", 5 | LOCATION="mssql:1433", 6 | AUTH_METHOD="BASIC", 7 | LOGIN="SA", 8 | DATABASE_NAME="master", 9 | PASSWORD_SECRET_NAME="mssql_local_password" 10 | ); 11 | -------------------------------------------------------------------------------- /scripts/debug/config/client/ydb.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: YDB 13 | endpoint { 14 | host: "localhost" 15 | port: 2136 16 | } 17 | database: "local" 18 | use_tls: false 19 | } 20 | 21 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.ydb.cloud.txt: -------------------------------------------------------------------------------- 1 | UPSERT OBJECT `my_sa_signature_name` (TYPE SECRET) WITH value=`my_sa_secret_value`; 2 | 3 | CREATE EXTERNAL DATA SOURCE ydb_dev WITH ( 4 | SOURCE_TYPE="Ydb", 5 | AUTH_METHOD="SERVICE_ACCOUNT", 6 | DATABASE_ID="etnra7druh6pvp7obhdr", 7 | SERVICE_ACCOUNT_ID="my_sa", 8 | SERVICE_ACCOUNT_SECRET_NAME="my_sa_signature_name", 9 | USE_TLS="TRUE" 10 | ); 11 | -------------------------------------------------------------------------------- /scripts/debug/config/client/ydb_no_cred.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: YDB 13 | endpoint { 14 | host: "localhost" 15 | port: 2136 16 | } 17 | database: "local" 18 | use_tls: false 19 | } 20 | 21 | -------------------------------------------------------------------------------- /scripts/parallel_read/yq_dev_20250810.csv: -------------------------------------------------------------------------------- 1 | Replicas,Tasks,Time 2 | 8,64,27 3 | 8,32,30 4 | 8,16,38 5 | 8,8,66 6 | 8,4,130 7 | 8,2,242 8 | 8,1,517 9 | 4,64,41 10 | 4,32,43 11 | 4,16,50 12 | 4,8,77 13 | 4,4,129 14 | 4,2,249 15 | 4,1,480 16 | 2,64,63 17 | 2,32,57 18 | 2,16,62 19 | 2,8,79 20 | 2,4,136 21 | 2,2,256 22 | 2,1,515 23 | 1,64,102 24 | 1,32,92 25 | 1,16,101 26 | 1,8,107 27 | 1,4,134 28 | 1,2,242 29 | 1,1,463 30 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.clickhouse.local.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT local_password (TYPE SECRET) WITH (value = "password"); 2 | 3 | CREATE EXTERNAL DATA SOURCE external_datasource WITH ( 4 | SOURCE_TYPE="ClickHouse", 5 | LOCATION="localhost:9000", 6 | AUTH_METHOD="BASIC", 7 | PROTOCOL="NATIVE", 8 | LOGIN="admin", 9 | DATABASE_NAME="connector", 10 | PASSWORD_SECRET_NAME="local_password" 11 | ); 12 | -------------------------------------------------------------------------------- /common/stacktrace.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | func PrintStackTrace() { 9 | var ( 10 | buf = make([]byte, 1024) 11 | n int 12 | ) 13 | 14 | for { 15 | n = runtime.Stack(buf, false) 16 | if n < len(buf) { 17 | break 18 | } 19 | 20 | buf = make([]byte, 2*len(buf)) 21 | } 22 | 23 | fmt.Printf("Stack Trace:\n%s\n", string(buf[:n])) 24 | } 25 | -------------------------------------------------------------------------------- /library/go/test/yatest/go.go: -------------------------------------------------------------------------------- 1 | package yatest 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | func PrepareGOPATH() error { 8 | return preparePath("GOPATH") 9 | } 10 | 11 | func PrepareGOCACHE() error { 12 | return preparePath("GOCACHE") 13 | } 14 | 15 | func preparePath(name string) error { 16 | p, err := os.MkdirTemp(WorkPath(""), "name") 17 | if err != nil { 18 | return err 19 | } 20 | return os.Setenv(name, p) 21 | } 22 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/consts.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | //nolint:unused 4 | const ( 5 | TypeNone = "none" 6 | TypeString = "string" 7 | TypeHash = "hash" 8 | TypeList = "list" 9 | TypeSet = "set" 10 | TypeZSet = "zset" 11 | TypeStream = "stream" 12 | 13 | KeyColumnName = "key" 14 | StringColumnName = "string_values" 15 | HashColumnName = "hash_values" 16 | 17 | scanBatchSize = 100000 18 | ) 19 | -------------------------------------------------------------------------------- /tests/infra/datasource/greenplum/init/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | /etc/init.d/ssh start 6 | sleep 1 7 | su - gpadmin bash -c 'gpstart -a' 8 | 9 | su - gpadmin -c 'bash /init_db.sh' 10 | 11 | trap "kill %1; su - gpadmin bash -c 'gpstop -a -M fast' && END=1" INT TERM 12 | 13 | tail -f `ls /data/master/gpsne-1/pg_log/gpdb-* | tail -n1` & 14 | 15 | #trap 16 | while [ "$END" == '' ]; do 17 | sleep 1 18 | done -------------------------------------------------------------------------------- /tests/utils/id_array_builders.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | import ( 4 | "github.com/apache/arrow/go/v13/arrow/array" 5 | "github.com/apache/arrow/go/v13/arrow/memory" 6 | ) 7 | 8 | func NewInt32IDArrayBuilder(pool memory.Allocator) *array.Int32Builder { 9 | return array.NewInt32Builder(pool) 10 | } 11 | 12 | func NewInt64IDArrayBuilder(pool memory.Allocator) *array.Int64Builder { 13 | return array.NewInt64Builder(pool) 14 | } 15 | -------------------------------------------------------------------------------- /Dockerfile.release: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/ydb-platform/fq-connector-go:base 2 | 3 | LABEL org.opencontainers.image.source=https://github.com/ydb-platform/fq-connector-go 4 | 5 | RUN apk add bind dnsmasq && apk cache clean 6 | COPY app/server/config/config.prod.yaml /opt/ydb/cfg/fq-connector-go.yaml 7 | COPY fq-connector-go /opt/ydb/bin/fq-connector-go 8 | 9 | ENTRYPOINT ["/opt/ydb/bin/fq-connector-go", "server", "-c", "/opt/ydb/cfg/fq-connector-go.yaml"] 10 | -------------------------------------------------------------------------------- /scripts/debug/config/server/server.no_tls.txt: -------------------------------------------------------------------------------- 1 | connector_server { 2 | endpoint { 3 | host: "0.0.0.0" 4 | port: 2130 5 | } 6 | } 7 | 8 | logger { 9 | log_level: TRACE 10 | enable_sql_query_logging: true 11 | } 12 | 13 | pprof_server { 14 | endpoint { 15 | host: "0.0.0.0" 16 | port: 6060 17 | } 18 | } 19 | 20 | paging { 21 | bytes_per_page: 4194304 22 | prefetch_queue_capacity: 2 23 | } 24 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/trino/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | trino: 5 | image: trinodb/trino:latest 6 | container_name: trino 7 | ports: 8 | - "8080:8080" 9 | volumes: 10 | - ./etc:/etc/trino 11 | - ./data:/var/trino/data 12 | depends_on: 13 | - valkey 14 | 15 | valkey: 16 | image: valkey/valkey:8.0.1 17 | container_name: valkey 18 | ports: 19 | - "6379:6379" -------------------------------------------------------------------------------- /library/go/test/yatest/env_test.go: -------------------------------------------------------------------------------- 1 | package yatest 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestContextParameters(t *testing.T) { 10 | val, ok := BuildFlag("AUTOCHECK") 11 | if ok { 12 | assert.Equal(t, "yes", val) 13 | } else { 14 | _, ok = BuildFlag("TESTS_REQUESTED") 15 | assert.Equal(t, true, ok) 16 | } 17 | 18 | assert.Equal(t, "library/go/test/yatest/gotest", ProjectPath()) 19 | } 20 | -------------------------------------------------------------------------------- /scripts/debug/config/client/mysql.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | data_source_instance { 7 | kind: MYSQL 8 | endpoint { 9 | host: "localhost" 10 | port: 3306 11 | } 12 | database: "fq" 13 | credentials { 14 | basic { 15 | username: "root" 16 | password: "password" 17 | } 18 | } 19 | protocol: NATIVE 20 | } 21 | -------------------------------------------------------------------------------- /scripts/timezone/trino/clickhouse/init/init_db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | clickhouse client -n <<-EOSQL 5 | DROP TABLE IF EXISTS db.datetime; 6 | CREATE TABLE db.datetime ( 7 | id Int32, 8 | datetime DateTime(3), 9 | datetime_explicit_tz DateTime(3, 'Asia/Tokyo') 10 | ) ENGINE = MergeTree ORDER BY id; 11 | INSERT INTO db.datetime (*) VALUES 12 | (1, '2024-01-01 00:00:00', '2024-01-01 00:00:00'); 13 | EOSQL 14 | -------------------------------------------------------------------------------- /scripts/debug/postgresql/maintain/list_schemas.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | export PGPASSWORD=qwerty12345 6 | 7 | SCRIPT=" 8 | select schema_name 9 | from information_schema.schemata; 10 | " 11 | 12 | sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}" 13 | 14 | 15 | SCRIPT=" 16 | select nspname 17 | from pg_catalog.pg_namespace; 18 | " 19 | 20 | sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}" 21 | -------------------------------------------------------------------------------- /tools/ydb/query_service_negative/init/01_basic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /ydb -p tests-ydb-client yql -s ' 4 | 5 | CREATE TABLE simple (id Int32 NOT NULL, col1 String, col2 Int32, PRIMARY KEY (id)); 6 | COMMIT; 7 | INSERT INTO simple (id, col1, col2) VALUES 8 | (1, "ydb_a", 10), 9 | (2, "ydb_b", 20), 10 | (3, "ydb_c", 30), 11 | (4, "ydb_d", 40), 12 | (5, "ydb_e", 50), 13 | (6, "ydb_g", NULL); 14 | COMMIT; 15 | ' 16 | -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/timer.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 8 | ) 9 | 10 | var _ metrics.Timer = (*Timer)(nil) 11 | 12 | // Timer measures gauge duration. 13 | type Timer struct { 14 | gg prometheus.Gauge 15 | } 16 | 17 | func (t Timer) RecordDuration(value time.Duration) { 18 | t.gg.Set(value.Seconds()) 19 | } 20 | -------------------------------------------------------------------------------- /library/go/core/xerrors/benchxerrors/benchxerrors.go: -------------------------------------------------------------------------------- 1 | package benchxerrors 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/ydb-platform/fq-connector-go/library/go/core/xerrors/internal/modes" 8 | ) 9 | 10 | func RunPerMode(b *testing.B, bench func(b *testing.B)) { 11 | for _, mode := range modes.KnownStackTraceModes() { 12 | b.Run(fmt.Sprintf("Mode%s", mode), func(b *testing.B) { 13 | modes.SetStackTraceMode(mode) 14 | bench(b) 15 | }) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.redis.local.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT secret_password (TYPE SECRET) WITH (value = "password"); 2 | 3 | CREATE EXTERNAL DATA SOURCE external_data_source WITH ( 4 | SOURCE_TYPE="Redis", 5 | LOCATION="localhost:6379", 6 | DATABASE_NAME="0", 7 | AUTH_METHOD="BASIC", 8 | LOGIN="default", 9 | PASSWORD_SECRET_NAME="secret_password", 10 | --PROTOCOL="NATIVE", 11 | USE_TLS="FALSE" 12 | ); 13 | -------------------------------------------------------------------------------- /scripts/timezone/trino/postgresql/init/init_db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL 5 | DROP TABLE IF EXISTS datetime; 6 | CREATE TABLE datetime ( 7 | id int, 8 | ts_without_tz TIMESTAMP WITHOUT TIME ZONE, 9 | ts_with_tz TIMESTAMP WITH TIME ZONE 10 | ); 11 | INSERT INTO datetime VALUES (1, '2024-01-01 00:00:00', '2024-01-01 00:00:00 Asia/Tokyo'); 12 | EOSQL 13 | -------------------------------------------------------------------------------- /tests/infra/datasource/ms_sql_server/init/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/mssql/server:2022-latest 2 | 3 | # Create a config directory 4 | USER root 5 | RUN mkdir -p /usr/config 6 | WORKDIR /usr/config 7 | 8 | # Bundle config source 9 | COPY . /usr/config 10 | 11 | # Grant permissions for to our scripts to be executable 12 | RUN chmod +x /usr/config/entrypoint.sh 13 | RUN chmod +x /usr/config/configure-db.sh 14 | 15 | USER mssql 16 | 17 | ENTRYPOINT ["./entrypoint.sh"] 18 | -------------------------------------------------------------------------------- /app/server/config/config.prod.txt: -------------------------------------------------------------------------------- 1 | connector_server { 2 | endpoint { 3 | host: "0.0.0.0" 4 | port: 2130 5 | } 6 | } 7 | 8 | logger { 9 | log_level: INFO 10 | enable_sql_query_logging: false 11 | } 12 | 13 | metrics_server { 14 | endpoint { 15 | host: "0.0.0.0" 16 | port: 8766 17 | } 18 | } 19 | 20 | paging { 21 | bytes_per_page: 4194304 22 | prefetch_queue_capacity: 2 23 | } 24 | 25 | conversion { 26 | use_unsafe_converters: true 27 | } 28 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/clickhouse/type_names.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | const ( 4 | typeBool = "Bool" 5 | typeInt8 = "Int8" 6 | typeInt16 = "Int16" 7 | typeInt32 = "Int32" 8 | typeInt64 = "Int64" 9 | typeUInt8 = "UInt8" 10 | typeUInt16 = "UInt16" 11 | typeUInt32 = "UInt32" 12 | typeUInt64 = "UInt64" 13 | typeFloat32 = "Float32" 14 | typeFload64 = "Float64" 15 | typeString = "String" 16 | typeDate = "Date" 17 | typeDate32 = "Date32" 18 | ) 19 | -------------------------------------------------------------------------------- /library/go/core/metrics/mock/timer.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 7 | "go.uber.org/atomic" 8 | ) 9 | 10 | var _ metrics.Timer = (*Timer)(nil) 11 | 12 | // Timer measures gauge duration. 13 | type Timer struct { 14 | Name string 15 | Tags map[string]string 16 | Value *atomic.Duration 17 | } 18 | 19 | func (t *Timer) RecordDuration(value time.Duration) { 20 | t.Value.Store(value) 21 | } 22 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.postgresql.local.txt: -------------------------------------------------------------------------------- 1 | -- SELECT * FROM external_datasource.`datetime`; 2 | 3 | -- SELECT * FROM external_datasource.lineitem WHERE l_linenumber > 0; 4 | 5 | SELECT col_27_numeric_int, col_28_numeric_rational FROM external_datasource.primitives WHERE col_27_numeric_int = Decimal("1", 10, 0); 6 | 7 | -- SELECT col_27_numeric_int, col_28_numeric_rational FROM external_datasource.primitives WHERE col_28_numeric_rational = Decimal("-22.22", 4, 2); 8 | 9 | 10 | -------------------------------------------------------------------------------- /tools/ydb/query_service_negative/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ydb: 3 | image: ghcr.io/ydb-platform/local-ydb:24.2.8 4 | container_name: ${USER}-fq-connector-go-tests-ydb 5 | hostname: localhost 6 | ports: 7 | - '2136:2136' 8 | - '8765:8765' 9 | environment: 10 | YDB_DEFAULT_LOG_LEVEL: ERROR 11 | POSTGRES_USER: admin 12 | POSTGRES_PASSWORD: password 13 | volumes: 14 | - ./init/init_ydb:/init_ydb 15 | - ./init/01_basic.sh:/01_basic.sh 16 | -------------------------------------------------------------------------------- /app/server/config/config.debug.txt: -------------------------------------------------------------------------------- 1 | connector_server { 2 | endpoint { 3 | host: "0.0.0.0" 4 | port: 2130 5 | } 6 | } 7 | 8 | logger { 9 | log_level: DEBUG 10 | enable_sql_query_logging: true 11 | } 12 | 13 | pprof_server { 14 | endpoint { 15 | host: "0.0.0.0" 16 | port: 6060 17 | } 18 | } 19 | 20 | paging { 21 | bytes_per_page: 4194304 22 | prefetch_queue_capacity: 2 23 | } 24 | 25 | conversion { 26 | use_unsafe_converters: true 27 | } 28 | -------------------------------------------------------------------------------- /common/duration.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | func DurationFromString(src string) (time.Duration, error) { 9 | out, err := time.ParseDuration(src) 10 | if err != nil { 11 | return 0, fmt.Errorf("parse duration: %v", err) 12 | } 13 | 14 | return out, nil 15 | } 16 | 17 | func MustDurationFromString(src string) time.Duration { 18 | out, err := DurationFromString(src) 19 | if err != nil { 20 | panic(err) 21 | } 22 | 23 | return out 24 | } 25 | -------------------------------------------------------------------------------- /library/go/core/xerrors/errorf_multiple_errors_test.go: -------------------------------------------------------------------------------- 1 | package xerrors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestErrorfMultipleErrors(t *testing.T) { 10 | err1 := New("error1") 11 | err2 := New("error2") 12 | err3 := New("error3") 13 | 14 | compositeErr := Errorf("errorf: %w, %w", err1, err2) 15 | 16 | require.True(t, Is(compositeErr, err1)) 17 | require.True(t, Is(compositeErr, err2)) 18 | require.False(t, Is(compositeErr, err3)) 19 | } 20 | -------------------------------------------------------------------------------- /library/go/core/metrics/nop/timer.go: -------------------------------------------------------------------------------- 1 | package nop 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 7 | ) 8 | 9 | var _ metrics.Timer = (*Timer)(nil) 10 | 11 | type Timer struct{} 12 | 13 | func (Timer) RecordDuration(_ time.Duration) {} 14 | 15 | var _ metrics.TimerVec = (*TimerVec)(nil) 16 | 17 | type TimerVec struct{} 18 | 19 | func (t TimerVec) With(_ map[string]string) metrics.Timer { 20 | return Timer{} 21 | } 22 | 23 | func (t TimerVec) Reset() {} 24 | -------------------------------------------------------------------------------- /tests/infra/datasource/opensearch/init/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | INIT_DIR="/usr/share/opensearch/docker-entrypoint-initdb.d" 5 | if [ -d "$INIT_DIR" ]; then 6 | echo "Executing initialization scripts..." 7 | for script in "$INIT_DIR"/*; do 8 | if [ -f "$script" ] && [ -x "$script" ]; then 9 | echo "Running $script" 10 | "$script" & 11 | fi 12 | done 13 | fi 14 | 15 | echo "Starting OpenSearch..." 16 | exec /usr/share/opensearch/bin/opensearch -E plugins.security.disabled=true -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ydb/split.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package NYql.Connector.App.Server.DataSource.RDBMS.Ydb; 4 | 5 | option go_package = "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/ydb/"; 6 | 7 | message TSplitDescription { 8 | message TDataShard { 9 | } 10 | 11 | message TColumnShard { 12 | repeated uint64 tablet_ids = 1; 13 | } 14 | 15 | oneof payload { 16 | TDataShard data_shard = 1; 17 | TColumnShard column_shard = 2; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /scripts/debug/clickhouse/config/z_log_disable.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /scripts/debug/config/client/s3.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: S3 13 | endpoint { 14 | host: "localhost" 15 | port: 9000 16 | } 17 | database: "dqrun" 18 | credentials { 19 | basic { 20 | username: "crab" 21 | password: "qwerty12345" 22 | } 23 | } 24 | protocol: HTTP 25 | } 26 | -------------------------------------------------------------------------------- /app/server/observation/storage.go: -------------------------------------------------------------------------------- 1 | package observation 2 | 3 | import ( 4 | "fmt" 5 | 6 | "go.uber.org/zap" 7 | 8 | "github.com/ydb-platform/fq-connector-go/app/config" 9 | ) 10 | 11 | func NewStorage(logger *zap.Logger, cfg *config.TObservationConfig) (Storage, error) { 12 | if cfg == nil { 13 | return &storageDummyImpl{}, nil 14 | } 15 | 16 | storage, err := newStorageSQLite(logger, cfg.Storage.GetSqlite()) 17 | if err != nil { 18 | return nil, fmt.Errorf("new storage SQLite: %w", err) 19 | } 20 | 21 | return storage, nil 22 | } 23 | -------------------------------------------------------------------------------- /scripts/debug/config/client/redis.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: REDIS 13 | endpoint { 14 | host: "localhost" 15 | port: 6379 16 | } 17 | database: "connector" 18 | credentials { 19 | basic { 20 | username: "default" 21 | password: "" 22 | } 23 | } 24 | protocol: NATIVE 25 | } 26 | -------------------------------------------------------------------------------- /scripts/debug/config/client/ch.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: CLICKHOUSE 13 | endpoint { 14 | host: "localhost" 15 | port: 9000 16 | } 17 | database: "connector" 18 | credentials { 19 | basic { 20 | username: "admin" 21 | password: "password" 22 | } 23 | } 24 | protocol: NATIVE 25 | } 26 | -------------------------------------------------------------------------------- /scripts/debug/config/client/opensearch.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: OPENSEARCH 13 | endpoint { 14 | host: "localhost" 15 | port: 9200 16 | } 17 | database: "connector" 18 | credentials { 19 | basic { 20 | username: "admin" 21 | password: "password" 22 | } 23 | } 24 | protocol: HTTP 25 | } -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/histogram.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 8 | ) 9 | 10 | var _ metrics.Histogram = (*Histogram)(nil) 11 | 12 | type Histogram struct { 13 | hm prometheus.Observer 14 | } 15 | 16 | func (h Histogram) RecordValue(value float64) { 17 | h.hm.Observe(value) 18 | } 19 | 20 | func (h Histogram) RecordDuration(value time.Duration) { 21 | h.hm.Observe(value.Seconds()) 22 | } 23 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.mongodb.local.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT mongodb_local_password (TYPE SECRET) WITH (value = "password"); 2 | 3 | CREATE EXTERNAL DATA SOURCE mongodb_external_datasource WITH ( 4 | SOURCE_TYPE="MongoDB", 5 | LOCATION="localhost:27017", 6 | AUTH_METHOD="BASIC", 7 | LOGIN="admin", 8 | DATABASE_NAME="connector", 9 | PASSWORD_SECRET_NAME="mongodb_local_password", 10 | READING_MODE="TABLE", 11 | UNSUPPORTED_TYPE_DISPLAY_MODE="UNSUPPORTED_OMIT", 12 | UNEXPECTED_TYPE_DISPLAY_MODE="UNEXPECTED_AS_NULL" 13 | ); 14 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/clickhouse/common.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | func rewriteQueryArgs(src []any) []any { 4 | dst := make([]any, len(src)) 5 | 6 | for i, arg := range src { 7 | switch v := arg.(type) { 8 | case []byte: 9 | // It's important to convert byte slice into a string 10 | // in order to distinguish `String` from `Array[Uint8]`. 11 | // TODO: in case if pushdown for the arrays is necessary, add more complicated logic here. 12 | dst[i] = string(v) 13 | default: 14 | dst[i] = arg 15 | } 16 | } 17 | 18 | return dst 19 | } 20 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/oracle/retry.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | "errors" 5 | "net" 6 | "strings" 7 | 8 | "github.com/ydb-platform/fq-connector-go/app/server/utils/retry" 9 | ) 10 | 11 | func ErrorCheckerMakeConnection(err error) bool { 12 | // For a some reason poll.ErrNetClosed is unexported 13 | var opError *net.OpError 14 | if errors.As(err, &opError) { 15 | if strings.Contains(opError.Err.Error(), "use of closed network connection") { 16 | return true 17 | } 18 | } 19 | 20 | return retry.ErrorCheckerMakeConnectionCommon(err) 21 | } 22 | -------------------------------------------------------------------------------- /common/call_stack.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import "runtime" 4 | 5 | func GetCallStackFunctionNames() []string { 6 | var functionNames []string 7 | 8 | pc := make([]uintptr, 20) 9 | 10 | n := runtime.Callers(2, pc) 11 | if n == 0 { 12 | return functionNames 13 | } 14 | 15 | pc = pc[:n] 16 | 17 | frames := runtime.CallersFrames(pc) 18 | 19 | for { 20 | frame, more := frames.Next() 21 | 22 | functionNames = append(functionNames, frame.Function) 23 | 24 | if !more { 25 | break 26 | } 27 | } 28 | 29 | return functionNames 30 | } 31 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ydb/table_metadata_cache/factory.go: -------------------------------------------------------------------------------- 1 | package table_metadata_cache 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/ydb-platform/fq-connector-go/app/config" 7 | ) 8 | 9 | func NewCache(cfg *config.TYdbConfig_TTableMetadataCache) (Cache, error) { 10 | if cfg == nil { 11 | return &noopCache{}, nil 12 | } 13 | 14 | switch cfg.GetStorage().(type) { 15 | case *config.TYdbConfig_TTableMetadataCache_Ristretto: 16 | return newRistrettoCache(cfg) 17 | default: 18 | return nil, fmt.Errorf("unknown storage: %v", cfg.GetStorage()) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /library/go/x/xreflect/assign.go: -------------------------------------------------------------------------------- 1 | package xreflect 2 | 3 | import "reflect" 4 | 5 | // Assign source's value to target's value it points to. Source must be value, target must be pointer to existing value. 6 | // Source must be assignable to target's value it points to. 7 | func Assign(source interface{}, target interface{}) bool { 8 | val := reflect.ValueOf(target) 9 | typ := val.Type() 10 | targetType := typ.Elem() 11 | if reflect.TypeOf(source).AssignableTo(targetType) { 12 | val.Elem().Set(reflect.ValueOf(source)) 13 | return true 14 | } 15 | 16 | return false 17 | } 18 | -------------------------------------------------------------------------------- /scripts/debug/config/client/ms_sql_server.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: MS_SQL_SERVER 13 | endpoint { 14 | host: "localhost" 15 | port: 1433 16 | } 17 | database: "test" 18 | credentials { 19 | basic { 20 | username: "SA" 21 | password: "Qwerty12345!" 22 | } 23 | } 24 | protocol: NATIVE 25 | use_tls: false 26 | } -------------------------------------------------------------------------------- /app/observation/discovery/factory.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/ydb-platform/fq-connector-go/app/config" 7 | ) 8 | 9 | func NewDiscovery(cfg *config.TObservationDiscoveryConfig) (Discovery, error) { 10 | switch t := cfg.GetPayload().(type) { 11 | case *config.TObservationDiscoveryConfig_Static: 12 | return newStaticDiscovery(t.Static), nil 13 | case *config.TObservationDiscoveryConfig_Kubernetes: 14 | return newKubernetesDiscovery(t.Kubernetes), nil 15 | default: 16 | return nil, fmt.Errorf("unknown discovery type: %T", t) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.YQ-4416.txt: -------------------------------------------------------------------------------- 1 | PRAGMA generic.UsePredicatePushdown="true"; 2 | 3 | SELECT 4 | MIN(l_comment), 5 | MIN(l_commitdate), 6 | MIN(l_discount), 7 | MIN(l_extendedprice), 8 | MIN(l_linenumber), 9 | MIN(l_linestatus), 10 | MIN(l_orderkey), 11 | MIN(l_partkey), 12 | MIN(l_quantity), 13 | MIN(l_receiptdate), 14 | MIN(l_returnflag), 15 | MIN(l_shipdate), 16 | MIN(l_shipinstruct), 17 | MIN(l_shipmode), 18 | MIN(l_suppkey), 19 | MIN(l_tax) 20 | FROM (SELECT * FROM external_datasource.olap_lineitem_s10 LIMIT 10000000); 21 | 22 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/trino/etc/jvm.config: -------------------------------------------------------------------------------- 1 | -server 2 | -Xmx16G 3 | -XX:InitialRAMPercentage=80 4 | -XX:MaxRAMPercentage=80 5 | -XX:G1HeapRegionSize=32M 6 | -XX:+ExplicitGCInvokesConcurrent 7 | -XX:+ExitOnOutOfMemoryError 8 | -XX:+HeapDumpOnOutOfMemoryError 9 | -XX:-OmitStackTraceInFastThrow 10 | -XX:ReservedCodeCacheSize=512M 11 | -XX:PerMethodRecompilationCutoff=10000 12 | -XX:PerBytecodeRecompilationCutoff=10000 13 | -Djdk.attach.allowAttachSelf=true 14 | -Djdk.nio.maxCachedBufferSize=2000000 15 | -Dfile.encoding=UTF-8 16 | # Allow loading dynamic agent used by JOL 17 | -XX:+EnableDynamicAgentLoading -------------------------------------------------------------------------------- /common/credentials.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import ( 4 | "os" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | ) 8 | 9 | func MaybeInjectTokenToDataSourceInstance(dsi *api_common.TGenericDataSourceInstance) { 10 | // securely override credentials 11 | if token := os.Getenv("IAM_TOKEN"); token != "" { 12 | dsi.Credentials = &api_common.TGenericCredentials{ 13 | Payload: &api_common.TGenericCredentials_Token{ 14 | Token: &api_common.TGenericCredentials_TToken{ 15 | Type: "IAM", 16 | Value: token, 17 | }, 18 | }, 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /scripts/debug/config/client/gp.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: GREENPLUM 13 | endpoint { 14 | host: "localhost" 15 | port: 32806 16 | } 17 | database: "template1" 18 | credentials { 19 | basic { 20 | username: "gpadmin" 21 | password: "123456" 22 | } 23 | } 24 | gp_options: { 25 | schema: "public" 26 | } 27 | protocol: NATIVE 28 | } 29 | -------------------------------------------------------------------------------- /scripts/debug/config/client/pg.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: POSTGRESQL 13 | endpoint { 14 | host: "localhost" 15 | port: 5432 16 | } 17 | database: "connector" 18 | credentials { 19 | basic { 20 | username: "admin" 21 | password: "password" 22 | } 23 | } 24 | pg_options: { 25 | schema: "public" 26 | } 27 | protocol: NATIVE 28 | } 29 | -------------------------------------------------------------------------------- /library/go/yson/decoder.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type DecoderOptions struct { 8 | SupportYPAPIMaps bool 9 | } 10 | 11 | type Decoder struct { 12 | opts *DecoderOptions 13 | 14 | R *Reader 15 | } 16 | 17 | func NewDecoder(r io.Reader) *Decoder { 18 | return &Decoder{R: NewReader(r)} 19 | } 20 | 21 | func NewDecoderFromBytes(b []byte) *Decoder { 22 | return &Decoder{R: NewReaderFromBytes(b)} 23 | } 24 | 25 | func (d *Decoder) Decode(v any) error { 26 | return decodeAny(d.R, v, d.opts) 27 | } 28 | 29 | func (d *Decoder) CheckFinish() error { 30 | return d.R.CheckFinish() 31 | } 32 | -------------------------------------------------------------------------------- /scripts/debug/config/client/ydb_iam.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | tls: {} 12 | 13 | data_source_instance { 14 | kind: YDB 15 | endpoint { 16 | host: "ydb.serverless.yandexcloud.net" 17 | port: 2135 18 | } 19 | database: "/ru-central1/b1gtl2kg13him37quoo6/etnejle6hb72cdr6aqps" 20 | credentials { 21 | token { 22 | type: "IAM" 23 | value: "" 24 | } 25 | } 26 | use_tls: false 27 | } 28 | 29 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ydb/table_metadata_cache/value.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package NYql.Connector.App.Server.DataSource.RDBMS.Ydb.TableMetadataCache; 4 | 5 | option go_package = "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/ydb/table_metadata_cache"; 6 | 7 | import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto"; 8 | 9 | enum EStoreType { 10 | STORE_TYPE_UNSPECIFIED = 0; 11 | STORE_TYPE_ROW = 1; 12 | STORE_TYPE_COLUMN = 2; 13 | } 14 | 15 | message TValue { 16 | NYql.NConnector.NApi.TSchema schema = 1; 17 | EStoreType store_type = 2; 18 | } -------------------------------------------------------------------------------- /tests/infra/datasource/opensearch/init/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM opensearchproject/opensearch:3.2.0 2 | 3 | USER root 4 | 5 | RUN mkdir -p /usr/share/opensearch/docker-entrypoint-initdb.d && \ 6 | chown opensearch:opensearch /usr/share/opensearch/docker-entrypoint-initdb.d 7 | 8 | COPY opensearch-init.sh /usr/share/opensearch/docker-entrypoint-initdb.d/ 9 | COPY entrypoint.sh /usr/share/opensearch/ 10 | 11 | RUN chmod +x /usr/share/opensearch/entrypoint.sh && \ 12 | chmod +x /usr/share/opensearch/docker-entrypoint-initdb.d/opensearch-init.sh 13 | 14 | USER opensearch 15 | 16 | ENTRYPOINT ["/usr/share/opensearch/entrypoint.sh"] 17 | -------------------------------------------------------------------------------- /scripts/debug/postgresql/maintain/show_table_size.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | export PGPASSWORD=qwerty12345 6 | 7 | SCRIPT=" 8 | DROP TABLE IF EXISTS benchmark_1g; 9 | CREATE TABLE benchmark_1g (id bigserial, col varchar(1024)); 10 | INSERT INTO benchmark_1g SELECT generate_series(1,1048576) AS id, REPEAT(md5(random()::text), 32) AS col; 11 | " 12 | 13 | sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}" 14 | 15 | SCRIPT=" 16 | SELECT pg_size_pretty(pg_total_relation_size('public.benchmark_1g')); 17 | " 18 | 19 | sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}" 20 | -------------------------------------------------------------------------------- /app/observation/discovery/static.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/app/config" 8 | ) 9 | 10 | type staticDiscovery struct { 11 | cfg *config.TObservationDiscoveryConfig_TStaticDiscoveryConfig 12 | } 13 | 14 | func (d *staticDiscovery) GetEndpoints(_ *zap.Logger) ([]*api_common.TGenericEndpoint, error) { 15 | return d.cfg.Endpoints, nil 16 | } 17 | 18 | func newStaticDiscovery(cfg *config.TObservationDiscoveryConfig_TStaticDiscoveryConfig) Discovery { 19 | return &staticDiscovery{cfg: cfg} 20 | } 21 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ydb/table_metadata_cache/noop.go: -------------------------------------------------------------------------------- 1 | package table_metadata_cache 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | ) 8 | 9 | var _ Cache = (*noopCache)(nil) 10 | 11 | type noopCache struct { 12 | } 13 | 14 | func (noopCache) Put(_ *zap.Logger, _ *api_common.TGenericDataSourceInstance, _ string, _ *TValue) bool { 15 | return true 16 | } 17 | 18 | func (noopCache) Get(_ *zap.Logger, _ *api_common.TGenericDataSourceInstance, _ string) (*TValue, bool) { 19 | return nil, false 20 | } 21 | 22 | func (noopCache) Metrics() *Metrics { 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/timer_test.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | dto "github.com/prometheus/client_model/go" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestTimer_RecordDuration(t *testing.T) { 13 | g := &Timer{gg: prometheus.NewGauge(prometheus.GaugeOpts{ 14 | Name: "test_timer_record_duration", 15 | })} 16 | 17 | g.RecordDuration(42 * time.Second) 18 | 19 | var res dto.Metric 20 | err := g.gg.Write(&res) 21 | 22 | assert.NoError(t, err) 23 | assert.Equal(t, float64(42), res.GetGauge().GetValue()) 24 | } 25 | -------------------------------------------------------------------------------- /library/go/core/xerrors/internal/modes/stack_frames_count.go: -------------------------------------------------------------------------------- 1 | package modes 2 | 3 | import "sync/atomic" 4 | 5 | type StackFramesCount = int32 6 | 7 | const ( 8 | StackFramesCount16 StackFramesCount = 16 9 | StackFramesCount32 StackFramesCount = 32 10 | StackFramesCount64 StackFramesCount = 64 11 | StackFramesCount128 StackFramesCount = 128 12 | ) 13 | 14 | var StackFramesCountMax = StackFramesCount32 15 | 16 | func SetStackFramesCountMax(count StackFramesCount) { 17 | atomic.StoreInt32(&StackFramesCountMax, count) 18 | } 19 | 20 | func GetStackFramesCountMax() StackFramesCount { 21 | return atomic.LoadInt32(&StackFramesCountMax) 22 | } 23 | -------------------------------------------------------------------------------- /library/go/yson/compat.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | // IsTrue is a helper function for compatibility with C++ ConvertTo(). 4 | // 5 | // It returns true, if value is bool(true) or string("true"). 6 | // 7 | // Early versions of YSON was missing separate wire type for bool. Instead, string "true" was used to specify true value. 8 | // C++ code still accepts string value where bool is expected, for compatibility reasons. 9 | func IsTrue(value any) bool { 10 | switch v := value.(type) { 11 | case bool: 12 | return v 13 | case string: 14 | return v == "true" 15 | case nil: 16 | return false 17 | default: 18 | return false 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /app/client/metrics/cmd.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | const configFlag = "config" 11 | 12 | func init() { 13 | Cmd.Flags().StringP(configFlag, "c", "", "path to server config file") 14 | 15 | if err := Cmd.MarkFlagRequired(configFlag); err != nil { 16 | fmt.Println(err) 17 | os.Exit(1) 18 | } 19 | } 20 | 21 | var Cmd = &cobra.Command{ 22 | Use: "metrics", 23 | Short: "Client for Solomon HTTP API", 24 | Run: func(cmd *cobra.Command, args []string) { 25 | if err := runClient(cmd, args); err != nil { 26 | fmt.Println(err) 27 | os.Exit(1) 28 | } 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /library/go/core/metrics/solomon/metrics_opts.go: -------------------------------------------------------------------------------- 1 | package solomon 2 | 3 | import "time" 4 | 5 | type MetricsOpts struct { 6 | useNameTag bool 7 | tags map[string]string 8 | timestamp *time.Time 9 | } 10 | 11 | type MetricOpt func(*MetricsOpts) 12 | 13 | func WithTags(tags map[string]string) func(*MetricsOpts) { 14 | return func(m *MetricsOpts) { 15 | m.tags = tags 16 | } 17 | } 18 | 19 | func WithUseNameTag() func(*MetricsOpts) { 20 | return func(m *MetricsOpts) { 21 | m.useNameTag = true 22 | } 23 | } 24 | 25 | func WithTimestamp(t time.Time) func(*MetricsOpts) { 26 | return func(m *MetricsOpts) { 27 | m.timestamp = &t 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /app/client/cmd.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/ydb-platform/fq-connector-go/app/client/connector" 7 | "github.com/ydb-platform/fq-connector-go/app/client/metrics" 8 | "github.com/ydb-platform/fq-connector-go/app/client/observation" 9 | "github.com/ydb-platform/fq-connector-go/app/client/ydb" 10 | ) 11 | 12 | var Cmd = &cobra.Command{ 13 | Use: "client", 14 | Short: "Client for various services working within fq-connector-go process", 15 | } 16 | 17 | func init() { 18 | Cmd.AddCommand(connector.Cmd) 19 | Cmd.AddCommand(metrics.Cmd) 20 | Cmd.AddCommand(observation.Cmd) 21 | Cmd.AddCommand(ydb.Cmd) 22 | } 23 | -------------------------------------------------------------------------------- /app/server/utils/counter.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | import "golang.org/x/exp/constraints" 4 | 5 | type number interface { 6 | constraints.Integer | constraints.Float 7 | } 8 | 9 | type Counter[T number] struct { 10 | parent *Counter[T] 11 | value T 12 | } 13 | 14 | func (c *Counter[T]) Add(delta T) { 15 | if c.parent != nil { 16 | c.parent.value += delta 17 | } 18 | 19 | c.value += delta 20 | } 21 | 22 | func (c *Counter[T]) Value() T { 23 | return c.value 24 | } 25 | 26 | func (c *Counter[T]) MakeChild() *Counter[T] { 27 | return &Counter[T]{parent: c} 28 | } 29 | 30 | func NewCounter[T number]() *Counter[T] { 31 | return &Counter[T]{} 32 | } 33 | -------------------------------------------------------------------------------- /app/bench/cpu_utilization_monitor_linux.go: -------------------------------------------------------------------------------- 1 | //go:build cgo && linux 2 | 3 | package bench 4 | 5 | // #include 6 | import "C" 7 | import "time" 8 | 9 | type cpuUtilizationMonitorLinux struct { 10 | startTime time.Time 11 | startTicks C.long 12 | } 13 | 14 | func (mon *cpuUtilizationMonitorLinux) getPercentage() float64 { 15 | clockSeconds := float64(C.clock()-mon.startTicks) / float64(C.CLOCKS_PER_SEC) 16 | realSeconds := time.Since(mon.startTime).Seconds() 17 | 18 | return clockSeconds / realSeconds * 100 19 | } 20 | 21 | func NewCPUUtilizationMonitor() cpuUtilizationMonitor { 22 | return &cpuUtilizationMonitorLinux{ 23 | startTime: time.Now(), 24 | startTicks: C.clock(), 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/clickhouse/table_metadata_query.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" 5 | rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" 6 | ) 7 | 8 | func TableMetadataQuery(request *api_service_protos.TDescribeTableRequest) (string, *rdbms_utils.QueryArgs) { 9 | query := `SELECT name, type, numeric_precision, toInt64(numeric_scale) 10 | FROM system.columns WHERE table = ? and database = ?` 11 | 12 | var args rdbms_utils.QueryArgs 13 | 14 | args.AddUntyped(request.Table) 15 | args.AddUntyped(request.DataSourceInstance.Database) 16 | 17 | return query, &args 18 | } 19 | -------------------------------------------------------------------------------- /library/go/core/metrics/nop/gauge.go: -------------------------------------------------------------------------------- 1 | package nop 2 | 3 | import "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 4 | 5 | var _ metrics.Gauge = (*Gauge)(nil) 6 | 7 | type Gauge struct{} 8 | 9 | func (Gauge) Set(_ float64) {} 10 | 11 | func (Gauge) Add(_ float64) {} 12 | 13 | var _ metrics.GaugeVec = (*GaugeVec)(nil) 14 | 15 | type GaugeVec struct{} 16 | 17 | func (t GaugeVec) With(_ map[string]string) metrics.Gauge { 18 | return Gauge{} 19 | } 20 | 21 | func (t GaugeVec) Reset() {} 22 | 23 | var _ metrics.FuncGauge = (*FuncGauge)(nil) 24 | 25 | type FuncGauge struct { 26 | function func() float64 27 | } 28 | 29 | func (g FuncGauge) Function() func() float64 { 30 | return g.function 31 | } 32 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/utils/split_provider.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | import ( 4 | "github.com/ydb-platform/fq-connector-go/app/server/datasource" 5 | ) 6 | 7 | var _ SplitProvider = (*defaultSplitProvider)(nil) 8 | 9 | type defaultSplitProvider struct{} 10 | 11 | func (defaultSplitProvider) ListSplits( 12 | params *ListSplitsParams, 13 | ) error { 14 | // By default we deny table splitting 15 | select { 16 | case params.ResultChan <- &datasource.ListSplitResult{Slct: params.Select, Description: nil}: 17 | case <-params.Ctx.Done(): 18 | return params.Ctx.Err() 19 | } 20 | 21 | return nil 22 | } 23 | 24 | func NewDefaultSplitProvider() SplitProvider { 25 | return &defaultSplitProvider{} 26 | } 27 | -------------------------------------------------------------------------------- /library/go/core/metrics/nop/counter.go: -------------------------------------------------------------------------------- 1 | package nop 2 | 3 | import "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 4 | 5 | var _ metrics.Counter = (*Counter)(nil) 6 | 7 | type Counter struct{} 8 | 9 | func (Counter) Inc() {} 10 | 11 | func (Counter) Add(_ int64) {} 12 | 13 | var _ metrics.CounterVec = (*CounterVec)(nil) 14 | 15 | type CounterVec struct{} 16 | 17 | func (t CounterVec) With(_ map[string]string) metrics.Counter { 18 | return Counter{} 19 | } 20 | 21 | func (t CounterVec) Reset() {} 22 | 23 | var _ metrics.FuncCounter = (*FuncCounter)(nil) 24 | 25 | type FuncCounter struct { 26 | function func() int64 27 | } 28 | 29 | func (c FuncCounter) Function() func() int64 { 30 | return c.function 31 | } 32 | -------------------------------------------------------------------------------- /library/go/core/metrics/mock/gauge.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 5 | "go.uber.org/atomic" 6 | ) 7 | 8 | var _ metrics.Gauge = (*Gauge)(nil) 9 | 10 | // Gauge tracks single float64 value. 11 | type Gauge struct { 12 | Name string 13 | Tags map[string]string 14 | Value *atomic.Float64 15 | } 16 | 17 | func (g *Gauge) Set(value float64) { 18 | g.Value.Store(value) 19 | } 20 | 21 | func (g *Gauge) Add(value float64) { 22 | g.Value.Add(value) 23 | } 24 | 25 | var _ metrics.FuncGauge = (*FuncGauge)(nil) 26 | 27 | type FuncGauge struct { 28 | function func() float64 29 | } 30 | 31 | func (g FuncGauge) Function() func() float64 { 32 | return g.function 33 | } 34 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/logging/consts.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | const ( 4 | clusterColumnName = "cluster" 5 | jsonPayloadColumnName = "json_payload" 6 | labelsColumnName = "labels" 7 | levelColumnName = "level" 8 | messageColumnName = "message" 9 | metaColumnName = "meta" 10 | projectColumnName = "project" 11 | serviceColumnName = "service" 12 | timestampColumnName = "timestamp" 13 | hostnameColumnName = "hostname" 14 | spanIDColumnName = "span.id" 15 | traceIDColumnName = "trace.id" 16 | 17 | levelTraceValue = "TRACE" 18 | levelDebugValue = "DEBUG" 19 | levelInfoValue = "INFO" 20 | levelWarnValue = "WARN" 21 | levelErrorValue = "ERROR" 22 | levelFatalValue = "FATAL" 23 | ) 24 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ydb/table_metadata_cache/interface.go: -------------------------------------------------------------------------------- 1 | package table_metadata_cache 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | ) 8 | 9 | // Metrics represents cache statistics 10 | type Metrics struct { 11 | Hits uint64 12 | Misses uint64 13 | Ratio float64 14 | KeysAdded uint64 15 | KeysEvicted uint64 16 | KeysDropped uint64 17 | Size uint64 18 | } 19 | 20 | type Cache interface { 21 | Put(logger *zap.Logger, dsi *api_common.TGenericDataSourceInstance, tableName string, value *TValue) bool 22 | Get(logger *zap.Logger, dsi *api_common.TGenericDataSourceInstance, tableName string) (*TValue, bool) 23 | Metrics() *Metrics 24 | } 25 | -------------------------------------------------------------------------------- /scripts/debug/config/client/mongodb.local.txt: -------------------------------------------------------------------------------- 1 | connector_server_endpoint { 2 | host: "localhost" 3 | port: 2130 4 | } 5 | 6 | metrics_server_endpoint { 7 | host: "localhost" 8 | port: 8766 9 | } 10 | 11 | data_source_instance { 12 | kind: MONGO_DB 13 | endpoint { 14 | host: "localhost" 15 | port: 27017 16 | } 17 | database: "connector" 18 | credentials { 19 | basic { 20 | username: "admin" 21 | password: "password" 22 | } 23 | } 24 | protocol: NATIVE 25 | mongodb_options { 26 | reading_mode: TABLE 27 | unsupported_type_display_mode: UNSUPPORTED_OMIT 28 | unexpected_type_display_mode: UNEXPECTED_AS_NULL 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/postgresql/table_metadata_query.go: -------------------------------------------------------------------------------- 1 | package postgresql 2 | 3 | import ( 4 | api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" 5 | rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" 6 | ) 7 | 8 | func TableMetadataQuery( 9 | request *api_service_protos.TDescribeTableRequest, 10 | schema string, 11 | ) (string, *rdbms_utils.QueryArgs) { 12 | query := "SELECT column_name, data_type, numeric_precision, numeric_scale " + 13 | "FROM information_schema.columns WHERE table_name = $1 AND table_schema = $2" 14 | 15 | var args rdbms_utils.QueryArgs 16 | 17 | args.AddUntyped(request.Table) 18 | args.AddUntyped(schema) 19 | 20 | return query, &args 21 | } 22 | -------------------------------------------------------------------------------- /app/server/datasource/prometheus/analysis/trino/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | prometheus: 3 | image: prom/prometheus 4 | container_name: prometheus 5 | ports: 6 | - "9090:9090" 7 | volumes: 8 | - ./prometheus.yml:/etc/prometheus/prometheus.yml 9 | command: 10 | - '--config.file=/etc/prometheus/prometheus.yml' 11 | - '--storage.tsdb.retention.time=12h' 12 | - '--storage.tsdb.wal-compression' 13 | 14 | trino: 15 | image: trinodb/trino:latest 16 | container_name: trino 17 | ports: 18 | - "8084:8080" 19 | depends_on: 20 | - prometheus 21 | volumes: 22 | - ./catalog/prometheus.properties:/etc/trino/catalog/prometheus.properties 23 | - ./info.trino:/etc/trino/info.trino -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/gauge.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 6 | ) 7 | 8 | var _ metrics.Gauge = (*Gauge)(nil) 9 | 10 | // Gauge tracks single float64 value. 11 | type Gauge struct { 12 | gg prometheus.Gauge 13 | } 14 | 15 | func (g Gauge) Set(value float64) { 16 | g.gg.Set(value) 17 | } 18 | 19 | func (g Gauge) Add(value float64) { 20 | g.gg.Add(value) 21 | } 22 | 23 | var _ metrics.FuncGauge = (*FuncGauge)(nil) 24 | 25 | type FuncGauge struct { 26 | ff prometheus.GaugeFunc 27 | function func() float64 28 | } 29 | 30 | func (g FuncGauge) Function() func() float64 { 31 | return g.function 32 | } 33 | -------------------------------------------------------------------------------- /library/go/core/metrics/mock/int_gauge.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 5 | "go.uber.org/atomic" 6 | ) 7 | 8 | var _ metrics.IntGauge = (*IntGauge)(nil) 9 | 10 | // IntGauge tracks single int64 value. 11 | type IntGauge struct { 12 | Name string 13 | Tags map[string]string 14 | Value *atomic.Int64 15 | } 16 | 17 | func (g *IntGauge) Set(value int64) { 18 | g.Value.Store(value) 19 | } 20 | 21 | func (g *IntGauge) Add(value int64) { 22 | g.Value.Add(value) 23 | } 24 | 25 | var _ metrics.FuncIntGauge = (*FuncIntGauge)(nil) 26 | 27 | type FuncIntGauge struct { 28 | function func() int64 29 | } 30 | 31 | func (g FuncIntGauge) Function() func() int64 { 32 | return g.function 33 | } 34 | -------------------------------------------------------------------------------- /library/go/core/metrics/nop/int_gauge.go: -------------------------------------------------------------------------------- 1 | package nop 2 | 3 | import "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 4 | 5 | var _ metrics.IntGauge = (*IntGauge)(nil) 6 | 7 | type IntGauge struct{} 8 | 9 | func (IntGauge) Set(_ int64) {} 10 | 11 | func (IntGauge) Add(_ int64) {} 12 | 13 | var _ metrics.IntGaugeVec = (*IntGaugeVec)(nil) 14 | 15 | type IntGaugeVec struct{} 16 | 17 | func (t IntGaugeVec) With(_ map[string]string) metrics.IntGauge { 18 | return IntGauge{} 19 | } 20 | 21 | func (t IntGaugeVec) Reset() {} 22 | 23 | var _ metrics.FuncIntGauge = (*FuncIntGauge)(nil) 24 | 25 | type FuncIntGauge struct { 26 | function func() int64 27 | } 28 | 29 | func (g FuncIntGauge) Function() func() int64 { 30 | return g.function 31 | } 32 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.postgresql.local.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT postgresql_local_password (TYPE SECRET) WITH (value = "password"); 2 | 3 | CREATE EXTERNAL DATA SOURCE external_datasource WITH ( 4 | SOURCE_TYPE="PostgreSQL", 5 | LOCATION="localhost:5433", 6 | AUTH_METHOD="BASIC", 7 | PROTOCOL="NATIVE", 8 | LOGIN="admin", 9 | DATABASE_NAME="connector", 10 | PASSWORD_SECRET_NAME="postgresql_local_password" 11 | ); 12 | 13 | -- CREATE EXTERNAL DATA SOURCE external_datasource WITH ( 14 | -- SOURCE_TYPE="PostgreSQL", 15 | -- LOCATION="localhost:6432", 16 | -- AUTH_METHOD="BASIC", 17 | -- PROTOCOL="NATIVE", 18 | -- LOGIN="crab", 19 | -- DATABASE_NAME="tpch", 20 | -- PASSWORD_SECRET_NAME="postgresql_local_password" 21 | -- ); 22 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/logging/split.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package NYql.Connector.App.Server.DataSource.RDBMS.Logging; 4 | 5 | import "yql/essentials/providers/common/proto/gateways_config.proto"; 6 | 7 | option go_package = "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/logging/"; 8 | 9 | message TSplitDescription { 10 | // TYdb is used to describe the column shards of the OLAP YDB database 11 | // that is used as an underlying storage for Cloud Logging. 12 | message TYdb { 13 | NYql.TGenericEndpoint endpoint = 1; 14 | string database_name = 2; 15 | string table_name = 3; 16 | repeated uint64 tablet_ids = 4; 17 | } 18 | 19 | oneof payload { 20 | TYdb ydb = 1; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ms_sql_server/table_metadata_query.go: -------------------------------------------------------------------------------- 1 | package ms_sql_server 2 | 3 | import ( 4 | _ "github.com/denisenkom/go-mssqldb" 5 | 6 | api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" 7 | rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" 8 | ) 9 | 10 | func TableMetadataQuery(request *api_service_protos.TDescribeTableRequest) (string, *rdbms_utils.QueryArgs) { 11 | // opts := request.GetDataSourceInstance().GetPgOptions().GetSchema() 12 | query := `SELECT column_name, data_type, numeric_precision, numeric_scale 13 | FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = @p1;` 14 | 15 | var args rdbms_utils.QueryArgs 16 | 17 | args.AddUntyped(request.Table) 18 | 19 | return query, &args 20 | } 21 | -------------------------------------------------------------------------------- /library/go/core/metrics/internal/pkg/metricsutil/buckets.go: -------------------------------------------------------------------------------- 1 | package metricsutil 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 7 | ) 8 | 9 | // BucketsBounds unwraps Buckets bounds to slice of float64. 10 | func BucketsBounds(b metrics.Buckets) []float64 { 11 | bkts := make([]float64, b.Size()) 12 | for i := range bkts { 13 | bkts[i] = b.UpperBound(i) 14 | } 15 | sort.Float64s(bkts) 16 | return bkts 17 | } 18 | 19 | // DurationBucketsBounds unwraps DurationBuckets bounds to slice of float64. 20 | func DurationBucketsBounds(b metrics.DurationBuckets) []float64 { 21 | bkts := make([]float64, b.Size()) 22 | for i := range bkts { 23 | bkts[i] = b.UpperBound(i).Seconds() 24 | } 25 | sort.Float64s(bkts) 26 | return bkts 27 | } 28 | -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/int_gauge.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 6 | ) 7 | 8 | var _ metrics.IntGauge = (*IntGauge)(nil) 9 | 10 | // IntGauge tracks single int64 value. 11 | type IntGauge struct { 12 | metrics.Gauge 13 | } 14 | 15 | func (i IntGauge) Set(value int64) { 16 | i.Gauge.Set(float64(value)) 17 | } 18 | 19 | func (i IntGauge) Add(value int64) { 20 | i.Gauge.Add(float64(value)) 21 | } 22 | 23 | var _ metrics.FuncIntGauge = (*FuncIntGauge)(nil) 24 | 25 | type FuncIntGauge struct { 26 | ff prometheus.GaugeFunc 27 | function func() int64 28 | } 29 | 30 | func (g FuncIntGauge) Function() func() int64 { 31 | return g.function 32 | } 33 | -------------------------------------------------------------------------------- /docs/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | 6 | import pandas as pd 7 | 8 | 9 | def make_table() -> str: 10 | df = pd.read_csv("types.csv") 11 | out = df.to_markdown(index=False) 12 | print(out) 13 | return out 14 | 15 | 16 | def read_template() -> str: 17 | with open("type_mapping_table.md.template", "r") as f: 18 | return f.read() 19 | 20 | 21 | def make_page(): 22 | out = read_template() + make_table() 23 | with open("type_mapping_table.md", "w") as f: 24 | f.write(out) 25 | 26 | 27 | def main(): 28 | if len(sys.argv) != 2: 29 | print("unexpected number of elements") 30 | exit(1) 31 | 32 | os.chdir(sys.argv[1]) 33 | make_page() 34 | 35 | 36 | if __name__ == "__main__": 37 | main() 38 | -------------------------------------------------------------------------------- /tests/utils/context.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "google.golang.org/grpc/metadata" 8 | 9 | "github.com/ydb-platform/fq-connector-go/common" 10 | ) 11 | 12 | func getTestName() string { 13 | functionNames := common.GetCallStackFunctionNames() 14 | if len(functionNames) == 0 { 15 | return "" 16 | } 17 | 18 | for _, functionName := range functionNames { 19 | if strings.Contains(functionName, "*Suite") { 20 | split := strings.Split(functionName, ".") 21 | 22 | return split[len(split)-1] 23 | } 24 | } 25 | 26 | return "" 27 | } 28 | 29 | func NewContextWithTestName() context.Context { 30 | md := metadata.Pairs(common.TestName, getTestName()) 31 | 32 | return metadata.NewOutgoingContext(context.Background(), md) 33 | } 34 | -------------------------------------------------------------------------------- /app/server/utils/counter_test.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestCounter(t *testing.T) { 10 | t.Run("simple", func(t *testing.T) { 11 | counter := NewCounter[int64]() 12 | counter.Add(1) 13 | counter.Add(-1) 14 | require.Equal(t, int64(0), counter.Value()) 15 | }) 16 | 17 | t.Run("hierarchy", func(t *testing.T) { 18 | parent := NewCounter[uint64]() 19 | require.Equal(t, uint64(0), parent.Value()) 20 | 21 | child1 := parent.MakeChild() 22 | child2 := parent.MakeChild() 23 | 24 | child1.Add(1) 25 | require.Equal(t, uint64(1), child1.Value()) 26 | 27 | child2.Add(1) 28 | require.Equal(t, uint64(1), child1.Value()) 29 | 30 | require.Equal(t, uint64(2), parent.Value()) 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/mysql/type_names.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | const ( 4 | typeInt = "int" 5 | typeMediumInt = "mediumint" 6 | typeBigInt = "bigint" 7 | typeFloat = "float" 8 | typeDouble = "double" 9 | typeTinyInt = "tinyint" 10 | typeSmallInt = "smallint" 11 | typeLongBlob = "longblob" 12 | typeBlob = "blob" 13 | typeMediumBlob = "mediumblob" 14 | typeTinyBlob = "tinyblob" 15 | typeBinary = "binary" 16 | typeVarBinary = "varbinary" 17 | typeChar = "char" 18 | typeVarChar = "varchar" 19 | typeText = "text" 20 | typeLongText = "longtext" 21 | typeTinyText = "tinytext" 22 | typeMediumText = "mediumtext" 23 | typeDate = "date" 24 | typeDatetime = "datetime" 25 | typeTimestamp = "timestamp" 26 | typeJSON = "json" 27 | ) 28 | -------------------------------------------------------------------------------- /library/go/httputil/headers/authorization.go: -------------------------------------------------------------------------------- 1 | package headers 2 | 3 | import "strings" 4 | 5 | const ( 6 | AuthorizationKey = "Authorization" 7 | 8 | TokenTypeBearer TokenType = "bearer" 9 | TokenTypeMAC TokenType = "mac" 10 | ) 11 | 12 | type TokenType string 13 | 14 | // String implements stringer interface 15 | func (tt TokenType) String() string { 16 | return string(tt) 17 | } 18 | 19 | func AuthorizationTokenType(token string) TokenType { 20 | if len(token) > len(TokenTypeBearer) && 21 | strings.ToLower(token[:len(TokenTypeBearer)]) == TokenTypeBearer.String() { 22 | return TokenTypeBearer 23 | } 24 | 25 | if len(token) > len(TokenTypeMAC) && 26 | strings.ToLower(token[:len(TokenTypeMAC)]) == TokenTypeMAC.String() { 27 | return TokenTypeMAC 28 | } 29 | 30 | return TokenType("unknown") 31 | } 32 | -------------------------------------------------------------------------------- /library/go/core/metrics/solomon/spack_compression_test.go: -------------------------------------------------------------------------------- 1 | package solomon 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func compress(t *testing.T, c uint8, s string) []byte { 11 | buf := bytes.Buffer{} 12 | w := newCompressedWriter(&buf, CompressionType(c)) 13 | _, err := w.Write([]byte(s)) 14 | assert.Equal(t, nil, err) 15 | assert.Equal(t, nil, w.Close()) 16 | return buf.Bytes() 17 | } 18 | 19 | func TestCompression_None(t *testing.T) { 20 | assert.Equal(t, []byte(nil), compress(t, uint8(CompressionNone), "")) 21 | assert.Equal(t, []byte{'a'}, compress(t, uint8(CompressionNone), "a")) 22 | } 23 | 24 | func TestCompression_Lz4(t *testing.T) { 25 | assert.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, compress(t, uint8(CompressionLz4), "")) 26 | } 27 | -------------------------------------------------------------------------------- /library/go/core/metrics/mock/counter.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 5 | "go.uber.org/atomic" 6 | ) 7 | 8 | var _ metrics.Counter = (*Counter)(nil) 9 | 10 | // Counter tracks monotonically increasing value. 11 | type Counter struct { 12 | Name string 13 | Tags map[string]string 14 | Value *atomic.Int64 15 | } 16 | 17 | // Inc increments counter by 1. 18 | func (c *Counter) Inc() { 19 | c.Add(1) 20 | } 21 | 22 | // Add adds delta to the counter. Delta must be >=0. 23 | func (c *Counter) Add(delta int64) { 24 | c.Value.Add(delta) 25 | } 26 | 27 | var _ metrics.FuncCounter = (*FuncCounter)(nil) 28 | 29 | type FuncCounter struct { 30 | function func() int64 31 | } 32 | 33 | func (c FuncCounter) Function() func() int64 { 34 | return c.function 35 | } 36 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/oracle/table_metadata_query.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" 5 | rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" 6 | ) 7 | 8 | func TableMetadataQuery(request *api_service_protos.TDescribeTableRequest) (string, *rdbms_utils.QueryArgs) { 9 | // TODO YQ-3413: synonym tables and from other users. 10 | // TODO YQ-3454: all capitalize 11 | // query := `SELECT column_name, data_type, DATA_PRECISION, DATA_SCALE 12 | // FROM user_tab_columns WHERE table_name = :1` 13 | query := `SELECT column_name, data_type 14 | FROM user_tab_columns WHERE table_name = :1` 15 | 16 | var args rdbms_utils.QueryArgs 17 | 18 | args.AddUntyped(request.Table) 19 | 20 | return query, &args 21 | } 22 | -------------------------------------------------------------------------------- /scripts/bench/ydb_sls_local.txt: -------------------------------------------------------------------------------- 1 | server_remote: { 2 | connector_server_endpoint: { 3 | host: "localhost" 4 | port: 2130 5 | } 6 | } 7 | 8 | data_source_instance { 9 | kind: YDB 10 | endpoint { 11 | host: "ydb.serverless.yandexcloud.net" 12 | port: 2135 13 | } 14 | database: "/ru-central1/b1gtl2kg13him37quoo6/etni8pumlo8mo3e0cemr" 15 | credentials { 16 | token { 17 | type: "IAM" 18 | value: "..." 19 | } 20 | } 21 | protocol: NATIVE 22 | use_tls: true 23 | } 24 | 25 | table: "lost_numbers" 26 | 27 | test_cases: [ 28 | { 29 | client_params { 30 | duration: "30s" 31 | queries_per_second: 5 32 | } 33 | } 34 | ] 35 | 36 | result_dir: "/home/vitalyisaev/projects/fq-connector-go/scripts/bench/ydb/results" 37 | -------------------------------------------------------------------------------- /.github/workflows/copilot-reminder.yml: -------------------------------------------------------------------------------- 1 | name: 'reminder: enable GitHub Copilot' 2 | 3 | on: 4 | pull_request: 5 | types: [opened] 6 | 7 | jobs: 8 | remind: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | pull-requests: write # This permission is required to post a comment 12 | steps: 13 | - name: Post Copilot Reminder Comment 14 | uses: actions/github-script@v7 15 | with: 16 | script: | 17 | github.rest.issues.createComment({ 18 | owner: context.repo.owner, 19 | repo: context.repo.repo, 20 | issue_number: context.issue.number, 21 | body: '👋 **Friendly Reminder:** Please enable GitHub Copilot for this pull request to get an AI-generated summary of the changes. You can find the "Enable" button in the right-hand sidebar.' 22 | }) -------------------------------------------------------------------------------- /app/server/paging/sink_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=sinkState -output=sink_string.go"; DO NOT EDIT. 2 | 3 | package paging 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[sinkOperational-1] 12 | _ = x[sinkFailed-2] 13 | _ = x[sinkFinished-3] 14 | } 15 | 16 | const _sinkState_name = "sinkOperationalsinkFailedsinkFinished" 17 | 18 | var _sinkState_index = [...]uint8{0, 15, 25, 37} 19 | 20 | func (i sinkState) String() string { 21 | i -= 1 22 | if i < 0 || i >= sinkState(len(_sinkState_index)-1) { 23 | return "sinkState(" + strconv.FormatInt(int64(i+1), 10) + ")" 24 | } 25 | return _sinkState_name[_sinkState_index[i]:_sinkState_index[i+1]] 26 | } 27 | -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/counter.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 6 | ) 7 | 8 | var _ metrics.Counter = (*Counter)(nil) 9 | 10 | // Counter tracks monotonically increasing value. 11 | type Counter struct { 12 | cnt prometheus.Counter 13 | } 14 | 15 | // Inc increments counter by 1. 16 | func (c Counter) Inc() { 17 | c.cnt.Inc() 18 | } 19 | 20 | // Add adds delta to the counter. Delta must be >=0. 21 | func (c Counter) Add(delta int64) { 22 | c.cnt.Add(float64(delta)) 23 | } 24 | 25 | var _ metrics.FuncCounter = (*FuncCounter)(nil) 26 | 27 | type FuncCounter struct { 28 | cnt prometheus.CounterFunc 29 | function func() int64 30 | } 31 | 32 | func (c FuncCounter) Function() func() int64 { 33 | return c.function 34 | } 35 | -------------------------------------------------------------------------------- /common/protobuf.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import ( 4 | "fmt" 5 | 6 | "google.golang.org/protobuf/encoding/protojson" 7 | "google.golang.org/protobuf/proto" 8 | ) 9 | 10 | func ProtobufToJSON(m proto.Message, multiline bool, indent string) ([]byte, error) { 11 | marshaler := protojson.MarshalOptions{ 12 | Multiline: multiline, 13 | Indent: indent, 14 | EmitUnpopulated: false, 15 | UseProtoNames: true, 16 | } 17 | 18 | prettyJSON, err := marshaler.Marshal(m) 19 | if err != nil { 20 | return nil, fmt.Errorf("marshal protobuf message to JSON: %v", err) 21 | } 22 | 23 | return prettyJSON, nil 24 | } 25 | 26 | func MustProtobufToJSONString(m proto.Message, multiline bool, indent string) string { 27 | json, err := ProtobufToJSON(m, multiline, indent) 28 | if err != nil { 29 | panic(err) 30 | } 31 | 32 | return string(json) 33 | } 34 | -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/stream.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/prometheus/common/expfmt" 9 | ) 10 | 11 | const ( 12 | StreamText expfmt.Format = expfmt.FmtText 13 | StreamCompact expfmt.Format = expfmt.FmtProtoCompact 14 | ) 15 | 16 | func (r *Registry) Stream(_ context.Context, w io.Writer) (int, error) { 17 | metrics, err := r.Gather() 18 | if err != nil { 19 | return 0, fmt.Errorf("cannot gather metrics: %w", err) 20 | } 21 | 22 | enc := expfmt.NewEncoder(w, r.streamFormat) 23 | for _, mf := range metrics { 24 | if err := enc.Encode(mf); err != nil { 25 | return 0, fmt.Errorf("cannot encode metric family: %w", err) 26 | } 27 | } 28 | 29 | // prometheus encoder does not report how much bytes have been written 30 | // so we indicate it by returning -1 instead 31 | return -1, nil 32 | } 33 | -------------------------------------------------------------------------------- /scripts/debug/clickhouse/maintain/show_table_size.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | URL='http://admin:password@localhost:8123/?database=dqrun' 6 | 7 | #echo "SELECT table, formatReadableSize(sum(bytes)) as size, min(min_date) as min_date, max(max_date) as max_date 8 | # FROM system.parts WHERE active GROUP BY table" | curl "${URL}" --data-binary @- 9 | 10 | echo "SELECT 11 | database, 12 | table, 13 | formatReadableSize(sum(data_compressed_bytes) AS size) AS compressed, 14 | formatReadableSize(sum(data_uncompressed_bytes) AS usize) AS uncompressed, 15 | round(usize / size, 2) AS compr_rate, 16 | sum(rows) AS rows, 17 | count() AS part_count 18 | FROM system.parts 19 | WHERE (active = 1) AND (database LIKE '%') AND (table LIKE '%') 20 | GROUP BY database, table ORDER BY size DESC;" | curl "${URL}" --data-binary @- 21 | -------------------------------------------------------------------------------- /library/go/yson/infer_example_test.go: -------------------------------------------------------------------------------- 1 | package yson_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/ydb-platform/fq-connector-go/library/go/yson" 8 | ) 9 | 10 | type Inner struct { 11 | G bool `yson:"g,attr"` 12 | } 13 | 14 | type anonymous struct { 15 | F *int `yson:"f,attr"` 16 | *Inner 17 | } 18 | 19 | type Anonymous struct { 20 | F *int `yson:"f,attr"` 21 | *Inner 22 | } 23 | 24 | type MyStruct struct { 25 | Value string `yson:",value"` 26 | Attr int64 `yson:"attr,attr"` 27 | NoName float64 `yson:",attr"` 28 | NoAttr string `yson:"no_attr"` 29 | NoTag string 30 | Inner *Inner `yson:"inner,attr"` 31 | anonymous 32 | Anonymous `yson:"a,attr"` 33 | } 34 | 35 | var Attrs = yson.MustInferAttrs(&MyStruct{}) 36 | 37 | func ExampleMustInferAttrs() { 38 | fmt.Println(strings.Join(Attrs, ",")) 39 | 40 | // Output: 41 | // attr,NoName,inner,f,g,a 42 | } 43 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.clickhouse.local.txt: -------------------------------------------------------------------------------- 1 | --SELECT * FROM external_datasource.datetime WHERE id > 1 AND col_04_datetime64 > Timestamp("2023-12-31T15:00:00.000000Z"); 2 | 3 | --SELECT * FROM external_datasource.datetime 4 | -- WHERE 5 | -- (id > 1 AND col_04_datetime64 > Timestamp("1980-01-01T00:00:00.000000Z")) 6 | -- AND 7 | -- (id < 10 AND col_04_datetime64 < Timestamp("2040-01-01T00:00:00.000000Z")); 8 | 9 | 10 | --SELECT * FROM external_datasource.datetime 11 | -- WHERE 12 | -- (id > 1 AND col_04_datetime64 > Timestamp("1980-01-01T00:00:00.000000Z")) 13 | -- OR 14 | -- (id < 10 AND col_04_datetime64 < Timestamp("2040-01-01T00:00:00.000000Z")); 15 | 16 | SELECT * FROM external_datasource.datetime 17 | WHERE id > 1 OR col_04_datetime64 > Timestamp("1980-01-01T00:00:00.000000Z"); 18 | 19 | --SELECT * FROM external_datasource.datetime; 20 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/demo/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | redis: 5 | image: redis:latest 6 | container_name: redis 7 | restart: unless-stopped 8 | ports: 9 | - "6379:6379" 10 | # No volumes defined for redis 11 | # Using default configuration without AOF persistence 12 | command: redis-server --requirepass password 13 | networks: 14 | - redis-network 15 | 16 | redis-commander: 17 | image: rediscommander/redis-commander:latest 18 | container_name: redis-commander 19 | restart: unless-stopped 20 | environment: 21 | - REDIS_HOSTS=local:redis:6379:0:password 22 | - HTTP_USER=admin 23 | - HTTP_PASSWORD=password 24 | ports: 25 | - "8081:8081" 26 | depends_on: 27 | - redis 28 | networks: 29 | - redis-network 30 | 31 | networks: 32 | redis-network: 33 | driver: bridge -------------------------------------------------------------------------------- /library/go/yson/reader_test.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestReadListFragment(t *testing.T) { 10 | for _, input := range []string{"1;{a=1}", "1;{a=1};"} { 11 | r := NewReaderKindFromBytes([]byte(input), StreamListFragment) 12 | 13 | for i := 0; i < 2; i++ { 14 | ok, err := r.NextListItem() 15 | require.True(t, ok) 16 | require.NoError(t, err) 17 | 18 | _, err = r.NextRawValue() 19 | require.NoError(t, err) 20 | } 21 | 22 | ok, err := r.NextListItem() 23 | require.False(t, ok) 24 | require.NoError(t, err) 25 | } 26 | } 27 | 28 | func TestReadEmpty(t *testing.T) { 29 | for _, input := range []string{"", " "} { 30 | r := NewReaderKindFromBytes([]byte(input), StreamListFragment) 31 | 32 | ok, err := r.NextListItem() 33 | require.NoError(t, err) 34 | require.False(t, ok) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /scripts/bench/ydb_sls_cloud.txt: -------------------------------------------------------------------------------- 1 | server_remote: { 2 | connector_server_endpoint: { 3 | host: "localhost" 4 | port: 50051 5 | } 6 | 7 | tls: { 8 | insecure_skip_verify: true 9 | } 10 | } 11 | 12 | data_source_instance { 13 | kind: YDB 14 | endpoint { 15 | host: "ydb.serverless.yandexcloud.net" 16 | port: 2135 17 | } 18 | database: "/ru-central1/b1gtl2kg13him37quoo6/etni8pumlo8mo3e0cemr" 19 | credentials { 20 | token { 21 | type: "IAM" 22 | value: "..." 23 | } 24 | } 25 | protocol: NATIVE 26 | use_tls: true 27 | } 28 | 29 | table: "lost_numbers" 30 | 31 | test_cases: [ 32 | { 33 | client_params { 34 | duration: "30s" 35 | queries_per_second: 5 36 | } 37 | } 38 | ] 39 | 40 | result_dir: "/home/vitalyisaev/projects/fq-connector-go/scripts/bench/ydb/results" 41 | -------------------------------------------------------------------------------- /app/server/datasource/nosql/redis/trino/etc/table-descriptions/example_table.json: -------------------------------------------------------------------------------- 1 | { 2 | "tableName": "example_table", 3 | "schemaName": "default", 4 | "key": { 5 | "dataFormat": "raw", 6 | "fields": [ 7 | { 8 | "name": "id", 9 | "type": "varchar" 10 | } 11 | ] 12 | }, 13 | "value": { 14 | "dataFormat": "hash", 15 | "fields": [ 16 | { 17 | "name": "field1", 18 | "type": "varchar", 19 | "mapping": "field1" 20 | }, 21 | { 22 | "name": "field2", 23 | "type": "varchar", 24 | "mapping": "field2" 25 | }, 26 | { 27 | "name": "field3", 28 | "type": "varchar", 29 | "mapping": "field3" 30 | } 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /scripts/bench/redis_columns.txt: -------------------------------------------------------------------------------- 1 | server_local: { 2 | endpoint: { 3 | host: "localhost" 4 | port: 2130 5 | } 6 | } 7 | 8 | data_source_instance { 9 | kind: REDIS 10 | endpoint { 11 | host: "localhost" 12 | port: 6379 13 | } 14 | database: "connector" 15 | credentials { 16 | basic { 17 | username: "default" 18 | password: "" 19 | } 20 | } 21 | protocol: NATIVE 22 | } 23 | 24 | table: "key:*" 25 | 26 | test_cases: [ 27 | { 28 | columns: [ 29 | "key", 30 | "string_values" 31 | ], 32 | server_params: { 33 | paging: { 34 | bytes_per_page: 4194304 35 | prefetch_queue_capacity: 2 36 | } 37 | } 38 | } 39 | ] 40 | 41 | result_dir: "/Users/gisolomennikov/itmo/diploma/fq-connector-go/scripts/bench/redis/results/columns" 42 | -------------------------------------------------------------------------------- /library/go/core/metrics/nop/histogram.go: -------------------------------------------------------------------------------- 1 | package nop 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 7 | ) 8 | 9 | var ( 10 | _ metrics.Histogram = (*Histogram)(nil) 11 | _ metrics.Timer = (*Histogram)(nil) 12 | ) 13 | 14 | type Histogram struct{} 15 | 16 | func (Histogram) RecordValue(_ float64) {} 17 | 18 | func (Histogram) RecordDuration(_ time.Duration) {} 19 | 20 | var _ metrics.HistogramVec = (*HistogramVec)(nil) 21 | 22 | type HistogramVec struct{} 23 | 24 | func (t HistogramVec) With(_ map[string]string) metrics.Histogram { 25 | return Histogram{} 26 | } 27 | 28 | func (t HistogramVec) Reset() {} 29 | 30 | var _ metrics.TimerVec = (*DurationHistogramVec)(nil) 31 | 32 | type DurationHistogramVec struct{} 33 | 34 | func (t DurationHistogramVec) With(_ map[string]string) metrics.Timer { 35 | return Histogram{} 36 | } 37 | 38 | func (t DurationHistogramVec) Reset() {} 39 | -------------------------------------------------------------------------------- /library/go/test/testhelpers/remove_lines.go: -------------------------------------------------------------------------------- 1 | package testhelpers 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | ) 8 | 9 | func RemoveLines(str string, lines ...int) (string, error) { 10 | if len(lines) == 0 { 11 | return str, nil 12 | } 13 | 14 | sort.Ints(lines) 15 | 16 | var b strings.Builder 17 | b.Grow(len(str)) 18 | 19 | var count int 20 | var start int 21 | var lineID int 22 | for i, s := range str { 23 | if s != '\n' { 24 | continue 25 | } 26 | 27 | if lines[lineID] != count { 28 | b.WriteString(str[start:i]) 29 | b.WriteString("\n") 30 | } else { 31 | lineID++ 32 | if len(lines) <= lineID { 33 | b.WriteString(str[i+1:]) 34 | break 35 | } 36 | } 37 | 38 | count++ 39 | start = i + 1 40 | } 41 | 42 | if len(lines) > lineID { 43 | return str, fmt.Errorf("not all lines were removed: processed line ids before %d for lines %d", lineID, lines) 44 | } 45 | 46 | return b.String(), nil 47 | } 48 | -------------------------------------------------------------------------------- /library/go/core/metrics/mock/histogram.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | "time" 7 | 8 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics" 9 | "go.uber.org/atomic" 10 | ) 11 | 12 | var ( 13 | _ metrics.Histogram = (*Histogram)(nil) 14 | _ metrics.Timer = (*Histogram)(nil) 15 | ) 16 | 17 | type Histogram struct { 18 | Name string 19 | Tags map[string]string 20 | BucketBounds []float64 21 | BucketValues []int64 22 | InfValue *atomic.Int64 23 | mutex sync.Mutex 24 | } 25 | 26 | func (h *Histogram) RecordValue(value float64) { 27 | boundIndex := sort.SearchFloat64s(h.BucketBounds, value) 28 | 29 | if boundIndex < len(h.BucketValues) { 30 | h.mutex.Lock() 31 | h.BucketValues[boundIndex] += 1 32 | h.mutex.Unlock() 33 | } else { 34 | h.InfValue.Inc() 35 | } 36 | } 37 | 38 | func (h *Histogram) RecordDuration(value time.Duration) { 39 | h.RecordValue(value.Seconds()) 40 | } 41 | -------------------------------------------------------------------------------- /app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/ydb-platform/fq-connector-go/app/bench" 10 | "github.com/ydb-platform/fq-connector-go/app/client" 11 | "github.com/ydb-platform/fq-connector-go/app/observation" 12 | "github.com/ydb-platform/fq-connector-go/app/server" 13 | "github.com/ydb-platform/fq-connector-go/app/validate" 14 | "github.com/ydb-platform/fq-connector-go/app/version" 15 | ) 16 | 17 | var rootCmd = &cobra.Command{ 18 | Use: "connector", 19 | Short: "Connector for external data sources", 20 | } 21 | 22 | func init() { 23 | rootCmd.AddCommand(bench.Cmd) 24 | rootCmd.AddCommand(client.Cmd) 25 | rootCmd.AddCommand(observation.Cmd) 26 | rootCmd.AddCommand(server.Cmd) 27 | rootCmd.AddCommand(validate.Cmd) 28 | rootCmd.AddCommand(version.Cmd) 29 | } 30 | 31 | func main() { 32 | if err := rootCmd.Execute(); err != nil { 33 | fmt.Println(err) 34 | os.Exit(1) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /app/observation/cmd.go: -------------------------------------------------------------------------------- 1 | package observation 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | const ( 11 | configFlag = "config" 12 | ) 13 | 14 | var Cmd = &cobra.Command{ 15 | Use: "observation", 16 | Short: "Observation GRPC API client", 17 | } 18 | 19 | // Track command 20 | var serverCmd = &cobra.Command{ 21 | Use: "server", 22 | Short: "Run server tracking running queries from multiple connectors", 23 | Run: func(cmd *cobra.Command, _ []string) { 24 | if err := StartAggregationServer(cmd); err != nil { 25 | fmt.Println(err) 26 | os.Exit(1) 27 | } 28 | }, 29 | } 30 | 31 | func init() { 32 | // Add track command to the root command 33 | Cmd.AddCommand(serverCmd) 34 | 35 | // Add flags for track command 36 | serverCmd.Flags().String(configFlag, "", "Path to configuration file") 37 | 38 | // Mark required flags 39 | if err := serverCmd.MarkFlagRequired(configFlag); err != nil { 40 | panic(err) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/utils/queryphase_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=QueryPhase"; DO NOT EDIT. 2 | 3 | package utils //nolint:revive 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[QueryPhaseUnspecified-0] 12 | _ = x[QueryPhaseDescribeTable-1] 13 | _ = x[QueryPhaseListSplits-2] 14 | _ = x[QueryPhaseReadSplits-3] 15 | } 16 | 17 | const _QueryPhase_name = "QueryPhaseUnspecifiedQueryPhaseDescribeTableQueryPhaseListSplitsQueryPhaseReadSplits" 18 | 19 | var _QueryPhase_index = [...]uint8{0, 21, 44, 64, 84} 20 | 21 | func (i QueryPhase) String() string { 22 | if i < 0 || i >= QueryPhase(len(_QueryPhase_index)-1) { 23 | return "QueryPhase(" + strconv.FormatInt(int64(i), 10) + ")" 24 | } 25 | return _QueryPhase_name[_QueryPhase_index[i]:_QueryPhase_index[i+1]] 26 | } 27 | -------------------------------------------------------------------------------- /scripts/bench/postgresql_datetime.txt: -------------------------------------------------------------------------------- 1 | server_local: { 2 | endpoint: { 3 | host: "localhost" 4 | port: 2130 5 | } 6 | } 7 | 8 | data_source_instance { 9 | kind: POSTGRESQL 10 | endpoint { 11 | host: "localhost" 12 | port: 5432 13 | } 14 | database: "tpch" 15 | credentials { 16 | basic { 17 | username: "admin" 18 | password: "password" 19 | } 20 | } 21 | protocol: NATIVE 22 | pg_options: { 23 | schema: "public" 24 | } 25 | } 26 | 27 | table: "lineitem" 28 | 29 | test_cases: [ 30 | { 31 | server_params: { 32 | paging: { 33 | bytes_per_page: 4194304 34 | prefetch_queue_capacity: 2 35 | } 36 | }, 37 | columns: [ 38 | "l_shipdate" 39 | ] 40 | } 41 | ] 42 | 43 | result_dir: "/home/vitalyisaev/projects/fq-connector-go/scripts/bench/postgresql/results/datetime" 44 | -------------------------------------------------------------------------------- /library/go/test/assertpb/assert.go: -------------------------------------------------------------------------------- 1 | package assertpb 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/golang/protobuf/proto" 7 | "github.com/google/go-cmp/cmp" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | type TestingT interface { 12 | Errorf(format string, args ...interface{}) 13 | FailNow() 14 | Helper() 15 | } 16 | 17 | func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { 18 | t.Helper() 19 | 20 | if cmp.Equal(expected, actual, cmp.Comparer(proto.Equal)) { 21 | return true 22 | } 23 | 24 | diff := cmp.Diff(expected, actual, cmp.Comparer(proto.Equal)) 25 | return assert.Fail(t, fmt.Sprintf("Not equal: \n"+ 26 | "expected: %s\n"+ 27 | "actual : %s\n"+ 28 | "diff : %s", expected, actual, diff), msgAndArgs) 29 | } 30 | 31 | func Equalf(t TestingT, expected, actual interface{}, msg string, args ...interface{}) bool { 32 | t.Helper() 33 | 34 | return Equal(t, expected, actual, append([]interface{}{msg}, args...)...) 35 | } 36 | -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/gauge_test.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | dto "github.com/prometheus/client_model/go" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestGauge_Add(t *testing.T) { 12 | g := &Gauge{gg: prometheus.NewGauge(prometheus.GaugeOpts{ 13 | Name: "test_gauge_add", 14 | })} 15 | 16 | var expectValue float64 = 42 17 | g.Add(expectValue) 18 | 19 | var res dto.Metric 20 | err := g.gg.Write(&res) 21 | 22 | assert.NoError(t, err) 23 | assert.Equal(t, expectValue, res.GetGauge().GetValue()) 24 | } 25 | 26 | func TestGauge_Set(t *testing.T) { 27 | g := &Gauge{gg: prometheus.NewGauge(prometheus.GaugeOpts{ 28 | Name: "test_gauge_set", 29 | })} 30 | 31 | var expectValue float64 = 42 32 | g.Set(expectValue) 33 | 34 | var res dto.Metric 35 | err := g.gg.Write(&res) 36 | 37 | assert.NoError(t, err) 38 | assert.Equal(t, expectValue, res.GetGauge().GetValue()) 39 | } 40 | -------------------------------------------------------------------------------- /library/go/core/metrics/prometheus/counter_test.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | dto "github.com/prometheus/client_model/go" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestCounter_Add(t *testing.T) { 12 | c := &Counter{cnt: prometheus.NewCounter(prometheus.CounterOpts{ 13 | Name: "test_counter_add", 14 | })} 15 | 16 | var expectValue int64 = 42 17 | c.Add(expectValue) 18 | 19 | var res dto.Metric 20 | err := c.cnt.Write(&res) 21 | 22 | assert.NoError(t, err) 23 | assert.Equal(t, expectValue, int64(res.GetCounter().GetValue())) 24 | } 25 | 26 | func TestCounter_Inc(t *testing.T) { 27 | c := &Counter{cnt: prometheus.NewCounter(prometheus.CounterOpts{ 28 | Name: "test_counter_inc", 29 | })} 30 | 31 | var res dto.Metric 32 | for i := 1; i <= 10; i++ { 33 | c.Inc() 34 | err := c.cnt.Write(&res) 35 | assert.NoError(t, err) 36 | assert.Equal(t, int64(i), int64(res.GetCounter().GetValue())) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /library/go/yson/error.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | ) 8 | 9 | var ErrInvalidNesting = errors.New("invalid YSON nesting") 10 | 11 | type UnsupportedTypeError struct { 12 | UserType reflect.Type 13 | } 14 | 15 | func (e *UnsupportedTypeError) Error() string { 16 | if e.UserType == nil { 17 | return "yson: nil is not supported" 18 | } 19 | return "yson: value of type " + e.UserType.String() + " is not supported" 20 | } 21 | 22 | type TypeError struct { 23 | UserType reflect.Type 24 | YSONType Type 25 | 26 | Struct string 27 | Field string 28 | } 29 | 30 | func (e *TypeError) Error() string { 31 | msg := "yson: cannot unmarshal " + e.YSONType.String() + " into value of type " + e.UserType.String() 32 | if e.Struct != "" { 33 | msg += " at " + e.Struct + "." + e.Field 34 | } 35 | return msg 36 | } 37 | 38 | type SyntaxError struct { 39 | Message string 40 | } 41 | 42 | func (e *SyntaxError) Error() string { 43 | return fmt.Sprintf("yson: syntax error: %s", e.Message) 44 | } 45 | -------------------------------------------------------------------------------- /library/go/httputil/headers/authorization_test.go: -------------------------------------------------------------------------------- 1 | package headers_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/ydb-platform/fq-connector-go/library/go/httputil/headers" 8 | ) 9 | 10 | func TestAuthorizationTokenType(t *testing.T) { 11 | testCases := []struct { 12 | name string 13 | token string 14 | expected headers.TokenType 15 | }{ 16 | {"bearer", "bearer ololo.trololo", headers.TokenTypeBearer}, 17 | {"Bearer", "Bearer ololo.trololo", headers.TokenTypeBearer}, 18 | {"BEARER", "BEARER ololo.trololo", headers.TokenTypeBearer}, 19 | {"mac", "mac ololo.trololo", headers.TokenTypeMAC}, 20 | {"Mac", "Mac ololo.trololo", headers.TokenTypeMAC}, 21 | {"MAC", "MAC ololo.trololo", headers.TokenTypeMAC}, 22 | {"unknown", "shimba ololo.trololo", headers.TokenType("unknown")}, 23 | } 24 | 25 | for _, tc := range testCases { 26 | t.Run(tc.name, func(t *testing.T) { 27 | assert.Equal(t, tc.expected, headers.AuthorizationTokenType(tc.token)) 28 | }) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /library/go/x/xruntime/stacktrace_benchmark_test.go: -------------------------------------------------------------------------------- 1 | package xruntime 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/ydb-platform/fq-connector-go/library/go/test/testhelpers" 8 | ) 9 | 10 | func BenchmarkNew(b *testing.B) { 11 | inputs := []struct { 12 | Name string 13 | Func func(skip int) *StackTrace 14 | }{ 15 | { 16 | Name: "Frame", 17 | Func: NewFrame, 18 | }, 19 | { 20 | Name: "StackTrace16", 21 | Func: NewStackTrace16, 22 | }, 23 | { 24 | Name: "StackTrace32", 25 | Func: NewStackTrace32, 26 | }, 27 | { 28 | Name: "StackTrace64", 29 | Func: NewStackTrace64, 30 | }, 31 | { 32 | Name: "StackTrace128", 33 | Func: NewStackTrace128, 34 | }, 35 | } 36 | 37 | for _, depth := range []int{1, 16, 32, 64, 128, 256} { 38 | for _, input := range inputs { 39 | b.Run(fmt.Sprintf("Depth%d_%s", depth, input.Name), func(b *testing.B) { 40 | for i := 0; i < b.N; i++ { 41 | testhelpers.Recurse(depth, func() { input.Func(0) }) 42 | } 43 | }) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /library/go/x/xsync/singleinflight.go: -------------------------------------------------------------------------------- 1 | package xsync 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | ) 7 | 8 | // SingleInflight allows only one execution of function at time. 9 | // For more exhaustive functionality see https://pkg.go.dev/golang.org/x/sync/singleflight. 10 | type SingleInflight struct { 11 | updatingOnce atomic.Value 12 | } 13 | 14 | // NewSingleInflight creates new SingleInflight. 15 | func NewSingleInflight() SingleInflight { 16 | var v atomic.Value 17 | v.Store(new(sync.Once)) 18 | return SingleInflight{updatingOnce: v} 19 | } 20 | 21 | // Do executes the given function, making sure that only one execution is in-flight. 22 | // If another caller comes in, it waits for the original to complete. 23 | func (i *SingleInflight) Do(f func()) { 24 | i.getOnce().Do(func() { 25 | f() 26 | i.setOnce() 27 | }) 28 | } 29 | 30 | func (i *SingleInflight) getOnce() *sync.Once { 31 | return i.updatingOnce.Load().(*sync.Once) 32 | } 33 | 34 | func (i *SingleInflight) setOnce() { 35 | i.updatingOnce.Store(new(sync.Once)) 36 | } 37 | -------------------------------------------------------------------------------- /library/go/core/xerrors/new.go: -------------------------------------------------------------------------------- 1 | package xerrors 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/ydb-platform/fq-connector-go/library/go/x/xruntime" 8 | ) 9 | 10 | type newError struct { 11 | msg string 12 | stacktrace *xruntime.StackTrace 13 | } 14 | 15 | var _ ErrorStackTrace = &newError{} 16 | 17 | func New(text string) error { 18 | return &newError{ 19 | msg: text, 20 | stacktrace: newStackTrace(1, nil), 21 | } 22 | } 23 | 24 | func (e *newError) Error() string { 25 | return e.msg 26 | } 27 | 28 | func (e *newError) Format(s fmt.State, v rune) { 29 | switch v { 30 | case 'v': 31 | if s.Flag('+') && e.stacktrace != nil { 32 | _, _ = io.WriteString(s, e.msg) 33 | _, _ = io.WriteString(s, "\n") 34 | writeStackTrace(s, e.stacktrace) 35 | return 36 | } 37 | 38 | fallthrough 39 | case 's': 40 | _, _ = io.WriteString(s, e.msg) 41 | case 'q': 42 | _, _ = fmt.Fprintf(s, "%q", e.msg) 43 | } 44 | } 45 | 46 | func (e *newError) StackTrace() *xruntime.StackTrace { 47 | return e.stacktrace 48 | } 49 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/mysql/table_metadata_query.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" 5 | rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" 6 | ) 7 | 8 | func TableMetadataQuery(request *api_service_protos.TDescribeTableRequest) (string, *rdbms_utils.QueryArgs) { 9 | // TODO: do not add 'unsigned' modifiers to column type and use the driver-provided fields instead. 10 | // 11 | // In MySQL schema and database are basically the same thing. So we can safely pass dbname as 12 | // `schema_name` when quering `information_schema`. 13 | // 14 | // TODO: learn how to extract numeric precision and scale from MySQL 15 | query := `SELECT 16 | column_name, 17 | column_type 18 | FROM information_schema.columns 19 | WHERE table_name = ? AND table_schema = ?` 20 | 21 | var args rdbms_utils.QueryArgs 22 | 23 | args.AddUntyped(request.Table) 24 | args.AddUntyped(request.GetDataSourceInstance().Database) 25 | 26 | return query, &args 27 | } 28 | -------------------------------------------------------------------------------- /app/config/client.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package NYql.Connector.App.Config; 3 | 4 | import "yql/essentials/providers/common/proto/gateways_config.proto"; 5 | 6 | option go_package = "github.com/ydb-platform/fq-connector-go/app/config"; 7 | 8 | // Connector client configuration 9 | message TClientConfig { 10 | // Connector GRPC API endpoint to connect 11 | NYql.TGenericEndpoint connector_server_endpoint = 4; 12 | // TLS credentials for Connector 13 | TClientTLSConfig tls = 2; 14 | // Data source instance we read data from 15 | NYql.TGenericDataSourceInstance data_source_instance = 3; 16 | 17 | // Solomon metrics endpoint to connect 18 | NYql.TGenericEndpoint metrics_server_endpoint = 5; 19 | 20 | reserved 1; 21 | } 22 | 23 | message TClientTLSConfig { 24 | // CA certificate path 25 | string ca = 1; 26 | // Disables certificate host name checking. 27 | // Should be used carefully only for debugging purposes. 28 | // See https://pkg.go.dev/crypto/tls#Config for more details. 29 | bool insecure_skip_verify = 2; 30 | } 31 | -------------------------------------------------------------------------------- /common/endpoint.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 9 | ) 10 | 11 | func EndpointToString(ep *api_common.TGenericEndpoint) string { 12 | host := ep.GetHost() 13 | 14 | // Check if this is an IPv6 address (contains colon) 15 | if strings.Contains(host, ":") { 16 | return fmt.Sprintf("[%s]:%d", host, ep.GetPort()) 17 | } 18 | 19 | return fmt.Sprintf("%s:%d", host, ep.GetPort()) 20 | } 21 | 22 | func StringToEndpoint(s string) (*api_common.TGenericEndpoint, error) { 23 | ss := strings.Split(s, ":") 24 | 25 | if len(ss) != 2 { 26 | return nil, fmt.Errorf("invalid endpoint format: %s", s) 27 | } 28 | 29 | port, err := strconv.ParseUint(ss[1], 10, 32) 30 | if err != nil { 31 | return nil, fmt.Errorf("invalid port: %s", ss[1]) 32 | } 33 | 34 | if port > 65535 { 35 | return nil, fmt.Errorf("invalid port: %s", ss[1]) 36 | } 37 | 38 | return &api_common.TGenericEndpoint{ 39 | Host: ss[0], 40 | Port: uint32(port), 41 | }, nil 42 | } 43 | -------------------------------------------------------------------------------- /scripts/timezone/trino/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | trino: 5 | image: trinodb/trino:472 6 | ports: 7 | - "8080:8080" 8 | volumes: 9 | - ./catalog:/etc/trino/catalog 10 | depends_on: 11 | - postgresql 12 | - clickhouse 13 | 14 | postgresql: 15 | image: postgres:17.4 16 | environment: 17 | POSTGRES_DB: db 18 | POSTGRES_USER: user 19 | POSTGRES_PASSWORD: password 20 | ports: 21 | - "5432:5432" 22 | volumes: 23 | - ./postgresql/init:/docker-entrypoint-initdb.d 24 | 25 | clickhouse: 26 | image: mirror.gcr.io/clickhouse/clickhouse-server:24.10.2.80 27 | ports: 28 | - "8123:8123" 29 | - "9000:9000" 30 | environment: 31 | CLICKHOUSE_DB: db 32 | CLICKHOUSE_USER: user 33 | CLICKHOUSE_PASSWORD: password 34 | CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 35 | volumes: 36 | - ./clickhouse/init:/docker-entrypoint-initdb.d 37 | ulimits: 38 | nproc: 65535 39 | nofile: 40 | soft: 262144 41 | hard: 262144 -------------------------------------------------------------------------------- /scripts/bench/postgresql_queue.txt: -------------------------------------------------------------------------------- 1 | server_local: { 2 | endpoint: { 3 | host: "localhost" 4 | port: 2130 5 | } 6 | } 7 | 8 | data_source_instance { 9 | kind: POSTGRESQL 10 | endpoint { 11 | host: "localhost" 12 | port: 5432 13 | } 14 | database: "tpch" 15 | credentials { 16 | basic { 17 | username: "admin" 18 | password: "password" 19 | } 20 | } 21 | protocol: NATIVE 22 | pg_options: { 23 | schema: "public" 24 | } 25 | } 26 | 27 | table: "lineitem" 28 | 29 | test_cases: [ 30 | { 31 | server_params: { 32 | paging: { 33 | bytes_per_page: 4194304 34 | prefetch_queue_capacity: 2 35 | } 36 | } 37 | }, 38 | { 39 | server_params: { 40 | paging: { 41 | bytes_per_page: 16777216 42 | prefetch_queue_capacity: 4 43 | } 44 | } 45 | } 46 | ] 47 | 48 | result_dir: "/home/vitalyisaev/projects/fq-connector-go/scripts/bench/postgresql/results" 49 | -------------------------------------------------------------------------------- /library/go/core/metrics/internal/pkg/registryutil/registryutil_test.go: -------------------------------------------------------------------------------- 1 | package registryutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestBuildFQName(t *testing.T) { 10 | testCases := []struct { 11 | name string 12 | parts []string 13 | sep string 14 | expected string 15 | }{ 16 | { 17 | name: "empty", 18 | parts: nil, 19 | sep: "_", 20 | expected: "", 21 | }, 22 | { 23 | name: "one part", 24 | parts: []string{"part"}, 25 | sep: "_", 26 | expected: "part", 27 | }, 28 | { 29 | name: "two parts", 30 | parts: []string{"part", "another"}, 31 | sep: "_", 32 | expected: "part_another", 33 | }, 34 | { 35 | name: "parts with sep", 36 | parts: []string{"abcde", "deabc"}, 37 | sep: "abc", 38 | expected: "deabcde", 39 | }, 40 | } 41 | 42 | for _, testCase := range testCases { 43 | c := testCase 44 | t.Run(c.name, func(t *testing.T) { 45 | assert.Equal(t, c.expected, BuildFQName(c.sep, c.parts...)) 46 | }) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/utils/select_query_args.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | import "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" 4 | 5 | type QueryArg struct { 6 | YdbType *Ydb.Type 7 | Value any 8 | } 9 | 10 | type QueryArgs struct { 11 | args []*QueryArg 12 | } 13 | 14 | func (q *QueryArgs) AddTyped(ydbType *Ydb.Type, arg any) *QueryArgs { 15 | q.args = append(q.args, &QueryArg{ydbType, arg}) 16 | 17 | return q 18 | } 19 | 20 | func (q *QueryArgs) AddUntyped(arg any) *QueryArgs { return q.AddTyped(nil, arg) } 21 | 22 | func (q *QueryArgs) Count() int { 23 | if q == nil { 24 | return 0 25 | } 26 | 27 | return len(q.args) 28 | } 29 | 30 | func (q *QueryArgs) Values() []any { 31 | if q == nil { 32 | return []any{} 33 | } 34 | 35 | args := make([]any, len(q.args)) 36 | for i, arg := range q.args { 37 | args[i] = arg.Value 38 | } 39 | 40 | return args 41 | } 42 | 43 | func (q *QueryArgs) Get(i int) *QueryArg { return q.args[i] } 44 | 45 | func (q *QueryArgs) GetAll() []*QueryArg { 46 | if q == nil { 47 | return nil 48 | } 49 | 50 | return q.args 51 | } 52 | -------------------------------------------------------------------------------- /library/go/yson/ya.make: -------------------------------------------------------------------------------- 1 | GO_LIBRARY() 2 | 3 | INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) 4 | 5 | SRCS( 6 | compat.go 7 | decoder.go 8 | error.go 9 | escape.go 10 | generic.go 11 | infer.go 12 | marshal.go 13 | reader.go 14 | reflect.go 15 | scanner.go 16 | struct_tag.go 17 | time.go 18 | unmarshal.go 19 | validate.go 20 | writer.go 21 | ypath.go 22 | ) 23 | 24 | GO_TEST_SRCS( 25 | benchmark_test.go 26 | decoder_test.go 27 | escape_test.go 28 | generic_test.go 29 | infer_test.go 30 | marshal_test.go 31 | reader_test.go 32 | reflect_test.go 33 | scanner_test.go 34 | struct_tag_test.go 35 | time_test.go 36 | unmarshal_test.go 37 | validate_test.go 38 | writer_test.go 39 | ypath_test.go 40 | ) 41 | 42 | GO_XTEST_SRCS( 43 | bugs_test.go 44 | infer_example_test.go 45 | ) 46 | 47 | END() 48 | 49 | RECURSE( 50 | gotest 51 | yson2json 52 | ) 53 | 54 | IF ( 55 | NOT 56 | OPENSOURCE 57 | ) 58 | # fuzz/crashers contain "a.yandex-team.ru" 59 | 60 | RECURSE(fuzz) 61 | ENDIF() 62 | -------------------------------------------------------------------------------- /library/go/yson/struct_tag.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | ) 7 | 8 | type Tag struct { 9 | Name string 10 | 11 | Omitempty bool 12 | Value bool 13 | Attr bool 14 | Attrs bool 15 | Key bool 16 | } 17 | 18 | // ParseTag parses yson annotation for struct field tag. 19 | func ParseTag(fieldName string, fieldTag reflect.StructTag) (tag *Tag, skip bool) { 20 | tag = &Tag{} 21 | 22 | tagValue, ok := fieldTag.Lookup("yson") 23 | if !ok { 24 | tag.Name = fieldName 25 | } 26 | 27 | parts := strings.Split(tagValue, ",") 28 | switch parts[0] { 29 | case "": 30 | tag.Name = fieldName 31 | case "-": 32 | return nil, true 33 | default: 34 | tag.Name = parts[0] 35 | } 36 | 37 | for _, part := range parts[1:] { 38 | switch part { 39 | case "omitempty": 40 | tag.Omitempty = true 41 | case "key": 42 | tag.Key = true 43 | case "value": 44 | tag.Value = true 45 | tag.Name = "" 46 | case "attr": 47 | tag.Attr = true 48 | case "attrs": 49 | tag.Attrs = true 50 | tag.Name = "" 51 | } 52 | } 53 | 54 | return tag, false 55 | } 56 | -------------------------------------------------------------------------------- /tests/infra/datasource/ms_sql_server/init/configure-db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | # Wait 60 seconds for SQL Server to start up by ensuring that 5 | # calling SQLCMD does not return an error code, which will ensure that sqlcmd is accessible 6 | # and that system and user databases return "0" which means all databases are in an "online" state 7 | # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-2017 8 | 9 | ERRCODE=1 10 | i=0 11 | 12 | while [[ $i -lt 60 ]] && [[ $ERRCODE -ne 0 ]]; do 13 | i=$i+1 14 | DBSTATUS=$(/opt/mssql-tools18/bin/sqlcmd -C -U sa -P $SA_PASSWORD -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases") 15 | ERRCODE=$? 16 | echo "$i $DBSTATUS $ERRCODE" 17 | sleep 1 18 | done 19 | 20 | if [[ "$ERRCODE" -ne 0 ]]; then 21 | echo "SQL Server took more than 60 seconds to start up or one or more databases are not in an ONLINE state" 22 | exit 1 23 | fi 24 | 25 | # Run the setup script to create the DB and the schema in the DB 26 | /opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P $SA_PASSWORD -d master -i setup.sql 27 | -------------------------------------------------------------------------------- /library/go/yson/struct_tag_test.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestStructTagParse(t *testing.T) { 11 | for _, testCase := range []struct { 12 | fieldName string 13 | tagValue reflect.StructTag 14 | tag *Tag 15 | skip bool 16 | }{ 17 | { 18 | "MyField", 19 | "", 20 | &Tag{Name: "MyField"}, 21 | false, 22 | }, 23 | { 24 | "MySkippedField", 25 | `yson:"-"`, 26 | nil, 27 | true, 28 | }, 29 | { 30 | "MyValue", 31 | `yson:",value"`, 32 | &Tag{Value: true}, 33 | false, 34 | }, 35 | { 36 | "MyKey", 37 | `yson:",omitempty,key"`, 38 | &Tag{Name: "MyKey", Omitempty: true, Key: true}, 39 | false, 40 | }, 41 | { 42 | "MyAttr", 43 | `yson:"format,attr"`, 44 | &Tag{Name: "format", Attr: true}, 45 | false, 46 | }, 47 | } { 48 | t.Run(testCase.fieldName, func(t *testing.T) { 49 | tag, skip := ParseTag(testCase.fieldName, testCase.tagValue) 50 | require.Equal(t, testCase.skip, skip) 51 | require.Equal(t, testCase.tag, tag) 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /library/go/yson/generic_test.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func runGenericTests(t *testing.T, tests []testCase) { 9 | t.Helper() 10 | 11 | for i, testCase := range tests { 12 | t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { 13 | t.Helper() 14 | 15 | testDecoder(t, []byte(testCase.input), testCase.expected, true) 16 | }) 17 | } 18 | } 19 | 20 | func TestDecodeInterface(t *testing.T) { 21 | var v1 any = map[string]any{ 22 | "a": int64(1), 23 | "b": "c", 24 | } 25 | 26 | runGenericTests(t, []testCase{ 27 | {"{a=1;b=c}", v1}, 28 | }) 29 | } 30 | 31 | func TestDecodeGenericList(t *testing.T) { 32 | var v1 any = []any{ 33 | int64(1), 34 | "foo", 35 | nil, 36 | } 37 | 38 | runGenericTests(t, []testCase{ 39 | {"[1;foo;#]", v1}, 40 | {"[1;foo;#;]", v1}, 41 | }) 42 | } 43 | 44 | func TestDecodeGenericAttrs(t *testing.T) { 45 | var v1 any = &ValueWithAttrs{ 46 | Attrs: map[string]any{ 47 | "foo": int64(1), 48 | "bar": "zog", 49 | }, 50 | Value: int64(1), 51 | } 52 | 53 | runGenericTests(t, []testCase{ 54 | {"1", v1}, 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /scripts/bench/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | clickhouse: 6 | image: clickhouse/clickhouse-server 7 | container_name: fq-connector-go-bench-clickhouse 8 | ports: 9 | - '8123:8123' 10 | - '9000:9000' 11 | environment: 12 | CLICKHOUSE_DB: tpch 13 | CLICKHOUSE_USER: admin 14 | CLICKHOUSE_PASSWORD: password 15 | CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 16 | volumes: 17 | - ./clickhouse/data/:/var/lib/clickhouse/ 18 | ulimits: 19 | nproc: 65535 20 | nofile: 21 | soft: 262144 22 | hard: 262144 23 | 24 | postgresql: 25 | image: postgres:15.3 26 | container_name: fq-connector-go-bench-postgresql 27 | ports: 28 | - '5432:5432' 29 | environment: 30 | POSTGRES_DB: tpch 31 | POSTGRES_USER: admin 32 | POSTGRES_PASSWORD: password 33 | PGDATA: /var/lib/postgresql/data/pgdata 34 | volumes: 35 | - ./postgresql/data/:/var/lib/postgresql/data/ 36 | 37 | redis: 38 | image: valkey/valkey:8.0.1 39 | container_name: fq-connector-go-bench-redis 40 | ports: 41 | - "6379:6379" -------------------------------------------------------------------------------- /scripts/debug/kqprun/script.ydb.local.txt: -------------------------------------------------------------------------------- 1 | PRAGMA generic.UsePredicatePushdown="true"; 2 | 3 | -- SELECT * FROM external_datasource.`datetime` WHERE col_03_timestamp = Timestamp("1988-11-20T12:55:28.123456Z"); 4 | -- SELECT * FROM external_datasource.primitives; 5 | 6 | -- SELECT * FROM external_datasource.pushdown_coalesce 7 | -- WHERE 8 | -- col_01_timestamp >= Timestamp("2021-01-01T00:00:00Z") 9 | -- AND 10 | -- col_01_timestamp <= Timestamp("2024-01-01T00:00:00Z") 11 | -- ; 12 | 13 | -- SELECT * FROM external_datasource.`yq-4224` WHERE hash = "6758ddf04f23be19dc7adf08356c697f21dc751aabc1c71b55d340ee920781ca"; 14 | -- SELECT * FROM external_datasource.`yq-4224` WHERE hash LIKE "6758%"; 15 | 16 | SELECT * FROM external_datasource.simple WHERE id = (2 + 3); 17 | 18 | -- SELECT * FROM external_datasource.primitives WHERE `col_13_utf8` LIKE Utf8("a%"); 19 | 20 | -- SELECT * FROM external_datasource.pushdown_regexp WHERE `col_01_string` REGEXP '\\d+'; 21 | 22 | -- SELECT * FROM external_datasource.pushdown_regexp WHERE `col_02_utf8` REGEXP '\\d+'; 23 | 24 | -- SELECT * FROM external_datasource.pushdown_regexp WHERE `col_01_string` LIKE 'a%b%d'; 25 | 26 | -------------------------------------------------------------------------------- /app/server/utils/retry/error_checker.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net" 7 | "os" 8 | "strings" 9 | "syscall" 10 | ) 11 | 12 | type ErrorChecker func(err error) bool 13 | 14 | func ErrorCheckerMakeConnectionCommon(err error) bool { 15 | // 'i/o timeout' 16 | if strings.Contains(err.Error(), "i/o timeout") { 17 | return true 18 | } 19 | 20 | if errors.Is(err, os.ErrDeadlineExceeded) { 21 | return true 22 | } 23 | 24 | if strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { 25 | return true 26 | } 27 | 28 | // 'connection refused' 29 | if errors.Is(err, syscall.ECONNREFUSED) { 30 | return true 31 | } 32 | 33 | if strings.Contains(err.Error(), "connection refused") { 34 | return true 35 | } 36 | 37 | // DNS errors typically caused by CI overload, like 'server misbehaving' and so on. 38 | var dnsError *net.DNSError 39 | if errors.As(err, &dnsError) { 40 | if dnsError.IsTemporary || dnsError.IsTimeout { 41 | return true 42 | } 43 | } 44 | 45 | if strings.Contains(err.Error(), "server misbehaving") { 46 | return true 47 | } 48 | 49 | return false 50 | } 51 | 52 | func ErrorCheckerNoop(_ error) bool { 53 | return false 54 | } 55 | -------------------------------------------------------------------------------- /app/server/datasource/prometheus/type_mapping.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/common/model" 7 | 8 | "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" 9 | ) 10 | 11 | const ( 12 | timestampColumn = "timestamp" 13 | valueColumn = "value" 14 | ) 15 | 16 | func metricToYdbSchema(labels []string) []*Ydb.Column { 17 | ydbColumns := make([]*Ydb.Column, 0, len(labels)) 18 | 19 | for _, label := range labels { 20 | ydbColumns = append(ydbColumns, &Ydb.Column{ 21 | Name: label, 22 | Type: &Ydb.Type{Type: &Ydb.Type_OptionalType{ 23 | OptionalType: &Ydb.OptionalType{ 24 | Item: &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_STRING}}, 25 | }, 26 | }}, 27 | }) 28 | } 29 | 30 | // All schemas contain timestamp and value 31 | ydbColumns = append(ydbColumns, []*Ydb.Column{{ 32 | Name: timestampColumn, 33 | Type: &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_TIMESTAMP}}, 34 | }, { 35 | Name: valueColumn, 36 | Type: &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_DOUBLE}}, 37 | }}...) 38 | 39 | return ydbColumns 40 | } 41 | 42 | func toPromTime(t time.Time) int64 { 43 | return int64(model.TimeFromUnixNano(t.UnixNano())) 44 | } 45 | -------------------------------------------------------------------------------- /scripts/debug/tls/grpc/localhost.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDFjCCAf4CCQCzrLIhrWa55zANBgkqhkiG9w0BAQsFADBCMQswCQYDVQQGEwJV 3 | UzETMBEGA1UECAwKQ2FsaWZvcm5pYTEPMA0GA1UECgwGR29vZ2xlMQ0wCwYDVQQL 4 | DARnUlBDMCAXDTE5MDYyNDIyMjIzM1oYDzIxMTkwNTMxMjIyMjMzWjBWMQswCQYD 5 | VQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEPMA0GA1UECgwGR29vZ2xlMQ0w 6 | CwYDVQQLDARnUlBDMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB 7 | AQUAA4IBDwAwggEKAoIBAQCtCW0TjugnIUu8BEVIYvdMP+/2GENQDjZhZ8eKR5C6 8 | toDGbgjsDtt/GxISAg4cg70fIvy0XolnGPZodvfHDM4lJ7yHBOdZD8TXQoE6okR7 9 | HZuLUJ20M0pXgWqtRewKRUjuYsSDXBnzLiZw1dcv9nGpo+Bqa8NonpiGRRpEkshF 10 | D6T9KU9Ts/x+wMQBIra2Gj0UMh79jPhUuxcYAQA0JQGivnOtdwuPiumpnUT8j8h6 11 | tWg5l01EsCZWJecCF85KnGpJEVYPyPqBqGsy0nGS9plGotOWF87+jyUQt+KD63xA 12 | aBmTro86mKDDKEK4JvzjVeMGz2UbVcLPiiZnErTFaiXJAgMBAAEwDQYJKoZIhvcN 13 | AQELBQADggEBAKsDgOPCWp5WCy17vJbRlgfgk05sVNIHZtzrmdswjBmvSg8MUpep 14 | XqcPNUpsljAXsf9UM5IFEMRdilUsFGWvHjBEtNAW8WUK9UV18WRuU//0w1Mp5HAN 15 | xUEKb4BoyZr65vlCnTR+AR5c9FfPvLibhr5qHs2RA8Y3GyLOcGqBWed87jhdQLCc 16 | P1bxB+96le5JeXq0tw215lxonI2/3ZYVK4/ok9gwXrQoWm8YieJqitk/ZQ4S17/4 17 | pynHtDfdxLn23EXeGx+UTxJGfpRmhEZdJ+MN7QGYoomzx5qS5XoYKxRNrDlirJpr 18 | OqXIn8E1it+6d5gOZfuHawcNGhRLplE/pfA= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /scripts/debug/tls/grpc/localhost.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDFjCCAf4CCQCzrLIhrWa55zANBgkqhkiG9w0BAQsFADBCMQswCQYDVQQGEwJV 3 | UzETMBEGA1UECAwKQ2FsaWZvcm5pYTEPMA0GA1UECgwGR29vZ2xlMQ0wCwYDVQQL 4 | DARnUlBDMCAXDTE5MDYyNDIyMjIzM1oYDzIxMTkwNTMxMjIyMjMzWjBWMQswCQYD 5 | VQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEPMA0GA1UECgwGR29vZ2xlMQ0w 6 | CwYDVQQLDARnUlBDMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB 7 | AQUAA4IBDwAwggEKAoIBAQCtCW0TjugnIUu8BEVIYvdMP+/2GENQDjZhZ8eKR5C6 8 | toDGbgjsDtt/GxISAg4cg70fIvy0XolnGPZodvfHDM4lJ7yHBOdZD8TXQoE6okR7 9 | HZuLUJ20M0pXgWqtRewKRUjuYsSDXBnzLiZw1dcv9nGpo+Bqa8NonpiGRRpEkshF 10 | D6T9KU9Ts/x+wMQBIra2Gj0UMh79jPhUuxcYAQA0JQGivnOtdwuPiumpnUT8j8h6 11 | tWg5l01EsCZWJecCF85KnGpJEVYPyPqBqGsy0nGS9plGotOWF87+jyUQt+KD63xA 12 | aBmTro86mKDDKEK4JvzjVeMGz2UbVcLPiiZnErTFaiXJAgMBAAEwDQYJKoZIhvcN 13 | AQELBQADggEBAKsDgOPCWp5WCy17vJbRlgfgk05sVNIHZtzrmdswjBmvSg8MUpep 14 | XqcPNUpsljAXsf9UM5IFEMRdilUsFGWvHjBEtNAW8WUK9UV18WRuU//0w1Mp5HAN 15 | xUEKb4BoyZr65vlCnTR+AR5c9FfPvLibhr5qHs2RA8Y3GyLOcGqBWed87jhdQLCc 16 | P1bxB+96le5JeXq0tw215lxonI2/3ZYVK4/ok9gwXrQoWm8YieJqitk/ZQ4S17/4 17 | pynHtDfdxLn23EXeGx+UTxJGfpRmhEZdJ+MN7QGYoomzx5qS5XoYKxRNrDlirJpr 18 | OqXIn8E1it+6d5gOZfuHawcNGhRLplE/pfA= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /app/client/metrics/client.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | "go.uber.org/zap" 8 | 9 | "github.com/ydb-platform/fq-connector-go/app/config" 10 | "github.com/ydb-platform/fq-connector-go/common" 11 | ) 12 | 13 | func runClient(cmd *cobra.Command, _ []string) error { 14 | configPath, err := cmd.Flags().GetString(configFlag) 15 | if err != nil { 16 | return fmt.Errorf("get config flag: %v", err) 17 | } 18 | 19 | var cfg config.TClientConfig 20 | 21 | if err := common.NewConfigFromPrototextFile[*config.TClientConfig](configPath, &cfg); err != nil { 22 | return fmt.Errorf("unknown instance: %w", err) 23 | } 24 | 25 | logger := common.NewDefaultLogger() 26 | 27 | if err := callServer(logger, &cfg); err != nil { 28 | return fmt.Errorf("call server: %w", err) 29 | } 30 | 31 | return nil 32 | } 33 | 34 | func callServer(_ *zap.Logger, cfg *config.TClientConfig) error { 35 | mp, err := common.NewMetricsSnapshot(cfg.MetricsServerEndpoint, false) 36 | if err != nil { 37 | return fmt.Errorf("new metrics provider: %w", err) 38 | } 39 | 40 | result := mp.FindStatusSensors("RATE", "DescribeTable", "status_total", "OK") 41 | 42 | fmt.Println(result) 43 | 44 | return nil 45 | } 46 | -------------------------------------------------------------------------------- /app/server/conversion/strftime.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Phus Lu. All rights reserved. 2 | // Use of this source code is governed by a MIT 3 | // license that can be found in the NOTICE file. 4 | 5 | package conversion 6 | 7 | const tab = "00010203040506070809" + 8 | "10111213141516171819" + 9 | "20212223242526272829" + 10 | "30313233343536373839" + 11 | "40414243444546474849" + 12 | "50515253545556575859" + 13 | "60616263646566676869" + 14 | "70717273747576777879" + 15 | "80818283848586878889" + 16 | "90919293949596979899" 17 | 18 | func formatNanoseconds(buf []byte, ns int) []byte { 19 | // fast transformation of nanoseconds 20 | var tmp [9]byte 21 | 22 | b := ns % 100 * 2 23 | 24 | tmp[8] = tab[b+1] 25 | tmp[7] = tab[b] 26 | 27 | ns /= 100 28 | 29 | b = ns % 100 * 2 30 | tmp[6] = tab[b+1] 31 | tmp[5] = tab[b] 32 | 33 | ns /= 100 34 | 35 | b = ns % 100 * 2 36 | tmp[4] = tab[b+1] 37 | tmp[3] = tab[b] 38 | 39 | ns /= 100 40 | 41 | b = ns % 100 * 2 42 | tmp[2] = tab[b+1] 43 | tmp[1] = tab[b] 44 | tmp[0] = byte(ns/100) + '0' 45 | 46 | // check for trailing zeroes 47 | i := 8 48 | for ; i >= 0; i-- { 49 | if tmp[i] != '0' { 50 | break 51 | } 52 | } 53 | 54 | buf = append(buf, tmp[:i+1]...) 55 | 56 | return buf 57 | } 58 | -------------------------------------------------------------------------------- /scripts/debug/tls/grpc/root.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDWTCCAkGgAwIBAgIJAPOConZMwykwMA0GCSqGSIb3DQEBCwUAMEIxCzAJBgNV 3 | BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMQ8wDQYDVQQKDAZHb29nbGUxDTAL 4 | BgNVBAsMBGdSUEMwIBcNMTkwNjI0MjIyMDA3WhgPMjExOTA1MzEyMjIwMDdaMEIx 5 | CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMQ8wDQYDVQQKDAZHb29n 6 | bGUxDTALBgNVBAsMBGdSUEMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB 7 | AQCwqei3TfyLidnQNDJ2lierMYo229K92DuORni7nSjJQ59Jc3dNMsmqGQJjCD8o 8 | 6mTlKM/oCbs27Wpx+OxcOLvT95j2kiDGca1fCvaMdguIod09SWiyMpv/hp0trLv7 9 | NJIKHznath6rHYX2Ii3fZ1yCPzyQbEPSAA+GNpoNm1v1ZWmWKke9v7vLlS3inNlW 10 | Mt9jepK7DrtbNZnVDjeItnppBSbVYRMxIyNHkepFbqXx5TpkCvl4M4XQZw9bfSxQ 11 | i3WZ3q+T1Tw//OUdPNc+OfMhu0MA0QoMwikskP0NaIC3dbJZ5Ogx0RcnaB4E+9C6 12 | O/znUEh3WuKVl5HXBF+UwWoFAgMBAAGjUDBOMB0GA1UdDgQWBBRm3JIgzgK4G97J 13 | fbMGatWMZc7V3jAfBgNVHSMEGDAWgBRm3JIgzgK4G97JfbMGatWMZc7V3jAMBgNV 14 | HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCNiV8x41if094ry2srS0YucpiN 15 | 3rTPk08FOLsENTMYai524TGXJti1P6ofGr5KXCL0uxTByHE3fEiMMud2TIY5iHQo 16 | Y4mzDTTcb+Q7yKHwYZMlcp6nO8W+NeY5t+S0JPHhb8deKWepcN2UpXBUYQLw7AiE 17 | l96T9Gi+vC9h/XE5IVwHFQXTxf5UYzXtW1nfapvrOONg/ms41dgmrRKIi+knWfiJ 18 | FdHpHX2sfDAoJtnpEISX+nxRGNVTLY64utXWm4yxaZJshvy2s8zWJgRg7rtwAhTT 19 | Np9E9MnihXLEmDI4Co9XlLPJyZFmqImsbmVuKFeQOCiLAoPJaMI2lbi7fiTo 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /scripts/debug/tls/grpc/root.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDWTCCAkGgAwIBAgIJAPOConZMwykwMA0GCSqGSIb3DQEBCwUAMEIxCzAJBgNV 3 | BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMQ8wDQYDVQQKDAZHb29nbGUxDTAL 4 | BgNVBAsMBGdSUEMwIBcNMTkwNjI0MjIyMDA3WhgPMjExOTA1MzEyMjIwMDdaMEIx 5 | CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMQ8wDQYDVQQKDAZHb29n 6 | bGUxDTALBgNVBAsMBGdSUEMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB 7 | AQCwqei3TfyLidnQNDJ2lierMYo229K92DuORni7nSjJQ59Jc3dNMsmqGQJjCD8o 8 | 6mTlKM/oCbs27Wpx+OxcOLvT95j2kiDGca1fCvaMdguIod09SWiyMpv/hp0trLv7 9 | NJIKHznath6rHYX2Ii3fZ1yCPzyQbEPSAA+GNpoNm1v1ZWmWKke9v7vLlS3inNlW 10 | Mt9jepK7DrtbNZnVDjeItnppBSbVYRMxIyNHkepFbqXx5TpkCvl4M4XQZw9bfSxQ 11 | i3WZ3q+T1Tw//OUdPNc+OfMhu0MA0QoMwikskP0NaIC3dbJZ5Ogx0RcnaB4E+9C6 12 | O/znUEh3WuKVl5HXBF+UwWoFAgMBAAGjUDBOMB0GA1UdDgQWBBRm3JIgzgK4G97J 13 | fbMGatWMZc7V3jAfBgNVHSMEGDAWgBRm3JIgzgK4G97JfbMGatWMZc7V3jAMBgNV 14 | HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCNiV8x41if094ry2srS0YucpiN 15 | 3rTPk08FOLsENTMYai524TGXJti1P6ofGr5KXCL0uxTByHE3fEiMMud2TIY5iHQo 16 | Y4mzDTTcb+Q7yKHwYZMlcp6nO8W+NeY5t+S0JPHhb8deKWepcN2UpXBUYQLw7AiE 17 | l96T9Gi+vC9h/XE5IVwHFQXTxf5UYzXtW1nfapvrOONg/ms41dgmrRKIi+knWfiJ 18 | FdHpHX2sfDAoJtnpEISX+nxRGNVTLY64utXWm4yxaZJshvy2s8zWJgRg7rtwAhTT 19 | Np9E9MnihXLEmDI4Co9XlLPJyZFmqImsbmVuKFeQOCiLAoPJaMI2lbi7fiTo 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /library/go/core/xerrors/internal/modes/stack_trace_mode.go: -------------------------------------------------------------------------------- 1 | package modes 2 | 3 | import "sync/atomic" 4 | 5 | type StackTraceMode int32 6 | 7 | const ( 8 | StackTraceModeFrames StackTraceMode = iota 9 | StackTraceModeStacks 10 | StackTraceModeStackThenFrames 11 | StackTraceModeStackThenNothing 12 | StackTraceModeNothing 13 | ) 14 | 15 | func (m StackTraceMode) String() string { 16 | return []string{"Frames", "Stacks", "StackThenFrames", "StackThenNothing", "Nothing"}[m] 17 | } 18 | 19 | const defaultStackTraceMode = StackTraceModeFrames 20 | 21 | var ( 22 | // Default mode 23 | stackTraceMode = defaultStackTraceMode 24 | // Known modes (used in tests) 25 | knownStackTraceModes = []StackTraceMode{ 26 | StackTraceModeFrames, 27 | StackTraceModeStacks, 28 | StackTraceModeStackThenFrames, 29 | StackTraceModeStackThenNothing, 30 | StackTraceModeNothing, 31 | } 32 | ) 33 | 34 | func SetStackTraceMode(v StackTraceMode) { 35 | atomic.StoreInt32((*int32)(&stackTraceMode), int32(v)) 36 | } 37 | 38 | func GetStackTraceMode() StackTraceMode { 39 | return StackTraceMode(atomic.LoadInt32((*int32)(&stackTraceMode))) 40 | } 41 | 42 | func DefaultStackTraceMode() { 43 | SetStackTraceMode(defaultStackTraceMode) 44 | } 45 | 46 | func KnownStackTraceModes() []StackTraceMode { 47 | return knownStackTraceModes 48 | } 49 | -------------------------------------------------------------------------------- /tests/infra/datasource/ydb/datasource.go: -------------------------------------------------------------------------------- 1 | package ydb 2 | 3 | import ( 4 | "fmt" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/tests/infra/datasource" 8 | "github.com/ydb-platform/fq-connector-go/tests/infra/docker_compose" 9 | ) 10 | 11 | const ( 12 | serviceName = "ydb" 13 | internalPort = 2136 14 | database = "local" 15 | ) 16 | 17 | func deriveDataSourceFromDockerCompose(ed *docker_compose.EndpointDeterminer) (*datasource.DataSource, error) { 18 | var ( 19 | dsi = &api_common.TGenericDataSourceInstance{ 20 | Kind: api_common.EGenericDataSourceKind_YDB, 21 | Database: database, 22 | UseTls: false, 23 | Protocol: api_common.EGenericProtocol_NATIVE, 24 | Credentials: &api_common.TGenericCredentials{ 25 | Payload: &api_common.TGenericCredentials_Basic{ 26 | Basic: &api_common.TGenericCredentials_TBasic{ 27 | Username: "admin", 28 | Password: "password", 29 | }, 30 | }, 31 | }, 32 | } 33 | 34 | err error 35 | ) 36 | 37 | dsi.Endpoint, err = ed.GetEndpoint(serviceName, internalPort) 38 | if err != nil { 39 | return nil, fmt.Errorf("derive endpoint: %w", err) 40 | } 41 | 42 | return &datasource.DataSource{ 43 | Instances: []*api_common.TGenericDataSourceInstance{dsi}, 44 | }, nil 45 | } 46 | -------------------------------------------------------------------------------- /app/server/utils/decimal/deserialize.go: -------------------------------------------------------------------------------- 1 | // Package decimal provides utilities for working with decimal numbers. 2 | package decimal 3 | 4 | import ( 5 | "math/big" 6 | "slices" 7 | 8 | "github.com/shopspring/decimal" 9 | ) 10 | 11 | // Deserialize converts a byte array representation to a decimal value 12 | func Deserialize( 13 | src []byte, // source byte array 14 | scale uint32, // scale factor 15 | ) *decimal.Decimal { 16 | // Make a copy of the source to avoid modifying the original 17 | buf := make([]byte, len(src)) 18 | copy(buf, src) 19 | 20 | // LittleEndian -> BigEndian 21 | slices.Reverse(buf) 22 | 23 | // Create a new big.Int from the bytes 24 | bigInt := new(big.Int).SetBytes(buf) 25 | 26 | // Check if the number is negative (most significant bit is set) 27 | isNegative := len(buf) > 0 && (buf[0]&0x80) != 0 28 | 29 | if isNegative { 30 | // For negative numbers: subtract from 2^{8*blobSize} to get the original negative value 31 | twoToThe128 := new(big.Int).Lsh(big.NewInt(1), uint(blobSize*8)) 32 | 33 | bigInt = new(big.Int).Sub(twoToThe128, bigInt) 34 | bigInt.Neg(bigInt) 35 | } 36 | 37 | // Create decimal from big.Int 38 | result := decimal.NewFromBigInt(bigInt, 0) 39 | 40 | // Only shift when scale > 0 41 | if scale > 0 { 42 | result = result.Shift(-int32(scale)) 43 | } 44 | 45 | return &result 46 | } 47 | -------------------------------------------------------------------------------- /app/server/datasource/mock.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/stretchr/testify/mock" 7 | "go.uber.org/zap" 8 | 9 | api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" 10 | "github.com/ydb-platform/fq-connector-go/app/server/paging" 11 | ) 12 | 13 | var _ DataSource[any] = (*DataSourceMock[any])(nil) 14 | 15 | //nolint:revive 16 | type DataSourceMock[T paging.Acceptor] struct { 17 | mock.Mock 18 | } 19 | 20 | func (*DataSourceMock[T]) DescribeTable( 21 | _ context.Context, 22 | _ *zap.Logger, 23 | _ *api_service_protos.TDescribeTableRequest, 24 | ) (*api_service_protos.TDescribeTableResponse, error) { 25 | panic("not implemented") // TODO: Implement 26 | } 27 | 28 | func (*DataSourceMock[T]) ListSplits( 29 | _ context.Context, 30 | _ *zap.Logger, 31 | _ *api_service_protos.TListSplitsRequest, 32 | _ *api_service_protos.TSelect, 33 | _ chan<- *ListSplitResult) error { 34 | panic("not implemented") // TODO: Implement 35 | } 36 | 37 | func (m *DataSourceMock[T]) ReadSplit( 38 | ctx context.Context, 39 | logger *zap.Logger, 40 | queryID string, 41 | request *api_service_protos.TReadSplitsRequest, 42 | split *api_service_protos.TSplit, 43 | sinkFactory paging.SinkFactory[T], 44 | ) error { 45 | return m.Called(ctx, logger, queryID, request, split, sinkFactory).Error(0) 46 | } 47 | -------------------------------------------------------------------------------- /app/server/conversion/interface.go: -------------------------------------------------------------------------------- 1 | package conversion 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ydb-platform/fq-connector-go/common" 7 | ) 8 | 9 | type ValuePtrConverter[IN common.ValueType, OUT common.ValueType] interface { 10 | Convert(in *IN) (OUT, error) 11 | } 12 | 13 | type Collection interface { 14 | Bool() ValuePtrConverter[bool, uint8] 15 | Int8() ValuePtrConverter[int8, int8] 16 | Int16() ValuePtrConverter[int16, int16] 17 | Int32() ValuePtrConverter[int32, int32] 18 | Int64() ValuePtrConverter[int64, int64] 19 | Uint8() ValuePtrConverter[uint8, uint8] 20 | Uint16() ValuePtrConverter[uint16, uint16] 21 | Uint32() ValuePtrConverter[uint32, uint32] 22 | Uint64() ValuePtrConverter[uint64, uint64] 23 | Float32() ValuePtrConverter[float32, float32] 24 | Float64() ValuePtrConverter[float64, float64] 25 | String() ValuePtrConverter[string, string] 26 | StringToBytes() ValuePtrConverter[string, []byte] 27 | Bytes() ValuePtrConverter[[]byte, []byte] 28 | BytesToString() ValuePtrConverter[[]byte, string] 29 | Date() ValuePtrConverter[time.Time, uint16] 30 | DateToString() ValuePtrConverter[time.Time, string] 31 | Datetime() ValuePtrConverter[time.Time, uint32] 32 | DatetimeToString() ValuePtrConverter[time.Time, string] 33 | Timestamp() ValuePtrConverter[time.Time, uint64] 34 | TimestampToString(utc bool) ValuePtrConverter[time.Time, string] 35 | } 36 | -------------------------------------------------------------------------------- /library/go/yson/time.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | import "time" 4 | 5 | const ytTimeLayout = "2006-01-02T15:04:05.000000Z" 6 | 7 | // Time is an alias for time.Time with YT specific time representation format. 8 | type Time time.Time 9 | 10 | func (t Time) IsZero() bool { 11 | return time.Time(t).IsZero() 12 | } 13 | 14 | func (t *Time) UnmarshalText(text []byte) error { 15 | ts, err := UnmarshalTime(string(text)) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | *t = ts 21 | return nil 22 | } 23 | 24 | func (t Time) MarshalText() (text []byte, err error) { 25 | s, err := MarshalTime(t) 26 | return []byte(s), err 27 | } 28 | 29 | // UnmarshalTime decodes time from YT-specific time format. 30 | // 31 | // Entity is decoded into zero time. 32 | func UnmarshalTime(in string) (t Time, err error) { 33 | if in == "#" { 34 | return Time{}, nil 35 | } 36 | var tt time.Time 37 | tt, err = time.Parse(ytTimeLayout, in) 38 | t = Time(tt) 39 | return 40 | } 41 | 42 | // MarshalTime encodes time to YT-specific time format. 43 | // 44 | // Zero time is encoded into entity. 45 | func MarshalTime(t Time) (s string, err error) { 46 | if time.Time(t).IsZero() { 47 | return "#", nil 48 | } 49 | return time.Time(t).UTC().Format(ytTimeLayout), nil 50 | } 51 | 52 | // Duration is an alias for time.Duration with YT specific time representation format. 53 | type Duration time.Duration 54 | -------------------------------------------------------------------------------- /tests/infra/datasource/mysql/datasource.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "fmt" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/tests/infra/datasource" 8 | "github.com/ydb-platform/fq-connector-go/tests/infra/docker_compose" 9 | ) 10 | 11 | const ( 12 | serviceName = "mysql" 13 | internalPort = 3306 14 | database = "fq" 15 | username = "root" 16 | password = "password" 17 | schema = "fq" 18 | ) 19 | 20 | func deriveDataSourceFromDockerCompose(ed *docker_compose.EndpointDeterminer) (*datasource.DataSource, error) { 21 | dsi := &api_common.TGenericDataSourceInstance{ 22 | Kind: api_common.EGenericDataSourceKind_MYSQL, 23 | Database: database, 24 | Credentials: &api_common.TGenericCredentials{ 25 | Payload: &api_common.TGenericCredentials_Basic{ 26 | Basic: &api_common.TGenericCredentials_TBasic{ 27 | Username: username, 28 | Password: password, 29 | }, 30 | }, 31 | }, 32 | Protocol: api_common.EGenericProtocol_NATIVE, 33 | UseTls: false, 34 | } 35 | 36 | var err error 37 | 38 | dsi.Endpoint, err = ed.GetEndpoint(serviceName, internalPort) 39 | if err != nil { 40 | return nil, fmt.Errorf("derive endpoint: %w", err) 41 | } 42 | 43 | return &datasource.DataSource{ 44 | Instances: []*api_common.TGenericDataSourceInstance{dsi}, 45 | }, nil 46 | } 47 | -------------------------------------------------------------------------------- /tests/infra/datasource/opensearch/datasource.go: -------------------------------------------------------------------------------- 1 | package opensearch 2 | 3 | import ( 4 | "fmt" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/tests/infra/datasource" 8 | "github.com/ydb-platform/fq-connector-go/tests/infra/docker_compose" 9 | ) 10 | 11 | const ( 12 | serviceName = "opensearch" 13 | internalPort = 9200 14 | database = "connector" 15 | username = "admin" 16 | password = "password" 17 | ) 18 | 19 | func deriveDataSourceFromDockerCompose(ed *docker_compose.EndpointDeterminer) (*datasource.DataSource, error) { 20 | dsi := &api_common.TGenericDataSourceInstance{ 21 | Kind: api_common.EGenericDataSourceKind_OPENSEARCH, 22 | Database: database, 23 | Credentials: &api_common.TGenericCredentials{ 24 | Payload: &api_common.TGenericCredentials_Basic{ 25 | Basic: &api_common.TGenericCredentials_TBasic{ 26 | Username: username, 27 | Password: password, 28 | }, 29 | }, 30 | }, 31 | Protocol: api_common.EGenericProtocol_HTTP, 32 | UseTls: false, 33 | } 34 | 35 | var err error 36 | 37 | dsi.Endpoint, err = ed.GetEndpoint(serviceName, internalPort) 38 | if err != nil { 39 | return nil, fmt.Errorf("derive endpoint: %w", err) 40 | } 41 | 42 | return &datasource.DataSource{ 43 | Instances: []*api_common.TGenericDataSourceInstance{dsi}, 44 | }, nil 45 | } 46 | -------------------------------------------------------------------------------- /tests/infra/datasource/ms_sql_server/datasource.go: -------------------------------------------------------------------------------- 1 | package ms_sql_server 2 | 3 | import ( 4 | "fmt" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/tests/infra/datasource" 8 | "github.com/ydb-platform/fq-connector-go/tests/infra/docker_compose" 9 | ) 10 | 11 | const ( 12 | serviceName = "ms_sql_server" 13 | internalPort = 1433 14 | database = "master" 15 | username = "sa" 16 | password = "Qwerty12345!" 17 | ) 18 | 19 | func deriveDataSourceFromDockerCompose(ed *docker_compose.EndpointDeterminer) (*datasource.DataSource, error) { 20 | dsi := &api_common.TGenericDataSourceInstance{ 21 | Kind: api_common.EGenericDataSourceKind_MS_SQL_SERVER, 22 | Database: database, 23 | Credentials: &api_common.TGenericCredentials{ 24 | Payload: &api_common.TGenericCredentials_Basic{ 25 | Basic: &api_common.TGenericCredentials_TBasic{ 26 | Username: username, 27 | Password: password, 28 | }, 29 | }, 30 | }, 31 | Protocol: api_common.EGenericProtocol_NATIVE, 32 | UseTls: false, 33 | } 34 | 35 | var err error 36 | 37 | dsi.Endpoint, err = ed.GetEndpoint(serviceName, internalPort) 38 | if err != nil { 39 | return nil, fmt.Errorf("derive endpoint: %w", err) 40 | } 41 | 42 | return &datasource.DataSource{ 43 | Instances: []*api_common.TGenericDataSourceInstance{dsi}, 44 | }, nil 45 | } 46 | -------------------------------------------------------------------------------- /tests/infra/datasource/redis/datasource.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "fmt" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/tests/infra/datasource" 8 | "github.com/ydb-platform/fq-connector-go/tests/infra/docker_compose" 9 | ) 10 | 11 | const ( 12 | serviceName = "redis" 13 | internalPort = 6379 14 | database = "0" 15 | username = "default" 16 | password = "" 17 | ) 18 | 19 | func deriveDataSourceFromDockerCompose(ed *docker_compose.EndpointDeterminer) (*datasource.DataSource, error) { 20 | dsi := &api_common.TGenericDataSourceInstance{ 21 | Kind: api_common.EGenericDataSourceKind_REDIS, 22 | Database: database, 23 | Credentials: &api_common.TGenericCredentials{ 24 | Payload: &api_common.TGenericCredentials_Basic{ 25 | Basic: &api_common.TGenericCredentials_TBasic{ 26 | Username: username, 27 | Password: password, 28 | }, 29 | }, 30 | }, 31 | Protocol: api_common.EGenericProtocol_NATIVE, 32 | UseTls: false, 33 | // Дополнительные опции можно передать при необходимости 34 | } 35 | 36 | var err error 37 | 38 | dsi.Endpoint, err = ed.GetEndpoint(serviceName, internalPort) 39 | if err != nil { 40 | return nil, fmt.Errorf("derive endpoint: %w", err) 41 | } 42 | 43 | return &datasource.DataSource{ 44 | Instances: []*api_common.TGenericDataSourceInstance{dsi}, 45 | }, nil 46 | } 47 | -------------------------------------------------------------------------------- /examples/docker_compose/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | ydb: 3 | image: ghcr.io/ydb-platform/local-ydb:25.1.4.7 4 | container_name: fq-example-ydb 5 | hostname: localhost 6 | ports: 7 | - '2136:2136' 8 | - '8765:8765' 9 | environment: 10 | GRPC_TLS_PORT: 2135 11 | GRPC_PORT: 2136 12 | MON_PORT: 8765 13 | YDB_DEFAULT_LOG_LEVEL: NOTICE 14 | FQ_CONNECTOR_ENDPOINT: grpc://fq-connector-go:2130 15 | 16 | postgresql: 17 | image: mirror.gcr.io/postgres 18 | container_name: fq-example-postgresql 19 | ports: 20 | - '5432:5432' 21 | environment: 22 | POSTGRES_DB: fq 23 | POSTGRES_USER: admin 24 | POSTGRES_PASSWORD: password 25 | PGDATA: /var/lib/postgresql/data/pgdata 26 | 27 | mysql: 28 | image: mirror.gcr.io/library/mysql:8.0 29 | container_name: fq-example-mysql 30 | environment: 31 | MYSQL_DATABASE: fq 32 | MYSQL_ROOT_PASSWORD: password 33 | ports: 34 | - '3306:3306' 35 | 36 | mssql: 37 | image: mcr.microsoft.com/mssql/server:2022-latest 38 | container_name: fq-example-mssql 39 | environment: 40 | ACCEPT_EULA: Y 41 | MSSQL_SA_PASSWORD: Qwerty12345! 42 | ports: 43 | - '1433:1433' 44 | 45 | fq-connector-go: 46 | image: ghcr.io/ydb-platform/fq-connector-go:latest 47 | container_name: fq-example-connector 48 | ports: 49 | - '2130:2130' 50 | -------------------------------------------------------------------------------- /scripts/debug/tls/oracle/generate_cert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | certsDir=./ 4 | 5 | function createDirIfNotExists { 6 | dirName=$1 7 | if [ ! -d $dirName ]; then 8 | echo "creating dir '$dirName'" 9 | mkdir -p $dirName 10 | fi 11 | } 12 | 13 | function setCertsPermissions { 14 | sudo chmod +r client.key 15 | } 16 | 17 | function createCerts { 18 | createDirIfNotExists $certsDir 19 | cd $certsDir 20 | 21 | echo "Generate RSA client.key 4096" 22 | 23 | openssl genrsa -out client.key 4096 24 | 25 | echo "Generate client.csr" 26 | 27 | openssl req -new -key client.key -out client.csr -subj '/CN=oracle' -addext "subjectAltName = DNS:oracle" 28 | 29 | echo "Generate RSA caCert.key 4096" 30 | 31 | openssl genrsa -out caCert.key 4096 32 | 33 | echo "Generate caCert.crt" 34 | 35 | openssl req -new -x509 -days 1826 -key caCert.key -out caCert.crt -subj '/CN=oracle/C=US/OU=Class 2 Public Primary Certification Authority/O=VeriSign' -addext "subjectAltName = DNS:oracle" 36 | 37 | echo "Generate cert.crt subscribing by caCert from client.csr" 38 | 39 | openssl x509 -req -days 730 -in client.csr -CA caCert.crt -CAkey caCert.key -extensions v3_ca -extfile ./extfile.cnf -set_serial 01 -out cert.crt 40 | 41 | setCertsPermissions 42 | 43 | cd .. 44 | } 45 | 46 | 47 | 48 | preRunDir=($PWD) 49 | scriptDir=$0 50 | cd $0 51 | 52 | createCerts 53 | 54 | cd $preRunDir -------------------------------------------------------------------------------- /app/client/utils/preset.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | "go.uber.org/zap" 8 | 9 | "github.com/ydb-platform/fq-connector-go/app/config" 10 | "github.com/ydb-platform/fq-connector-go/common" 11 | ) 12 | 13 | type Preset struct { 14 | Logger *zap.Logger 15 | Cfg *config.TClientConfig 16 | TableName string 17 | } 18 | 19 | func (p *Preset) Close() { 20 | if err := p.Logger.Sync(); err != nil { 21 | fmt.Println("failed to sync logger", err) 22 | } 23 | } 24 | 25 | func MakePreset(cmd *cobra.Command) (*Preset, error) { 26 | configPath, err := cmd.Flags().GetString(ConfigFlag) 27 | if err != nil { 28 | return nil, fmt.Errorf("get config flag: %v", err) 29 | } 30 | 31 | tableName, err := cmd.Flags().GetString(TableFlag) 32 | if err != nil { 33 | return nil, fmt.Errorf("get table flag: %v", err) 34 | } 35 | 36 | var cfg config.TClientConfig 37 | 38 | if err = common.NewConfigFromPrototextFile[*config.TClientConfig](configPath, &cfg); err != nil { 39 | return nil, fmt.Errorf("unknown instance: %w", err) 40 | } 41 | 42 | // override credentials if IAM-token provided 43 | common.MaybeInjectTokenToDataSourceInstance(cfg.DataSourceInstance) 44 | 45 | logger := common.AnnotateLoggerWithDataSourceInstance(common.NewDefaultLogger(), cfg.DataSourceInstance) 46 | 47 | return &Preset{ 48 | Logger: logger, 49 | Cfg: &cfg, 50 | TableName: tableName, 51 | }, nil 52 | } 53 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/ms_sql_server/connection.go: -------------------------------------------------------------------------------- 1 | package ms_sql_server 2 | 3 | import ( 4 | "database/sql" 5 | 6 | _ "github.com/denisenkom/go-mssqldb" 7 | "go.uber.org/zap" 8 | 9 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 10 | rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" 11 | "github.com/ydb-platform/fq-connector-go/common" 12 | ) 13 | 14 | var _ rdbms_utils.Connection = (*Connection)(nil) 15 | 16 | type Connection struct { 17 | db *sql.DB 18 | queryLogger common.QueryLogger 19 | dataSourceInstance *api_common.TGenericDataSourceInstance 20 | tableName string 21 | } 22 | 23 | func (c *Connection) Close() error { 24 | return c.db.Close() 25 | } 26 | 27 | func (c *Connection) DataSourceInstance() *api_common.TGenericDataSourceInstance { 28 | return c.dataSourceInstance 29 | } 30 | 31 | func (c *Connection) TableName() string { 32 | return c.tableName 33 | } 34 | 35 | func (c *Connection) Query(params *rdbms_utils.QueryParams) (*rdbms_utils.QueryResult, error) { 36 | c.queryLogger.Dump(params.QueryText, params.QueryArgs.Values()...) 37 | 38 | out, err := c.db.QueryContext(params.Ctx, params.QueryText, params.QueryArgs.Values()...) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return &rdbms_utils.QueryResult{ 44 | Rows: rows{out}, 45 | }, nil 46 | } 47 | 48 | func (c *Connection) Logger() *zap.Logger { 49 | return c.queryLogger.Logger 50 | } 51 | -------------------------------------------------------------------------------- /common/time.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | var ( 9 | // According to https://ydb.tech/en/docs/yql/reference/types/primitive#datetime 10 | minYDBTime = time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) 11 | maxYDBTime = time.Date(2106, time.January, 1, 0, 0, 0, 0, time.UTC) 12 | ) 13 | 14 | func TimeToYDBDate(t *time.Time) (uint16, error) { 15 | if t.Before(minYDBTime) || t.After(maxYDBTime) { 16 | return 0, fmt.Errorf("convert '%v' to YDB Date: %w", t, ErrValueOutOfTypeBounds) 17 | } 18 | 19 | days := t.Sub(minYDBTime).Hours() / 24 20 | 21 | return uint16(days), nil 22 | } 23 | 24 | func TimeToYDBDatetime(t *time.Time) (uint32, error) { 25 | if t.Before(minYDBTime) || t.After(maxYDBTime) { 26 | return 0, fmt.Errorf("convert '%v' to YDB Date: %w", t, ErrValueOutOfTypeBounds) 27 | } 28 | 29 | seconds := t.Unix() 30 | 31 | return uint32(seconds), nil 32 | } 33 | 34 | func TimeToYDBTimestamp(t *time.Time) (uint64, error) { 35 | if t.Before(minYDBTime) || t.After(maxYDBTime) { 36 | return 0, fmt.Errorf("convert '%v' to YDB Date: %w", t, ErrValueOutOfTypeBounds) 37 | } 38 | 39 | seconds := t.UnixMicro() 40 | 41 | return uint64(seconds), nil 42 | } 43 | 44 | type ydbTime interface { 45 | uint16 | uint32 | uint64 46 | } 47 | 48 | func MustTimeToYDBType[OUT ydbTime](f func(t *time.Time) (OUT, error), t time.Time) OUT { 49 | res, err := f(&t) 50 | if err != nil { 51 | panic(err) 52 | } 53 | 54 | return res 55 | } 56 | -------------------------------------------------------------------------------- /app/server/datasource/rdbms/utils/unit_test_helpers_test.go: -------------------------------------------------------------------------------- 1 | package utils //nolint:revive 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestDataConverter(t *testing.T) { 11 | dc := DataConverter{} 12 | 13 | type testCase struct { 14 | src [][]any 15 | dst [][][]any 16 | rowsPerBlock int 17 | } 18 | 19 | testCases := []testCase{ 20 | { 21 | src: [][]any{ 22 | {int32(1), "a"}, 23 | {int32(2), "b"}, 24 | {int32(3), "c"}, 25 | {int32(4), "d"}, 26 | }, 27 | dst: [][][]any{ 28 | { 29 | {int32(1), int32(2)}, 30 | {"a", "b"}, 31 | }, 32 | { 33 | {int32(3), int32(4)}, 34 | {"c", "d"}, 35 | }, 36 | }, 37 | rowsPerBlock: 2, 38 | }, 39 | { 40 | src: [][]any{ 41 | {int32(1), "a"}, 42 | {int32(2), "b"}, 43 | {int32(3), "c"}, 44 | {int32(4), "d"}, 45 | {int32(5), "e"}, 46 | }, 47 | dst: [][][]any{ 48 | { 49 | {int32(1), int32(2)}, 50 | {"a", "b"}, 51 | }, 52 | { 53 | {int32(3), int32(4)}, 54 | {"c", "d"}, 55 | }, 56 | { 57 | {int32(5)}, 58 | {"e"}, 59 | }, 60 | }, 61 | rowsPerBlock: 2, 62 | }, 63 | } 64 | 65 | for _, tc := range testCases { 66 | tc := tc 67 | t.Run(fmt.Sprintf("rowsPerRecord_%d", tc.rowsPerBlock), func(t *testing.T) { 68 | actual := dc.RowsToColumnBlocks(tc.src, tc.rowsPerBlock) 69 | require.Equal(t, tc.dst, actual) 70 | }) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /library/go/x/xruntime/stacktrace.go: -------------------------------------------------------------------------------- 1 | package xruntime 2 | 3 | import ( 4 | "runtime" 5 | ) 6 | 7 | type StackTrace struct { 8 | frames []uintptr 9 | full bool 10 | } 11 | 12 | func NewStackTrace16(skip int) *StackTrace { 13 | var pcs [16]uintptr 14 | return newStackTrace(skip+2, pcs[:]) 15 | } 16 | 17 | func NewStackTrace32(skip int) *StackTrace { 18 | var pcs [32]uintptr 19 | return newStackTrace(skip+2, pcs[:]) 20 | } 21 | 22 | func NewStackTrace64(skip int) *StackTrace { 23 | var pcs [64]uintptr 24 | return newStackTrace(skip+2, pcs[:]) 25 | } 26 | 27 | func NewStackTrace128(skip int) *StackTrace { 28 | var pcs [128]uintptr 29 | return newStackTrace(skip+2, pcs[:]) 30 | } 31 | 32 | func newStackTrace(skip int, pcs []uintptr) *StackTrace { 33 | n := runtime.Callers(skip+1, pcs) 34 | return &StackTrace{frames: pcs[:n], full: true} 35 | } 36 | 37 | func NewFrame(skip int) *StackTrace { 38 | var pcs [3]uintptr 39 | n := runtime.Callers(skip+1, pcs[:]) 40 | return &StackTrace{frames: pcs[:n]} 41 | } 42 | 43 | func (st *StackTrace) Frames() []runtime.Frame { 44 | frames := runtime.CallersFrames(st.frames[:]) 45 | if !st.full { 46 | if _, ok := frames.Next(); !ok { 47 | return nil 48 | } 49 | 50 | fr, ok := frames.Next() 51 | if !ok { 52 | return nil 53 | } 54 | 55 | return []runtime.Frame{fr} 56 | } 57 | 58 | var res []runtime.Frame 59 | for { 60 | frame, more := frames.Next() 61 | if !more { 62 | break 63 | } 64 | 65 | res = append(res, frame) 66 | } 67 | 68 | return res 69 | } 70 | -------------------------------------------------------------------------------- /examples/docker_compose/README.md: -------------------------------------------------------------------------------- 1 | # YDB Federated Query dockerized setup 2 | 3 | Run at least three containers (the YDB itself, the connector and at least one external datasource - PostgreSQL in this case): 4 | 5 | ```bash 6 | docker compose pull 7 | docker compose up -d ydb fq-connector-go postgresql 8 | ``` 9 | 10 | Enter the PostgreSQL container: 11 | ```bash 12 | docker compose exec -it postgresql psql -d fq -U admin 13 | ``` 14 | 15 | Initialize PostgreSQL with some table: 16 | ``` 17 | CREATE TABLE example ( 18 | id int, 19 | col_01_int int, 20 | col_02_text text 21 | ); 22 | 23 | INSERT INTO example (id, col_01_int, col_02_text) VALUES 24 | (1, 10, 'a'), 25 | (2, 20, 'b'), 26 | (3, 30, 'c'), 27 | (4, NULL, NULL); 28 | ``` 29 | 30 | Visit http://localhost:8765 in your browser. Click: `Databases` -> `/local`, and you'll see a query editor. Execute the following queries: 31 | 32 | ```sql 33 | CREATE OBJECT postgresql_datasource_user_password (TYPE SECRET) WITH (value = "password"); 34 | ``` 35 | 36 | ```sql 37 | CREATE EXTERNAL DATA SOURCE postgresql_datasource WITH ( 38 | SOURCE_TYPE="PostgreSQL", 39 | LOCATION="postgresql:5432", 40 | DATABASE_NAME="fq", 41 | AUTH_METHOD="BASIC", 42 | LOGIN="admin", 43 | PASSWORD_SECRET_NAME="postgresql_datasource_user_password", 44 | PROTOCOL="NATIVE", 45 | USE_TLS="FALSE", 46 | SCHEMA="public" 47 | ); 48 | ``` 49 | 50 | Finally, you'll be able to query the external table: 51 | ```sql 52 | SELECT * FROM postgresql_datasource.example; 53 | ``` -------------------------------------------------------------------------------- /library/go/core/metrics/mock/registry_opts.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/ydb-platform/fq-connector-go/library/go/core/metrics/internal/pkg/registryutil" 5 | ) 6 | 7 | type RegistryOpts struct { 8 | Separator rune 9 | Prefix string 10 | Tags map[string]string 11 | AllowLoadRegisteredMetrics bool 12 | } 13 | 14 | // NewRegistryOpts returns new initialized instance of RegistryOpts 15 | func NewRegistryOpts() *RegistryOpts { 16 | return &RegistryOpts{ 17 | Separator: '.', 18 | Tags: make(map[string]string), 19 | } 20 | } 21 | 22 | // SetTags overrides existing tags 23 | func (o *RegistryOpts) SetTags(tags map[string]string) *RegistryOpts { 24 | o.Tags = tags 25 | return o 26 | } 27 | 28 | // AddTags merges given tags with existing 29 | func (o *RegistryOpts) AddTags(tags map[string]string) *RegistryOpts { 30 | for k, v := range tags { 31 | o.Tags[k] = v 32 | } 33 | return o 34 | } 35 | 36 | // SetPrefix overrides existing prefix 37 | func (o *RegistryOpts) SetPrefix(prefix string) *RegistryOpts { 38 | o.Prefix = prefix 39 | return o 40 | } 41 | 42 | // AppendPrefix adds given prefix as postfix to existing using separator 43 | func (o *RegistryOpts) AppendPrefix(prefix string) *RegistryOpts { 44 | o.Prefix = registryutil.BuildFQName(string(o.Separator), o.Prefix, prefix) 45 | return o 46 | } 47 | 48 | // SetSeparator overrides existing separator 49 | func (o *RegistryOpts) SetSeparator(separator rune) *RegistryOpts { 50 | o.Separator = separator 51 | return o 52 | } 53 | -------------------------------------------------------------------------------- /tools/docker_compose_update/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | 7 | "go.uber.org/zap" 8 | 9 | "github.com/ydb-platform/fq-connector-go/common" 10 | ) 11 | 12 | var pathesToComposes = [3]string{"/ydb/library/yql/providers/generic/connector/tests/datasource", 13 | "/ydb/library/yql/providers/generic/connector/tests/join", 14 | "/ydb/tests/fq/generic"} 15 | 16 | func main() { 17 | logger := common.NewDefaultLogger() 18 | 19 | err := run(logger) 20 | if err != nil { 21 | logger.Error("run", zap.Error(err)) 22 | } 23 | } 24 | 25 | func run(logger *zap.Logger) error { 26 | path := flag.String("path", "path", "Specify the path to ydb file.") 27 | 28 | flag.Parse() 29 | 30 | if err := checkFileExistance(*path); err != nil { 31 | return fmt.Errorf("check existence of file %v: %w", *path, err) 32 | } 33 | 34 | tag, err := getLatestVersion() 35 | if err != nil { 36 | return fmt.Errorf("get latest version %w", err) 37 | } 38 | 39 | checksum, err := getChecksum(tag) 40 | if err != nil { 41 | return fmt.Errorf("get check sum %w", err) 42 | } 43 | 44 | logger.Info("values", zap.String("path", *path), zap.String("tag", tag), zap.String("checksum", checksum)) 45 | 46 | for _, pathToComposes := range pathesToComposes { 47 | fullPath := *path + pathToComposes 48 | 49 | newImage := fmt.Sprintf("ghcr.io/ydb-platform/fq-connector-go:%s@%s", tag, checksum) 50 | 51 | if err = walkDockerCompose(logger, fullPath, newImage); err != nil { 52 | return fmt.Errorf("walk docker compose %w", err) 53 | } 54 | } 55 | 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /app/config/observation.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package NYql.Connector.App.Config; 3 | 4 | import "yql/essentials/providers/common/proto/gateways_config.proto"; 5 | import "app/config/client.proto"; 6 | import "app/config/server.proto"; 7 | 8 | option go_package = "github.com/ydb-platform/fq-connector-go/app/config"; 9 | 10 | // TObservationServerConfig describe the configuration of the observation service 11 | // that helps to track the state of the queries running across different Connector instances 12 | message TObservationServerConfig { 13 | // Endpoint for the HTTP 14 | NYql.TGenericEndpoint endpoint = 1; 15 | 16 | // Discovery service provides the list of Observation API endpoints 17 | // to retrieve data from 18 | TObservationDiscoveryConfig discovery = 2; 19 | 20 | // Time interval between polling Observation API endpoints. 21 | // Valid values should satisfy `time.ParseDuration` (e. g. '5s', '100ms', '3h'). 22 | string polling_interval = 3; 23 | } 24 | 25 | message TObservationDiscoveryConfig { 26 | // TStaticDiscoveryConfig configures the static list of Observation API endpoints 27 | message TStaticDiscoveryConfig { 28 | repeated NYql.TGenericEndpoint endpoints = 1; 29 | } 30 | 31 | // TKubernetesDiscoveryConfig configures Kubernetes API 32 | message TKubernetesDiscoveryConfig { 33 | string label_selector = 1; 34 | uint32 target_port = 2; 35 | } 36 | 37 | oneof payload { 38 | TStaticDiscoveryConfig static = 1; 39 | TKubernetesDiscoveryConfig kubernetes = 2; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /tests/infra/datasource/greenplum/datasource.go: -------------------------------------------------------------------------------- 1 | package greenplum 2 | 3 | import ( 4 | "fmt" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/tests/infra/datasource" 8 | "github.com/ydb-platform/fq-connector-go/tests/infra/docker_compose" 9 | ) 10 | 11 | const ( 12 | serviceName = "greenplum" 13 | internalPort = 5432 14 | database = "template1" 15 | username = "gpadmin" 16 | password = "123456" 17 | schema = "public" 18 | ) 19 | 20 | func deriveDataSourceFromDockerCompose(ed *docker_compose.EndpointDeterminer) (*datasource.DataSource, error) { 21 | dsi := &api_common.TGenericDataSourceInstance{ 22 | Kind: api_common.EGenericDataSourceKind_GREENPLUM, 23 | Database: database, 24 | Credentials: &api_common.TGenericCredentials{ 25 | Payload: &api_common.TGenericCredentials_Basic{ 26 | Basic: &api_common.TGenericCredentials_TBasic{ 27 | Username: username, 28 | Password: password, 29 | }, 30 | }, 31 | }, 32 | Protocol: api_common.EGenericProtocol_NATIVE, 33 | UseTls: false, 34 | Options: &api_common.TGenericDataSourceInstance_GpOptions{ 35 | GpOptions: &api_common.TGreenplumDataSourceOptions{ 36 | Schema: schema, 37 | }, 38 | }, 39 | } 40 | 41 | var err error 42 | 43 | dsi.Endpoint, err = ed.GetEndpoint(serviceName, internalPort) 44 | if err != nil { 45 | return nil, fmt.Errorf("derive endpoint: %w", err) 46 | } 47 | 48 | return &datasource.DataSource{ 49 | Instances: []*api_common.TGenericDataSourceInstance{dsi}, 50 | }, nil 51 | } 52 | -------------------------------------------------------------------------------- /app/bench/cpu_utilization_monitor_darwin.go: -------------------------------------------------------------------------------- 1 | //go:build cgo && darwin 2 | 3 | package bench 4 | 5 | // #include 6 | // #include 7 | import "C" 8 | import "time" 9 | 10 | // cpuUtilizationMonitorDarwin реализует интерфейс cpuUtilizationMonitor для macOS. 11 | type cpuUtilizationMonitorDarwin struct { 12 | startTime time.Time 13 | startCPUTime float64 14 | } 15 | 16 | // getCPUTime получает общее процессорное время (user + sys) для текущего процесса. 17 | func getCPUTime() float64 { 18 | var usage C.struct_rusage 19 | if ret := C.getrusage(C.RUSAGE_SELF, &usage); ret != 0 { 20 | // В случае ошибки возвращаем 0, хотя можно обработать ошибку более детально. 21 | return 0.0 22 | } 23 | userTime := float64(usage.ru_utime.tv_sec) + float64(usage.ru_utime.tv_usec)/1e6 24 | sysTime := float64(usage.ru_stime.tv_sec) + float64(usage.ru_stime.tv_usec)/1e6 25 | return userTime + sysTime 26 | } 27 | 28 | // getPercentage вычисляет процент использования CPU, сравнивая затраченное процессорное время с реальным временем. 29 | func (mon *cpuUtilizationMonitorDarwin) getPercentage() float64 { 30 | currentCPUTime := getCPUTime() 31 | cpuTimeDiff := currentCPUTime - mon.startCPUTime 32 | realSeconds := time.Since(mon.startTime).Seconds() 33 | return (cpuTimeDiff / realSeconds) * 100 34 | } 35 | 36 | // NewCPUUtilizationMonitor возвращает новый экземпляр мониторинга использования CPU. 37 | func NewCPUUtilizationMonitor() cpuUtilizationMonitor { 38 | return &cpuUtilizationMonitorDarwin{ 39 | startTime: time.Now(), 40 | startCPUTime: getCPUTime(), 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /tests/infra/datasource/postgresql/datasource.go: -------------------------------------------------------------------------------- 1 | package postgresql 2 | 3 | import ( 4 | "fmt" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/tests/infra/datasource" 8 | "github.com/ydb-platform/fq-connector-go/tests/infra/docker_compose" 9 | ) 10 | 11 | const ( 12 | serviceName = "postgresql" 13 | internalPort = 5432 14 | database = "connector" 15 | username = "admin" 16 | password = "password" 17 | schema = "public" 18 | ) 19 | 20 | func deriveDataSourceFromDockerCompose(ed *docker_compose.EndpointDeterminer) (*datasource.DataSource, error) { 21 | dsi := &api_common.TGenericDataSourceInstance{ 22 | Kind: api_common.EGenericDataSourceKind_POSTGRESQL, 23 | Database: database, 24 | Credentials: &api_common.TGenericCredentials{ 25 | Payload: &api_common.TGenericCredentials_Basic{ 26 | Basic: &api_common.TGenericCredentials_TBasic{ 27 | Username: username, 28 | Password: password, 29 | }, 30 | }, 31 | }, 32 | Protocol: api_common.EGenericProtocol_NATIVE, 33 | UseTls: false, 34 | Options: &api_common.TGenericDataSourceInstance_PgOptions{ 35 | PgOptions: &api_common.TPostgreSQLDataSourceOptions{ 36 | Schema: schema, 37 | }, 38 | }, 39 | } 40 | 41 | var err error 42 | 43 | dsi.Endpoint, err = ed.GetEndpoint(serviceName, internalPort) 44 | if err != nil { 45 | return nil, fmt.Errorf("derive endpoint: %w", err) 46 | } 47 | 48 | return &datasource.DataSource{ 49 | Instances: []*api_common.TGenericDataSourceInstance{dsi}, 50 | }, nil 51 | } 52 | -------------------------------------------------------------------------------- /library/go/core/metrics/solomon/timer_test.go: -------------------------------------------------------------------------------- 1 | package solomon 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "go.uber.org/atomic" 10 | ) 11 | 12 | func TestTimer_RecordDuration(t *testing.T) { 13 | c := &Timer{ 14 | name: "mytimer", 15 | metricType: typeGauge, 16 | tags: map[string]string{"ololo": "trololo"}, 17 | } 18 | 19 | c.RecordDuration(1 * time.Second) 20 | assert.Equal(t, 1*time.Second, c.value.Load()) 21 | 22 | c.RecordDuration(42 * time.Millisecond) 23 | assert.Equal(t, 42*time.Millisecond, c.value.Load()) 24 | } 25 | 26 | func TestTimerRated_MarshalJSON(t *testing.T) { 27 | c := &Timer{ 28 | name: "mytimer", 29 | metricType: typeRated, 30 | tags: map[string]string{"ololo": "trololo"}, 31 | value: *atomic.NewDuration(42 * time.Millisecond), 32 | } 33 | 34 | b, err := json.Marshal(c) 35 | assert.NoError(t, err) 36 | 37 | expected := []byte(`{"type":"RATE","labels":{"ololo":"trololo","sensor":"mytimer"},"value":0.042}`) 38 | assert.Equal(t, expected, b) 39 | } 40 | 41 | func TestNameTagTimer_MarshalJSON(t *testing.T) { 42 | c := &Timer{ 43 | name: "mytimer", 44 | metricType: typeRated, 45 | tags: map[string]string{"ololo": "trololo"}, 46 | value: *atomic.NewDuration(42 * time.Millisecond), 47 | 48 | useNameTag: true, 49 | } 50 | 51 | b, err := json.Marshal(c) 52 | assert.NoError(t, err) 53 | 54 | expected := []byte(`{"type":"RATE","labels":{"name":"mytimer","ololo":"trololo"},"value":0.042}`) 55 | assert.Equal(t, expected, b) 56 | } 57 | -------------------------------------------------------------------------------- /common/config.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "google.golang.org/protobuf/encoding/protojson" 8 | "google.golang.org/protobuf/encoding/prototext" 9 | "google.golang.org/protobuf/proto" 10 | "sigs.k8s.io/yaml" 11 | 12 | "github.com/ydb-platform/fq-connector-go/app/config" 13 | ) 14 | 15 | type AppConfig interface { 16 | *config.TServerConfig | 17 | *config.TClientConfig | 18 | *config.TBenchmarkConfig | 19 | *config.TObservationServerConfig 20 | proto.Message 21 | } 22 | 23 | func NewConfigFromPrototextFile[T AppConfig](configPath string, dst T) error { 24 | data, err := os.ReadFile(configPath) 25 | if err != nil { 26 | return fmt.Errorf("read file %v: %w", configPath, err) 27 | } 28 | 29 | if err := prototext.Unmarshal(data, dst); err != nil { 30 | return fmt.Errorf("prototext unmarshal `%v`: %w", string(data), err) 31 | } 32 | 33 | return nil 34 | } 35 | 36 | func NewConfigFromYAMLFile[T AppConfig](configPath string, dst T) error { 37 | dataYAML, err := os.ReadFile(configPath) 38 | if err != nil { 39 | return fmt.Errorf("read file %v: %w", configPath, err) 40 | } 41 | 42 | // convert YAML to JSON 43 | dataJSON, err := yaml.YAMLToJSON(dataYAML) 44 | if err != nil { 45 | return fmt.Errorf("convert YAML to JSON: %w", err) 46 | } 47 | 48 | // than parse JSON 49 | 50 | unmarshaller := protojson.UnmarshalOptions{ 51 | DiscardUnknown: true, 52 | } 53 | 54 | if err := unmarshaller.Unmarshal(dataJSON, dst); err != nil { 55 | return fmt.Errorf("protojson unmarshal `%v`: %w", string(dataJSON), err) 56 | } 57 | 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /scripts/debug/kqprun/scheme.join.txt: -------------------------------------------------------------------------------- 1 | CREATE OBJECT postgresql_local_password (TYPE SECRET) WITH (value = "password"); 2 | 3 | CREATE EXTERNAL DATA SOURCE postgresql_external_datasource WITH ( 4 | SOURCE_TYPE="PostgreSQL", 5 | LOCATION="localhost:5432", 6 | AUTH_METHOD="BASIC", 7 | SCHEMA="public", 8 | LOGIN="admin", 9 | DATABASE_NAME="connector", 10 | PASSWORD_SECRET_NAME="postgresql_local_password", 11 | PROTOCOL="NATIVE" 12 | ); 13 | 14 | CREATE OBJECT clickhouse_local_password (TYPE SECRET) WITH (value = "password"); 15 | 16 | CREATE EXTERNAL DATA SOURCE clickhouse_external_datasource WITH ( 17 | SOURCE_TYPE="ClickHouse", 18 | LOCATION="localhost:9000", 19 | AUTH_METHOD="BASIC", 20 | LOGIN="admin", 21 | DATABASE_NAME="connector", 22 | PASSWORD_SECRET_NAME="clickhouse_local_password", 23 | PROTOCOL="NATIVE" 24 | ); 25 | 26 | CREATE OBJECT ydb_local_password (TYPE SECRET) WITH (value = password); 27 | 28 | CREATE EXTERNAL DATA SOURCE ydb_external_datasource WITH ( 29 | SOURCE_TYPE="Ydb", 30 | LOCATION="localhost:2136", 31 | AUTH_METHOD="BASIC", 32 | LOGIN="admin", 33 | DATABASE_NAME="local", 34 | PASSWORD_SECRET_NAME="ydb_local_password" 35 | ); 36 | 37 | 38 | CREATE OBJECT greenplum_local_password (TYPE SECRET) WITH (value = "123456"); 39 | 40 | CREATE EXTERNAL DATA SOURCE greenplum_external_datasource WITH ( 41 | SOURCE_TYPE="Greenplum", 42 | LOCATION="localhost:6432", 43 | AUTH_METHOD="BASIC", 44 | SCHEMA="public", 45 | LOGIN="gpadmin", 46 | DATABASE_NAME="template1", 47 | PASSWORD_SECRET_NAME="greenplum_local_password" 48 | ); 49 | -------------------------------------------------------------------------------- /tests/infra/datasource/oracle/datasource.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | "fmt" 5 | 6 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 7 | "github.com/ydb-platform/fq-connector-go/tests/infra/datasource" 8 | "github.com/ydb-platform/fq-connector-go/tests/infra/docker_compose" 9 | ) 10 | 11 | const ( 12 | endpointServiceName = "oracle" 13 | internalPort = 1521 14 | database = "C##ADMIN" 15 | username = "C##ADMIN" 16 | password = "password" 17 | dbServiceName = "FREE" 18 | ) 19 | 20 | func deriveDataSourceFromDockerCompose(ed *docker_compose.EndpointDeterminer) (*datasource.DataSource, error) { 21 | dsi := &api_common.TGenericDataSourceInstance{ 22 | Kind: api_common.EGenericDataSourceKind_ORACLE, 23 | Database: database, 24 | Credentials: &api_common.TGenericCredentials{ 25 | Payload: &api_common.TGenericCredentials_Basic{ 26 | Basic: &api_common.TGenericCredentials_TBasic{ 27 | Username: username, 28 | Password: password, 29 | }, 30 | }, 31 | }, 32 | Options: &api_common.TGenericDataSourceInstance_OracleOptions{ 33 | OracleOptions: &api_common.TOracleDataSourceOptions{ 34 | ServiceName: dbServiceName, 35 | }, 36 | }, 37 | Protocol: api_common.EGenericProtocol_NATIVE, 38 | UseTls: false, 39 | } 40 | 41 | var err error 42 | 43 | dsi.Endpoint, err = ed.GetEndpoint(endpointServiceName, internalPort) 44 | if err != nil { 45 | return nil, fmt.Errorf("derive endpoint: %w", err) 46 | } 47 | 48 | return &datasource.DataSource{ 49 | Instances: []*api_common.TGenericDataSourceInstance{dsi}, 50 | }, nil 51 | } 52 | -------------------------------------------------------------------------------- /app/server/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | _ "embed" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/ydb-platform/fq-connector-go/app/config" 11 | ) 12 | 13 | var ( 14 | //go:embed config.debug.txt 15 | prototextConfig string 16 | //go:embed config.debug.yaml 17 | yamlConfig string 18 | ) 19 | 20 | func TestConfig(t *testing.T) { 21 | files := map[string]string{ 22 | "prototext": prototextConfig, 23 | "yaml": yamlConfig, 24 | } 25 | 26 | for key, body := range files { 27 | key := key 28 | body := body 29 | 30 | t.Run(key, func(t *testing.T) { 31 | f, err := os.CreateTemp("", "test-config") 32 | require.NoError(t, err) 33 | 34 | path := f.Name() 35 | 36 | _, err = f.WriteString(body) 37 | require.NoError(t, err) 38 | 39 | err = f.Close() 40 | require.NoError(t, err) 41 | 42 | defer os.Remove(path) 43 | 44 | cfg, err := NewConfigFromFile(path) 45 | require.NoError(t, err) 46 | require.NotNil(t, cfg) 47 | 48 | require.Equal(t, "0.0.0.0", cfg.ConnectorServer.Endpoint.Host) 49 | require.Equal(t, uint32(2130), cfg.ConnectorServer.Endpoint.Port) 50 | require.Equal(t, config.ELogLevel_DEBUG, cfg.Logger.LogLevel) 51 | require.Equal(t, true, cfg.Logger.EnableSqlQueryLogging) 52 | require.Equal(t, "0.0.0.0", cfg.PprofServer.Endpoint.Host) 53 | require.Equal(t, uint32(6060), cfg.PprofServer.Endpoint.Port) 54 | require.Equal(t, uint64(4*(1<<20)), cfg.Paging.BytesPerPage) 55 | require.Equal(t, uint32(2), cfg.Paging.PrefetchQueueCapacity) 56 | require.Equal(t, true, cfg.Conversion.UseUnsafeConverters) 57 | }) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /app/server/utils/decimal/serialize.go: -------------------------------------------------------------------------------- 1 | // Package decimal provides utilities for working with decimal numbers. 2 | package decimal 3 | 4 | import ( 5 | "math/big" 6 | "slices" 7 | 8 | "github.com/shopspring/decimal" 9 | ) 10 | 11 | const ( 12 | blobSize = 16 13 | ) 14 | 15 | // Serializer provides methods for serializing decimal values with reusable big.Int objects 16 | type Serializer struct { 17 | input *big.Int 18 | negative *big.Int 19 | twoToThe128 *big.Int 20 | } 21 | 22 | // NewSerializer creates a new Serializer with initialized big.Int objects 23 | func NewSerializer() *Serializer { 24 | s := &Serializer{ 25 | input: new(big.Int), 26 | negative: new(big.Int), 27 | } 28 | 29 | // Pre-calculate 2^128 for negative number handling 30 | s.twoToThe128 = new(big.Int).Lsh(big.NewInt(1), uint(blobSize*8)) 31 | 32 | return s 33 | } 34 | 35 | // Serialize converts a decimal value to a byte array representation 36 | func (s *Serializer) Serialize( 37 | val *decimal.Decimal, 38 | scale uint32, 39 | dst []byte, // acceptor 40 | ) { 41 | // Reset the bigInt to avoid carrying over previous values 42 | s.input.SetInt64(0) 43 | 44 | // Only shift when scale > 0 45 | if scale > 0 { 46 | scaled := val.Shift(int32(scale)) 47 | s.input.Set(scaled.BigInt()) 48 | } else { 49 | // Directly use the original value's BigInt 50 | s.input.Set(val.BigInt()) 51 | } 52 | 53 | if s.input.Sign() >= 0 { 54 | s.input.FillBytes(dst) 55 | } else { 56 | // For negative numbers: add 2^128 to make it positive 57 | s.negative.Add(s.twoToThe128, s.input) 58 | s.negative.FillBytes(dst) 59 | } 60 | 61 | // BigEndian -> LittleEndian 62 | slices.Reverse(dst) 63 | } 64 | -------------------------------------------------------------------------------- /app/client/ydb/cmd.go: -------------------------------------------------------------------------------- 1 | package ydb 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var Cmd = &cobra.Command{ 11 | Use: "ydb", 12 | Short: "Client for YDB", 13 | } 14 | 15 | var columnShardsDataDistributionCmd = &cobra.Command{ 16 | Use: "cs_data_distribution", 17 | Short: "Estimation of data distribution across column shards", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | if err := columnShardsDataDistribution(cmd, args); err != nil { 20 | fmt.Println(err) 21 | os.Exit(1) 22 | } 23 | }, 24 | } 25 | 26 | var columnShardBenchmarkSelectCmd = &cobra.Command{ 27 | Use: "cs_benchmark_select", 28 | Short: "Benchmark speed of a `SELECT * from table` query with columnar table", 29 | Run: func(cmd *cobra.Command, args []string) { 30 | if err := columnShardBenchmarkSelect(cmd, args); err != nil { 31 | fmt.Println(err) 32 | os.Exit(1) 33 | } 34 | }, 35 | } 36 | 37 | const ( 38 | configFlag = "config" 39 | tableFlag = "table" 40 | ) 41 | 42 | func init() { 43 | Cmd.AddCommand(columnShardsDataDistributionCmd) 44 | Cmd.AddCommand(columnShardBenchmarkSelectCmd) 45 | 46 | Cmd.Flags().StringP(configFlag, "c", "", "path to server config file") 47 | 48 | if err := Cmd.MarkFlagRequired(configFlag); err != nil { 49 | fmt.Println(err) 50 | os.Exit(1) 51 | } 52 | 53 | Cmd.Flags().StringP(tableFlag, "t", "", "table to read") 54 | 55 | if err := Cmd.MarkFlagRequired(tableFlag); err != nil { 56 | fmt.Println(err) 57 | os.Exit(1) 58 | } 59 | 60 | // inherit parent flags 61 | columnShardsDataDistributionCmd.Flags().AddFlagSet(Cmd.Flags()) 62 | columnShardBenchmarkSelectCmd.Flags().AddFlagSet(Cmd.Flags()) 63 | } 64 | -------------------------------------------------------------------------------- /common/client_basic.go: -------------------------------------------------------------------------------- 1 | package common //nolint:revive 2 | 3 | import ( 4 | "context" 5 | 6 | "go.uber.org/zap" 7 | "google.golang.org/grpc" 8 | 9 | api_common "github.com/ydb-platform/fq-connector-go/api/common" 10 | api_service "github.com/ydb-platform/fq-connector-go/api/service" 11 | api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" 12 | ) 13 | 14 | type clientBasic struct { 15 | client api_service.ConnectorClient 16 | conn *grpc.ClientConn 17 | logger *zap.Logger 18 | } 19 | 20 | func (c *clientBasic) DescribeTable( 21 | ctx context.Context, 22 | dsi *api_common.TGenericDataSourceInstance, 23 | typeMappingSettings *api_service_protos.TTypeMappingSettings, 24 | tableName string, 25 | ) (*api_service_protos.TDescribeTableResponse, error) { 26 | request := &api_service_protos.TDescribeTableRequest{ 27 | DataSourceInstance: dsi, 28 | Table: tableName, 29 | TypeMappingSettings: typeMappingSettings, 30 | } 31 | 32 | return c.client.DescribeTable(ctx, request) 33 | } 34 | 35 | type ReadSplitsOption interface { 36 | apply(request *api_service_protos.TReadSplitsRequest) 37 | } 38 | 39 | type readSplitsFilteringOption struct { 40 | filtering api_service_protos.TReadSplitsRequest_EFiltering 41 | } 42 | 43 | func (o readSplitsFilteringOption) apply(request *api_service_protos.TReadSplitsRequest) { 44 | request.Filtering = o.filtering 45 | } 46 | 47 | func WithFiltering(filtering api_service_protos.TReadSplitsRequest_EFiltering) ReadSplitsOption { 48 | return readSplitsFilteringOption{filtering: filtering} 49 | } 50 | 51 | func (c *clientBasic) Close() { 52 | LogCloserError(c.logger, c.conn, "client GRPC connection") 53 | } 54 | -------------------------------------------------------------------------------- /library/go/core/metrics/solomon/func_gauge_test.go: -------------------------------------------------------------------------------- 1 | package solomon 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "go.uber.org/atomic" 9 | ) 10 | 11 | func TestFuncGauge_Value(t *testing.T) { 12 | val := new(atomic.Float64) 13 | c := &FuncGauge{ 14 | name: "mygauge", 15 | metricType: typeGauge, 16 | tags: map[string]string{"ololo": "trololo"}, 17 | function: func() float64 { 18 | return val.Load() 19 | }, 20 | } 21 | 22 | val.Store(1) 23 | assert.Equal(t, float64(1), c.Snapshot().(*Gauge).value.Load()) 24 | 25 | val.Store(42) 26 | assert.Equal(t, float64(42), c.Snapshot().(*Gauge).value.Load()) 27 | 28 | } 29 | 30 | func TestFunGauge_MarshalJSON(t *testing.T) { 31 | c := &FuncGauge{ 32 | name: "mygauge", 33 | metricType: typeGauge, 34 | tags: map[string]string{"ololo": "trololo"}, 35 | function: func() float64 { 36 | return 42.18 37 | }, 38 | } 39 | 40 | b, err := json.Marshal(c) 41 | assert.NoError(t, err) 42 | 43 | expected := []byte(`{"type":"DGAUGE","labels":{"ololo":"trololo","sensor":"mygauge"},"value":42.18}`) 44 | assert.Equal(t, expected, b) 45 | } 46 | 47 | func TestNameTagFunGauge_MarshalJSON(t *testing.T) { 48 | c := &FuncGauge{ 49 | name: "mygauge", 50 | metricType: typeGauge, 51 | tags: map[string]string{"ololo": "trololo"}, 52 | function: func() float64 { 53 | return 42.18 54 | }, 55 | 56 | useNameTag: true, 57 | } 58 | 59 | b, err := json.Marshal(c) 60 | assert.NoError(t, err) 61 | 62 | expected := []byte(`{"type":"DGAUGE","labels":{"name":"mygauge","ololo":"trololo"},"value":42.18}`) 63 | assert.Equal(t, expected, b) 64 | } 65 | -------------------------------------------------------------------------------- /app/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var ( 11 | tag string 12 | author string 13 | commitHash string 14 | branch string 15 | commitDate string 16 | commitMessage string 17 | username string 18 | buildLocation string 19 | hostname string 20 | hostInfo string 21 | pathToGo string 22 | goVersion string 23 | ) 24 | 25 | var Cmd = &cobra.Command{ 26 | Use: "version", 27 | Short: "version of current build", 28 | Run: func(_ *cobra.Command, _ []string) { 29 | fmt.Println(GetInfo()) 30 | }, 31 | } 32 | 33 | func GetInfo() string { 34 | sb := strings.Builder{} 35 | 36 | sb.WriteString("VCS info:\n\n\n") 37 | sb.WriteString(fmt.Sprintf("\tBranch: %s\n", branch)) 38 | sb.WriteString(fmt.Sprintf("\tCommit: %s\n", commitHash)) 39 | sb.WriteString(fmt.Sprintf("\tTag: %s\n", tag)) 40 | sb.WriteString(fmt.Sprintf("\tAuthor: %s\n", author)) 41 | sb.WriteString(fmt.Sprintf("\tSummary: %s\n", commitMessage)) 42 | sb.WriteString(fmt.Sprintf("\tCommit Date: %s\n\n", commitDate)) 43 | sb.WriteString("Other info:\n") 44 | sb.WriteString(fmt.Sprintf("\tBuilt by: %s\n", username)) 45 | sb.WriteString(fmt.Sprintf("\tBuilding location: %s\n", buildLocation)) 46 | sb.WriteString(fmt.Sprintf("\tHostname: %s\n", hostname)) 47 | sb.WriteString("\tHost information:\n") 48 | sb.WriteString(fmt.Sprintf("\t\t%s\n\n", hostInfo)) 49 | sb.WriteString("Build info:\n") 50 | sb.WriteString(fmt.Sprintf("\tCompiler: %s\n", pathToGo)) 51 | sb.WriteString("\tCompiler version:\n") 52 | sb.WriteString(fmt.Sprintf("\t\t%s\n", goVersion)) 53 | 54 | return sb.String() 55 | } 56 | -------------------------------------------------------------------------------- /library/go/yson/infer.go: -------------------------------------------------------------------------------- 1 | package yson 2 | 3 | import ( 4 | "reflect" 5 | 6 | "golang.org/x/xerrors" 7 | ) 8 | 9 | // InferAttrs infers attribute names from struct. 10 | // 11 | // Function returns attribute names of struct fields 12 | // including exported fields of yson-untagged anonymous fields 13 | func InferAttrs(v any) ([]string, error) { 14 | if v == nil { 15 | return nil, xerrors.New("can't infer attrs from nil value") 16 | } 17 | 18 | uv := reflect.ValueOf(v) 19 | if uv.Kind() == reflect.Ptr { 20 | uv = uv.Elem() 21 | } 22 | 23 | if uv.Kind() != reflect.Struct { 24 | return nil, xerrors.Errorf("can't infer attrs from value of type %v", uv.Type()) 25 | } 26 | 27 | var attrs []string 28 | 29 | var inferFields func(typ reflect.Type) 30 | inferFields = func(typ reflect.Type) { 31 | if typ.Kind() == reflect.Ptr { 32 | typ = typ.Elem() 33 | } 34 | 35 | if typ.Kind() != reflect.Struct { 36 | return 37 | } 38 | 39 | for i := 0; i < typ.NumField(); i++ { 40 | field := typ.Field(i) 41 | 42 | _, tagged := field.Tag.Lookup("yson") 43 | 44 | decodedTag, skip := ParseTag(field.Name, field.Tag) 45 | if skip { 46 | continue 47 | } 48 | attr := decodedTag.Name 49 | 50 | if field.Anonymous && !tagged { 51 | inferFields(field.Type) 52 | continue 53 | } 54 | 55 | if decodedTag.Attr { 56 | attrs = append(attrs, attr) 57 | } 58 | } 59 | } 60 | 61 | inferFields(uv.Type()) 62 | 63 | return attrs, nil 64 | } 65 | 66 | // MustInferAttrs is like InferAttrs but panics in case of an error. 67 | func MustInferAttrs(v any) []string { 68 | attrs, err := InferAttrs(v) 69 | if err != nil { 70 | panic(err) 71 | } 72 | return attrs 73 | } 74 | -------------------------------------------------------------------------------- /library/go/core/metrics/solomon/func_int_gauge_test.go: -------------------------------------------------------------------------------- 1 | package solomon 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "go.uber.org/atomic" 9 | ) 10 | 11 | func TestFuncIntGauge_Value(t *testing.T) { 12 | val := new(atomic.Int64) 13 | c := &FuncIntGauge{ 14 | name: "myintgauge", 15 | metricType: typeIGauge, 16 | tags: map[string]string{"ololo": "trololo"}, 17 | function: func() int64 { 18 | return val.Load() 19 | }, 20 | } 21 | 22 | val.Store(1) 23 | assert.Equal(t, int64(1), c.Snapshot().(*IntGauge).value.Load()) 24 | 25 | val.Store(42) 26 | assert.Equal(t, int64(42), c.Snapshot().(*IntGauge).value.Load()) 27 | 28 | } 29 | 30 | func TestFunIntGauge_MarshalJSON(t *testing.T) { 31 | c := &FuncIntGauge{ 32 | name: "myintgauge", 33 | metricType: typeIGauge, 34 | tags: map[string]string{"ololo": "trololo"}, 35 | function: func() int64 { 36 | return 42 37 | }, 38 | } 39 | 40 | b, err := json.Marshal(c) 41 | assert.NoError(t, err) 42 | 43 | expected := []byte(`{"type":"IGAUGE","labels":{"ololo":"trololo","sensor":"myintgauge"},"value":42}`) 44 | assert.Equal(t, expected, b) 45 | } 46 | 47 | func TestNameTagFunIntGauge_MarshalJSON(t *testing.T) { 48 | c := &FuncIntGauge{ 49 | name: "myintgauge", 50 | metricType: typeIGauge, 51 | tags: map[string]string{"ololo": "trololo"}, 52 | function: func() int64 { 53 | return 42 54 | }, 55 | 56 | useNameTag: true, 57 | } 58 | 59 | b, err := json.Marshal(c) 60 | assert.NoError(t, err) 61 | 62 | expected := []byte(`{"type":"IGAUGE","labels":{"name":"myintgauge","ololo":"trololo"},"value":42}`) 63 | assert.Equal(t, expected, b) 64 | } 65 | -------------------------------------------------------------------------------- /scripts/debug/tls/oracle/client.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIEejCCAmICAQAwETEPMA0GA1UEAwwGb3JhY2xlMIICIjANBgkqhkiG9w0BAQEF 3 | AAOCAg8AMIICCgKCAgEAx5SGvUroP3LDhh/z1mEXb2b2llxZE8ZIFCPH2NjlX3VJ 4 | sIidngJstmsIUVV9xuExE/GaKg9Xx1U4Y06jlwlPfIYFgSvrSN5YW+EhdLO6/IBD 5 | nTeaZ2wijO4mY/tKLz5nGrYV3VgiIqwT4HUR7Y/YGKcqKoTGWiWlZdluMKquBS6+ 6 | amVXwTAuAnVHozA1vVeJO9bSyhl+h7sBJn6qRyIwoE0Skgivw6w4JcFAgvhRSsM6 7 | DYBnkEUY/MyZG9dddQ7E9YL9Cx8IWOjVwrB4eT7esX2VzUCAKbtBz5QQEQTslP4z 8 | W2i54qn9ONAeiM7lGhEfNy/1rBIwtlOmh5VynetcIpPixxd0FzZj1tQue8yKVHZs 9 | rZzksgnI6RBamUaZsH9AjEFXmjTTkLC8Rjdcjm7XdSuo7JHG+6ofzEXQ3sGcRVQM 10 | ehsWG70mXJjYfKRdG/+aRYxdHYc1FLsmYXX4AHSqUcTLqZ7P/UAuahXraplz/BtB 11 | RgfDCGKtQkRpFjDbAiI0scF4yfqOUI3TX4mL/VKU1FFIyBbcRe2PESoj1gIG7O07 12 | S3DSVoiGHAhaCFY02fFlREOaYVyUeptyjYOc/Eg0HwuR+rjw2EShlDgSzQTlifwN 13 | i2racT/ta0Y8qhlmGEibp0NCuOiUwf024436vH+smv1zaZi1bAOzJrTcojDs2z8C 14 | AwEAAaAkMCIGCSqGSIb3DQEJDjEVMBMwEQYDVR0RBAowCIIGb3JhY2xlMA0GCSqG 15 | SIb3DQEBCwUAA4ICAQBC0JSrPEmqzTJ/Cx5CnMClKgrfcin5A/gZb81g7Q8dJRaw 16 | j7NoAIOHApdQ0zkSfif+/1DXYk7m7Ku/TLgpTtaOVA4JhP2D/A/0kejc3jw1yP2E 17 | +R2KH71+nchqjUrgOWZyP0BTZIjZWzZensymqBg5EKeWxY0LP1dlDdFH4MYSOIcU 18 | RoM8sQjD6C47KNfryBtO3P0MK517V8U28THWP2lP3BosFqI+lJ6+Xg4z1W0aA3cB 19 | HQRbpP38oqDm1MJ48T1kETXAL6LZaWUgRPzU7Mv+iZbz028oCRUbZwAhW9Bzxhkg 20 | 4up80lPynZWwL45h1vwXpO8RNpdv1Cq7IRX4UX8vJ06WFAkqW+Dw7W24KTTwTuti 21 | 3vM5ibq20K/UBmmfFhqud3aQwhOrF5nV4Bx+UVujnocIf2Fi58DHp+xCqCw4OkFr 22 | yPyBDgbXIkHQo8a2y1wDEIwkxWRpbDY1OQ3LcBZ/fLlN8GkVFmJ5ccOO4bA6lIi8 23 | 0UbZHsFzNQpbI+fQaPkU7BKargZ4225W491x9iYvJe6D0pmu9hodCgAh1ZLez06J 24 | +36pY7f3JXZqnjQFX63WiCC0LNz51wWTCR+9bQF2SvsnByqOXoVo8gHDD0bGwZTJ 25 | tJ8+CbIELsBXIHKpQEr91KVDeOrZxUFsy/PovHJbQV93Mpgh2WAqgT7mTN6iug== 26 | -----END CERTIFICATE REQUEST----- 27 | -------------------------------------------------------------------------------- /app/server/config/config.prod.yaml: -------------------------------------------------------------------------------- 1 | connector_server: 2 | endpoint: 3 | host: "0.0.0.0" 4 | port: 2130 5 | 6 | logger: 7 | log_level: INFO 8 | enable_sql_query_logging: false 9 | 10 | metrics_server: 11 | endpoint: 12 | host: "0.0.0.0" 13 | port: 8767 14 | 15 | paging: 16 | bytes_per_page: 4194304 17 | prefetch_queue_capacity: 2 18 | 19 | conversion: 20 | use_unsafe_converters: true 21 | 22 | data_source_default: &data_source_default_var 23 | open_connection_timeout: 5s 24 | ping_connection_timeout: 5s 25 | exponential_backoff: 26 | initial_interval: 500ms 27 | randomization_factor: 0.5 28 | multiplier: 1.5 29 | max_interval: 20s 30 | max_elapsed_time: 1m 31 | 32 | datasources: 33 | clickhouse: 34 | <<: *data_source_default_var 35 | pushdown: 36 | enable_timestamp_pushdown: false # YQ-4063 37 | 38 | greenplum: 39 | <<: *data_source_default_var 40 | pushdown: 41 | enable_timestamp_pushdown: true 42 | 43 | ms_sql_server: 44 | <<: *data_source_default_var 45 | pushdown: 46 | enable_timestamp_pushdown: false # YQ-4062 47 | 48 | mysql: 49 | <<: *data_source_default_var 50 | result_chan_capacity: 1024 51 | pushdown: 52 | enable_timestamp_pushdown: true 53 | 54 | postgresql: 55 | <<: *data_source_default_var 56 | pushdown: 57 | enable_timestamp_pushdown: true 58 | splitting: 59 | enabled: true 60 | table_physical_size_threshold_bytes: 104857600 #100 MB 61 | 62 | ydb: 63 | <<: *data_source_default_var 64 | use_underlay_network_for_dedicated_databases: false 65 | mode: MODE_QUERY_SERVICE_NATIVE 66 | pushdown: 67 | enable_timestamp_pushdown: true 68 | splitting: 69 | enabled_on_column_shards: true 70 | --------------------------------------------------------------------------------