├── .git-blame-ignore-revs ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug.yml │ ├── config.yml │ ├── docs.yml │ └── feature.yml ├── codeql-config.yml ├── dependabot.yml ├── semantic.yml ├── stale.yml └── workflows │ ├── api-changes.yml │ ├── codeql-analysis.yml │ ├── codspeed.yml │ ├── cookiecutter-e2e.yml │ ├── dependency-review.yml │ ├── pr-preview-links.yml │ ├── release.yml │ ├── resources │ └── requirements.txt │ ├── test.yml │ ├── version_bump.yml │ └── zizmor.yml ├── .gitignore ├── .http_cache ├── 005b9221b5ae5f7c.json ├── 04f8e4f2843bd9b6.json ├── 08093cee53a5f8d7.json ├── 0926adaa00cbab1f.json ├── 098527f72da11ff2.json ├── 0b994e7ff3e250d9.json ├── 0b9cc7f7352df521.json ├── 0c16170a3cc01357.json ├── 0c2e7a7269b518f9.json ├── 0e14c7cd76fc3256.json ├── 0f554cb1a81b27b8.json ├── 108c4601940971bc.json ├── 12dc15b497f62699.json ├── 15682ccbb2ff5226.json ├── 160face28baf3cf6.json ├── 162e0cf1d9617eb5.json ├── 17768b855ebcf27f.json ├── 182b49aa9b828421.json ├── 18c60e1889ad1ce8.json ├── 1bc7d78d6657032e.json ├── 1db19d2861ba6e76.json ├── 1edab926189d24c6.json ├── 2221ecd7321360fd.json ├── 22f5ac720120f546.json ├── 23f223ded671e5d9.json ├── 25c2bb3d015f04ec.json ├── 2738537ab86ff424.json ├── 28e33ffc8ac3eb04.json ├── 2925edf06cbcc9f1.json ├── 29efb39e51a3e697.json ├── 29f3a31c5c2a432c.json ├── 2acd22896288275e.json ├── 2c0494d4f1958de3.json ├── 2fe15dd746560eda.json ├── 3061f81ccfd0eee6.json ├── 3076412eed8f8a25.json ├── 30c381daf213db58.json ├── 394d7b5c409191dd.json ├── 3a4e005c16d493b3.json ├── 3d8eace95bd40e5e.json ├── 3e7d5eedd0fbce96.json ├── 48e9045917c16a66.json ├── 4da15cf0697dbe8f.json ├── 4dfb7cba50251fec.json ├── 51f5762f30a79127.json ├── 5212b48018ed61de.json ├── 53eeb9d7a6cd12a7.json ├── 545c8ff6eaa7353d.json ├── 57ad39431fd0e3b3.json ├── 57b6cba610603287.json ├── 59b3570f212d3915.json ├── 60d00668e685e44d.json ├── 6c9e8d328d8945b1.json ├── 718c94b2fa2fdf5e.json ├── 74974e563cf60690.json ├── 74ec626c4163696b.json ├── 792ba9f75edd5350.json ├── 7e824d1f22c30cfc.json ├── 8019efbdbb3abdea.json ├── 806d2562f3ba9e03.json ├── 80b9590505b9d4aa.json ├── 83cbc7472f14f82c.json ├── 86caaa61a9ff1c13.json ├── 8801296965a5ba5c.json ├── 881c6cbfeb207636.json ├── 90a097d8f243bc5c.json ├── 92095ac60795aaa6.json ├── 97f390cd7700be39.json ├── 9e6bceedc53bf79d.json ├── a082be16fb2c317d.json ├── a2a55225db0d3c09.json ├── aa3e8c826c682182.json ├── afd3e9805838f127.json ├── b04f614678d3d489.json ├── b3815de0f3152810.json ├── b591d06ad3d042c8.json ├── baa77140af0e7d7c.json ├── bda1d8348f679f18.json ├── bed520161da8e3ab.json ├── bff7643cbc7ba501.json ├── c013ffe32a4017dc.json ├── c6ff89af32df2354.json ├── c958252a14f2ca1c.json ├── cb51897a84ac6fb7.json ├── cca49e2f68f2330f.json ├── d34d13adc04708d5.json ├── d5633d082f497f12.json ├── d67461d61168f832.json ├── d91746d68d0ec25f.json ├── db8c38dcdc823a36.json ├── dc102fb8f61aeac4.json ├── dd63bf6e6cd23500.json ├── de5578202ac7a187.json ├── de5dea9afb38aa10.json ├── e086a6c5ea3d836b.json ├── e0c8e734ddb66ee5.json ├── e173d5879160ae4b.json ├── e277ff32f91ee019.json ├── e381b7ebf1844772.json ├── e3d798836464fdf9.json ├── e45368adb89009e9.json ├── e7c3533ca8ed5940.json ├── e8083d0ca17f6078.json ├── e8eab2657236b4d7.json ├── ed6a1ab65495bdd9.json ├── f1525bc8ae1dcaf2.json ├── f39293bd356ca0ae.json ├── f4004f8ecb68f7b5.json ├── f672c53ef898e5b7.json ├── f9f2cf11ac7b37cb.json ├── fc7fe10af9a4ab3b.json ├── ff221b1aded73b94.json └── redirects.sqlite ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── codecov.yml ├── cookiecutter ├── .gitignore ├── mapper-template │ ├── README.md │ ├── cookiecutter.json │ ├── hooks │ │ └── post_gen_project.py │ └── {{cookiecutter.mapper_id}} │ │ ├── .github │ │ ├── dependabot.yml │ │ └── workflows │ │ │ ├── build.yml │ │ │ └── test.yml │ │ ├── .gitignore │ │ ├── .pre-commit-config.yaml │ │ ├── .secrets │ │ └── .gitignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── meltano.yml │ │ ├── output │ │ └── .gitignore │ │ ├── pyproject.toml │ │ ├── tests │ │ └── __init__.py │ │ ├── tox.ini │ │ └── {{cookiecutter.library_name}} │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── mapper.py ├── tap-template │ ├── README.md │ ├── cookiecutter.json │ ├── cookiecutter.tests.yml │ ├── hooks │ │ └── post_gen_project.py │ └── {{cookiecutter.tap_id}} │ │ ├── .github │ │ ├── dependabot.yml │ │ └── workflows │ │ │ ├── build.yml │ │ │ └── test.yml │ │ ├── .gitignore │ │ ├── .pre-commit-config.yaml │ │ ├── .secrets │ │ └── .gitignore │ │ ├── .vscode │ │ └── launch.json │ │ ├── LICENSE │ │ ├── README.md │ │ ├── meltano.yml │ │ ├── output │ │ └── .gitignore │ │ ├── pyproject.toml │ │ ├── tests │ │ ├── __init__.py │ │ └── test_core.py │ │ ├── tox.ini │ │ └── {{cookiecutter.library_name}} │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── auth.py │ │ ├── graphql-client.py │ │ ├── other-client.py │ │ ├── rest-client.py │ │ ├── schemas │ │ └── __init__.py │ │ ├── sql-client.py │ │ ├── streams.py │ │ └── tap.py └── target-template │ ├── README.md │ ├── cookiecutter.json │ ├── cookiecutter.tests.yml │ ├── hooks │ └── post_gen_project.py │ └── {{cookiecutter.target_id}} │ ├── .github │ ├── dependabot.yml │ └── workflows │ │ ├── build.yml │ │ └── test.yml │ ├── .gitignore │ ├── .pre-commit-config.yaml │ ├── .secrets │ └── .gitignore │ ├── LICENSE │ ├── README.md │ ├── meltano.yml │ ├── pyproject.toml │ ├── tests │ ├── __init__.py │ └── {{ 'test' }}_core.py │ ├── tox.ini │ └── {{cookiecutter.library_name}} │ ├── __init__.py │ ├── __main__.py │ ├── sinks.py │ └── target.py ├── docs ├── CONTRIBUTING.md ├── _static │ ├── .gitkeep │ ├── css │ │ └── custom.css │ ├── fonts │ │ ├── HankenGrotesk-Regular.woff │ │ ├── HankenGrotesk-Regular.woff2 │ │ ├── PlusJakartaSans-Regular.woff │ │ └── PlusJakartaSans-Regular.woff2 │ └── img │ │ ├── favicon.png │ │ ├── logo-light.svg │ │ └── logo.svg ├── _templates │ ├── base.html │ ├── class.rst │ ├── plugin_class.rst │ └── stream_class.rst ├── batch.md ├── capabilities.rst ├── classes │ ├── singer_sdk.BatchSink.rst │ ├── singer_sdk.GraphQLStream.rst │ ├── singer_sdk.InlineMapper.rst │ ├── singer_sdk.RESTStream.rst │ ├── singer_sdk.RecordSink.rst │ ├── singer_sdk.SQLConnector.rst │ ├── singer_sdk.SQLSink.rst │ ├── singer_sdk.SQLStream.rst │ ├── singer_sdk.SQLTap.rst │ ├── singer_sdk.SQLTarget.rst │ ├── singer_sdk.Sink.rst │ ├── singer_sdk.Stream.rst │ ├── singer_sdk.Tap.rst │ ├── singer_sdk.Target.rst │ ├── singer_sdk.authenticators.APIAuthenticatorBase.rst │ ├── singer_sdk.authenticators.APIKeyAuthenticator.rst │ ├── singer_sdk.authenticators.BasicAuthenticator.rst │ ├── singer_sdk.authenticators.BearerTokenAuthenticator.rst │ ├── singer_sdk.authenticators.OAuthAuthenticator.rst │ ├── singer_sdk.authenticators.OAuthJWTAuthenticator.rst │ ├── singer_sdk.authenticators.SimpleAuthenticator.rst │ ├── singer_sdk.batch.BaseBatcher.rst │ ├── singer_sdk.batch.JSONLinesBatcher.rst │ ├── singer_sdk.connectors.sql.JSONSchemaToSQL.rst │ ├── singer_sdk.connectors.sql.SQLToJSONSchema.rst │ ├── singer_sdk.exceptions.ConfigValidationError.rst │ ├── singer_sdk.exceptions.FatalAPIError.rst │ ├── singer_sdk.exceptions.InvalidStreamSortException.rst │ ├── singer_sdk.exceptions.MapExpressionError.rst │ ├── singer_sdk.exceptions.MaxRecordsLimitException.rst │ ├── singer_sdk.exceptions.RecordsWithoutSchemaException.rst │ ├── singer_sdk.exceptions.RetriableAPIError.rst │ ├── singer_sdk.exceptions.StreamMapConfigError.rst │ ├── singer_sdk.exceptions.TapStreamConnectionFailure.rst │ ├── singer_sdk.exceptions.TooManyRecordsException.rst │ ├── singer_sdk.pagination.BaseAPIPaginator.rst │ ├── singer_sdk.pagination.BaseHATEOASPaginator.rst │ ├── singer_sdk.pagination.BaseOffsetPaginator.rst │ ├── singer_sdk.pagination.BasePageNumberPaginator.rst │ ├── singer_sdk.pagination.HeaderLinkPaginator.rst │ ├── singer_sdk.pagination.JSONPathPaginator.rst │ ├── singer_sdk.pagination.LegacyPaginatedStreamProtocol.rst │ ├── singer_sdk.pagination.LegacyStreamPaginator.rst │ ├── singer_sdk.pagination.SimpleHeaderPaginator.rst │ ├── singer_sdk.pagination.SinglePagePaginator.rst │ └── typing │ │ ├── singer_sdk.typing.ArrayType.rst │ │ ├── singer_sdk.typing.BooleanType.rst │ │ ├── singer_sdk.typing.Constant.rst │ │ ├── singer_sdk.typing.CustomType.rst │ │ ├── singer_sdk.typing.DateTimeType.rst │ │ ├── singer_sdk.typing.DateType.rst │ │ ├── singer_sdk.typing.DiscriminatedUnion.rst │ │ ├── singer_sdk.typing.DurationType.rst │ │ ├── singer_sdk.typing.EmailType.rst │ │ ├── singer_sdk.typing.HostnameType.rst │ │ ├── singer_sdk.typing.IPv4Type.rst │ │ ├── singer_sdk.typing.IPv6Type.rst │ │ ├── singer_sdk.typing.IntegerType.rst │ │ ├── singer_sdk.typing.JSONPointerType.rst │ │ ├── singer_sdk.typing.NumberType.rst │ │ ├── singer_sdk.typing.ObjectType.rst │ │ ├── singer_sdk.typing.OneOf.rst │ │ ├── singer_sdk.typing.PropertiesList.rst │ │ ├── singer_sdk.typing.Property.rst │ │ ├── singer_sdk.typing.RegexType.rst │ │ ├── singer_sdk.typing.RelativeJSONPointerType.rst │ │ ├── singer_sdk.typing.StringType.rst │ │ ├── singer_sdk.typing.TimeType.rst │ │ ├── singer_sdk.typing.URITemplateType.rst │ │ ├── singer_sdk.typing.URIType.rst │ │ └── singer_sdk.typing.UUIDType.rst ├── cli_commands.md ├── code_samples.md ├── conf.py ├── context_object.md ├── deprecation.md ├── dev_guide.md ├── faq.md ├── guides │ ├── config-schema.md │ ├── custom-clis.md │ ├── index.md │ ├── migrate-to-uv.md │ ├── pagination-classes.md │ ├── performance.md │ ├── porting.md │ ├── sql-tap.md │ └── sql-target.md ├── images │ └── 200.png ├── implementation │ ├── at_least_once.md │ ├── catalog_metadata.md │ ├── cli.md │ ├── discovery.md │ ├── index.md │ ├── logging.md │ ├── metrics.md │ ├── record_metadata.md │ └── state.md ├── incremental_replication.md ├── index.md_ ├── index.rst ├── make.bat ├── parent_streams.md ├── partitioning.md ├── python_tips.md ├── reference.rst ├── release_process.md ├── sinks.md ├── stream_maps.md ├── testing.md └── typing.rst ├── e2e-tests └── cookiecutters │ ├── README.md │ ├── mapper-base.json │ ├── tap-faker.json │ ├── tap-graphql-jwt.json │ ├── tap-no-license.json │ ├── tap-other-custom.json │ ├── tap-rest-api_key-github.json │ ├── tap-rest-basic_auth.json │ ├── tap-rest-bearer_token.json │ ├── tap-rest-custom.json │ ├── tap-rest-jwt.json │ ├── tap-rest-oauth2.json │ ├── tap-sql-custom.json │ ├── target-per_record.json │ └── target-sql.json ├── fixtures └── csv │ ├── customers.csv │ └── employees.csv ├── noxfile.py ├── pyproject.toml ├── samples ├── aapl │ ├── AAPL.json │ ├── README.md │ ├── __init__.py │ ├── __main__.py │ ├── aapl.py │ └── fundamentals.json ├── sample_custom_sql_adapter │ ├── __init__.py │ └── connector.py ├── sample_duckdb │ ├── __init__.py │ └── connector.py ├── sample_mapper │ ├── __init__.py │ ├── __main__.py │ └── mapper.py ├── sample_tap_bigquery │ ├── __init__.py │ └── __main__.py ├── sample_tap_countries │ ├── __init__.py │ ├── __main__.py │ ├── countries_streams.py │ ├── countries_tap.py │ └── schemas │ │ └── continents.json ├── sample_tap_csv │ ├── __init__.py │ ├── __main__.py │ ├── client.py │ └── sample_tap_csv.py ├── sample_tap_dummy_json │ ├── .github │ │ ├── dependabot.yml │ │ └── workflows │ │ │ ├── build.yml │ │ │ └── test.yml │ ├── .gitignore │ ├── .pre-commit-config.yaml │ ├── .secrets │ │ └── .gitignore │ ├── LICENSE │ ├── README.md │ ├── __init__.py │ ├── meltano.yml │ ├── output │ │ └── .gitignore │ ├── plugins │ │ └── loaders │ │ │ └── target-jsonl--andyh1203.lock │ ├── pyproject.toml │ ├── ruff.toml │ ├── tap_dummyjson │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── auth.py │ │ ├── client.py │ │ ├── schemas │ │ │ └── __init__.py │ │ ├── streams.py │ │ └── tap.py │ ├── tests │ │ ├── __init__.py │ │ └── test_core.py │ └── tox.ini ├── sample_tap_fake_people │ ├── __init__.py │ ├── __main__.py │ └── tap.py ├── sample_tap_gitlab │ ├── __init__.py │ ├── __main__.py │ ├── gitlab-config.sample.json │ ├── gitlab_graphql_streams.py │ ├── gitlab_rest_streams.py │ ├── gitlab_tap.py │ └── schemas │ │ ├── commits.json │ │ ├── currentuser.json │ │ ├── epic_issues.json │ │ ├── issues.json │ │ ├── projects-graphql.json │ │ ├── projects.json │ │ └── releases.json ├── sample_tap_hostile │ ├── __init__.py │ ├── __main__.py │ ├── hostile_streams.py │ └── hostile_tap.py ├── sample_tap_sqlite │ ├── __init__.py │ └── __main__.py ├── sample_target_csv │ ├── __init__.py │ ├── __main__.py │ ├── csv_target.py │ └── csv_target_sink.py ├── sample_target_parquet │ ├── __init__.py │ ├── __main__.py │ ├── parquet_target.py │ └── parquet_target_sink.py └── sample_target_sqlite │ ├── __init__.py │ └── __main__.py ├── singer_sdk ├── __init__.py ├── _singerlib │ ├── __init__.py │ ├── catalog.py │ ├── encoding.py │ ├── exceptions.py │ ├── json.py │ ├── messages.py │ └── utils.py ├── about.py ├── authenticators.py ├── batch.py ├── cli │ ├── __init__.py │ └── common_options.py ├── configuration │ ├── __init__.py │ └── _dict_config.py ├── connectors │ ├── __init__.py │ └── sql.py ├── contrib │ ├── __init__.py │ ├── batch_encoder_jsonl.py │ ├── batch_encoder_parquet.py │ ├── filesystem │ │ ├── __init__.py │ │ ├── config.py │ │ ├── stream.py │ │ └── tap.py │ └── msgspec.py ├── exceptions.py ├── helpers │ ├── __init__.py │ ├── _batch.py │ ├── _catalog.py │ ├── _classproperty.py │ ├── _compat.py │ ├── _conformers.py │ ├── _flattening.py │ ├── _secrets.py │ ├── _state.py │ ├── _typing.py │ ├── _util.py │ ├── capabilities.py │ ├── jsonpath.py │ └── types.py ├── internal │ └── __init__.py ├── io_base.py ├── mapper.py ├── mapper_base.py ├── metrics.py ├── pagination.py ├── plugin_base.py ├── py.typed ├── singerlib │ ├── __init__.py │ ├── catalog.py │ ├── encoding │ │ ├── __init__.py │ │ ├── base.py │ │ └── simple.py │ ├── exceptions.py │ ├── json.py │ ├── messages.py │ ├── schema.py │ └── utils.py ├── sinks │ ├── __init__.py │ ├── batch.py │ ├── core.py │ ├── record.py │ └── sql.py ├── streams │ ├── __init__.py │ ├── core.py │ ├── graphql.py │ ├── rest.py │ └── sql.py ├── tap_base.py ├── target_base.py ├── testing │ ├── __init__.py │ ├── config.py │ ├── factory.py │ ├── legacy.py │ ├── pytest_plugin.py │ ├── runners.py │ ├── suites.py │ ├── tap_tests.py │ ├── target_test_streams │ │ ├── __init__.py │ │ ├── array_data.singer │ │ ├── camelcase.singer │ │ ├── camelcase_complex_schema.singer │ │ ├── duplicate_records.singer │ │ ├── encoded_string_data.singer │ │ ├── invalid_schema.singer │ │ ├── multiple_state_messages.singer │ │ ├── no_primary_keys.singer │ │ ├── no_primary_keys_append.singer │ │ ├── optional_attributes.singer │ │ ├── pk_updates.singer │ │ ├── record_before_schema.singer │ │ ├── record_missing_fields.singer │ │ ├── record_missing_key_property.singer │ │ ├── record_missing_required_property.singer │ │ ├── schema_no_properties.singer │ │ ├── schema_updates.singer │ │ ├── special_chars_in_attributes.singer │ │ ├── user_location_data.singer │ │ └── user_location_upsert_data.singer │ ├── target_tests.py │ └── templates.py └── typing.py ├── tests ├── __init__.py ├── conftest.py ├── contrib │ ├── __init__.py │ └── test_batch_encoder_parquet.py ├── core │ ├── __init__.py │ ├── configuration │ │ ├── __init__.py │ │ └── test_dict_config.py │ ├── conftest.py │ ├── resources │ │ ├── batch.1.jsonl.gz │ │ ├── batch.2.jsonl.gz │ │ ├── continents.parquet.gz │ │ ├── countries.parquet.gz │ │ └── testfile.parquet │ ├── rest │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_authenticators.py │ │ ├── test_failure.py │ │ └── test_pagination.py │ ├── sinks │ │ ├── __init__.py │ │ ├── test_sdc_metadata.py │ │ ├── test_sql_sink.py │ │ ├── test_type_checker.py │ │ └── test_validation.py │ ├── snapshots │ │ └── test_parent_child │ │ │ ├── test_child_deselected_parent │ │ │ ├── singer.jsonl │ │ │ └── stderr.log │ │ │ ├── test_deselected_child │ │ │ ├── singer.jsonl │ │ │ └── stderr.log │ │ │ ├── test_one_parent_many_children │ │ │ ├── singer.jsonl │ │ │ └── stderr.log │ │ │ ├── test_parent_context_fields_in_child │ │ │ ├── singer.jsonl │ │ │ └── stderr.log │ │ │ └── test_skip_deleted_parent_child_streams │ │ │ └── stderr.log │ ├── targets │ │ ├── __init__.py │ │ └── test_target_sql.py │ ├── test_about.py │ ├── test_batch.py │ ├── test_capabilities.py │ ├── test_catalog_selection.py │ ├── test_connector_sql.py │ ├── test_flattening.py │ ├── test_io.py │ ├── test_jsonschema_helpers.py │ ├── test_mapper.py │ ├── test_mapper_class.py │ ├── test_metrics.py │ ├── test_parent_child.py │ ├── test_plugin_base.py │ ├── test_plugin_config.py │ ├── test_record_typing.py │ ├── test_schema.py │ ├── test_singer_messages.py │ ├── test_sql_typing.py │ ├── test_state_handling.py │ ├── test_streams.py │ ├── test_tap_class.py │ ├── test_target_base.py │ ├── test_target_class.py │ ├── test_testing.py │ └── test_typing.py ├── external │ ├── __init__.py │ ├── conftest.py │ ├── test_tap_dummyjson.py │ └── test_tap_gitlab.py ├── samples │ ├── __init__.py │ ├── conftest.py │ ├── resources │ │ └── messages.jsonl │ ├── snapshots │ │ └── test_target_csv │ │ │ ├── test_countries_to_csv │ │ │ ├── singer.log │ │ │ ├── tap.jsonl │ │ │ └── target.jsonl │ │ │ ├── test_countries_to_csv_mapped │ │ │ ├── mapper.jsonl │ │ │ ├── mapper.log │ │ │ ├── tap.jsonl │ │ │ ├── tap.log │ │ │ ├── target.jsonl │ │ │ └── target.log │ │ │ └── test_fake_people_to_csv │ │ │ ├── singer.log │ │ │ ├── tap.jsonl │ │ │ └── target.jsonl │ ├── test_tap_countries.py │ ├── test_tap_csv.py │ ├── test_tap_sqlite.py │ ├── test_target_csv.py │ ├── test_target_parquet.py │ └── test_target_sqlite.py ├── singerlib │ ├── __init__.py │ ├── encoding │ │ ├── conftest.py │ │ ├── test_msgspec.py │ │ └── test_simple.py │ ├── test_catalog.py │ ├── test_messages.py │ ├── test_schema.py │ └── test_utils.py └── snapshots │ ├── about_format │ ├── json.snap.json │ ├── markdown.snap.md │ └── text.snap.txt │ ├── countries_write_schemas │ └── countries_write_schemas │ ├── jsonschema │ ├── additional_properties.json │ ├── base.json │ ├── duplicates.json │ ├── duplicates_additional_properties.json │ ├── duplicates_no_additional_properties.json │ ├── no_additional_properties.json │ ├── pattern_properties.json │ ├── required.json │ ├── required_additional_properties.json │ ├── required_duplicates.json │ ├── required_duplicates_additional_properties.json │ ├── required_duplicates_no_additional_properties.json │ └── required_no_additional_properties.json │ └── mapped_stream │ ├── aliased_stream.jsonl │ ├── aliased_stream_batch.jsonl │ ├── aliased_stream_not_expr.jsonl │ ├── aliased_stream_quoted.jsonl │ ├── builtin_variable_self.jsonl │ ├── builtin_variable_stream_name.jsonl │ ├── builtin_variable_stream_name_alias.jsonl │ ├── builtin_variable_stream_name_alias_expr.jsonl │ ├── builtin_variable_underscore.jsonl │ ├── changed_key_properties.jsonl │ ├── dates.jsonl │ ├── drop_property.jsonl │ ├── drop_property_null_string.jsonl │ ├── fake_credit_card_number.jsonl │ ├── fake_email_seed_class.jsonl │ ├── fake_email_seed_instance.jsonl │ ├── flatten_all.jsonl │ ├── flatten_depth_0.jsonl │ ├── flatten_depth_1.jsonl │ ├── json_dumps.jsonl │ ├── keep_all_fields.jsonl │ ├── map_and_flatten.jsonl │ ├── no_map.jsonl │ ├── non_pk_passthrough.jsonl │ ├── only_mapped_fields.jsonl │ ├── only_mapped_fields_null_string.jsonl │ ├── record_to_column.jsonl │ ├── sourced_stream_1.jsonl │ ├── sourced_stream_1_null_string.jsonl │ └── sourced_stream_2.jsonl └── uv.lock /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # chore: Enforce `import typing as t` by configuring `ICN003` 2 | 54222bb2dc1903c0816347952c6a77c30267f30f 3 | 4 | # docs: Auto-format markdown with `mdformat` 5 | e91be649cb3494479ae4096729f21a55b0034917 6 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners 2 | 3 | # Ending a path in a `/` will specify the code owners for every file 4 | # nested in that directory, on any level 5 | 6 | # Default owners 7 | * @edgarrmondragon 8 | 9 | # CI/CD 10 | /.github/workflows/ @edgarrmondragon @meltano/engineering 11 | 12 | # Docs (General) 13 | /docs/ @meltano/engineering @meltano/marketing 14 | /README.md @tayloramurphy @meltano/engineering @meltano/marketing 15 | 16 | # Docs (Contributing) 17 | /docs/CONTRIBUTING.md @tayloramurphy @meltano/engineering 18 | 19 | # Release Ops (see `/.pyproject.toml` for list of bumped files) 20 | /cookiecutter/*/*/pyproject.toml @meltano/engineering 21 | /docs/conf.py @meltano/engineering 22 | /pyproject.toml @meltano/engineering 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Meltano Community 4 | url: https://meltano.com/slack 5 | about: Join us on Slack. 6 | - name: Start a discussion 7 | url: https://github.com/meltano/sdk/discussions/new 8 | about: Start a GitHub discussion. 9 | - name: Singer SDK Documentation 10 | url: https://sdk.meltano.com 11 | about: Learn more about the Singer SDK. 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/docs.yml: -------------------------------------------------------------------------------- 1 | name: Documentation change 2 | description: Request a documentation change 3 | title: "docs: " 4 | type: Docs 5 | labels: ["Documentation", "valuestream/SDK"] 6 | assignees: 7 | - edgarrmondragon 8 | 9 | body: 10 | - type: markdown 11 | attributes: 12 | value: | 13 | Thanks for taking the time to fill out this documentation request! 14 | - type: dropdown 15 | id: scope 16 | attributes: 17 | label: Documentation type 18 | description: What kind of documentation change are you requesting? 19 | options: 20 | - Tutorials 21 | - How-to guides 22 | - Reference 23 | - Explanation 24 | validations: 25 | required: true 26 | - type: textarea 27 | id: what-you-want 28 | attributes: 29 | label: Description 30 | description: Describe what you want to see in the documentation 31 | placeholder: "I was trying to do X, but the documentation didn't tell me how to do it, or it was unclear." 32 | validations: 33 | required: true 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature.yml: -------------------------------------------------------------------------------- 1 | name: Feature request 2 | description: Request a new feature 3 | title: "feat: <title>" 4 | labels: ["kind/Feature", "valuestream/SDK"] 5 | type: Feat 6 | assignees: 7 | - edgarrmondragon 8 | 9 | body: 10 | - type: markdown 11 | attributes: 12 | value: | 13 | Thanks for taking the time to request a new feature! 14 | - type: dropdown 15 | id: scope 16 | attributes: 17 | label: Feature scope 18 | description: Functionality this new feature would impact 19 | options: 20 | - Taps (catalog, state, tests, etc.) 21 | - Inline mapping (stream maps, flattening, etc.) 22 | - Targets (data type handling, batching, SQL object generation, tests, etc.) 23 | - Configuration (settings parsing, validation, etc.) 24 | - CLI (options, error messages, logging, etc.) 25 | - Cookiecutter templates 26 | - Other 27 | validations: 28 | required: true 29 | - type: textarea 30 | id: what 31 | attributes: 32 | label: Description 33 | description: Describe the feature you would like to see 34 | validations: 35 | required: true 36 | -------------------------------------------------------------------------------- /.github/codeql-config.yml: -------------------------------------------------------------------------------- 1 | paths: 2 | - samples 3 | - singer_sdk 4 | - tests 5 | paths-ignore: 6 | - cookiecutter 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | enable-beta-ecosystems: true 3 | updates: 4 | - package-ecosystem: "uv" 5 | directory: "/" 6 | schedule: 7 | interval: weekly 8 | time: "12:00" 9 | labels: [Dependencies] 10 | groups: 11 | development-dependencies: 12 | dependency-type: development 13 | runtime-dependencies: 14 | dependency-type: production 15 | versioning-strategy: lockfile-only 16 | - package-ecosystem: pip 17 | directory: "/.github/workflows/resources" 18 | schedule: 19 | interval: weekly 20 | time: "12:00" 21 | labels: [Dependencies] 22 | groups: 23 | ci: 24 | patterns: 25 | - "*" 26 | - package-ecosystem: github-actions 27 | directory: "/" 28 | schedule: 29 | interval: weekly 30 | labels: [Dependencies] 31 | groups: 32 | actions: 33 | patterns: 34 | - "*" 35 | -------------------------------------------------------------------------------- /.github/semantic.yml: -------------------------------------------------------------------------------- 1 | # Config ref: https://github.com/Ezard/semantic-prs 2 | 3 | # Validate the PR title, and ignore all commit messages 4 | titleOnly: true 5 | 6 | # Provides a custom URL for the "Details" link, which appears next to the success/failure message from the app: 7 | targetUrl: https://sdk.meltano.com/en/latest/CONTRIBUTING.html#semantic-pull-requests 8 | 9 | # The values allowed for the "type" part of the PR title/commit message. 10 | # e.g. for a PR title/commit message of "feat: add some stuff", the type would be "feat" 11 | types: 12 | - ci 13 | - chore 14 | - build 15 | - docs 16 | - feat 17 | - fix 18 | - packaging 19 | - perf 20 | - refactor 21 | - revert 22 | - style 23 | - test 24 | 25 | # The values allowed for the "scope" part of the PR title/commit message. 26 | # e.g. for a PR title/commit message of "feat(awesome-feature): add some stuff", 27 | # the scope would be "awesome-feature" 28 | scopes: 29 | - taps # tap SDK only 30 | - targets # target SDK only 31 | - mappers # mappers only 32 | - templates # cookiecutters 33 | - deps # production dependencies 34 | - deps-dev # development dependencies (testing, linting, etc.) 35 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # This config file extends the shared Meltano GitHub org stale bot config: 2 | # https://github.com/meltano/.github/blob/main/.github/stale.yml 3 | 4 | _extends: .github 5 | 6 | # In most cases, this file should not be updated. 7 | # Updates to the stale bot config should be shared by all Meltano GitHub repositories. 8 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | name: Dependency Review 2 | 3 | on: 4 | pull_request: {} 5 | workflow_dispatch: 6 | inputs: {} 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | dependency-review: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 16 | with: 17 | persist-credentials: false 18 | - uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9 # v4.7.1 19 | if: ${{ github.event_name == 'pull_request' }} 20 | with: 21 | fail-on-severity: high 22 | -------------------------------------------------------------------------------- /.github/workflows/pr-preview-links.yml: -------------------------------------------------------------------------------- 1 | name: Read the Docs Pull Request Preview 2 | 3 | on: 4 | pull_request_target: # zizmor: ignore[dangerous-triggers] 5 | types: 6 | - opened 7 | 8 | permissions: {} 9 | 10 | jobs: 11 | pr-preview-links: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | pull-requests: write 15 | steps: 16 | - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1.5 17 | with: 18 | project-slug: "meltano-sdk" 19 | -------------------------------------------------------------------------------- /.github/workflows/resources/requirements.txt: -------------------------------------------------------------------------------- 1 | griffe~=1.7 2 | nox==2025.5.1 3 | pre-commit==4.2.0 4 | twine==6.1.0 5 | -------------------------------------------------------------------------------- /.github/workflows/zizmor.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Actions Security Analysis with zizmor 🌈 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: 7 | branches: ["**"] 8 | 9 | permissions: {} 10 | 11 | jobs: 12 | zizmor: 13 | name: zizmor latest via PyPI 14 | runs-on: ubuntu-latest 15 | permissions: 16 | security-events: write 17 | # required for workflows in private repositories 18 | contents: read 19 | actions: read 20 | steps: 21 | - name: Checkout repository 22 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 23 | with: 24 | persist-credentials: false 25 | 26 | - name: Install the latest version of uv 27 | uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0 28 | with: 29 | version: ">=0.6,<0.7" 30 | 31 | - name: Run zizmor 🌈 32 | run: uvx zizmor --pedantic --format sarif .github > results.sarif 33 | 34 | - name: Upload SARIF file 35 | uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 36 | with: 37 | sarif_file: results.sarif 38 | category: zizmor 39 | -------------------------------------------------------------------------------- /.http_cache/redirects.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/.http_cache/redirects.sqlite -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-24.04 5 | tools: 6 | python: "3.13" 7 | jobs: 8 | create_environment: 9 | - asdf plugin add uv 10 | - asdf install uv latest 11 | - asdf global uv latest 12 | install: 13 | - uv sync --frozen --no-dev --group docs 14 | build: 15 | html: 16 | - uv run sphinx-build -T -b html docs $READTHEDOCS_OUTPUT/html 17 | 18 | sphinx: 19 | builder: html 20 | configuration: docs/conf.py 21 | fail_on_warning: true 22 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | precision: 2 3 | range: "70...100" 4 | round: down 5 | status: 6 | project: 7 | default: 8 | target: auto 9 | threshold: 1% 10 | base: auto 11 | patch: 12 | default: 13 | target: 100% 14 | threshold: 1% 15 | base: auto 16 | -------------------------------------------------------------------------------- /cookiecutter/.gitignore: -------------------------------------------------------------------------------- 1 | # Include template VSCode directory 2 | !*/*/.vscode/ 3 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/README.md: -------------------------------------------------------------------------------- 1 | # Singer Mapper Template 2 | 3 | To use this cookie cutter template: 4 | 5 | ```bash 6 | pip3 install pipx 7 | pipx ensurepath 8 | # You may need to reopen your shell at this point 9 | pipx install cookiecutter 10 | ``` 11 | 12 | Initialize Cookiecutter template directly from Git: 13 | 14 | ```bash 15 | cookiecutter https://github.com/meltano/sdk --directory="cookiecutter/mapper-template" 16 | ``` 17 | 18 | Or locally from an already-cloned `sdk` repo: 19 | 20 | ```bash 21 | cookiecutter ./sdk/cookiecutter/mapper-template 22 | ``` 23 | 24 | See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html). 25 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MyMapperName", 3 | "admin_name": "FirstName LastName", 4 | "admin_email": "firstname.lastname@example.com", 5 | "mapper_id": "mapper-{{ cookiecutter.name.lower() }}", 6 | "library_name": "{{ cookiecutter.mapper_id.replace('-', '_') }}", 7 | "variant": "None (Skip)", 8 | "faker_extra": false, 9 | "include_ci_files": ["GitHub", "None (Skip)"], 10 | "license": ["Apache-2.0"], 11 | "ide": ["VSCode", "None"], 12 | "__prompts__": { 13 | "name": "The name of the mapper, in CamelCase", 14 | "admin_name": "Provide your [bold yellow]full name[/]", 15 | "admin_email": "Provide your [bold yellow]email[/]", 16 | "mapper_id": "The ID of the tap, in kebab-case", 17 | "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.", 18 | "faker_extra": "Add [bold orange1][link=https://faker.readthedocs.io/en/master/]Faker[/link][/] as an extra dependency to support generating fake data in stream maps?", 19 | "include_ci_files": "Whether to include CI files for a common CI services", 20 | "license": "The license for the project", 21 | "ide": "Add configuration files for your preferred IDE" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/hooks/post_gen_project.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from pathlib import Path 3 | import shutil 4 | 5 | 6 | BASE_PATH = Path("{{cookiecutter.library_name}}") 7 | 8 | 9 | if __name__ == "__main__": 10 | if "{{ cookiecutter.license }}" != "Apache-2.0": 11 | Path("LICENSE").unlink() 12 | 13 | if "{{ cookiecutter.include_ci_files }}" != "GitHub": 14 | shutil.rmtree(Path(".github")) 15 | 16 | if "{{ cookiecutter.ide }}" != "VSCode": 17 | shutil.rmtree(".vscode", ignore_errors=True) 18 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: pip 9 | directory: "/" 10 | schedule: 11 | interval: weekly 12 | commit-message: 13 | prefix: "chore(deps): " 14 | prefix-development: "chore(deps-dev): " 15 | groups: 16 | development-dependencies: 17 | dependency-type: development 18 | runtime-dependencies: 19 | dependency-type: production 20 | update-types: 21 | - "patch" 22 | versioning-strategy: increase-if-necessary 23 | - package-ecosystem: github-actions 24 | directory: "/" 25 | schedule: 26 | interval: weekly 27 | commit-message: 28 | prefix: "ci: " 29 | groups: 30 | actions: 31 | patterns: 32 | - "*" 33 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autofix_prs: true 3 | autoupdate_schedule: weekly 4 | autoupdate_commit_msg: 'chore: pre-commit autoupdate' 5 | skip: 6 | - uv-lock 7 | 8 | repos: 9 | - repo: https://github.com/pre-commit/pre-commit-hooks 10 | rev: v5.0.0 11 | hooks: 12 | - id: check-json 13 | exclude: | 14 | (?x)^( 15 | .*/launch.json 16 | )$ 17 | - id: check-toml 18 | - id: check-yaml 19 | - id: end-of-file-fixer 20 | - id: trailing-whitespace 21 | 22 | - repo: https://github.com/python-jsonschema/check-jsonschema 23 | rev: 0.33.0 24 | hooks: 25 | - id: check-dependabot 26 | - id: check-github-workflows 27 | - id: check-meltano 28 | 29 | - repo: https://github.com/astral-sh/ruff-pre-commit 30 | rev: v0.11.11 31 | hooks: 32 | - id: ruff 33 | args: [--fix, --exit-non-zero-on-fix, --show-fixes] 34 | - id: ruff-format 35 | 36 | - repo: https://github.com/pre-commit/mirrors-mypy 37 | rev: v1.15.0 38 | hooks: 39 | - id: mypy 40 | 41 | - repo: https://github.com/astral-sh/uv-pre-commit 42 | rev: 0.7.8 43 | hooks: 44 | - id: uv-lock 45 | - id: uv-sync 46 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.secrets/.gitignore: -------------------------------------------------------------------------------- 1 | # IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, 2 | # make sure those are never staged for commit into your git repo. You can store them here or another 3 | # secure location. 4 | # 5 | # Note: This may be redundant with the global .gitignore for, and is provided 6 | # for redundancy. If the `.secrets` folder is not needed, you may delete it 7 | # from the project. 8 | 9 | * 10 | !.gitignore 11 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | send_anonymous_usage_stats: true 3 | project_id: "{{cookiecutter.mapper_id}}" 4 | default_environment: test 5 | venv: 6 | backend: uv 7 | environments: 8 | - name: test 9 | plugins: 10 | extractors: 11 | - name: tap-smoke-test 12 | variant: meltano 13 | pip_url: git+https://github.com/meltano/tap-smoke-test.git 14 | config: 15 | streams: 16 | - stream_name: animals 17 | input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl 18 | 19 | loaders: 20 | - name: target-jsonl 21 | variant: andyh1203 22 | pip_url: target-jsonl 23 | 24 | mappers: 25 | - name: "{{cookiecutter.mapper_id}}" 26 | namespace: "{{cookiecutter.library_name}}" 27 | pip_url: -e . 28 | 29 | # TODO: Declare settings and their types here: 30 | settings: 31 | - name: example_config 32 | kind: string 33 | label: Example Config 34 | description: An example configuration setting 35 | 36 | # TODO: Declare mapping instances here: 37 | # https://docs.meltano.com/guide/mappers/#example-1 38 | mappings: 39 | - name: example 40 | config: 41 | example_config: foo 42 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/output/.gitignore: -------------------------------------------------------------------------------- 1 | # This directory is used as a target by target-jsonl, so ignore all files 2 | 3 | * 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Test suite for {{ cookiecutter.mapper_id }}.""" 2 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini: -------------------------------------------------------------------------------- 1 | # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy 2 | 3 | [tox] 4 | envlist = py3{9,10,11,12,13} 5 | minversion = 4.22 6 | requires = 7 | tox>=4.22 8 | 9 | [testenv] 10 | pass_env = 11 | {%- if cookiecutter.include_ci_files == "GitHub" %} 12 | GITHUB_* 13 | {%- endif %} 14 | {{cookiecutter.mapper_id.replace('-', '_').upper()}}_* 15 | dependency_groups = 16 | test 17 | commands = 18 | pytest {posargs} 19 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__init__.py: -------------------------------------------------------------------------------- 1 | """{{ cookiecutter.name }} Mapper.""" 2 | -------------------------------------------------------------------------------- /cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__main__.py: -------------------------------------------------------------------------------- 1 | """{{ cookiecutter.name }} entry point.""" 2 | 3 | from __future__ import annotations 4 | 5 | from {{ cookiecutter.library_name }}.mapper import {{ cookiecutter.name }}Mapper 6 | 7 | {{ cookiecutter.name }}Mapper.cli() 8 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/README.md: -------------------------------------------------------------------------------- 1 | # Singer Tap Template 2 | 3 | To use this cookie cutter template: 4 | 5 | ```bash 6 | pip3 install pipx 7 | pipx ensurepath 8 | # You may need to reopen your shell at this point 9 | pipx install cookiecutter 10 | ``` 11 | 12 | Initialize Cookiecutter template directly from Git: 13 | 14 | ```bash 15 | cookiecutter https://github.com/meltano/sdk --directory="cookiecutter/tap-template" 16 | ``` 17 | 18 | Or locally from an already-cloned `sdk` repo: 19 | 20 | ```bash 21 | cookiecutter ./sdk/cookiecutter/tap-template 22 | ``` 23 | 24 | See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html). 25 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/hooks/post_gen_project.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from pathlib import Path 3 | import shutil 4 | 5 | 6 | PACKAGE_PATH = Path("{{cookiecutter.library_name}}") 7 | 8 | 9 | if __name__ == "__main__": 10 | # Rename stream type client and delete others 11 | target = PACKAGE_PATH / "client.py" 12 | raw_client_py = PACKAGE_PATH / "{{cookiecutter.stream_type|lower}}-client.py" 13 | raw_client_py.rename(target) 14 | 15 | for client_py in PACKAGE_PATH.rglob("*-client.py"): 16 | client_py.unlink() 17 | 18 | if "{{ cookiecutter.stream_type }}" != "REST": 19 | shutil.rmtree(PACKAGE_PATH.joinpath("schemas"), ignore_errors=True) 20 | 21 | if "{{ cookiecutter.auth_method }}" not in ("OAuth2", "JWT"): 22 | PACKAGE_PATH.joinpath("auth.py").unlink() 23 | 24 | if "{{ cookiecutter.stream_type }}" == "SQL": 25 | PACKAGE_PATH.joinpath("streams.py").unlink() 26 | 27 | if "{{ cookiecutter.license }}" == "None": 28 | Path("LICENSE").unlink() 29 | 30 | if "{{ cookiecutter.include_ci_files }}" != "GitHub": 31 | shutil.rmtree(".github") 32 | 33 | if "{{ cookiecutter.ide }}" != "VSCode": 34 | shutil.rmtree(".vscode", ignore_errors=True) 35 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: pip 9 | directory: "/" 10 | schedule: 11 | interval: weekly 12 | commit-message: 13 | prefix: "chore(deps): " 14 | prefix-development: "chore(deps-dev): " 15 | groups: 16 | development-dependencies: 17 | dependency-type: development 18 | runtime-dependencies: 19 | dependency-type: production 20 | update-types: 21 | - "patch" 22 | versioning-strategy: increase-if-necessary 23 | - package-ecosystem: github-actions 24 | directory: "/" 25 | schedule: 26 | interval: weekly 27 | commit-message: 28 | prefix: "ci: " 29 | groups: 30 | actions: 31 | patterns: 32 | - "*" 33 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autofix_prs: true 3 | autoupdate_schedule: weekly 4 | autoupdate_commit_msg: 'chore: pre-commit autoupdate' 5 | skip: 6 | - uv-lock 7 | 8 | repos: 9 | - repo: https://github.com/pre-commit/pre-commit-hooks 10 | rev: v5.0.0 11 | hooks: 12 | - id: check-json 13 | exclude: | 14 | (?x)^( 15 | \.vscode/.*\.json 16 | )$ 17 | - id: check-toml 18 | - id: check-yaml 19 | - id: end-of-file-fixer 20 | - id: trailing-whitespace 21 | 22 | - repo: https://github.com/python-jsonschema/check-jsonschema 23 | rev: 0.33.0 24 | hooks: 25 | - id: check-dependabot 26 | - id: check-github-workflows 27 | - id: check-meltano 28 | 29 | - repo: https://github.com/astral-sh/ruff-pre-commit 30 | rev: v0.11.11 31 | hooks: 32 | - id: ruff 33 | args: [--fix, --exit-non-zero-on-fix, --show-fixes] 34 | - id: ruff-format 35 | 36 | - repo: https://github.com/pre-commit/mirrors-mypy 37 | rev: v1.15.0 38 | hooks: 39 | - id: mypy 40 | additional_dependencies: 41 | {%- if cookiecutter.stream_type == "SQL" %} 42 | - sqlalchemy-stubs 43 | {%- else %} 44 | - types-requests 45 | {%- endif %} 46 | 47 | - repo: https://github.com/astral-sh/uv-pre-commit 48 | rev: 0.7.8 49 | hooks: 50 | - id: uv-lock 51 | - id: uv-sync 52 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/.secrets/.gitignore: -------------------------------------------------------------------------------- 1 | # IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, 2 | # make sure those are never staged for commit into your git repo. You can store them here or another 3 | # secure location. 4 | # 5 | # Note: This may be redundant with the global .gitignore for, and is provided 6 | # for redundancy. If the `.secrets` folder is not needed, you may delete it 7 | # from the project. 8 | 9 | * 10 | !.gitignore 11 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "{{ cookiecutter.tap_id }}", 9 | "type": "debugpy", 10 | "request": "launch", 11 | "cwd": "${workspaceFolder}", 12 | "program": "{{ cookiecutter.library_name }}", 13 | "justMyCode": false, 14 | "args": [ 15 | "--config", 16 | ".secrets/config.json", 17 | ], 18 | }, 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/meltano.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | send_anonymous_usage_stats: true 3 | project_id: "{{cookiecutter.tap_id}}" 4 | default_environment: test 5 | venv: 6 | backend: uv 7 | environments: 8 | - name: test 9 | plugins: 10 | extractors: 11 | - name: "{{cookiecutter.tap_id}}" 12 | namespace: "{{cookiecutter.library_name}}" 13 | pip_url: -e . 14 | capabilities: 15 | - state 16 | - catalog 17 | - discover 18 | - about 19 | - stream-maps 20 | 21 | # TODO: Declare settings and their types here: 22 | settings: 23 | - name: username 24 | label: Username 25 | description: The username to use for authentication 26 | 27 | - name: password 28 | kind: password 29 | label: Password 30 | description: The password to use for authentication 31 | sensitive: true 32 | 33 | - name: start_date 34 | kind: date_iso8601 35 | label: Start Date 36 | description: Initial date to start extracting data from 37 | 38 | # TODO: Declare required settings here: 39 | settings_group_validation: 40 | - [username, password] 41 | 42 | # TODO: Declare default configuration values here: 43 | config: 44 | start_date: '2010-01-01T00:00:00Z' 45 | 46 | loaders: 47 | - name: target-jsonl 48 | variant: andyh1203 49 | pip_url: target-jsonl 50 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/output/.gitignore: -------------------------------------------------------------------------------- 1 | # This directory is used as a target by target-jsonl, so ignore all files 2 | 3 | * 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Test suite for {{ cookiecutter.tap_id }}.""" 2 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/test_core.py: -------------------------------------------------------------------------------- 1 | """Tests standard tap features using the built-in SDK tests library.""" 2 | 3 | import datetime 4 | 5 | from singer_sdk.testing import get_tap_test_class 6 | 7 | from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }} 8 | 9 | SAMPLE_CONFIG = { 10 | "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), 11 | # TODO: Initialize minimal tap config 12 | } 13 | 14 | 15 | # Run standard built-in tap tests from the SDK: 16 | TestTap{{ cookiecutter.source_name }} = get_tap_test_class( 17 | tap_class=Tap{{ cookiecutter.source_name }}, 18 | config=SAMPLE_CONFIG, 19 | ) 20 | 21 | 22 | # TODO: Create additional tests as appropriate for your tap. 23 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini: -------------------------------------------------------------------------------- 1 | # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy 2 | 3 | [tox] 4 | envlist = py3{9,10,11,12,13} 5 | minversion = 4.22 6 | requires = 7 | tox>=4.22 8 | 9 | [testenv] 10 | pass_env = 11 | {%- if cookiecutter.include_ci_files == "GitHub" %} 12 | GITHUB_* 13 | {%- endif %} 14 | {{cookiecutter.tap_id.replace('-', '_').upper()}}_* 15 | dependency_groups = 16 | test 17 | commands = 18 | pytest {posargs} 19 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py: -------------------------------------------------------------------------------- 1 | """Tap for {{ cookiecutter.source_name }}.""" 2 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__main__.py: -------------------------------------------------------------------------------- 1 | """{{ cookiecutter.source_name }} entry point.""" 2 | 3 | from __future__ import annotations 4 | 5 | from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }} 6 | 7 | Tap{{ cookiecutter.source_name }}.cli() 8 | -------------------------------------------------------------------------------- /cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/schemas/__init__.py: -------------------------------------------------------------------------------- 1 | """JSON schema files for the REST API.""" 2 | -------------------------------------------------------------------------------- /cookiecutter/target-template/README.md: -------------------------------------------------------------------------------- 1 | # SDK Target Template 2 | 3 | To use this cookie cutter template: 4 | 5 | ```bash 6 | pip3 install pipx 7 | pipx ensurepath 8 | # You may need to reopen your shell at this point 9 | pipx install cookiecutter 10 | ``` 11 | 12 | Initialize Cookiecutter template directly from Git: 13 | 14 | ```bash 15 | cookiecutter https://github.com/meltano/sdk --directory="cookiecutter/target-template" 16 | ``` 17 | 18 | Or locally from an already-cloned `sdk` repo: 19 | 20 | ```bash 21 | cookiecutter ./sdk/cookiecutter/target-template 22 | ``` 23 | 24 | See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html). 25 | -------------------------------------------------------------------------------- /cookiecutter/target-template/cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "destination_name": "MyDestinationName", 3 | "admin_name": "FirstName LastName", 4 | "admin_email": "firstname.lastname@example.com", 5 | "target_id": "target-{{ cookiecutter.destination_name.lower() }}", 6 | "library_name": "{{ cookiecutter.target_id.replace('-', '_') }}", 7 | "variant": "None (Skip)", 8 | "serialization_method": ["Per record", "Per batch", "SQL"], 9 | "faker_extra": false, 10 | "include_ci_files": ["GitHub", "None (Skip)"], 11 | "license": ["Apache-2.0"], 12 | "ide": ["VSCode", "None"], 13 | "__prompts__": { 14 | "name": "The name of the mapper, in CamelCase", 15 | "admin_name": "Provide your [bold yellow]full name[/]", 16 | "admin_email": "Provide your [bold yellow]email[/]", 17 | "mapper_id": "The ID of the tap, in kebab-case", 18 | "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.", 19 | "serialization_method": "The serialization method to use for loading data", 20 | "faker_extra": "Add [bold orange1][link=https://faker.readthedocs.io/en/master/]Faker[/link][/] as an extra dependency to support generating fake data in stream maps?", 21 | "include_ci_files": "Whether to include CI files for a common CI services", 22 | "license": "The license for the project", 23 | "ide": "Add configuration files for your preferred IDE" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cookiecutter/target-template/cookiecutter.tests.yml: -------------------------------------------------------------------------------- 1 | tests: 2 | - destination_name: TargetBatchSinkTest 3 | target_id: target-batchsink-test 4 | serialization_method: Per batch 5 | - destination_name: TargetRecordSinkTest 6 | target_id: target-recordsink-test 7 | serialization_method: Per record 8 | - destination_name: SQLSinkTest 9 | target_id: target-sqlsink-test 10 | serialization_method: SQL 11 | - destination_name: TargetRecordSinkTest 12 | target_id: target-recordsink-test 13 | variant: meltanolabs 14 | serialization_method: Per record -------------------------------------------------------------------------------- /cookiecutter/target-template/hooks/post_gen_project.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from pathlib import Path 3 | import shutil 4 | 5 | 6 | BASE_PATH = Path("{{cookiecutter.library_name}}") 7 | 8 | 9 | if __name__ == "__main__": 10 | if "{{ cookiecutter.license }}" != "Apache-2.0": 11 | Path("LICENSE").unlink() 12 | 13 | if "{{ cookiecutter.include_ci_files }}" != "GitHub": 14 | shutil.rmtree(Path(".github")) 15 | 16 | if "{{ cookiecutter.ide }}" != "VSCode": 17 | shutil.rmtree(".vscode", ignore_errors=True) 18 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: pip 9 | directory: "/" 10 | schedule: 11 | interval: weekly 12 | commit-message: 13 | prefix: "chore(deps): " 14 | prefix-development: "chore(deps-dev): " 15 | groups: 16 | development-dependencies: 17 | dependency-type: development 18 | runtime-dependencies: 19 | dependency-type: production 20 | update-types: 21 | - "patch" 22 | versioning-strategy: increase-if-necessary 23 | - package-ecosystem: github-actions 24 | directory: "/" 25 | schedule: 26 | interval: weekly 27 | commit-message: 28 | prefix: "ci: " 29 | groups: 30 | actions: 31 | patterns: 32 | - "*" 33 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autofix_prs: true 3 | autoupdate_schedule: weekly 4 | autoupdate_commit_msg: 'chore: pre-commit autoupdate' 5 | skip: 6 | - uv-lock 7 | 8 | repos: 9 | - repo: https://github.com/pre-commit/pre-commit-hooks 10 | rev: v5.0.0 11 | hooks: 12 | - id: check-json 13 | exclude: | 14 | (?x)^( 15 | .*/launch.json 16 | )$ 17 | - id: check-toml 18 | - id: check-yaml 19 | - id: end-of-file-fixer 20 | - id: trailing-whitespace 21 | 22 | - repo: https://github.com/python-jsonschema/check-jsonschema 23 | rev: 0.33.0 24 | hooks: 25 | - id: check-dependabot 26 | - id: check-github-workflows 27 | - id: check-meltano 28 | 29 | - repo: https://github.com/astral-sh/ruff-pre-commit 30 | rev: v0.11.11 31 | hooks: 32 | - id: ruff 33 | args: [--fix, --exit-non-zero-on-fix, --show-fixes] 34 | - id: ruff-format 35 | 36 | - repo: https://github.com/pre-commit/mirrors-mypy 37 | rev: v1.15.0 38 | hooks: 39 | - id: mypy 40 | additional_dependencies: 41 | {%- if cookiecutter.serialization_method != "SQL" %} 42 | - sqlalchemy-stubs 43 | {%- else %} 44 | - types-requests 45 | {%- endif %} 46 | 47 | - repo: https://github.com/astral-sh/uv-pre-commit 48 | rev: 0.7.8 49 | hooks: 50 | - id: uv-lock 51 | - id: uv-sync 52 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/.secrets/.gitignore: -------------------------------------------------------------------------------- 1 | # IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, 2 | # make sure those are never staged for commit into your git repo. You can store them here or another 3 | # secure location. 4 | # 5 | # Note: This may be redundant with the global .gitignore for, and is provided 6 | # for redundancy. If the `.secrets` folder is not needed, you may delete it 7 | # from the project. 8 | 9 | * 10 | !.gitignore 11 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/meltano.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | send_anonymous_usage_stats: true 3 | project_id: "{{cookiecutter.target_id}}" 4 | default_environment: test 5 | venv: 6 | backend: uv 7 | environments: 8 | - name: test 9 | plugins: 10 | extractors: 11 | - name: tap-smoke-test 12 | variant: meltano 13 | pip_url: git+https://github.com/meltano/tap-smoke-test.git 14 | config: 15 | streams: 16 | - stream_name: animals 17 | input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl 18 | 19 | loaders: 20 | - name: "{{cookiecutter.target_id}}" 21 | namespace: "{{cookiecutter.library_name}}" 22 | pip_url: -e . 23 | capabilities: 24 | - about 25 | - stream-maps 26 | - schema-flattening 27 | 28 | # TODO: Declare settings and their types here: 29 | settings: 30 | - name: username 31 | label: Username 32 | description: The username to use for authentication 33 | 34 | - name: password 35 | kind: password 36 | label: Password 37 | description: The password to use for authentication 38 | sensitive: true 39 | 40 | # TODO: Declare required settings here: 41 | settings_group_validation: 42 | - [username, password] 43 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Test suite for {{ cookiecutter.target_id }}.""" 2 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/tests/{{ 'test' }}_core.py: -------------------------------------------------------------------------------- 1 | """Tests standard target features using the built-in SDK tests library.""" 2 | 3 | from __future__ import annotations 4 | 5 | import typing as t 6 | 7 | import pytest 8 | from singer_sdk.testing import get_target_test_class 9 | 10 | from {{ cookiecutter.library_name }}.target import Target{{ cookiecutter.destination_name }} 11 | 12 | # TODO: Initialize minimal target config 13 | SAMPLE_CONFIG: dict[str, t.Any] = {} 14 | 15 | 16 | # Run standard built-in target tests from the SDK: 17 | StandardTargetTests = get_target_test_class( 18 | target_class=Target{{ cookiecutter.destination_name }}, 19 | config=SAMPLE_CONFIG, 20 | ) 21 | 22 | 23 | class TestTarget{{ cookiecutter.destination_name }}(StandardTargetTests): # type: ignore[misc, valid-type] 24 | """Standard Target Tests.""" 25 | 26 | @pytest.fixture(scope="class") 27 | def resource(self): # noqa: ANN201 28 | """Generic external resource. 29 | 30 | This fixture is useful for setup and teardown of external resources, 31 | such output folders, tables, buckets etc. for use during testing. 32 | 33 | Example usage can be found in the SDK samples test suite: 34 | https://github.com/meltano/sdk/tree/main/tests/samples 35 | """ 36 | return "resource" 37 | 38 | 39 | # TODO: Create additional tests as appropriate for your target. 40 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini: -------------------------------------------------------------------------------- 1 | # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy 2 | 3 | [tox] 4 | envlist = py3{9,10,11,12,13} 5 | minversion = 4.22 6 | requires = 7 | tox>=4.22 8 | 9 | [testenv] 10 | pass_env = 11 | {%- if cookiecutter.include_ci_files == "GitHub" %} 12 | GITHUB_* 13 | {%- endif %} 14 | {{cookiecutter.target_id.replace('-', '_').upper()}}_* 15 | dependency_groups = 16 | test 17 | commands = 18 | pytest {posargs} 19 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/__init__.py: -------------------------------------------------------------------------------- 1 | """Target for {{ cookiecutter.destination_name }}.""" 2 | -------------------------------------------------------------------------------- /cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/__main__.py: -------------------------------------------------------------------------------- 1 | """{{ cookiecutter.destination_name }} entry point.""" 2 | 3 | from __future__ import annotations 4 | 5 | from {{ cookiecutter.library_name }}.target import Target{{ cookiecutter.destination_name }} 6 | 7 | Target{{ cookiecutter.destination_name }}.cli() 8 | -------------------------------------------------------------------------------- /docs/_static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/docs/_static/.gitkeep -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: 'Hanken Grotesk'; 3 | src: url('../fonts/HankenGrotesk-Regular.woff2') format('woff2'), 4 | url('../fonts/HankenGrotesk-Regular.woff') format('woff'); 5 | font-weight: normal; 6 | font-style: normal; 7 | font-display: swap; 8 | } 9 | 10 | @font-face { 11 | font-family: 'Plus Jakarta Sans'; 12 | src: url('../fonts/PlusJakartaSans-Regular.woff2') format('woff2'), 13 | url('../fonts/PlusJakartaSans-Regular.woff') format('woff'); 14 | font-weight: normal; 15 | font-style: normal; 16 | font-display: swap; 17 | } 18 | 19 | h1 { 20 | color: var(--color-brand-content); 21 | } 22 | 23 | h1, h2, h3, h4, h5, h6 { 24 | font-family: "Plus Jakarta Sans"; 25 | } 26 | -------------------------------------------------------------------------------- /docs/_static/fonts/HankenGrotesk-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/docs/_static/fonts/HankenGrotesk-Regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/HankenGrotesk-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/docs/_static/fonts/HankenGrotesk-Regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/PlusJakartaSans-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/docs/_static/fonts/PlusJakartaSans-Regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/PlusJakartaSans-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/docs/_static/fonts/PlusJakartaSans-Regular.woff2 -------------------------------------------------------------------------------- /docs/_static/img/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/docs/_static/img/favicon.png -------------------------------------------------------------------------------- /docs/_templates/base.html: -------------------------------------------------------------------------------- 1 | {% extends '!base.html' %} 2 | {% block extrahead %} 3 | <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=IBM+Plex+Sans&display=swap"> 4 | <link rel="apple-touch-icon" sizes="180x180" href="{{ pathto('_static/img/favicon.png', 1) }}"> 5 | <link rel="icon" type="image/png" sizes="32x32" href="{{ pathto('_static/img/favicon.png', 1) }}"> 6 | <link rel="icon" type="image/png" sizes="192x192" href="{{ pathto('_static/img/favicon.png', 1) }}"> 7 | <!-- Google tag (gtag.js) --> 8 | <script async src="https://www.googletagmanager.com/gtag/js?id=GTM-WHJMBX2"></script> 9 | <script> 10 | window.dataLayer = window.dataLayer || []; 11 | function gtag(){dataLayer.push(arguments);} 12 | gtag('js', new Date()); 13 | 14 | gtag('config', 'GTM-WHJMBX2'); 15 | </script> 16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /docs/_templates/class.rst: -------------------------------------------------------------------------------- 1 | {{ fullname }} 2 | {{ "=" * fullname|length }} 3 | 4 | .. currentmodule:: {{ module }} 5 | 6 | .. autoclass:: {{ name }} 7 | :members: 8 | :special-members: __init__, __call__ 9 | {%- if name in ('IntegerType', 'NumberType') %} 10 | :inherited-members: JSONTypeHelper 11 | {%- endif %} 12 | -------------------------------------------------------------------------------- /docs/_templates/plugin_class.rst: -------------------------------------------------------------------------------- 1 | {{ fullname }} 2 | {{ "=" * fullname|length }} 3 | 4 | .. currentmodule:: {{ module }} 5 | 6 | .. autoclass:: {{ name }} 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: 10 | :special-members: __init__ 11 | -------------------------------------------------------------------------------- /docs/_templates/stream_class.rst: -------------------------------------------------------------------------------- 1 | {{ fullname }} 2 | {{ "=" * fullname|length }} 3 | 4 | .. currentmodule:: {{ module }} 5 | 6 | .. autoclass:: {{ name }} 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: Stream 10 | :special-members: __init__ 11 | -------------------------------------------------------------------------------- /docs/classes/singer_sdk.BatchSink.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.BatchSink 2 | ==================== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: BatchSink 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.GraphQLStream.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.GraphQLStream 2 | ======================== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: GraphQLStream 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: Stream 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.InlineMapper.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.InlineMapper 2 | ======================= 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: InlineMapper 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.RESTStream.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.RESTStream 2 | ===================== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: RESTStream 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: Stream 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.RecordSink.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.RecordSink 2 | ===================== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: RecordSink 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.SQLConnector.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.SQLConnector 2 | ======================= 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: SQLConnector 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.SQLSink.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.SQLSink 2 | ================== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: SQLSink 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.SQLStream.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.SQLStream 2 | ==================== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: SQLStream 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: Stream 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.SQLTap.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.SQLTap 2 | ================= 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: SQLTap 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.SQLTarget.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.SQLTarget 2 | ==================== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: SQLTarget 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.Sink.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.Sink 2 | =============== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: Sink 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.Stream.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.Stream 2 | ================= 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: Stream 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: Stream 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.Tap.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.Tap 2 | ============== 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: Tap 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.Target.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.Target 2 | ================= 3 | 4 | .. currentmodule:: singer_sdk 5 | 6 | .. autoclass:: Target 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: 10 | :special-members: __init__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.authenticators.APIAuthenticatorBase 2 | ============================================== 3 | 4 | .. currentmodule:: singer_sdk.authenticators 5 | 6 | .. autoclass:: APIAuthenticatorBase 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.authenticators.APIKeyAuthenticator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.authenticators.APIKeyAuthenticator 2 | ============================================= 3 | 4 | .. currentmodule:: singer_sdk.authenticators 5 | 6 | .. autoclass:: APIKeyAuthenticator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.authenticators.BasicAuthenticator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.authenticators.BasicAuthenticator 2 | ============================================ 3 | 4 | .. currentmodule:: singer_sdk.authenticators 5 | 6 | .. autoclass:: BasicAuthenticator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.authenticators.BearerTokenAuthenticator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.authenticators.BearerTokenAuthenticator 2 | ================================================== 3 | 4 | .. currentmodule:: singer_sdk.authenticators 5 | 6 | .. autoclass:: BearerTokenAuthenticator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.authenticators.OAuthAuthenticator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.authenticators.OAuthAuthenticator 2 | ============================================ 3 | 4 | .. currentmodule:: singer_sdk.authenticators 5 | 6 | .. autoclass:: OAuthAuthenticator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.authenticators.OAuthJWTAuthenticator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.authenticators.OAuthJWTAuthenticator 2 | =============================================== 3 | 4 | .. currentmodule:: singer_sdk.authenticators 5 | 6 | .. autoclass:: OAuthJWTAuthenticator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.authenticators.SimpleAuthenticator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.authenticators.SimpleAuthenticator 2 | ============================================= 3 | 4 | .. currentmodule:: singer_sdk.authenticators 5 | 6 | .. autoclass:: SimpleAuthenticator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.batch.BaseBatcher.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.batch.BaseBatcher 2 | ============================ 3 | 4 | .. currentmodule:: singer_sdk.batch 5 | 6 | .. autoclass:: BaseBatcher 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.batch.JSONLinesBatcher.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.batch.JSONLinesBatcher 2 | ================================= 3 | 4 | .. currentmodule:: singer_sdk.batch 5 | 6 | .. autoclass:: JSONLinesBatcher 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.connectors.sql.JSONSchemaToSQL.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.connectors.sql.JSONSchemaToSQL 2 | ========================================= 3 | 4 | .. currentmodule:: singer_sdk.connectors.sql 5 | 6 | .. autoclass:: JSONSchemaToSQL 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.connectors.sql.SQLToJSONSchema.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.connectors.sql.SQLToJSONSchema 2 | ========================================= 3 | 4 | .. currentmodule:: singer_sdk.connectors.sql 5 | 6 | .. autoclass:: SQLToJSONSchema 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.ConfigValidationError.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.ConfigValidationError 2 | =========================================== 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: ConfigValidationError 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.FatalAPIError.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.FatalAPIError 2 | =================================== 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: FatalAPIError 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.InvalidStreamSortException.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.InvalidStreamSortException 2 | ================================================ 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: InvalidStreamSortException 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.MapExpressionError.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.MapExpressionError 2 | ======================================== 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: MapExpressionError 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.MaxRecordsLimitException.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.MaxRecordsLimitException 2 | ============================================== 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: MaxRecordsLimitException 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.RecordsWithoutSchemaException.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.RecordsWithoutSchemaException 2 | =================================================== 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: RecordsWithoutSchemaException 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.RetriableAPIError.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.RetriableAPIError 2 | ======================================= 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: RetriableAPIError 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.StreamMapConfigError.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.StreamMapConfigError 2 | ========================================== 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: StreamMapConfigError 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.TapStreamConnectionFailure.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.TapStreamConnectionFailure 2 | ================================================ 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: TapStreamConnectionFailure 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.exceptions.TooManyRecordsException.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.exceptions.TooManyRecordsException 2 | ============================================= 3 | 4 | .. currentmodule:: singer_sdk.exceptions 5 | 6 | .. autoclass:: TooManyRecordsException 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.BaseAPIPaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.BaseAPIPaginator 2 | ====================================== 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: BaseAPIPaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.BaseHATEOASPaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.BaseHATEOASPaginator 2 | ========================================== 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: BaseHATEOASPaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.BaseOffsetPaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.BaseOffsetPaginator 2 | ========================================= 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: BaseOffsetPaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.BasePageNumberPaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.BasePageNumberPaginator 2 | ============================================= 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: BasePageNumberPaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.HeaderLinkPaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.HeaderLinkPaginator 2 | ========================================= 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: HeaderLinkPaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.JSONPathPaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.JSONPathPaginator 2 | ======================================= 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: JSONPathPaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.LegacyPaginatedStreamProtocol.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.LegacyPaginatedStreamProtocol 2 | =================================================== 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: LegacyPaginatedStreamProtocol 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.LegacyStreamPaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.LegacyStreamPaginator 2 | =========================================== 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: LegacyStreamPaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.SimpleHeaderPaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.SimpleHeaderPaginator 2 | =========================================== 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: SimpleHeaderPaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/singer_sdk.pagination.SinglePagePaginator.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.pagination.SinglePagePaginator 2 | ========================================= 3 | 4 | .. currentmodule:: singer_sdk.pagination 5 | 6 | .. autoclass:: SinglePagePaginator 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.ArrayType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.ArrayType 2 | =========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: ArrayType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.BooleanType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.BooleanType 2 | ============================= 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: BooleanType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.Constant.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.Constant 2 | ========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: Constant 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.CustomType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.CustomType 2 | ============================ 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: CustomType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.DateTimeType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.DateTimeType 2 | ============================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: DateTimeType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.DateType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.DateType 2 | ========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: DateType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.DiscriminatedUnion.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.DiscriminatedUnion 2 | ==================================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: DiscriminatedUnion 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.DurationType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.DurationType 2 | ============================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: DurationType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.EmailType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.EmailType 2 | =========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: EmailType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.HostnameType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.HostnameType 2 | ============================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: HostnameType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.IPv4Type.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.IPv4Type 2 | ========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: IPv4Type 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.IPv6Type.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.IPv6Type 2 | ========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: IPv6Type 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.IntegerType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.IntegerType 2 | ============================= 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: IntegerType 7 | :members: 8 | :special-members: __init__, __call__ 9 | :inherited-members: JSONTypeHelper -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.JSONPointerType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.JSONPointerType 2 | ================================= 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: JSONPointerType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.NumberType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.NumberType 2 | ============================ 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: NumberType 7 | :members: 8 | :special-members: __init__, __call__ 9 | :inherited-members: JSONTypeHelper -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.ObjectType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.ObjectType 2 | ============================ 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: ObjectType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.OneOf.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.OneOf 2 | ======================= 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: OneOf 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.PropertiesList.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.PropertiesList 2 | ================================ 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: PropertiesList 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.Property.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.Property 2 | ========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: Property 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.RegexType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.RegexType 2 | =========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: RegexType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.RelativeJSONPointerType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.RelativeJSONPointerType 2 | ========================================= 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: RelativeJSONPointerType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.StringType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.StringType 2 | ============================ 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: StringType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.TimeType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.TimeType 2 | ========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: TimeType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.URITemplateType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.URITemplateType 2 | ================================= 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: URITemplateType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.URIType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.URIType 2 | ========================= 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: URIType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/classes/typing/singer_sdk.typing.UUIDType.rst: -------------------------------------------------------------------------------- 1 | singer_sdk.typing.UUIDType 2 | ========================== 3 | 4 | .. currentmodule:: singer_sdk.typing 5 | 6 | .. autoclass:: UUIDType 7 | :members: 8 | :special-members: __init__, __call__ -------------------------------------------------------------------------------- /docs/context_object.md: -------------------------------------------------------------------------------- 1 | # The Context Object 2 | 3 | Many of the methods in the [Stream](classes/singer_sdk.Stream) class and its subclasses accept 4 | a `context` parameter, which is a dictionary that contains information about the stream 5 | partition or parent stream. 6 | 7 | ## Best practices for using context 8 | 9 | - The context object MUST NOT contain any sensitive information, such as API keys or secrets. 10 | This is because the context is<br><br> 11 | 12 | 1. sent to the target, 13 | 1. stored in the state file and 14 | 1. logged to the console as a tag in metrics and logs.<br><br> 15 | 16 | - The context object SHOULD NOT be mutated during the stream's lifecycle. This is because the 17 | context is stored in the state file, and mutating it will cause the state file to be 18 | inconsistent with the actual state of the stream. 19 | -------------------------------------------------------------------------------- /docs/deprecation.md: -------------------------------------------------------------------------------- 1 | # Deprecation Timeline 2 | 3 | This page outlines when various features of the Singer SDK will be removed or changed in a backward 4 | incompatible way, following their deprecation, as indicated in the 5 | [deprecation policy](./release_process.md#deprecation-policy). 6 | 7 | ## 1.0 8 | 9 | - The `RESTStream.get_next_page_token` method will no longer be called 10 | as part of the stream pagination process. It is replaced by the 11 | [`RESTStream.get_new_paginator`](singer_sdk.RESTStream.get_new_paginator). 12 | 13 | See the [migration guide](./guides/pagination-classes.md) for more information. 14 | 15 | - The `singer_sdk.testing.get_standard_tap_tests` and `singer_sdk.testing.get_standard_target_tests` functions will be removed. Replace them with `singer_sdk.testing.get_tap_test_class` and `singer_sdk.testing.get_target_test_class` functions respective to generate a richer test suite. 16 | 17 | - The `PyJWT` and `cryptography` libraries will be no longer be installed by default. If you are using the `OAuthJWTAuthenticator` you will need to install [`singer-sdk[jwt]`](./dev_guide.md#extra-features). 18 | -------------------------------------------------------------------------------- /docs/guides/index.md: -------------------------------------------------------------------------------- 1 | # In-depth Guides 2 | 3 | The following pages contain useful information for developers building on top of the Singer SDK. 4 | 5 | ```{toctree} 6 | :maxdepth: 2 7 | 8 | porting 9 | pagination-classes 10 | custom-clis 11 | config-schema 12 | performance 13 | sql-tap 14 | sql-target 15 | 16 | migrate-to-uv 17 | ``` 18 | -------------------------------------------------------------------------------- /docs/guides/migrate-to-uv.md: -------------------------------------------------------------------------------- 1 | # Migrating a plugin from Poetry to uv 2 | 3 | This guide will help you migrate your Singer plugin from Poetry to uv. 4 | 5 | ## Prerequisites 6 | 7 | - [uv](https://docs.astral.sh/uv/) 8 | 9 | ## Steps 10 | 11 | 1. Use the [migrate-to-uv](https://github.com/mkniewallner/migrate-to-uv) tool to migrate your project from Poetry to uv. Run the following command in your project directory: 12 | 13 | ```bash 14 | uvx migrate-to-uv 15 | ``` 16 | 17 | 2. Update the `build-system` section in your `pyproject.toml` file to use `hatchling`: 18 | 19 | ```toml 20 | [build-system] 21 | build-backend = "hatchling.build" 22 | requires = ["hatchling>=1,<2"] 23 | ``` 24 | 25 | 3. Update your CI/CD to use `uv` commands instead of `poetry` commands. For example, replace `poetry install` with `uv sync`. You may also want to use the [official `setup-uv`](https://github.com/astral-sh/setup-uv/) GitHub Action to install `uv` in your CI/CD workflow. 26 | 27 | ## Example 28 | 29 | For an example of a migration to `uv` for a Singer tap, see [how it was done for tap-stackexchange](https://github.com/MeltanoLabs/tap-stackexchange/pull/507). 30 | -------------------------------------------------------------------------------- /docs/images/200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/docs/images/200.png -------------------------------------------------------------------------------- /docs/implementation/catalog_metadata.md: -------------------------------------------------------------------------------- 1 | # Catalog Metadata 2 | 3 | The SDK automatically generates catalog metadata during catalog discovery. Selection rules overridden by a user will be respected. 4 | 5 | Primary key properties may not be deselected, as these are required for `key_properties` to be declared in stream messages. 6 | 7 | ## Additional Singer Metadata References 8 | 9 | - [Singer Spec: Metadata](https://hub.meltano.com/singer/spec#metadata) 10 | -------------------------------------------------------------------------------- /docs/implementation/discovery.md: -------------------------------------------------------------------------------- 1 | # Catalog Discovery 2 | 3 | All taps developed using the SDK will automatically support `discovery` as a base 4 | capability, which is the process of generating and emitting a catalog that describes the 5 | available streams and stream types. 6 | 7 | The catalog generated is automatically populated by a small number of developer inputs. Most 8 | importantly: 9 | 10 | - `Tap.discover_streams()` - Should return a list of available "discovered" streams. 11 | - `Stream.schema` or `Stream.schema_filepath` - The JSON Schema definition of each stream, 12 | provided either directly as a Python `dict` or indirectly as a `.json` filepath. 13 | - `Stream.primary_keys` - a list of strings indicating the primary key(s) of the stream. 14 | - `Stream.replication_key` - a single string indicating the name of the stream's replication 15 | key (if applicable). 16 | 17 | ## Additional Discovery Mode References 18 | 19 | - See the [Dev Guide](../dev_guide.md) and [Code Samples](../code_samples.md) for more 20 | information on working with dynamic stream schemas. 21 | - [Singer Spec: Discovery (meltano.com)](https://hub.meltano.com/singer/spec#discovery-mode) 22 | - [Singer Spec: Discovery (singer-io)](https://github.com/singer-io/getting-started/blob/master/docs/DISCOVERY_MODE.md) 23 | -------------------------------------------------------------------------------- /docs/implementation/index.md: -------------------------------------------------------------------------------- 1 | # Singer Implementation Details 2 | 3 | This section documents certain behaviors and expectations of the SDK framework. 4 | 5 | ```{toctree} 6 | :caption: Implementation Details 7 | 8 | cli 9 | discovery 10 | catalog_metadata 11 | record_metadata 12 | metrics 13 | logging 14 | state 15 | at_least_once 16 | ``` 17 | 18 | ## How to use the implementation reference material 19 | 20 | _**Note:** You should not need to master all of the details here in order 21 | to build your tap, and the behaviors described here should be automatic 22 | and/or intuitive. For general guidance on tap development, please instead refer to our 23 | [Dev Guide](../dev_guide.md)._ 24 | 25 | The specifications provided in this section are documented primarily to support 26 | advanced use cases, behavior overrides, backwards compatibility with legacy taps, 27 | debugging unexpected behaviors, or contributing back to the SDK itself. 28 | -------------------------------------------------------------------------------- /docs/implementation/metrics.md: -------------------------------------------------------------------------------- 1 | # Tap and Target Metrics 2 | 3 | Metrics logging is specified in the 4 | [Singer Spec](https://hub.meltano.com/singer/spec#metrics). 5 | 6 | The SDK will automatically emit the following metrics: 7 | 8 | - `record_count`: The number of records processed by the tap or target. 9 | - `http_request_duration`: The duration of HTTP requests made by the tap. 10 | - `sync_duration`: The duration of the sync operation. 11 | - `batch_processing_time`: The duration of processing a batch of records. 12 | 13 | ## Customization options 14 | 15 | ### `metrics_log_level` 16 | 17 | Metrics are logged at the `INFO` level. Developers may optionally add a 18 | `metrics_log_level` config option to their taps, `WARNING` or `ERROR` to disable 19 | metrics logging. 20 | 21 | ### `SINGER_SDK_LOG_CONFIG` 22 | 23 | Metrics are written by the `singer_sdk.metrics` logger, so the end user can set 24 | `SINGER_SDK_LOG_CONFIG` to a logging config file that defines the format and output 25 | for metrics. See the [logging docs](./logging.md) for an example file. 26 | 27 | ## Additional Singer Metrics References 28 | 29 | - [Singer Spec: Metrics](https://hub.meltano.com/singer/spec#metrics) 30 | -------------------------------------------------------------------------------- /docs/implementation/record_metadata.md: -------------------------------------------------------------------------------- 1 | # Record Metadata 2 | 3 | The SDK can automatically generate `_sdc_` ("Singer Data Capture") metadata properties when 4 | performing data loads in SDK-based targets. 5 | 6 | If `add_record_metadata` is defined as 7 | a config option by the developer, and if the user sets `add_record_metadata=True` within 8 | their own configuration, the following columns will be automatically added to each record: 9 | 10 | - `_sdc_extracted_at` - Timestamp indicating when the record was extracted the record from the source. 11 | - `_sdc_received_at` - Timestamp indicating when the record was received by the target for loading. 12 | - `_sdc_batched_at` - Timestamp indicating when the record's batch was initiated. 13 | - `_sdc_deleted_at` - Passed from a Singer tap if DELETE events are able to be tracked. In general, this is populated when the tap is synced LOG_BASED replication. If not sent from the tap, this field will be null. 14 | - `_sdc_sequence` - The epoch (milliseconds) that indicates the order in which the record was queued for loading. 15 | - `_sdc_table_version` - Indicates the version of the table. This column is used to determine when to issue TRUNCATE commands during loading, where applicable. 16 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/typing.rst: -------------------------------------------------------------------------------- 1 | JSON Schema Helpers 2 | =================== 3 | 4 | .. automodule:: singer_sdk.typing 5 | :noindex: 6 | :members: 7 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/README.md: -------------------------------------------------------------------------------- 1 | # CI for Empty Cookiecutters 2 | 3 | Cookiecutters for taps and targets include two kinds of tests: linting and end-to-end testing with pytest. When a new project is created with the cookiecutter we expect: 4 | 5 | - linting tests should pass 6 | - integration tests may fail (because no integration has been implemented yet) 7 | 8 | To automate creation of cookiecutter test projects, we use a [replay file](https://cookiecutter.readthedocs.io/en/stable/advanced/replay.html) generated by cookiecutter. 9 | 10 | ## Running Manually 11 | 12 | Run a test against tap-template cookiecutter against the `tap-rest-api_key-github.json` replay file, execute: 13 | 14 | ```bash 15 | bash test_cookiecutter.sh ../../cookiecutter/tap-template ./tap-rest-api_key-github.json 16 | bash test_cookiecutter.sh ../../cookiecutter/target-template ./target-per_record.json 17 | ``` 18 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/mapper-base.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "name": "MyMapperName", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "mapper_id": "mapper-base", 7 | "library_name": "mapper_base", 8 | "variant": "None (Skip)", 9 | "faker_extra": false, 10 | "include_ci_files": "None (Skip)", 11 | "license": "Apache-2.0", 12 | "ide": "VSCode", 13 | "_template": "../mapper-template/", 14 | "_output_dir": "." 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-faker.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-faker", 7 | "library_name": "tap_faker", 8 | "variant": "None (Skip)", 9 | "stream_type": "REST", 10 | "auth_method": "Bearer Token", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": true, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-graphql-jwt.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "GraphQLJWTTemplateTest", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-graphql-jwt", 7 | "library_name": "tap_graphql_jwt", 8 | "variant": "None (Skip)", 9 | "stream_type": "GraphQL", 10 | "auth_method": "JWT", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-no-license.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-no-license", 7 | "library_name": "tap_no_license", 8 | "variant": "None (Skip)", 9 | "stream_type": "REST", 10 | "auth_method": "Basic Auth", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "None", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-other-custom.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-other-custom", 7 | "library_name": "tap_other_custom", 8 | "variant": "None (Skip)", 9 | "stream_type": "Other", 10 | "auth_method": "Custom or N/A", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-rest-api_key-github.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-rest-api_key-github", 7 | "library_name": "tap_rest_api_key_github", 8 | "variant": "None (Skip)", 9 | "stream_type": "REST", 10 | "auth_method": "API Key", 11 | "include_ci_files": "GitHub", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-rest-basic_auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-rest-basic_auth", 7 | "library_name": "tap_rest_basic_auth", 8 | "variant": "None (Skip)", 9 | "stream_type": "REST", 10 | "auth_method": "Basic Auth", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-rest-bearer_token.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-rest-bearer_token", 7 | "library_name": "tap_rest_bearer_token", 8 | "variant": "None (Skip)", 9 | "stream_type": "REST", 10 | "auth_method": "Bearer Token", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-rest-custom.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-rest-custom", 7 | "library_name": "tap_rest_custom", 8 | "variant": "None (Skip)", 9 | "stream_type": "REST", 10 | "auth_method": "Custom or N/A", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-rest-jwt.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-rest-jwt", 7 | "library_name": "tap_rest_jwt", 8 | "variant": "None (Skip)", 9 | "stream_type": "REST", 10 | "auth_method": "JWT", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-rest-oauth2.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-rest-oauth2", 7 | "library_name": "tap_rest_oauth2", 8 | "variant": "None (Skip)", 9 | "stream_type": "REST", 10 | "auth_method": "OAuth2", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/tap-sql-custom.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "source_name": "AutomaticTestTap", 4 | "admin_name": "Automatic Tester", 5 | "admin_email": "auto.tester@example.com", 6 | "tap_id": "tap-sql-custom", 7 | "library_name": "tap_sql_custom", 8 | "variant": "None (Skip)", 9 | "stream_type": "SQL", 10 | "auth_method": "Custom or N/A", 11 | "include_ci_files": "None (Skip)", 12 | "faker_extra": false, 13 | "license": "Apache-2.0", 14 | "ide": "VSCode", 15 | "_template": "../tap-template/", 16 | "_output_dir": "." 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/target-per_record.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "destination_name": "MyDestinationName", 4 | "admin_name": "FirstName LastName", 5 | "admin_email": "firstname.lastname@example.com", 6 | "target_id": "target-per_record", 7 | "library_name": "target_per_record", 8 | "variant": "None (Skip)", 9 | "serialization_method": "Per record", 10 | "include_ci_files": "None (Skip)", 11 | "faker_extra": false, 12 | "license": "Apache-2.0", 13 | "ide": "VSCode", 14 | "_template": "./sdk/cookiecutter/target-template", 15 | "_output_dir": "." 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /e2e-tests/cookiecutters/target-sql.json: -------------------------------------------------------------------------------- 1 | { 2 | "cookiecutter": { 3 | "destination_name": "MyDestinationName", 4 | "admin_name": "FirstName LastName", 5 | "admin_email": "firstname.lastname@example.com", 6 | "target_id": "target-sql", 7 | "library_name": "target_sql", 8 | "variant": "None (Skip)", 9 | "serialization_method": "SQL", 10 | "include_ci_files": "None (Skip)", 11 | "faker_extra": false, 12 | "license": "Apache-2.0", 13 | "ide": "VSCode", 14 | "_template": "./sdk/cookiecutter/target-template", 15 | "_output_dir": "." 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /samples/aapl/README.md: -------------------------------------------------------------------------------- 1 | # Tap sample with large catalog 2 | 3 | This tap sample helps in evaluating the performance of catalog parsing and stream maps for taps with large schemas and records. 4 | 5 | ## Execution 6 | 7 | ```shell 8 | uv run python samples/aapl 9 | ``` 10 | 11 | Or if you want to trace the execution 12 | 13 | ```shell 14 | uv run viztracer samples/aapl/__main__.py 15 | uv run vizviewer result.json 16 | ``` 17 | -------------------------------------------------------------------------------- /samples/aapl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/samples/aapl/__init__.py -------------------------------------------------------------------------------- /samples/aapl/__main__.py: -------------------------------------------------------------------------------- 1 | """Executable for fundamentals tap. 2 | 3 | Run: 4 | 5 | $ uv run python samples/aapl 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | from aapl import Fundamentals 11 | 12 | Fundamentals.cli() 13 | -------------------------------------------------------------------------------- /samples/aapl/aapl.py: -------------------------------------------------------------------------------- 1 | """A simple tap with one big record and schema.""" 2 | 3 | from __future__ import annotations 4 | 5 | import importlib.resources 6 | import json 7 | 8 | from singer_sdk import Stream, Tap 9 | 10 | PROJECT_DIR = importlib.resources.files("samples.aapl") 11 | 12 | 13 | class AAPL(Stream): 14 | """An AAPL stream.""" 15 | 16 | name = "aapl" 17 | schema_filepath = PROJECT_DIR / "fundamentals.json" 18 | 19 | def get_records(self, _): # noqa: PLR6301 20 | """Generate a single record.""" 21 | with PROJECT_DIR.joinpath("AAPL.json").open() as f: 22 | record = json.load(f) 23 | 24 | yield record 25 | 26 | 27 | class Fundamentals(Tap): 28 | """Singer tap for fundamentals.""" 29 | 30 | name = "fundamentals" 31 | 32 | def discover_streams(self): 33 | """Get financial streams.""" 34 | return [AAPL(self)] 35 | -------------------------------------------------------------------------------- /samples/sample_custom_sql_adapter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/samples/sample_custom_sql_adapter/__init__.py -------------------------------------------------------------------------------- /samples/sample_custom_sql_adapter/connector.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import typing as t 4 | 5 | from sqlalchemy.engine.default import DefaultDialect 6 | 7 | if t.TYPE_CHECKING: 8 | from types import ModuleType 9 | 10 | 11 | class CustomSQLDialect(DefaultDialect): 12 | """Custom SQLite dialect that supports JSON.""" 13 | 14 | name = "myrdbms" 15 | 16 | def __init__(self, *args, **kwargs): 17 | super().__init__(*args, **kwargs) 18 | 19 | @classmethod 20 | def import_dbapi(cls): 21 | """Import the sqlite3 DBAPI.""" 22 | import sqlite3 # noqa: PLC0415 23 | 24 | return sqlite3 25 | 26 | @classmethod 27 | def dbapi(cls) -> ModuleType: # type: ignore[override] 28 | """Return the DBAPI module. 29 | 30 | NOTE: This is a legacy method that will stop being used by SQLAlchemy at some point. 31 | """ # noqa: E501 32 | return cls.import_dbapi() 33 | -------------------------------------------------------------------------------- /samples/sample_duckdb/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .connector import DuckDBConnector 4 | 5 | __all__ = [ 6 | "DuckDBConnector", 7 | ] 8 | -------------------------------------------------------------------------------- /samples/sample_duckdb/connector.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sqlalchemy as sa 4 | 5 | from singer_sdk.connectors import SQLConnector 6 | 7 | 8 | class DuckDBConnector(SQLConnector): 9 | allow_column_alter = True 10 | 11 | @staticmethod 12 | def get_column_alter_ddl( 13 | table_name: str, 14 | column_name: str, 15 | column_type: sa.types.TypeEngine, 16 | ) -> sa.DDL: 17 | return sa.DDL( 18 | "ALTER TABLE %(table_name)s ALTER COLUMN %(column_name)s TYPE %(column_type)s", # noqa: E501 19 | { 20 | "table_name": table_name, 21 | "column_name": column_name, 22 | "column_type": column_type, 23 | }, 24 | ) 25 | -------------------------------------------------------------------------------- /samples/sample_mapper/__init__.py: -------------------------------------------------------------------------------- 1 | """Stream maps transformer.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /samples/sample_mapper/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_mapper.mapper import StreamTransform 4 | 5 | StreamTransform.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_tap_bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | """A sample implementation for BigQuery.""" 2 | 3 | from __future__ import annotations 4 | 5 | from singer_sdk import SQLConnector, SQLStream, SQLTap 6 | from singer_sdk import typing as th # JSON schema typing helpers 7 | 8 | 9 | class BigQueryConnector(SQLConnector): 10 | """Connects to the BigQuery SQL source.""" 11 | 12 | def get_sqlalchemy_url(self, config: dict) -> str: # noqa: PLR6301 13 | """Concatenate a SQLAlchemy URL for use in connecting to the source.""" 14 | return f"bigquery://{config['project_id']}" 15 | 16 | 17 | class BigQueryStream(SQLStream): 18 | """Stream class for BigQuery streams.""" 19 | 20 | connector_class = BigQueryConnector 21 | 22 | 23 | class TapBigQuery(SQLTap): 24 | """BigQuery tap class.""" 25 | 26 | name = "tap-bigquery" 27 | 28 | config_jsonschema = th.PropertiesList( 29 | th.Property( 30 | "project_id", 31 | th.StringType, 32 | required=True, 33 | title="Project ID", 34 | description="GCP Project", 35 | ), 36 | ).to_dict() 37 | 38 | default_stream_class: type[SQLStream] = BigQueryStream 39 | 40 | 41 | __all__ = ["BigQueryConnector", "BigQueryStream", "TapBigQuery"] 42 | -------------------------------------------------------------------------------- /samples/sample_tap_bigquery/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_tap_bigquery import TapBigQuery 4 | 5 | TapBigQuery.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_tap_countries/__init__.py: -------------------------------------------------------------------------------- 1 | """Countries API Sample.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /samples/sample_tap_countries/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_tap_countries.countries_tap import SampleTapCountries 4 | 5 | SampleTapCountries.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_tap_countries/countries_tap.py: -------------------------------------------------------------------------------- 1 | """Sample tap test for tap-countries. 2 | 3 | This uses a free "Countries API" which does not require authentication. 4 | 5 | See the online explorer and query builder here: 6 | - https://countries.trevorblades.com/ 7 | """ 8 | 9 | from __future__ import annotations 10 | 11 | from samples.sample_tap_countries.countries_streams import ( 12 | ContinentsStream, 13 | CountriesStream, 14 | ) 15 | from singer_sdk import Stream, Tap 16 | from singer_sdk.contrib.msgspec import MsgSpecWriter 17 | from singer_sdk.typing import PropertiesList 18 | 19 | 20 | class SampleTapCountries(Tap): 21 | """Sample tap for Countries GraphQL API.""" 22 | 23 | name: str = "sample-tap-countries" 24 | config_jsonschema = PropertiesList().to_dict() 25 | 26 | message_writer_class = MsgSpecWriter 27 | 28 | def discover_streams(self) -> list[Stream]: 29 | """Return a list of discovered streams.""" 30 | return [ 31 | CountriesStream(tap=self), 32 | ContinentsStream(tap=self), 33 | ] 34 | 35 | 36 | if __name__ == "__main__": 37 | SampleTapCountries.cli() 38 | -------------------------------------------------------------------------------- /samples/sample_tap_countries/schemas/continents.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "code": { "type": ["string", "null"] }, 5 | "name": { "type": ["string", "null"] } 6 | } 7 | } -------------------------------------------------------------------------------- /samples/sample_tap_csv/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/samples/sample_tap_csv/__init__.py -------------------------------------------------------------------------------- /samples/sample_tap_csv/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_tap_csv.sample_tap_csv import SampleTapCSV 4 | 5 | SampleTapCSV.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: pip 9 | directory: "/" 10 | schedule: 11 | interval: weekly 12 | commit-message: 13 | prefix: "chore(deps): " 14 | prefix-development: "chore(deps-dev): " 15 | groups: 16 | development-dependencies: 17 | dependency-type: development 18 | runtime-dependencies: 19 | dependency-type: production 20 | update-types: 21 | - "patch" 22 | - package-ecosystem: pip 23 | directory: "/.github/workflows" 24 | schedule: 25 | interval: weekly 26 | commit-message: 27 | prefix: "ci: " 28 | groups: 29 | ci: 30 | patterns: 31 | - "*" 32 | - package-ecosystem: github-actions 33 | directory: "/" 34 | schedule: 35 | interval: weekly 36 | commit-message: 37 | prefix: "ci: " 38 | groups: 39 | actions: 40 | patterns: 41 | - "*" 42 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | ### A CI workflow template that runs linting and python testing 2 | ### TODO: Modify as needed or as desired. 3 | 4 | name: Test tap-dummyjson 5 | 6 | on: [push] 7 | 8 | jobs: 9 | pytest: 10 | runs-on: ubuntu-latest 11 | env: 12 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 17 | steps: 18 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Set up uv 24 | uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 25 | with: 26 | version: ">=0.5.19" 27 | - name: Install dependencies 28 | run: | 29 | poetry env use ${{ matrix.python-version }} 30 | poetry install 31 | - name: Test with pytest 32 | run: | 33 | uv run -p ${{ matrix.python-version }} pytest 34 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autofix_prs: true 3 | autoupdate_schedule: weekly 4 | autoupdate_commit_msg: 'chore: pre-commit autoupdate' 5 | 6 | repos: 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v5.0.0 9 | hooks: 10 | - id: check-json 11 | exclude: | 12 | (?x)^( 13 | \.vscode/.*\.json 14 | )$ 15 | - id: check-toml 16 | - id: check-yaml 17 | - id: end-of-file-fixer 18 | - id: trailing-whitespace 19 | 20 | - repo: https://github.com/python-jsonschema/check-jsonschema 21 | rev: 0.30.0 22 | hooks: 23 | - id: check-dependabot 24 | - id: check-github-workflows 25 | 26 | - repo: https://github.com/astral-sh/ruff-pre-commit 27 | rev: v0.8.1 28 | hooks: 29 | - id: ruff 30 | args: [--fix, --exit-non-zero-on-fix, --show-fixes] 31 | - id: ruff-format 32 | 33 | - repo: https://github.com/pre-commit/mirrors-mypy 34 | rev: v1.13.0 35 | hooks: 36 | - id: mypy 37 | additional_dependencies: 38 | - types-requests 39 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/.secrets/.gitignore: -------------------------------------------------------------------------------- 1 | # IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, 2 | # make sure those are never staged for commit into your git repo. You can store them here or another 3 | # secure location. 4 | # 5 | # Note: This may be redundant with the global .gitignore for, and is provided 6 | # for redundancy. If the `.secrets` folder is not needed, you may delete it 7 | # from the project. 8 | 9 | * 10 | !.gitignore 11 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/samples/sample_tap_dummy_json/__init__.py -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/meltano.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | send_anonymous_usage_stats: true 3 | project_id: "tap-dummyjson" 4 | default_environment: test 5 | environments: 6 | - name: test 7 | plugins: 8 | extractors: 9 | - name: "tap-dummyjson" 10 | namespace: "tap_dummyjson" 11 | 12 | pip_url: -e . 13 | 14 | capabilities: 15 | - state 16 | - catalog 17 | - discover 18 | - about 19 | - stream-maps 20 | 21 | settings: 22 | - name: username 23 | label: Username 24 | - name: password 25 | label: Password 26 | kind: password 27 | sensitive: true 28 | - name: api_url 29 | label: API URL 30 | description: The base URL for the API 31 | - name: start_date 32 | kind: date_iso8601 33 | 34 | settings_group_validation: 35 | - [username, password] 36 | 37 | config: 38 | start_date: '2024-01-01T00:00:00Z' 39 | 40 | loaders: 41 | - name: target-jsonl 42 | variant: andyh1203 43 | pip_url: target-jsonl 44 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/output/.gitignore: -------------------------------------------------------------------------------- 1 | # This directory is used as a target by target-jsonl, so ignore all files 2 | 3 | * 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "tap-dummyjson" 3 | version = "0.0.1" 4 | description = "Singer tap for DummyJSON, built with the Meltano Singer SDK." 5 | readme = "README.md" 6 | authors = [{ name = "Edgar Ramírez-Mondragón", email = "edgar@arch.dev" }] 7 | keywords = [ 8 | "ELT", 9 | "DummyJSON", 10 | ] 11 | classifiers = [ 12 | "Intended Audience :: Developers", 13 | "Operating System :: OS Independent", 14 | "Programming Language :: Python :: 3.9", 15 | "Programming Language :: Python :: 3.10", 16 | "Programming Language :: Python :: 3.11", 17 | "Programming Language :: Python :: 3.12", 18 | "Programming Language :: Python :: 3.13", 19 | ] 20 | license = "Apache-2.0" 21 | license-files = ["LICENSE"] 22 | requires-python = ">=3.9" 23 | dependencies = [ 24 | "requests~=2.32.3", 25 | "requests-cache~=1.2.1", 26 | "singer-sdk", 27 | ] 28 | optional-dependencies.s3 = [ 29 | "s3fs~=2025.5.0", 30 | ] 31 | 32 | [project.scripts] 33 | # CLI declaration 34 | tap-dummyjson = 'tap_dummyjson.tap:TapDummyJSON.cli' 35 | 36 | [dependency-groups] 37 | dev = [ 38 | "pytest>=8", 39 | "singer-sdk[testing]", 40 | ] 41 | 42 | [tool.mypy] 43 | python_version = "3.12" 44 | warn_unused_configs = true 45 | 46 | [tool.uv.sources] 47 | singer-sdk = { path = "../../", editable = true } 48 | 49 | [build-system] 50 | requires = ["hatchling"] 51 | build-backend = "hatchling.build" 52 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/ruff.toml: -------------------------------------------------------------------------------- 1 | src = ["tap_dummyjson"] 2 | target-version = "py39" 3 | 4 | [lint] 5 | ignore = [ 6 | "ANN", 7 | "ARG", 8 | "D", 9 | "COM812", # missing-trailing-comma 10 | "ISC001", # single-line-implicit-string-concatenation 11 | "S113", # request-without-timeout 12 | "PLR2004", # magic-value-comparison 13 | "RUF012", # mutable-class-default 14 | ] 15 | select = ["ALL"] 16 | 17 | [lint.flake8-annotations] 18 | allow-star-arg-any = true 19 | 20 | [lint.isort] 21 | known-first-party = ["tap_dummyjson"] 22 | 23 | [lint.pydocstyle] 24 | convention = "google" 25 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/tap_dummyjson/__init__.py: -------------------------------------------------------------------------------- 1 | """Tap for DummyJSON.""" 2 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/tap_dummyjson/__main__.py: -------------------------------------------------------------------------------- 1 | """DummyJSON entry point.""" 2 | 3 | from __future__ import annotations 4 | 5 | from tap_dummyjson.tap import TapDummyJSON 6 | 7 | TapDummyJSON.cli() 8 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/tap_dummyjson/schemas/__init__.py: -------------------------------------------------------------------------------- 1 | """JSON schema files for the REST API.""" 2 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Test suite for tap-dummyjson.""" 2 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/tests/test_core.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_tap_dummy_json.tap_dummyjson.tap import TapDummyJSON 4 | from singer_sdk.testing import get_tap_test_class 5 | 6 | CONFIG = { 7 | "username": "emilys", 8 | "password": "emilyspass", 9 | } 10 | 11 | TestTapDummyJSON = get_tap_test_class(tap_class=TapDummyJSON, config=CONFIG) 12 | -------------------------------------------------------------------------------- /samples/sample_tap_dummy_json/tox.ini: -------------------------------------------------------------------------------- 1 | # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy 2 | 3 | [tox] 4 | envlist = py3{9,10,11,12,13} 5 | requires = 6 | tox>=4.19 7 | 8 | [testenv] 9 | deps = 10 | pytest 11 | commands = 12 | pytest {posargs} 13 | -------------------------------------------------------------------------------- /samples/sample_tap_fake_people/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/samples/sample_tap_fake_people/__init__.py -------------------------------------------------------------------------------- /samples/sample_tap_fake_people/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_tap_fake_people.tap import SampleTapFakePeople 4 | 5 | SampleTapFakePeople.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_tap_gitlab/__init__.py: -------------------------------------------------------------------------------- 1 | """Gitlab API Sample.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /samples/sample_tap_gitlab/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from sample_tap_gitlab.gitlab_tap import SampleTapGitlab 4 | 5 | SampleTapGitlab.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_tap_gitlab/gitlab-config.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "auth_token": "dummy_key", 3 | "project_ids": [ 4 | "meltano/sdk", 5 | "meltano/meltano" 6 | ] 7 | } -------------------------------------------------------------------------------- /samples/sample_tap_gitlab/schemas/currentuser.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "name": { "type": ["null", "string"] } 5 | } 6 | } -------------------------------------------------------------------------------- /samples/sample_tap_gitlab/schemas/projects-graphql.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "name": { "type": ["null", "string"] } 5 | } 6 | } -------------------------------------------------------------------------------- /samples/sample_tap_hostile/__init__.py: -------------------------------------------------------------------------------- 1 | """A sample tap for testing SQL target property name transformations.""" 2 | 3 | from __future__ import annotations 4 | 5 | from .hostile_tap import SampleTapHostile 6 | 7 | __all__ = [ 8 | "SampleTapHostile", 9 | ] 10 | -------------------------------------------------------------------------------- /samples/sample_tap_hostile/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_tap_hostile import SampleTapHostile 4 | 5 | SampleTapHostile.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_tap_hostile/hostile_tap.py: -------------------------------------------------------------------------------- 1 | """A sample tap for testing SQL target property name transformations.""" 2 | 3 | from __future__ import annotations 4 | 5 | from samples.sample_tap_hostile.hostile_streams import HostilePropertyNamesStream 6 | from singer_sdk import Stream, Tap 7 | from singer_sdk.typing import PropertiesList 8 | 9 | 10 | class SampleTapHostile(Tap): 11 | """Sample tap for for testing SQL target property name transformations.""" 12 | 13 | name: str = "sample-tap-hostile" 14 | config_jsonschema = PropertiesList().to_dict() 15 | 16 | def discover_streams(self) -> list[Stream]: 17 | """Return a list of discovered streams.""" 18 | return [ 19 | HostilePropertyNamesStream(tap=self), 20 | ] 21 | 22 | 23 | if __name__ == "__main__": 24 | SampleTapHostile.cli() 25 | -------------------------------------------------------------------------------- /samples/sample_tap_sqlite/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_tap_sqlite import SQLiteTap 4 | 5 | SQLiteTap.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_target_csv/__init__.py: -------------------------------------------------------------------------------- 1 | """Module test for target-csv functionality.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /samples/sample_target_csv/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_target_csv.csv_target import SampleTargetCSV 4 | 5 | SampleTargetCSV.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_target_csv/csv_target.py: -------------------------------------------------------------------------------- 1 | """Sample target test for target-csv.""" 2 | 3 | from __future__ import annotations 4 | 5 | from samples.sample_target_csv.csv_target_sink import SampleCSVTargetSink 6 | from singer_sdk import typing as th 7 | from singer_sdk.target_base import Target 8 | 9 | 10 | class SampleTargetCSV(Target): 11 | """Sample target for CSV.""" 12 | 13 | name = "target-csv" 14 | config_jsonschema = th.PropertiesList( 15 | th.Property( 16 | "target_folder", 17 | th.StringType, 18 | default="output", 19 | required=False, # Default value will be used if not provided 20 | title="Target Folder", 21 | ), 22 | th.Property("file_naming_scheme", th.StringType, title="File Naming Scheme"), 23 | ).to_dict() 24 | default_sink_class = SampleCSVTargetSink 25 | -------------------------------------------------------------------------------- /samples/sample_target_parquet/__init__.py: -------------------------------------------------------------------------------- 1 | """Module test for target-parquet functionality.""" 2 | 3 | from __future__ import annotations 4 | 5 | # Reuse the tap connection rather than create a new target connection: 6 | from samples.sample_target_parquet.parquet_target import SampleTargetParquet 7 | from samples.sample_target_parquet.parquet_target_sink import SampleParquetTargetSink 8 | 9 | __all__ = [ 10 | "SampleParquetTargetSink", 11 | "SampleTargetParquet", 12 | ] 13 | -------------------------------------------------------------------------------- /samples/sample_target_parquet/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from sample_target_parquet.parquet_target import SampleTargetParquet 4 | 5 | SampleTargetParquet.cli() 6 | -------------------------------------------------------------------------------- /samples/sample_target_parquet/parquet_target.py: -------------------------------------------------------------------------------- 1 | """Sample target test for target-parquet.""" 2 | 3 | from __future__ import annotations 4 | 5 | from samples.sample_target_parquet.parquet_target_sink import SampleParquetTargetSink 6 | from singer_sdk import typing as th 7 | from singer_sdk.target_base import Target 8 | 9 | 10 | class SampleTargetParquet(Target): 11 | """Sample target for Parquet.""" 12 | 13 | name = "sample-target-parquet" 14 | config_jsonschema = th.PropertiesList( 15 | th.Property("filepath", th.StringType, title="Output File Path"), 16 | th.Property("file_naming_scheme", th.StringType, title="File Naming Scheme"), 17 | ).to_dict() 18 | default_sink_class = SampleParquetTargetSink 19 | 20 | 21 | if __name__ == "__main__": 22 | SampleTargetParquet.cli() 23 | -------------------------------------------------------------------------------- /samples/sample_target_sqlite/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_target_sqlite import SQLiteTarget 4 | 5 | SQLiteTarget.cli() 6 | -------------------------------------------------------------------------------- /singer_sdk/__init__.py: -------------------------------------------------------------------------------- 1 | """SDK for building Singer taps.""" 2 | 3 | from __future__ import annotations 4 | 5 | from singer_sdk import streams 6 | from singer_sdk.connectors import SQLConnector 7 | from singer_sdk.mapper_base import InlineMapper 8 | from singer_sdk.plugin_base import PluginBase 9 | from singer_sdk.sinks import BatchSink, RecordSink, Sink, SQLSink 10 | from singer_sdk.streams import GraphQLStream, RESTStream, SQLStream, Stream 11 | from singer_sdk.tap_base import SQLTap, Tap 12 | from singer_sdk.target_base import SQLTarget, Target 13 | 14 | __all__ = [ 15 | "BatchSink", 16 | "GraphQLStream", 17 | "InlineMapper", 18 | "PluginBase", 19 | "RESTStream", 20 | "RecordSink", 21 | "SQLConnector", 22 | "SQLSink", 23 | "SQLStream", 24 | "SQLTap", 25 | "SQLTarget", 26 | "Sink", 27 | "Stream", 28 | "Tap", 29 | "Target", 30 | "streams", 31 | ] 32 | -------------------------------------------------------------------------------- /singer_sdk/_singerlib/__init__.py: -------------------------------------------------------------------------------- 1 | """Deprecated.""" 2 | 3 | from __future__ import annotations 4 | 5 | import typing as t 6 | import warnings 7 | 8 | from singer_sdk.helpers._compat import SingerSDKDeprecationWarning 9 | 10 | if t.TYPE_CHECKING: 11 | from singer_sdk.singerlib import ( 12 | Catalog, # noqa: F401 13 | CatalogEntry, # noqa: F401 14 | Metadata, # noqa: F401 15 | Schema, # noqa: F401 16 | SelectionMask, # noqa: F401 17 | StreamMetadata, # noqa: F401 18 | resolve_schema_references, # noqa: F401 19 | ) 20 | 21 | 22 | def __getattr__(name: str): # noqa: ANN202 23 | import singer_sdk.singerlib # noqa: PLC0415 24 | 25 | warnings.warn( 26 | "The module `singer_sdk._singerlib` is deprecated and will be removed " 27 | "by August 2025. " 28 | "Please use `singer_sdk.singerlib` instead.", 29 | SingerSDKDeprecationWarning, 30 | stacklevel=2, 31 | ) 32 | 33 | return getattr(singer_sdk.singerlib, name) 34 | -------------------------------------------------------------------------------- /singer_sdk/_singerlib/catalog.py: -------------------------------------------------------------------------------- 1 | """Deprecated.""" 2 | 3 | from __future__ import annotations 4 | 5 | import warnings 6 | 7 | from singer_sdk.helpers._compat import SingerSDKDeprecationWarning 8 | 9 | 10 | def __getattr__(name: str): # noqa: ANN202 11 | from singer_sdk.singerlib import catalog # noqa: PLC0415 12 | 13 | warnings.warn( 14 | "The module `singer_sdk._singerlib.catalog` is deprecated and will be removed " 15 | "by August 2025. " 16 | "Please use `singer_sdk.singerlib.catalog` instead.", 17 | SingerSDKDeprecationWarning, 18 | stacklevel=2, 19 | ) 20 | 21 | return getattr(catalog, name) 22 | -------------------------------------------------------------------------------- /singer_sdk/_singerlib/encoding.py: -------------------------------------------------------------------------------- 1 | """Deprecated.""" 2 | 3 | from __future__ import annotations 4 | 5 | import warnings 6 | 7 | from singer_sdk.helpers._compat import SingerSDKDeprecationWarning 8 | 9 | 10 | def __getattr__(name: str): # noqa: ANN202 11 | import singer_sdk.singerlib.encoding # noqa: PLC0415 12 | 13 | warnings.warn( 14 | "The module `singer_sdk._singerlib.encoding` is deprecated and will be removed " 15 | "by August 2025. " 16 | "Please use `singer_sdk.singerlib.encoding` instead.", 17 | SingerSDKDeprecationWarning, 18 | stacklevel=2, 19 | ) 20 | 21 | return getattr(singer_sdk.singerlib.encoding, name) 22 | -------------------------------------------------------------------------------- /singer_sdk/_singerlib/exceptions.py: -------------------------------------------------------------------------------- 1 | """Deprecated.""" 2 | 3 | from __future__ import annotations 4 | 5 | import warnings 6 | 7 | from singer_sdk.helpers._compat import SingerSDKDeprecationWarning 8 | 9 | 10 | def __getattr__(name: str): # noqa: ANN202 11 | import singer_sdk.singerlib.exceptions # noqa: PLC0415 12 | 13 | warnings.warn( 14 | "The module `singer_sdk._singerlib.exceptions` is deprecated and will be " 15 | "removed by August 2025. " 16 | "Please use `singer_sdk.singerlib.exceptions` instead.", 17 | SingerSDKDeprecationWarning, 18 | stacklevel=2, 19 | ) 20 | 21 | return getattr(singer_sdk.singerlib.exceptions, name) 22 | -------------------------------------------------------------------------------- /singer_sdk/_singerlib/json.py: -------------------------------------------------------------------------------- 1 | """Deprecated.""" 2 | 3 | from __future__ import annotations 4 | 5 | import warnings 6 | 7 | from singer_sdk.helpers._compat import SingerSDKDeprecationWarning 8 | 9 | 10 | def __getattr__(name: str): # noqa: ANN202 11 | import singer_sdk.singerlib.json # noqa: PLC0415 12 | 13 | warnings.warn( 14 | "The module `singer_sdk._singerlib.json` is deprecated and will be removed " 15 | "by August 2025. " 16 | "Please use `singer_sdk.singerlib.json` instead.", 17 | SingerSDKDeprecationWarning, 18 | stacklevel=2, 19 | ) 20 | 21 | return getattr(singer_sdk.singerlib.json, name) 22 | -------------------------------------------------------------------------------- /singer_sdk/_singerlib/messages.py: -------------------------------------------------------------------------------- 1 | """Deprecated.""" 2 | 3 | from __future__ import annotations 4 | 5 | import warnings 6 | 7 | from singer_sdk.helpers._compat import SingerSDKDeprecationWarning 8 | 9 | 10 | def __getattr__(name: str): # noqa: ANN202 11 | import singer_sdk.singerlib.messages # noqa: PLC0415 12 | 13 | warnings.warn( 14 | "The module `singer_sdk._singerlib.messages` is deprecated and will be " 15 | "removed by August 2025. " 16 | "Please use `singer_sdk.singerlib.messages` instead.", 17 | SingerSDKDeprecationWarning, 18 | stacklevel=2, 19 | ) 20 | 21 | return getattr(singer_sdk.singerlib.messages, name) 22 | -------------------------------------------------------------------------------- /singer_sdk/_singerlib/utils.py: -------------------------------------------------------------------------------- 1 | """Deprecated.""" 2 | 3 | from __future__ import annotations 4 | 5 | import warnings 6 | 7 | from singer_sdk.helpers._compat import SingerSDKDeprecationWarning 8 | 9 | 10 | def __getattr__(name: str): # noqa: ANN202 11 | import singer_sdk.singerlib.utils # noqa: PLC0415 12 | 13 | warnings.warn( 14 | "The module `singer_sdk._singerlib.utils` is deprecated and will be removed " 15 | "by August 2025. " 16 | "Please use `singer_sdk.singerlib.utils` instead.", 17 | SingerSDKDeprecationWarning, 18 | stacklevel=2, 19 | ) 20 | 21 | return getattr(singer_sdk.singerlib.utils, name) 22 | -------------------------------------------------------------------------------- /singer_sdk/cli/__init__.py: -------------------------------------------------------------------------------- 1 | """Helpers for the tap, target and mapper CLIs.""" 2 | 3 | from __future__ import annotations 4 | 5 | import typing as t 6 | 7 | if t.TYPE_CHECKING: 8 | import click 9 | 10 | _T = t.TypeVar("_T") 11 | 12 | 13 | class plugin_cli: # noqa: N801 14 | """Decorator to create a plugin CLI.""" 15 | 16 | def __init__(self, method: t.Callable[..., click.Command]) -> None: 17 | """Create a new plugin CLI. 18 | 19 | Args: 20 | method: The method to call to get the command. 21 | """ 22 | self.method = method 23 | self.name: str | None = None 24 | 25 | def __get__(self, instance: _T, owner: type[_T]) -> click.Command: 26 | """Get the command. 27 | 28 | Args: 29 | instance: The instance of the plugin. 30 | owner: The plugin class. 31 | 32 | Returns: 33 | The CLI entrypoint. 34 | """ 35 | return self.method(owner) 36 | -------------------------------------------------------------------------------- /singer_sdk/cli/common_options.py: -------------------------------------------------------------------------------- 1 | """Common CLI options for plugins.""" 2 | 3 | from __future__ import annotations 4 | 5 | import typing as t 6 | 7 | import click 8 | 9 | PLUGIN_VERSION: t.Callable[..., t.Any] = click.option( 10 | "--version", 11 | is_flag=True, 12 | help="Display the package version.", 13 | ) 14 | 15 | PLUGIN_ABOUT: t.Callable[..., t.Any] = click.option( 16 | "--about", 17 | is_flag=True, 18 | help="Display package metadata and settings.", 19 | ) 20 | 21 | PLUGIN_ABOUT_FORMAT: t.Callable[..., t.Any] = click.option( 22 | "--format", 23 | "about_format", 24 | help="Specify output style for --about", 25 | type=click.Choice(["json", "markdown"], case_sensitive=False), 26 | default=None, 27 | ) 28 | 29 | PLUGIN_CONFIG: t.Callable[..., t.Any] = click.option( 30 | "--config", 31 | multiple=True, 32 | help="Configuration file location or 'ENV' to use environment variables.", 33 | type=click.STRING, 34 | default=(), 35 | ) 36 | 37 | PLUGIN_FILE_INPUT: t.Callable[..., t.Any] = click.option( 38 | "--input", 39 | "file_input", 40 | help="A path to read messages from instead of from standard in.", 41 | type=click.File("r"), 42 | ) 43 | -------------------------------------------------------------------------------- /singer_sdk/configuration/__init__.py: -------------------------------------------------------------------------------- 1 | """Configuration parsing and handling.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /singer_sdk/connectors/__init__.py: -------------------------------------------------------------------------------- 1 | """Module for SQL-related operations.""" 2 | 3 | from __future__ import annotations 4 | 5 | from .sql import SQLConnector 6 | 7 | __all__ = ["SQLConnector"] 8 | -------------------------------------------------------------------------------- /singer_sdk/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | """Singer SDK contrib modules.""" 2 | -------------------------------------------------------------------------------- /singer_sdk/contrib/filesystem/__init__.py: -------------------------------------------------------------------------------- 1 | """Filesystem interfaces for the Singer SDK.""" 2 | 3 | from __future__ import annotations 4 | 5 | from singer_sdk.contrib.filesystem.stream import FileStream 6 | from singer_sdk.contrib.filesystem.tap import FolderTap 7 | 8 | __all__ = ["FileStream", "FolderTap"] 9 | -------------------------------------------------------------------------------- /singer_sdk/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | """Helper library for the SDK.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /singer_sdk/helpers/_classproperty.py: -------------------------------------------------------------------------------- 1 | """Defines the `classproperty` decorator.""" 2 | 3 | from __future__ import annotations 4 | 5 | 6 | class classproperty(property): # noqa: N801 7 | """Class property decorator.""" 8 | 9 | def __get__(self, obj, objtype=None): # noqa: ANN001, ANN204 10 | return super().__get__(objtype) 11 | 12 | def __set__(self, obj, value): # noqa: ANN001, ANN204 13 | super().__set__(type(obj), value) 14 | 15 | def __delete__(self, obj): # noqa: ANN001, ANN204 16 | super().__delete__(type(obj)) 17 | -------------------------------------------------------------------------------- /singer_sdk/helpers/_compat.py: -------------------------------------------------------------------------------- 1 | """Compatibility helpers.""" 2 | 3 | from __future__ import annotations 4 | 5 | import datetime 6 | import sys 7 | from importlib import resources as importlib_resources 8 | 9 | if sys.version_info < (3, 12): 10 | from importlib.abc import Traversable 11 | else: 12 | from importlib.resources.abc import Traversable 13 | 14 | if sys.version_info < (3, 12): 15 | from importlib_metadata import entry_points 16 | else: 17 | from importlib.metadata import entry_points 18 | 19 | if sys.version_info < (3, 11): 20 | from backports.datetime_fromisoformat import MonkeyPatch 21 | 22 | MonkeyPatch.patch_fromisoformat() 23 | 24 | datetime_fromisoformat = datetime.datetime.fromisoformat 25 | date_fromisoformat = datetime.date.fromisoformat 26 | time_fromisoformat = datetime.time.fromisoformat 27 | 28 | 29 | class SingerSDKDeprecationWarning(DeprecationWarning): 30 | """Custom deprecation warning for the Singer SDK.""" 31 | 32 | 33 | __all__ = [ 34 | "SingerSDKDeprecationWarning", 35 | "Traversable", 36 | "date_fromisoformat", 37 | "datetime_fromisoformat", 38 | "entry_points", 39 | "importlib_resources", 40 | "time_fromisoformat", 41 | ] 42 | -------------------------------------------------------------------------------- /singer_sdk/helpers/_conformers.py: -------------------------------------------------------------------------------- 1 | """Helper functions for conforming identifiers.""" 2 | 3 | from __future__ import annotations 4 | 5 | import re 6 | from string import ascii_lowercase, digits 7 | 8 | 9 | def snakecase(string: str) -> str: 10 | """Convert string into snake case. 11 | 12 | Args: 13 | string: String to convert. 14 | 15 | Returns: 16 | string: Snake cased string. 17 | """ 18 | string = re.sub(r"[\-\.\s]", "_", string) 19 | string = ( 20 | ( 21 | string[0].lower() 22 | + re.sub( 23 | r"[A-Z]", 24 | lambda matched: f"_{matched.group(0).lower()!s}", 25 | string[1:], 26 | ) 27 | ) 28 | if string 29 | else string 30 | ) 31 | return re.sub(r"_{2,}", "_", string).rstrip("_") 32 | 33 | 34 | def replace_leading_digit(string: str) -> str: 35 | """Replace leading numeric character with equivalent letter. 36 | 37 | Args: 38 | string: String to process. 39 | 40 | Returns: 41 | A modified string if original starts with a number, 42 | else the unmodified original. 43 | """ 44 | if string[0] in digits: 45 | letters = list(ascii_lowercase) 46 | numbers = [int(d) for d in digits] 47 | digit_map = {n: letters[n] for n in numbers} 48 | return digit_map[int(string[0])] + string[1:] 49 | return string 50 | -------------------------------------------------------------------------------- /singer_sdk/helpers/_secrets.py: -------------------------------------------------------------------------------- 1 | """Helpers function for secrets management.""" 2 | 3 | from __future__ import annotations 4 | 5 | COMMON_SECRET_KEYS = [ 6 | "db_password", 7 | "password", 8 | "access_key", 9 | "private_key", 10 | "client_id", 11 | "client_secret", 12 | "refresh_token", 13 | "access_token", 14 | ] 15 | COMMON_SECRET_KEY_SUFFIXES = ["access_key_id"] 16 | 17 | 18 | def is_common_secret_key(key_name: str) -> bool: 19 | """Return true if the key_name value matches a known secret name or pattern.""" 20 | if key_name in COMMON_SECRET_KEYS: 21 | return True 22 | return any( 23 | key_name.lower().endswith(key_suffix) 24 | for key_suffix in COMMON_SECRET_KEY_SUFFIXES 25 | ) 26 | 27 | 28 | class SecretString(str): 29 | """For now, this class wraps a sensitive string to be identified as such later.""" 30 | 31 | def __init__(self, contents: str) -> None: 32 | """Initialize secret string.""" 33 | self.contents = contents 34 | 35 | def __repr__(self) -> str: 36 | """Render secret contents.""" 37 | return str(self.contents.__repr__()) 38 | 39 | def __str__(self) -> str: 40 | """Render secret contents.""" 41 | return str(self.contents.__str__()) 42 | -------------------------------------------------------------------------------- /singer_sdk/helpers/jsonpath.py: -------------------------------------------------------------------------------- 1 | """JSONPath helpers.""" 2 | 3 | from __future__ import annotations 4 | 5 | import logging 6 | import typing as t 7 | from functools import lru_cache 8 | 9 | from jsonpath_ng.ext import parse 10 | 11 | if t.TYPE_CHECKING: 12 | import jsonpath_ng 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def extract_jsonpath( 19 | expression: str, 20 | input: dict | list, # noqa: A002 21 | ) -> t.Generator[t.Any, None, None]: 22 | """Extract records from an input based on a JSONPath expression. 23 | 24 | Args: 25 | expression: JSONPath expression to match against the input. 26 | input: JSON object or array to extract records from. 27 | 28 | Yields: 29 | Records matched with JSONPath expression. 30 | """ 31 | compiled_jsonpath = _compile_jsonpath(expression) 32 | 33 | match: jsonpath_ng.DatumInContext 34 | matches = compiled_jsonpath.find(input) 35 | 36 | logger.info("JSONPath %s match count: %d", expression, len(matches)) 37 | 38 | for match in matches: 39 | yield match.value 40 | 41 | 42 | @lru_cache 43 | def _compile_jsonpath(expression: str) -> jsonpath_ng.JSONPath: 44 | """Parse a JSONPath expression and cache the result. 45 | 46 | Args: 47 | expression: A string representing a JSONPath expression. 48 | 49 | Returns: 50 | A compiled JSONPath object. 51 | """ 52 | return parse(expression) 53 | -------------------------------------------------------------------------------- /singer_sdk/helpers/types.py: -------------------------------------------------------------------------------- 1 | """Type aliases for use in the SDK.""" 2 | 3 | from __future__ import annotations 4 | 5 | import sys 6 | import typing as t 7 | from collections.abc import Mapping 8 | 9 | import requests 10 | 11 | if sys.version_info < (3, 10): 12 | from typing_extensions import TypeAlias 13 | else: 14 | from typing import TypeAlias # noqa: ICN003 15 | 16 | 17 | __all__ = [ 18 | "Context", 19 | "Record", 20 | ] 21 | 22 | Context: TypeAlias = Mapping[str, t.Any] 23 | Record: TypeAlias = dict[str, t.Any] 24 | Auth: TypeAlias = t.Callable[[requests.PreparedRequest], requests.PreparedRequest] 25 | 26 | 27 | class TapState(t.TypedDict, total=False): 28 | """Tap state.""" 29 | 30 | bookmarks: dict[str, dict[str, t.Any]] 31 | -------------------------------------------------------------------------------- /singer_sdk/internal/__init__.py: -------------------------------------------------------------------------------- 1 | """Internal utilities for the Singer SDK.""" 2 | -------------------------------------------------------------------------------- /singer_sdk/io_base.py: -------------------------------------------------------------------------------- 1 | """Abstract base classes for all Singer messages IO operations.""" 2 | 3 | from __future__ import annotations 4 | 5 | from singer_sdk.singerlib.encoding import ( 6 | GenericSingerReader, 7 | GenericSingerWriter, 8 | SingerMessageType, 9 | ) 10 | from singer_sdk.singerlib.encoding import SimpleSingerReader as SingerReader 11 | from singer_sdk.singerlib.encoding import SimpleSingerWriter as SingerWriter 12 | 13 | __all__ = [ 14 | "GenericSingerReader", 15 | "GenericSingerWriter", 16 | "SingerMessageType", 17 | "SingerReader", 18 | "SingerWriter", 19 | ] 20 | -------------------------------------------------------------------------------- /singer_sdk/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/singer_sdk/py.typed -------------------------------------------------------------------------------- /singer_sdk/singerlib/__init__.py: -------------------------------------------------------------------------------- 1 | """Low-level Singer components for building taps and targets.""" 2 | 3 | from __future__ import annotations 4 | 5 | from singer_sdk.singerlib import exceptions 6 | from singer_sdk.singerlib.catalog import ( 7 | Catalog, 8 | CatalogEntry, 9 | Metadata, 10 | MetadataMapping, 11 | SelectionMask, 12 | StreamMetadata, 13 | ) 14 | from singer_sdk.singerlib.messages import ( 15 | ActivateVersionMessage, 16 | Message, 17 | RecordMessage, 18 | SchemaMessage, 19 | SingerMessageType, 20 | StateMessage, 21 | exclude_null_dict, 22 | format_message, 23 | write_message, 24 | ) 25 | from singer_sdk.singerlib.schema import Schema, resolve_schema_references 26 | from singer_sdk.singerlib.utils import strftime, strptime_to_utc 27 | 28 | __all__ = [ 29 | "ActivateVersionMessage", 30 | "Catalog", 31 | "CatalogEntry", 32 | "Message", 33 | "Metadata", 34 | "MetadataMapping", 35 | "RecordMessage", 36 | "Schema", 37 | "SchemaMessage", 38 | "SelectionMask", 39 | "SingerMessageType", 40 | "StateMessage", 41 | "StreamMetadata", 42 | "exceptions", 43 | "exclude_null_dict", 44 | "format_message", 45 | "resolve_schema_references", 46 | "strftime", 47 | "strptime_to_utc", 48 | "write_message", 49 | ] 50 | -------------------------------------------------------------------------------- /singer_sdk/singerlib/encoding/__init__.py: -------------------------------------------------------------------------------- 1 | """Singer IO reader and writer classes.""" 2 | 3 | from __future__ import annotations 4 | 5 | from .base import GenericSingerReader, GenericSingerWriter, SingerMessageType 6 | from .simple import SimpleSingerReader, SimpleSingerWriter 7 | 8 | __all__ = [ 9 | "GenericSingerReader", 10 | "GenericSingerWriter", 11 | "SimpleSingerReader", 12 | "SimpleSingerWriter", 13 | "SingerMessageType", 14 | ] 15 | -------------------------------------------------------------------------------- /singer_sdk/singerlib/exceptions.py: -------------------------------------------------------------------------------- 1 | """Singer exceptions.""" 2 | 3 | from __future__ import annotations 4 | 5 | __all__ = [ 6 | "InvalidInputLine", 7 | ] 8 | 9 | 10 | class InvalidInputLine(Exception): 11 | """Raised when an input line is not a valid Singer message.""" 12 | -------------------------------------------------------------------------------- /singer_sdk/singerlib/messages.py: -------------------------------------------------------------------------------- 1 | """Singer message types and utilities.""" 2 | 3 | from __future__ import annotations 4 | 5 | from .encoding import SimpleSingerWriter as SingerWriter 6 | from .encoding.base import SingerMessageType 7 | from .encoding.simple import ( 8 | ActivateVersionMessage, 9 | Message, 10 | RecordMessage, 11 | SchemaMessage, 12 | StateMessage, 13 | exclude_null_dict, 14 | ) 15 | 16 | __all__ = [ 17 | "ActivateVersionMessage", 18 | "Message", 19 | "RecordMessage", 20 | "SchemaMessage", 21 | "SingerMessageType", 22 | "StateMessage", 23 | "exclude_null_dict", 24 | "format_message", 25 | "write_message", 26 | ] 27 | 28 | WRITER = SingerWriter() 29 | format_message = WRITER.format_message 30 | write_message = WRITER.write_message 31 | -------------------------------------------------------------------------------- /singer_sdk/sinks/__init__.py: -------------------------------------------------------------------------------- 1 | """Sink classes for targets.""" 2 | 3 | from __future__ import annotations 4 | 5 | from singer_sdk.sinks.batch import BatchSink 6 | from singer_sdk.sinks.core import Sink 7 | from singer_sdk.sinks.record import RecordSink 8 | from singer_sdk.sinks.sql import SQLSink 9 | 10 | __all__ = ["BatchSink", "RecordSink", "SQLSink", "Sink"] 11 | -------------------------------------------------------------------------------- /singer_sdk/streams/__init__.py: -------------------------------------------------------------------------------- 1 | """SDK for building Singer taps.""" 2 | 3 | from __future__ import annotations 4 | 5 | from singer_sdk.streams.core import Stream 6 | from singer_sdk.streams.graphql import GraphQLStream 7 | from singer_sdk.streams.rest import RESTStream 8 | from singer_sdk.streams.sql import SQLStream 9 | 10 | __all__ = ["GraphQLStream", "RESTStream", "SQLStream", "Stream"] 11 | -------------------------------------------------------------------------------- /singer_sdk/testing/config.py: -------------------------------------------------------------------------------- 1 | """Test config classes.""" 2 | 3 | from __future__ import annotations 4 | 5 | from dataclasses import dataclass, field 6 | 7 | 8 | @dataclass 9 | class SuiteConfig: 10 | """Test Suite Config, passed to each test. 11 | 12 | Args: 13 | max_records_limit: Max records to fetch during tap testing. 14 | ignore_no_records: Ignore stream test failures if stream returns no records, 15 | for all streams. 16 | ignore_no_records_for_streams: Ignore stream test failures if stream returns 17 | no records, for named streams. 18 | """ 19 | 20 | max_records_limit: int | None = 150 21 | ignore_no_records: bool = False 22 | ignore_no_records_for_streams: list[str] = field(default_factory=list) 23 | -------------------------------------------------------------------------------- /singer_sdk/testing/pytest_plugin.py: -------------------------------------------------------------------------------- 1 | """Pytest Plugin.""" 2 | 3 | from __future__ import annotations 4 | 5 | import pytest 6 | 7 | 8 | def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: 9 | """Pytest Hook, responsible for parameterizing tests. 10 | 11 | Called once per each test function, this hook will check if the function name is 12 | registered in the parent classes 'params' dict, and if so will parameterize 13 | the given test function with the values therein. 14 | 15 | Args: 16 | metafunc: Pytest MetaFunc instance, representing a test function or method. 17 | """ 18 | if metafunc.cls and hasattr(metafunc.cls, "params"): 19 | func_arg_list = metafunc.cls.params.get(metafunc.definition.name) 20 | func_arg_ids = ( 21 | metafunc.cls.param_ids.get(metafunc.definition.name) 22 | if hasattr(metafunc.cls, "param_ids") 23 | else None 24 | ) 25 | if func_arg_list: 26 | arg_names = list(func_arg_list[0].keys()) 27 | parameters = [ 28 | pytest.param(*tuple(func_args[name] for name in arg_names)) 29 | for func_args in func_arg_list 30 | ] 31 | metafunc.parametrize( 32 | ",".join(arg_names), 33 | parameters, 34 | ids=func_arg_ids, 35 | ) 36 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/__init__.py: -------------------------------------------------------------------------------- 1 | """Singer output samples, used for testing target behavior.""" 2 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/array_data.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_array_data", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "fruits": {"type": "array","items": {"type": "string"}}}}} 2 | {"type": "RECORD", "stream": "test_array_data", "record": {"id": 1, "fruits": [ "apple", "orange", "pear" ]}} 3 | {"type": "RECORD", "stream": "test_array_data", "record": {"id": 2, "fruits": [ "banana", "apple" ]}} 4 | {"type": "RECORD", "stream": "test_array_data", "record": {"id": 3, "fruits": [ "pear" ]}} 5 | {"type": "RECORD", "stream": "test_array_data", "record": {"id": 4, "fruits": [ "orange", "banana", "apple", "pear" ]}} 6 | {"type": "STATE", "value": {"test_array_data": 4}} 7 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/camelcase.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "TestCamelcase", "schema": {"type": "object", "properties": { "Id": {"type": "string"}, "clientName": {"type": "string"} }}, "key_properties": ["Id"]} 2 | {"type": "RECORD", "stream": "TestCamelcase", "record": {"Id": "1", "clientName": "Gitter Windows Desktop App"}} 3 | {"type": "RECORD", "stream": "TestCamelcase", "record": {"Id": "2", "clientName": "Gitter iOS App"}} 4 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/duplicate_records.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_duplicate_records", "key_properties": ["id"], "schema": {"required": ["id", "metric"], "type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}} 2 | {"type": "RECORD", "stream": "test_duplicate_records", "record": {"id": 1, "metric": 1}} 3 | {"type": "RECORD", "stream": "test_duplicate_records", "record": {"id": 2, "metric": 2}} 4 | {"type": "RECORD", "stream": "test_duplicate_records", "record": {"id": 1, "metric": 10}} 5 | {"type": "RECORD", "stream": "test_duplicate_records", "record": {"id": 2, "metric": 20}} 6 | {"type": "RECORD", "stream": "test_duplicate_records", "record": {"id": 1, "metric": 100}} 7 | {"type": "STATE", "value": {"test_duplicate_records": 2}} 8 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/invalid_schema.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_invalid_schema", "schema": {"type": "object"}, "key_properties": []} 2 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/no_primary_keys.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_no_pk", "key_properties": [], "schema": { "type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}} 2 | {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 1, "metric": 11}} 3 | {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 2, "metric": 22}} 4 | {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 3, "metric": 33}} 5 | {"type": "STATE", "value": {"test_no_pk": 3}} 6 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/no_primary_keys_append.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_no_pk", "key_properties": [], "schema": { "type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}} 2 | {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 1, "metric": 101}} 3 | {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 2, "metric": 202}} 4 | {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 3, "metric": 303}} 5 | {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 4, "metric": 404}} 6 | {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 5, "metric": 505}} 7 | {"type": "STATE", "value": {"test_no_pk": 5}} 8 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/optional_attributes.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_optional_attributes", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "optional": {"type": "string"}}}} 2 | {"type": "RECORD", "stream": "test_optional_attributes", "record": {"id": 1, "optional": "This is optional"}} 3 | {"type": "RECORD", "stream": "test_optional_attributes", "record": {"id": 2}} 4 | {"type": "RECORD", "stream": "test_optional_attributes", "record": {"id": 3, "optional": "Also optional"}} 5 | {"type": "RECORD", "stream": "test_optional_attributes", "record": {"id": 4}} 6 | {"type": "STATE", "value": {"test_optional_attributes": 4}} 7 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/pk_updates.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "example_stream", "schema": {"properties": {"id": {"type": "integer"}, "name": {"type": "string"}, "email": {"type": "string"}}, "key_properties": ["id"]}} 2 | {"type": "RECORD", "stream": "example_stream", "record": {"id": 1, "name": "Alice", "email": "alice@example.com"}} 3 | {"type": "SCHEMA", "stream": "example_stream", "schema": {"properties": {"id": {"type": "integer"}, "name": {"type": "string"}, "email": {"type": "string"}}, "key_properties": ["email"]}} 4 | {"type": "RECORD", "stream": "example_stream", "record": {"id": 2, "name": "Bob", "email": "bob@example.com"}} 5 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/record_before_schema.singer: -------------------------------------------------------------------------------- 1 | {"type": "RECORD", "stream": "test_record_before_schema", "record": {"id": 1, "metric": 6719}} 2 | {"type": "SCHEMA", "stream": "test_record_before_schema", "key_properties": ["id"], "schema": {"type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}} 3 | {"type": "RECORD", "stream": "test_record_before_schema", "record": {"id": 2, "metric": 3728}} 4 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/record_missing_fields.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "record_missing_fields", "key_properties": ["id"], "schema": {"type": "object", "properties": {"id": {"type": "integer"}, "optional": {"type": "string"}}, "required": ["id"]}} 2 | {"type": "RECORD", "stream": "record_missing_fields", "record": {"id": 1, "optional": "now you see me"}} 3 | {"type": "RECORD", "stream": "record_missing_fields", "record": {"id": 2}} 4 | {"type": "STATE", "value": {}} 5 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/record_missing_key_property.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_record_missing_key_property", "key_properties": ["id"], "schema": {"type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}} 2 | {"type": "RECORD", "stream": "test_record_missing_key_property", "record": {"metric": 8214}} 3 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/record_missing_required_property.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_record_missing_required_property", "key_properties": [], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}} 2 | {"type": "RECORD", "stream": "test_record_missing_required_property", "record": {"metric": 3215}} 3 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/schema_no_properties.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test_object_schema_with_properties", "key_properties": [], "schema": {"type": "object", "properties": { "object_store": {"type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}}}} 2 | {"type": "RECORD", "stream": "test_object_schema_with_properties", "record": {"object_store": {"id": 1, "metric": 187}}} 3 | {"type": "RECORD", "stream": "test_object_schema_with_properties", "record": {"object_store": {"id": 2, "metric": 203}}} 4 | {"type": "SCHEMA", "stream": "test_object_schema_no_properties", "key_properties": [], "schema": {"type": "object", "properties": { "object_store": {"type": "object"}}}} 5 | {"type": "RECORD", "stream": "test_object_schema_no_properties", "record": {"object_store": {"id": 1, "metric": 1}}} 6 | {"type": "RECORD", "stream": "test_object_schema_no_properties", "record": {"object_store": {"id": 2, "metric": 2}}} 7 | -------------------------------------------------------------------------------- /singer_sdk/testing/target_test_streams/special_chars_in_attributes.singer: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "test:SpecialChars!in?attributes", "schema": {"type": "object", "properties": {"_id": {"type": "string"}, "d": {"type": "object", "properties": {"env": {"type": "string"}, "agent:type": {"type": "string"}, "agent:os:version": {"type": "string"}}}}}, "key_properties": ["_id"]} 2 | {"type": "RECORD", "stream": "test:SpecialChars!in?attributes", "record": {"_id": "a2e98886", "d": {"env": "prod", "agent:type": "desktop", "agent:os:version": "10.13.1"}}, "version": 1541199424491, "time_extracted": "2018-11-02T22:57:04.841020Z"} 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """SDK tests.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /tests/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/contrib/__init__.py -------------------------------------------------------------------------------- /tests/core/__init__.py: -------------------------------------------------------------------------------- 1 | """SDK core tests.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /tests/core/configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/configuration/__init__.py -------------------------------------------------------------------------------- /tests/core/resources/batch.1.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/resources/batch.1.jsonl.gz -------------------------------------------------------------------------------- /tests/core/resources/batch.2.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/resources/batch.2.jsonl.gz -------------------------------------------------------------------------------- /tests/core/resources/continents.parquet.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/resources/continents.parquet.gz -------------------------------------------------------------------------------- /tests/core/resources/countries.parquet.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/resources/countries.parquet.gz -------------------------------------------------------------------------------- /tests/core/resources/testfile.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/resources/testfile.parquet -------------------------------------------------------------------------------- /tests/core/rest/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/rest/__init__.py -------------------------------------------------------------------------------- /tests/core/sinks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/sinks/__init__.py -------------------------------------------------------------------------------- /tests/core/snapshots/test_parent_child/test_child_deselected_parent/stderr.log: -------------------------------------------------------------------------------- 1 | INFO my-tap.parent Beginning full_table sync of 'parent' 2 | INFO my-tap.child Beginning full_table sync of 'child' with context: {'pid': 1} 3 | INFO my-tap.child Beginning full_table sync of 'child' with context: {'pid': 2} 4 | INFO my-tap.child Beginning full_table sync of 'child' with context: {'pid': 3} 5 | -------------------------------------------------------------------------------- /tests/core/snapshots/test_parent_child/test_deselected_child/singer.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"parent","schema":{"properties":{"id":{"type":"integer"}},"type":"object"},"key_properties":[]} 2 | {"type":"RECORD","stream":"parent","record":{"id":1},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"parent","record":{"id":2},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"parent","record":{"id":3},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"parent":{}}}} 6 | -------------------------------------------------------------------------------- /tests/core/snapshots/test_parent_child/test_deselected_child/stderr.log: -------------------------------------------------------------------------------- 1 | INFO my-tap Skipping deselected stream 'child'. 2 | INFO my-tap.parent Beginning full_table sync of 'parent' 3 | -------------------------------------------------------------------------------- /tests/core/snapshots/test_parent_child/test_one_parent_many_children/stderr.log: -------------------------------------------------------------------------------- 1 | INFO my-tap-many Skipping parse of env var settings... 2 | -------------------------------------------------------------------------------- /tests/core/snapshots/test_parent_child/test_parent_context_fields_in_child/stderr.log: -------------------------------------------------------------------------------- 1 | INFO my-tap.parent Beginning full_table sync of 'parent' 2 | INFO my-tap.child Beginning full_table sync of 'child' with context: {'pid': 1} 3 | INFO my-tap.child Beginning full_table sync of 'child' with context: {'pid': 2} 4 | INFO my-tap.child Beginning full_table sync of 'child' with context: {'pid': 3} 5 | -------------------------------------------------------------------------------- /tests/core/snapshots/test_parent_child/test_skip_deleted_parent_child_streams/stderr.log: -------------------------------------------------------------------------------- 1 | WARNING my-tap.parent Context for child streams of 'parent' is null, skipping sync of any child streams 2 | -------------------------------------------------------------------------------- /tests/core/targets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/core/targets/__init__.py -------------------------------------------------------------------------------- /tests/core/test_capabilities.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import warnings 4 | from inspect import currentframe, getframeinfo 5 | 6 | import pytest 7 | 8 | from singer_sdk.helpers.capabilities import CapabilitiesEnum 9 | 10 | 11 | class DummyCapabilitiesEnum(CapabilitiesEnum): 12 | """Simple capabilities enumeration.""" 13 | 14 | MY_SUPPORTED_FEATURE = "supported" 15 | MY_DEPRECATED_FEATURE = "deprecated", "No longer supported." 16 | 17 | 18 | def test_deprecated_capabilities(): 19 | with warnings.catch_warnings(): 20 | warnings.simplefilter("error") 21 | _ = DummyCapabilitiesEnum.MY_SUPPORTED_FEATURE 22 | 23 | with pytest.warns( 24 | DeprecationWarning, 25 | match="is deprecated. No longer supported", 26 | ) as record: 27 | _ = DummyCapabilitiesEnum.MY_DEPRECATED_FEATURE 28 | 29 | warning = record.list[0] 30 | frameinfo = getframeinfo(currentframe()) 31 | assert warning.lineno == frameinfo.lineno - 3 32 | assert warning.filename.endswith("test_capabilities.py") 33 | 34 | with pytest.warns( 35 | DeprecationWarning, 36 | match="is deprecated. No longer supported", 37 | ) as record: 38 | DummyCapabilitiesEnum("deprecated") 39 | 40 | warning = record.list[0] 41 | frameinfo = getframeinfo(currentframe()) 42 | assert warning.lineno == frameinfo.lineno - 3 43 | assert warning.filename.endswith("test_capabilities.py") 44 | -------------------------------------------------------------------------------- /tests/core/test_plugin_config.py: -------------------------------------------------------------------------------- 1 | """Test plugin config functions.""" 2 | 3 | from __future__ import annotations 4 | 5 | import typing as t 6 | 7 | from singer_sdk.tap_base import Tap 8 | from singer_sdk.typing import BooleanType, PropertiesList, Property 9 | 10 | if t.TYPE_CHECKING: 11 | from singer_sdk.streams.core import Stream 12 | 13 | SAMPLE_CONFIG: dict[str, t.Any] = {} 14 | 15 | 16 | class TapConfigTest(Tap): 17 | """Tap class for use in testing config operations.""" 18 | 19 | name = "tap-config-test" 20 | config_jsonschema = PropertiesList( 21 | Property("default_true", BooleanType, default=True), 22 | Property("default_false", BooleanType, default=False), 23 | ).to_dict() 24 | 25 | def discover_streams(self) -> list[Stream]: 26 | """Noop.""" 27 | return [] 28 | 29 | 30 | def test_tap_config_defaults(): 31 | """Run standard tap tests from the SDK.""" 32 | tap = TapConfigTest(config=SAMPLE_CONFIG, parse_env_config=True) 33 | assert "default_true" in tap.config 34 | assert "default_false" in tap.config 35 | assert tap.config["default_true"] is True 36 | assert tap.config["default_false"] is False 37 | -------------------------------------------------------------------------------- /tests/core/test_testing.py: -------------------------------------------------------------------------------- 1 | """Test the plugin testing helpers.""" 2 | 3 | from __future__ import annotations 4 | 5 | import pytest 6 | 7 | from singer_sdk.testing.factory import BaseTestClass 8 | 9 | 10 | def test_module_deprecations(): 11 | with pytest.deprecated_call(): 12 | from singer_sdk.testing import get_standard_tap_tests # noqa: F401, PLC0415 13 | 14 | with pytest.deprecated_call(): 15 | from singer_sdk.testing import get_standard_target_tests # noqa: F401, PLC0415 16 | 17 | from singer_sdk import testing # noqa: PLC0415 18 | 19 | with pytest.raises( 20 | AttributeError, 21 | match="module singer_sdk\\.testing has no attribute", 22 | ): 23 | testing.foo # noqa: B018 24 | 25 | 26 | def test_test_class_mro(): 27 | class PluginTestClass(BaseTestClass): 28 | pass 29 | 30 | PluginTestClass.params["x"] = 1 31 | 32 | class AnotherPluginTestClass(BaseTestClass): 33 | pass 34 | 35 | AnotherPluginTestClass.params["x"] = 2 36 | AnotherPluginTestClass.params["y"] = 3 37 | 38 | class SubPluginTestClass(PluginTestClass): 39 | pass 40 | 41 | assert PluginTestClass.params == {"x": 1} 42 | assert AnotherPluginTestClass.params == {"x": 2, "y": 3} 43 | assert SubPluginTestClass.params == {"x": 1} 44 | -------------------------------------------------------------------------------- /tests/external/__init__.py: -------------------------------------------------------------------------------- 1 | """SDK external system tests.""" 2 | 3 | from __future__ import annotations 4 | -------------------------------------------------------------------------------- /tests/external/conftest.py: -------------------------------------------------------------------------------- 1 | """External tests fixtures.""" 2 | 3 | from __future__ import annotations 4 | 5 | import json 6 | from pathlib import Path 7 | 8 | import pytest 9 | 10 | 11 | def gitlab_config() -> dict | None: 12 | """Create a tap-gitlab config object.""" 13 | 14 | path = Path("singer_sdk/tests/external/.secrets/gitlab-config.json") 15 | if not path.exists(): 16 | # local testing relative path 17 | path = Path("tests/external/.secrets/gitlab-config.json") 18 | 19 | if path.exists(): 20 | return json.loads(path.read_text()) 21 | 22 | return None 23 | 24 | 25 | @pytest.fixture(name="gitlab_config") 26 | def gitlab_config_fixture() -> dict | None: 27 | return gitlab_config() 28 | 29 | 30 | def ga_config() -> dict | None: 31 | """Create a tap-google-analytics config object.""" 32 | path = Path("singer_sdk/tests/external/.secrets/google-analytics-config.json") 33 | 34 | if path.exists(): 35 | return json.loads(path.read_text()) 36 | 37 | return None 38 | 39 | 40 | @pytest.fixture(name="ga_config") 41 | def ga_config_fixture() -> dict | None: 42 | return ga_config() 43 | -------------------------------------------------------------------------------- /tests/external/test_tap_dummyjson.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from samples.sample_tap_dummy_json.tap_dummyjson.tap import TapDummyJSON 4 | from singer_sdk.testing import SuiteConfig, get_tap_test_class 5 | 6 | CONFIG = { 7 | "username": "emilys", 8 | "password": "emilyspass", 9 | } 10 | 11 | TestTapDummyJSON = get_tap_test_class( 12 | tap_class=TapDummyJSON, 13 | config=CONFIG, 14 | suite_config=SuiteConfig(max_records_limit=15), 15 | ) 16 | -------------------------------------------------------------------------------- /tests/samples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/samples/__init__.py -------------------------------------------------------------------------------- /tests/samples/resources/messages.jsonl: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "users", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "name":{"type":["null","string"]}}}} 2 | {"type": "RECORD", "stream": "users", "record": {"id": 1, "name": "Chris"}} 3 | {"type": "RECORD", "stream": "users", "record": {"id": 2, "name": "Mike"}} 4 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_countries_to_csv/singer.log: -------------------------------------------------------------------------------- 1 | INFO sample-tap-countries Skipping parse of env var settings... 2 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_countries_to_csv/target.jsonl: -------------------------------------------------------------------------------- 1 | {"bookmarks": {"continents": {}, "countries": {}}} 2 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_countries_to_csv_mapped/mapper.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"continents","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"key_properties":["code"],"bookmark_properties":[]} 2 | {"type":"RECORD","stream":"continents","record":{"code":"AF","name":"Africa"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"continents","record":{"code":"AN","name":"Antarctica"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"continents","record":{"code":"AS","name":"Asia"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"RECORD","stream":"continents","record":{"code":"EU","name":"Europe"},"time_extracted":"2022-01-01T00:00:00+00:00"} 6 | {"type":"RECORD","stream":"continents","record":{"code":"NA","name":"North America"},"time_extracted":"2022-01-01T00:00:00+00:00"} 7 | {"type":"RECORD","stream":"continents","record":{"code":"OC","name":"Oceania"},"time_extracted":"2022-01-01T00:00:00+00:00"} 8 | {"type":"RECORD","stream":"continents","record":{"code":"SA","name":"South America"},"time_extracted":"2022-01-01T00:00:00+00:00"} 9 | {"type":"STATE","value":{"bookmarks":{"continents":{}}}} 10 | {"type":"STATE","value":{"bookmarks":{"continents":{},"countries":{}}}} 11 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_countries_to_csv_mapped/mapper.log: -------------------------------------------------------------------------------- 1 | INFO sample-tap-countries Skipping parse of env var settings... 2 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_countries_to_csv_mapped/tap.log: -------------------------------------------------------------------------------- 1 | INFO sample-tap-countries Skipping parse of env var settings... 2 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_countries_to_csv_mapped/target.jsonl: -------------------------------------------------------------------------------- 1 | {"bookmarks": {"continents": {}, "countries": {}}} 2 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_countries_to_csv_mapped/target.log: -------------------------------------------------------------------------------- 1 | INFO sample-tap-countries Skipping parse of env var settings... 2 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_fake_people_to_csv/singer.log: -------------------------------------------------------------------------------- 1 | INFO sample-tap-fake-people Skipping parse of env var settings... 2 | -------------------------------------------------------------------------------- /tests/samples/snapshots/test_target_csv/test_fake_people_to_csv/target.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/samples/snapshots/test_target_csv/test_fake_people_to_csv/target.jsonl -------------------------------------------------------------------------------- /tests/samples/test_target_parquet.py: -------------------------------------------------------------------------------- 1 | """Test class creation.""" 2 | 3 | from __future__ import annotations 4 | 5 | import shutil 6 | import uuid 7 | from pathlib import Path 8 | 9 | import pytest 10 | 11 | from samples.sample_target_parquet.parquet_target import SampleTargetParquet 12 | from singer_sdk.testing import get_target_test_class 13 | 14 | SAMPLE_FILEPATH = Path(f".output/test_{uuid.uuid4()}/") 15 | SAMPLE_FILENAME = SAMPLE_FILEPATH / "testfile.parquet" 16 | SAMPLE_CONFIG = { 17 | "filepath": str(SAMPLE_FILENAME), 18 | } 19 | 20 | StandardTests = get_target_test_class( 21 | target_class=SampleTargetParquet, 22 | config=SAMPLE_CONFIG, 23 | ) 24 | 25 | 26 | class TestSampleTargetParquet(StandardTests): 27 | """Standard Target Tests.""" 28 | 29 | @pytest.fixture(scope="class") 30 | def test_output_dir(self): 31 | return SAMPLE_FILEPATH 32 | 33 | @pytest.fixture(scope="class") 34 | def resource(self, test_output_dir): 35 | test_output_dir.mkdir(parents=True, exist_ok=True) 36 | yield test_output_dir 37 | shutil.rmtree(test_output_dir) 38 | -------------------------------------------------------------------------------- /tests/singerlib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meltano/sdk/9497d9ba6b330ea65cfaef0a8c4b4345344efcd0/tests/singerlib/__init__.py -------------------------------------------------------------------------------- /tests/singerlib/encoding/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | -------------------------------------------------------------------------------- /tests/singerlib/test_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from datetime import datetime, timezone 4 | 5 | import pytest 6 | import pytz 7 | 8 | from singer_sdk._singerlib import strftime, strptime_to_utc 9 | from singer_sdk._singerlib.utils import NonUTCDatetimeError 10 | 11 | 12 | def test_small_years(): 13 | assert ( 14 | strftime(datetime(90, 1, 1, tzinfo=pytz.UTC)) == "0090-01-01T00:00:00.000000Z" 15 | ) 16 | 17 | 18 | def test_round_trip(): 19 | now = datetime.now(tz=pytz.UTC) 20 | dtime = strftime(now) 21 | parsed_datetime = strptime_to_utc(dtime) 22 | formatted_datetime = strftime(parsed_datetime) 23 | assert dtime == formatted_datetime 24 | 25 | 26 | @pytest.mark.parametrize( 27 | "dtimestr", 28 | [ 29 | "2021-01-01T00:00:00.000000Z", 30 | "2021-01-01T00:00:00.000000+00:00", 31 | "2021-01-01T00:00:00.000000+06:00", 32 | "2021-01-01T00:00:00.000000-04:00", 33 | "2021-01-01T00:00:00.000000", 34 | ], 35 | ids=["Z", "offset+0", "offset+6", "offset-4", "no offset"], 36 | ) 37 | def test_strptime_to_utc(dtimestr): 38 | assert strptime_to_utc(dtimestr).tzinfo == timezone.utc 39 | 40 | 41 | def test_stftime_non_utc(): 42 | now = datetime.now(tz=pytz.timezone("America/New_York")) 43 | with pytest.raises(NonUTCDatetimeError): 44 | strftime(now) 45 | -------------------------------------------------------------------------------- /tests/snapshots/about_format/json.snap.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tap-example", 3 | "description": "Example tap for Singer SDK", 4 | "version": "0.1.1", 5 | "sdk_version": "1.0.0", 6 | "supported_python_versions": [ 7 | "3.8", 8 | "3.9", 9 | "3.10", 10 | "3.11", 11 | "3.12", 12 | "3.13" 13 | ], 14 | "capabilities": [ 15 | "catalog", 16 | "discover", 17 | "state" 18 | ], 19 | "settings": { 20 | "properties": { 21 | "start_date": { 22 | "type": "string", 23 | "format": "date-time", 24 | "description": "Start date for the tap to extract data from." 25 | }, 26 | "api_key": { 27 | "type": "string", 28 | "description": "API key for the tap to use." 29 | }, 30 | "complex_setting": { 31 | "type": "object", 32 | "description": "A complex setting, with sub-settings.", 33 | "properties": { 34 | "sub_setting": { 35 | "type": "string", 36 | "description": "A sub-setting." 37 | } 38 | } 39 | } 40 | }, 41 | "required": [ 42 | "api_key" 43 | ] 44 | } 45 | } -------------------------------------------------------------------------------- /tests/snapshots/about_format/markdown.snap.md: -------------------------------------------------------------------------------- 1 | # `tap-example` 2 | 3 | Example tap for Singer SDK 4 | 5 | Built with the [Meltano Singer SDK](https://sdk.meltano.com). 6 | 7 | ## Capabilities 8 | 9 | * `catalog` 10 | * `discover` 11 | * `state` 12 | 13 | ## Supported Python Versions 14 | 15 | * 3.8 16 | * 3.9 17 | * 3.10 18 | * 3.11 19 | * 3.12 20 | * 3.13 21 | 22 | ## Settings 23 | 24 | | Setting | Required | Default | Description | 25 | |:--------|:--------:|:-------:|:------------| 26 | | start_date | False | None | Start date for the tap to extract data from. | 27 | | api_key | True | None | API key for the tap to use. | 28 | | complex_setting | False | None | A complex setting, with sub-settings. | 29 | | complex_setting.sub_setting | False | None | A sub-setting. | 30 | 31 | A full list of supported settings and capabilities is available by running: `tap-example --about` 32 | -------------------------------------------------------------------------------- /tests/snapshots/about_format/text.snap.txt: -------------------------------------------------------------------------------- 1 | Name: tap-example 2 | Description: Example tap for Singer SDK 3 | Version: 0.1.1 4 | SDK Version: 1.0.0 5 | Supported Python Versions: 6 | - 3.8 7 | - 3.9 8 | - 3.10 9 | - 3.11 10 | - 3.12 11 | - 3.13 12 | Capabilities: 13 | - catalog 14 | - discover 15 | - state 16 | Settings: 17 | - Name: start_date 18 | Type: string 19 | Environment Variable: TAP_EXAMPLE_START_DATE 20 | - Name: api_key 21 | Type: string 22 | Environment Variable: TAP_EXAMPLE_API_KEY 23 | - Name: complex_setting 24 | Type: object 25 | Environment Variable: TAP_EXAMPLE_COMPLEX_SETTING 26 | -------------------------------------------------------------------------------- /tests/snapshots/countries_write_schemas/countries_write_schemas: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"continents","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"key_properties":["code"]} 2 | {"type":"SCHEMA","stream":"countries","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]},"native":{"type":["string","null"]},"phone":{"type":["string","null"]},"capital":{"type":["string","null"]},"currency":{"type":["string","null"]},"emoji":{"type":["string","null"]},"continent":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":["object","null"]},"languages":{"items":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"type":["array","null"]}},"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":["code"]} 3 | -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/additional_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string", 13 | "null" 14 | ] 15 | }, 16 | "username": { 17 | "type": [ 18 | "string", 19 | "null" 20 | ] 21 | }, 22 | "phone_number": { 23 | "type": [ 24 | "string", 25 | "null" 26 | ] 27 | } 28 | }, 29 | "additionalProperties": { 30 | "type": [ 31 | "string" 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/base.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string", 13 | "null" 14 | ] 15 | }, 16 | "username": { 17 | "type": [ 18 | "string", 19 | "null" 20 | ] 21 | }, 22 | "phone_number": { 23 | "type": [ 24 | "string", 25 | "null" 26 | ] 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/duplicates.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string", 13 | "null" 14 | ] 15 | }, 16 | "username": { 17 | "type": [ 18 | "string", 19 | "null" 20 | ] 21 | }, 22 | "phone_number": { 23 | "type": [ 24 | "string", 25 | "null" 26 | ] 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/duplicates_additional_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string", 13 | "null" 14 | ] 15 | }, 16 | "username": { 17 | "type": [ 18 | "string", 19 | "null" 20 | ] 21 | }, 22 | "phone_number": { 23 | "type": [ 24 | "string", 25 | "null" 26 | ] 27 | } 28 | }, 29 | "additionalProperties": { 30 | "type": [ 31 | "string" 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/duplicates_no_additional_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string", 13 | "null" 14 | ] 15 | }, 16 | "username": { 17 | "type": [ 18 | "string", 19 | "null" 20 | ] 21 | }, 22 | "phone_number": { 23 | "type": [ 24 | "string", 25 | "null" 26 | ] 27 | } 28 | }, 29 | "additionalProperties": false 30 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/no_additional_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string", 13 | "null" 14 | ] 15 | }, 16 | "username": { 17 | "type": [ 18 | "string", 19 | "null" 20 | ] 21 | }, 22 | "phone_number": { 23 | "type": [ 24 | "string", 25 | "null" 26 | ] 27 | } 28 | }, 29 | "additionalProperties": false 30 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/pattern_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string", 13 | "null" 14 | ] 15 | }, 16 | "username": { 17 | "type": [ 18 | "string", 19 | "null" 20 | ] 21 | }, 22 | "phone_number": { 23 | "type": [ 24 | "string", 25 | "null" 26 | ] 27 | } 28 | }, 29 | "patternProperties": { 30 | "^attr_[a-z]+$": { 31 | "type": [ 32 | "string" 33 | ] 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/required.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string" 13 | ] 14 | }, 15 | "username": { 16 | "type": [ 17 | "string" 18 | ] 19 | }, 20 | "phone_number": { 21 | "type": [ 22 | "string", 23 | "null" 24 | ] 25 | } 26 | }, 27 | "required": [ 28 | "email", 29 | "username" 30 | ] 31 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/required_additional_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string" 13 | ] 14 | }, 15 | "username": { 16 | "type": [ 17 | "string" 18 | ] 19 | }, 20 | "phone_number": { 21 | "type": [ 22 | "string", 23 | "null" 24 | ] 25 | } 26 | }, 27 | "required": [ 28 | "email", 29 | "username" 30 | ], 31 | "additionalProperties": { 32 | "type": [ 33 | "string" 34 | ] 35 | } 36 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/required_duplicates.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string" 13 | ] 14 | }, 15 | "username": { 16 | "type": [ 17 | "string" 18 | ] 19 | }, 20 | "phone_number": { 21 | "type": [ 22 | "string", 23 | "null" 24 | ] 25 | } 26 | }, 27 | "required": [ 28 | "email", 29 | "username" 30 | ] 31 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/required_duplicates_additional_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string" 13 | ] 14 | }, 15 | "username": { 16 | "type": [ 17 | "string" 18 | ] 19 | }, 20 | "phone_number": { 21 | "type": [ 22 | "string", 23 | "null" 24 | ] 25 | } 26 | }, 27 | "required": [ 28 | "email", 29 | "username" 30 | ], 31 | "additionalProperties": { 32 | "type": [ 33 | "string" 34 | ] 35 | } 36 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/required_duplicates_no_additional_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string" 13 | ] 14 | }, 15 | "username": { 16 | "type": [ 17 | "string" 18 | ] 19 | }, 20 | "phone_number": { 21 | "type": [ 22 | "string", 23 | "null" 24 | ] 25 | } 26 | }, 27 | "required": [ 28 | "email", 29 | "username" 30 | ], 31 | "additionalProperties": false 32 | } -------------------------------------------------------------------------------- /tests/snapshots/jsonschema/required_no_additional_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "id": { 5 | "type": [ 6 | "string", 7 | "null" 8 | ] 9 | }, 10 | "email": { 11 | "type": [ 12 | "string" 13 | ] 14 | }, 15 | "username": { 16 | "type": [ 17 | "string" 18 | ] 19 | }, 20 | "phone_number": { 21 | "type": [ 22 | "string", 23 | "null" 24 | ] 25 | } 26 | }, 27 | "required": [ 28 | "email", 29 | "username" 30 | ], 31 | "additionalProperties": false 32 | } -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/aliased_stream.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/aliased_stream_batch.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"BATCH","stream":"aliased_stream","encoding":{"format":"jsonl","compression":"gzip"},"manifest":["file:///tmp/stream.json.gz"]} 3 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 4 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/aliased_stream_not_expr.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"aliased.stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"aliased.stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"aliased.stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"aliased.stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/aliased_stream_quoted.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"__stream_name__","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"__stream_name__","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"__stream_name__","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"__stream_name__","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/builtin_variable_self.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"ALICE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"BOB@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"CHARLIE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/builtin_variable_stream_name_alias_expr.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"MYSTREAM","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"MYSTREAM","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"MYSTREAM","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"MYSTREAM","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/builtin_variable_underscore.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"ALICE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"BOB@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"CHARLIE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/changed_key_properties.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":["email_hash"]} 2 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/drop_property.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":[],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/drop_property_null_string.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":[],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/fake_credit_card_number.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"cc":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"cc":"4201040137208265027"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"cc":"675987782884"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"cc":"502011811259"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/fake_email_seed_class.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"zwells@example.org"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"josephcunningham@example.com"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"lydia62@example.net"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/fake_email_seed_instance.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"zwells@example.org"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"josephcunningham@example.com"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"lydia62@example.net"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/flatten_all.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__sub__custom_obj":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[3.14,2.718]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__sub__custom_obj":"obj-world","user__some_numbers":"[10.32,1.618]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[1.414,1.732]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/flatten_depth_0.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/flatten_depth_1.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub":"{\"num\":1,\"custom_obj\":\"obj-hello\"}","user__some_numbers":"[3.14,2.718]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub":"{\"num\":2,\"custom_obj\":\"obj-world\"}","user__some_numbers":"[10.32,1.618]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub":"{\"num\":3,\"custom_obj\":\"obj-hello\"}","user__some_numbers":"[1.414,1.732]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/json_dumps.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":"{\"id\": 1, \"sub\": {\"num\": 1, \"custom_obj\": \"obj-hello\"}, \"some_numbers\": [\"3.14\", \"2.718\"]}","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":"{\"id\": 2, \"sub\": {\"num\": 2, \"custom_obj\": \"obj-world\"}, \"some_numbers\": [\"10.32\", \"1.618\"]}","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":"{\"id\": 3, \"sub\": {\"num\": 3, \"custom_obj\": \"obj-hello\"}, \"some_numbers\": [\"1.414\", \"1.732\"]}","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/no_map.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/non_pk_passthrough.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"count":{"type":["integer","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"count":21},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"count":13},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"count":19},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/only_mapped_fields.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]},"email_hash_sha256":{"type":["string","null"]},"fixed_count":{"type":["integer","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060","email_hash_sha256":"ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976","fixed_count":20},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d","email_hash_sha256":"5ff860bf1190596c7188ab851db691f0f3169c453936e9e1eba2f9a47f7a0018","fixed_count":12},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f","email_hash_sha256":"add7232b65bb559f896cbcfa9a600170a7ca381a0366789dcf59ad986bdf4a98","fixed_count":18},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]},"fixed_count":{"type":["integer","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060","fixed_count":20},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d","fixed_count":12},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f","fixed_count":18},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/sourced_stream_1.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | -------------------------------------------------------------------------------- /tests/snapshots/mapped_stream/sourced_stream_2.jsonl: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"sourced_stream_2","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} 2 | {"type":"RECORD","stream":"sourced_stream_2","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 3 | {"type":"RECORD","stream":"sourced_stream_2","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 4 | {"type":"RECORD","stream":"sourced_stream_2","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} 5 | {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} 6 | --------------------------------------------------------------------------------