├── .coveragerc ├── .dockerignore ├── .github ├── CODEOWNERS ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── tw-rules.yaml └── workflows │ ├── connectors.yml │ ├── dockerhub.yml │ ├── e2e_tests.yml │ ├── lint_unit_tests.yml │ └── publish_doc.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .style.yapf ├── .yapfignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.barebone ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── bin └── pipelinewise-docker ├── dev-project ├── .env.template ├── README.md ├── docker-compose.yml ├── entrypoint.sh ├── mongo │ ├── Dockerfile │ ├── create-pipelinewise-user.sh │ ├── initiate-replica-set.sh │ ├── mongodb.pem │ ├── replica.key │ └── rootCA.pem ├── my │ ├── ca-cert.pem │ ├── server-cert.pem │ └── server-key.pem ├── pg │ ├── Dockerfile │ ├── pg_hba.conf │ ├── server.crt │ ├── server.key │ └── server.req └── pipelinewise-config │ ├── config.yml │ ├── tap_mongodb_to_pg.yaml │ ├── tap_mysql_mariadb.yml │ ├── tap_postgres.yml │ ├── tap_postgres_logical.yml │ └── target_postgres.yml ├── docs ├── Makefile ├── README.md ├── _static │ └── css │ │ └── custom.css ├── concept │ ├── fastsync.rst │ ├── linux_pipes.rst │ ├── replication_methods.rst │ └── singer.rst ├── conf.py ├── connectors │ ├── taps.rst │ ├── taps │ │ ├── github.rst │ │ ├── jira.rst │ │ ├── kafka.rst │ │ ├── mixpanel.rst │ │ ├── mongodb.rst │ │ ├── mysql.rst │ │ ├── postgres.rst │ │ ├── s3_csv.rst │ │ ├── slack.rst │ │ ├── snowflake.rst │ │ └── zendesk.rst │ ├── targets.rst │ └── targets │ │ ├── bigquery.rst │ │ ├── postgres.rst │ │ ├── redshift.rst │ │ ├── s3_csv.rst │ │ └── snowflake.rst ├── img │ ├── adwords-logo.png │ ├── bigquery-logo.png │ ├── github-logo.png │ ├── google-analytics-logo.png │ ├── jira-logo.png │ ├── kafka-logo.png │ ├── mariadb-logo.png │ ├── mixpanel-logo.png │ ├── mongodb-logo.png │ ├── mysql-logo.png │ ├── oracle-logo.png │ ├── partial_sync_case_1.png │ ├── partial_sync_case_2.png │ ├── partial_sync_case_3.png │ ├── partial_sync_case_4.png │ ├── partial_sync_case_5.png │ ├── partial_sync_case_all.png │ ├── pipelinewise-diagram-circle-bold.png │ ├── pipelinewise-with-text.png │ ├── pipelinewise.png │ ├── postgres-logo.png │ ├── redshift-logo.png │ ├── s3-logo.png │ ├── salesforce-logo.png │ ├── shopify-logo.png │ ├── slack-logo.png │ ├── snowflake-logo.png │ ├── twilio-logo.png │ ├── zendesk-logo.png │ └── zuora-logo.png ├── index.rst ├── installation_guide │ ├── creating_pipelines.rst │ ├── installation.rst │ └── running_pipelines.rst ├── make.bat ├── project │ ├── about.rst │ ├── contribution.rst │ └── licenses.rst ├── snowflake.png └── user_guide │ ├── alerts.rst │ ├── cli.rst │ ├── encrypting_passwords.rst │ ├── logging.rst │ ├── metadata_columns.rst │ ├── partial_sync.rst │ ├── resync.rst │ ├── scheduling.rst │ ├── schema_changes.rst │ ├── transformations.rst │ └── yaml_config.rst ├── entrypoint.sh ├── motd ├── pipelinewise ├── __init__.py ├── cli │ ├── __init__.py │ ├── alert_handlers │ │ ├── __init__.py │ │ ├── base_alert_handler.py │ │ ├── errors.py │ │ ├── slack_alert_handler.py │ │ └── victorops_alert_handler.py │ ├── alert_sender.py │ ├── commands.py │ ├── config.py │ ├── constants.py │ ├── errors.py │ ├── multiprocess.py │ ├── pipelinewise.py │ ├── samples │ │ ├── README.md │ │ ├── config.yml │ │ ├── tap_github.yml.sample │ │ ├── tap_google_analytics.yml.sample │ │ ├── tap_jira.yml.sample │ │ ├── tap_kafka.yml.sample │ │ ├── tap_mixpanel.yml.sample │ │ ├── tap_mongodb.yml.sample │ │ ├── tap_mysql_mariadb.yml.sample │ │ ├── tap_oracle.yml.sample │ │ ├── tap_postgres.yml.sample │ │ ├── tap_s3_csv.yml.sample │ │ ├── tap_salesforce.yml.sample │ │ ├── tap_shopify.yml.sample │ │ ├── tap_slack.yml.sample │ │ ├── tap_snowflake.yml.sample │ │ ├── tap_twilio.yml.sample │ │ ├── tap_zendesk.yml.sample │ │ ├── tap_zuora.yml.sample │ │ ├── target_postgres.yml.sample │ │ ├── target_redshift.yml.sample │ │ ├── target_s3_csv.yml.sample │ │ └── target_snowflake.yml.sample │ ├── schemas │ │ ├── config.json │ │ ├── tap.json │ │ └── target.json │ ├── tap_properties.py │ └── utils.py ├── fastsync │ ├── README.md │ ├── __init__.py │ ├── commons │ │ ├── __init__.py │ │ ├── errors.py │ │ ├── split_gzip.py │ │ ├── tap_mongodb.py │ │ ├── tap_mysql.py │ │ ├── tap_postgres.py │ │ ├── target_postgres.py │ │ ├── target_snowflake.py │ │ ├── transform_utils.py │ │ └── utils.py │ ├── mongodb_to_postgres.py │ ├── mongodb_to_snowflake.py │ ├── mysql_to_postgres.py │ ├── mysql_to_snowflake.py │ ├── partialsync │ │ ├── __init__.py │ │ ├── mysql_to_snowflake.py │ │ ├── postgres_to_snowflake.py │ │ └── utils.py │ ├── postgres_to_postgres.py │ └── postgres_to_snowflake.py ├── logger.py ├── logging.conf ├── logging_debug.conf └── utils.py ├── pylintrc ├── pytest.ini ├── scripts ├── check_any_file_changed.py ├── ci_check_no_file_changes.sh └── publish_docs.sh ├── setup.py ├── singer-connectors ├── tap-github │ ├── .gitignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── MANIFEST.in │ ├── README.md │ ├── config.sample.json │ ├── setup.py │ ├── tap_github │ │ ├── __init__.py │ │ └── schemas │ │ │ ├── assignees.json │ │ │ ├── collaborators.json │ │ │ ├── comments.json │ │ │ ├── commit_comments.json │ │ │ ├── commits.json │ │ │ ├── events.json │ │ │ ├── issue_events.json │ │ │ ├── issue_labels.json │ │ │ ├── issue_milestones.json │ │ │ ├── issues.json │ │ │ ├── project_cards.json │ │ │ ├── project_columns.json │ │ │ ├── projects.json │ │ │ ├── pull_requests.json │ │ │ ├── releases.json │ │ │ ├── review_comments.json │ │ │ ├── reviews.json │ │ │ ├── stargazers.json │ │ │ ├── team_members.json │ │ │ ├── team_memberships.json │ │ │ └── teams.json │ └── tests │ │ ├── __init__.py │ │ ├── base.py │ │ ├── test_github_discovery.py │ │ ├── test_github_pagination.py │ │ ├── test_github_start_date.py │ │ ├── test_github_sync.py │ │ └── unittests │ │ ├── test_exception_handling.py │ │ ├── test_formatting_dates.py │ │ ├── test_get_repos_from_config.py │ │ ├── test_key_error.py │ │ ├── test_pagination.py │ │ ├── test_rate_limit.py │ │ ├── test_repos_by_org.py │ │ ├── test_repos_by_org_matchers.py │ │ ├── test_stargazers_full_table.py │ │ ├── test_start_date_bookmark.py │ │ ├── test_sub_streams_selection.py │ │ └── test_verify_access.py ├── tap-jira │ └── requirements.txt ├── tap-kafka │ ├── .gitignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── docker-compose.yml │ ├── sample_logging.conf │ ├── setup.py │ ├── tap_kafka │ │ ├── __init__.py │ │ ├── common.py │ │ ├── errors.py │ │ ├── serialization │ │ │ ├── __init__.py │ │ │ ├── json_with_no_schema.py │ │ │ └── protobuf.py │ │ └── sync.py │ └── tests │ │ ├── __init__.py │ │ ├── integration │ │ ├── __init__.py │ │ ├── resources │ │ │ ├── json_messages_to_produce.json │ │ │ ├── json_messages_to_produce_2.json │ │ │ └── protobuf │ │ │ │ ├── messages_to_produce.json │ │ │ │ └── user.proto │ │ ├── test_consumer.py │ │ ├── test_protobuf.py │ │ └── utils.py │ │ └── unit │ │ ├── __init__.py │ │ ├── helper │ │ ├── __init__.py │ │ └── kafka_consumer_mock.py │ │ ├── resources │ │ ├── kafka-messages-from-multiple-partitions.json │ │ ├── state-with-bookmark-with-version.json │ │ └── state-with-bookmark.json │ │ ├── test_serializers.py │ │ └── test_tap_kafka.py ├── tap-mixpanel │ ├── .gitignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── MANIFEST.in │ ├── Makefile │ ├── README.md │ ├── config.json.example │ ├── setup.cfg │ ├── setup.py │ ├── state.json.example │ ├── tap_mixpanel │ │ ├── __init__.py │ │ ├── client.py │ │ ├── discover.py │ │ ├── schema.py │ │ ├── schemas │ │ │ ├── annotations.json │ │ │ ├── cohort_members.json │ │ │ ├── cohorts.json │ │ │ ├── engage.json │ │ │ ├── export.json │ │ │ ├── funnels.json │ │ │ └── revenue.json │ │ ├── streams.py │ │ ├── sync.py │ │ └── transform.py │ └── tests │ │ ├── __init__.py │ │ ├── configuration │ │ ├── __init__.py │ │ └── fixtures.py │ │ ├── tap_tester │ │ ├── __init__.py │ │ ├── tap_combined_test.py │ │ └── test_configuration.py │ │ └── unittests │ │ ├── __init__.py │ │ └── test_medium_client.py ├── tap-mongodb │ ├── .gitignore │ ├── CHANGELOG.md │ ├── CODEOWNERS │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── bin │ │ ├── populate_test_database.py │ │ └── setup_local_db.sh │ ├── docker-compose.yml │ ├── pylintrc │ ├── sample_config.json │ ├── sample_logging.conf │ ├── setup.py │ ├── tap_mongodb │ │ ├── __init__.py │ │ ├── config_utils.py │ │ ├── db_utils.py │ │ ├── errors.py │ │ ├── stream_utils.py │ │ └── sync_strategies │ │ │ ├── change_streams.py │ │ │ ├── common.py │ │ │ ├── full_table.py │ │ │ └── incremental.py │ └── tests │ │ ├── sync_strategies │ │ ├── test_change_streams.py │ │ └── test_common.py │ │ └── test_connection_string.py ├── tap-mysql │ ├── .gitignore │ ├── .noserc │ ├── .pylintrc │ ├── CHANGELOG.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── config.json.sample │ ├── dev │ │ ├── init.sql │ │ └── mariadb_logical_replication_notes.md │ ├── docker-compose.yml │ ├── sample_logging.conf │ ├── setup.py │ ├── tap_mysql │ │ ├── __init__.py │ │ ├── connection.py │ │ ├── discover_utils.py │ │ ├── stream_utils.py │ │ └── sync_strategies │ │ │ ├── __init__.py │ │ │ ├── binlog.py │ │ │ ├── common.py │ │ │ ├── full_table.py │ │ │ └── incremental.py │ └── tests │ │ ├── integration │ │ ├── test_full_table_interruption.py │ │ ├── test_tap_mysql.py │ │ └── utils.py │ │ └── unit │ │ ├── sync_strategies │ │ ├── __init__.py │ │ └── test_binlog.py │ │ ├── test__init__.py │ │ └── test_connection.py ├── tap-postgres │ ├── .gitignore │ ├── .pylintrc │ ├── CHANGELOG.md │ ├── Dockerfile │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── db_setup │ │ └── config │ │ │ └── postgresql.conf │ ├── docker-compose.yml │ ├── sample_logging.conf │ ├── setup.py │ ├── tap_postgres │ │ ├── __init__.py │ │ ├── db.py │ │ ├── discovery_utils.py │ │ ├── stream_utils.py │ │ └── sync_strategies │ │ │ ├── __init__.py │ │ │ ├── common.py │ │ │ ├── full_table.py │ │ │ ├── incremental.py │ │ │ └── logical_replication.py │ └── tests │ │ ├── __init__.py │ │ ├── integration │ │ ├── __init__.py │ │ ├── test_discovery.py │ │ ├── test_logical_replication.py │ │ ├── test_streams_utils.py │ │ └── test_unsupported_pk.py │ │ ├── unit │ │ ├── __init__.py │ │ ├── test_clear_state_on_replication_change.py │ │ ├── test_db.py │ │ ├── test_full_table.py │ │ ├── test_incremental.py │ │ └── test_logical_replication.py │ │ └── utils.py ├── tap-s3-csv │ ├── .gitignore │ ├── .pylintrc │ ├── CHANGELOG.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── config.sample.json │ ├── docker-compose.yml │ ├── sample_logging.conf │ ├── setup.py │ ├── tap_s3_csv │ │ ├── __init__.py │ │ ├── config.py │ │ ├── discover.py │ │ ├── s3.py │ │ └── sync.py │ └── tests │ │ ├── __init__.py │ │ ├── integration │ │ ├── __init__.py │ │ ├── mock_data.csv │ │ ├── test_s3_connection.py │ │ ├── test_tap_s3_csv.py │ │ └── utils.py │ │ └── unit │ │ ├── __init__.py │ │ ├── test__init.py │ │ └── test_s3.py ├── tap-salesforce │ ├── .gitignore │ ├── Blacklisting.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── setup.py │ └── tap_salesforce │ │ ├── __init__.py │ │ ├── salesforce │ │ ├── __init__.py │ │ ├── bulk.py │ │ ├── exceptions.py │ │ └── rest.py │ │ └── sync.py ├── tap-slack │ ├── .gitignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── setup.py │ ├── tap_slack │ │ ├── __init__.py │ │ ├── catalog.py │ │ ├── client.py │ │ ├── schemas │ │ │ ├── channel_members.json │ │ │ ├── channels.json │ │ │ ├── files.json │ │ │ ├── messages.json │ │ │ ├── remote_files.json │ │ │ ├── teams.json │ │ │ ├── threads.json │ │ │ ├── user_groups.json │ │ │ └── users.json │ │ ├── streams.py │ │ └── transform.py │ └── tests │ │ ├── __init__.py │ │ └── test_schemas.py ├── tap-snowflake │ ├── .gitignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── config_sample.json │ ├── pylintrc │ ├── sample_logging.conf │ ├── setup.py │ ├── tap_snowflake │ │ ├── __init__.py │ │ ├── connection.py │ │ └── sync_strategies │ │ │ ├── __init__.py │ │ │ ├── common.py │ │ │ ├── full_table.py │ │ │ └── incremental.py │ └── tests │ │ ├── __init__.py │ │ └── integration │ │ ├── test_tap_snowflake.py │ │ └── utils.py ├── tap-twilio │ ├── .gitignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── MANIFEST.in │ ├── Makefile │ ├── README.md │ ├── config.json.example │ ├── setup.py │ ├── state.json.example │ ├── tap_twilio │ │ ├── __init__.py │ │ ├── client.py │ │ ├── discover.py │ │ ├── schema.py │ │ ├── schemas │ │ │ ├── account_balance.json │ │ │ ├── accounts.json │ │ │ ├── activities.json │ │ │ ├── addresses.json │ │ │ ├── alerts.json │ │ │ ├── applications.json │ │ │ ├── archive │ │ │ │ └── _scema_template.json │ │ │ ├── available_phone_number_countries.json │ │ │ ├── available_phone_numbers_local.json │ │ │ ├── available_phone_numbers_mobile.json │ │ │ ├── available_phone_numbers_toll_free.json │ │ │ ├── calls.json │ │ │ ├── channels.json │ │ │ ├── chat_channels.json │ │ │ ├── chat_messages.json │ │ │ ├── conference_participants.json │ │ │ ├── conferences.json │ │ │ ├── cumulative_statistics.json │ │ │ ├── dependent_phone_numbers.json │ │ │ ├── events.json │ │ │ ├── incoming_phone_numbers.json │ │ │ ├── keys.json │ │ │ ├── members.json │ │ │ ├── message_media.json │ │ │ ├── messages.json │ │ │ ├── outgoing_caller_ids.json │ │ │ ├── queues.json │ │ │ ├── recordings.json │ │ │ ├── roles.json │ │ │ ├── services.json │ │ │ ├── task_channels.json │ │ │ ├── task_queues.json │ │ │ ├── tasks.json │ │ │ ├── transcriptions.json │ │ │ ├── usage.json │ │ │ ├── usage_records.json │ │ │ ├── usage_triggers.json │ │ │ ├── users.json │ │ │ ├── workers.json │ │ │ ├── workflows.json │ │ │ └── workspaces.json │ │ ├── streams.py │ │ ├── sync.py │ │ └── transform.py │ └── tests │ │ ├── __init__.py │ │ ├── test_sync.py │ │ └── unit │ │ └── test_transform.py ├── tap-zendesk │ ├── CHANGELOG.md │ ├── LICENSE │ ├── MANIFEST.in │ ├── README.md │ ├── sample_logging.conf │ ├── setup.py │ ├── spikes │ │ └── ticket_events.py │ ├── tap_zendesk │ │ ├── __init__.py │ │ ├── default_config.json │ │ ├── discover.py │ │ ├── metrics.py │ │ ├── schemas │ │ │ ├── group_memberships.json │ │ │ ├── groups.json │ │ │ ├── macros.json │ │ │ ├── organizations.json │ │ │ ├── satisfaction_ratings.json │ │ │ ├── shared │ │ │ │ ├── attachments.json │ │ │ │ ├── metadata.json │ │ │ │ └── via.json │ │ │ ├── sla_policies.json │ │ │ ├── tags.json │ │ │ ├── ticket_audits.json │ │ │ ├── ticket_comments.json │ │ │ ├── ticket_fields.json │ │ │ ├── ticket_forms.json │ │ │ ├── ticket_metrics.json │ │ │ ├── tickets.json │ │ │ └── users.json │ │ ├── streams.py │ │ └── sync.py │ └── test │ │ ├── __init__.py │ │ ├── helper │ │ └── zenpymock.py │ │ ├── test_catalog.json │ │ ├── test_do_sync.py │ │ ├── test_init.py │ │ └── test_state.json ├── target-postgres │ ├── CHANGELOG.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── docker-compose.yml │ ├── sample_logging.conf │ ├── setup.py │ ├── target_postgres │ │ ├── __init__.py │ │ └── db_sync.py │ └── tests │ │ ├── integration │ │ ├── resources │ │ │ ├── invalid-json.json │ │ │ ├── invalid-message-order.json │ │ │ ├── messages-pg-logical-streams-no-records.json │ │ │ ├── messages-pg-logical-streams.json │ │ │ ├── messages-with-invalid-records.json │ │ │ ├── messages-with-long-texts.json │ │ │ ├── messages-with-multiple-streams-modified-column.json │ │ │ ├── messages-with-multiple-streams.json │ │ │ ├── messages-with-nested-schema.json │ │ │ ├── messages-with-non-db-friendly-columns.json │ │ │ ├── messages-with-reserved-name-as-table-name.json │ │ │ ├── messages-with-space-in-table-name.json │ │ │ └── messages-with-unicode-characters.json │ │ ├── test_target_postgres.py │ │ └── utils.py │ │ └── unit │ │ ├── resources │ │ └── logical-streams.json │ │ ├── test_db_sync.py │ │ └── test_target_postgres.py ├── target-s3-csv │ ├── .gitignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── config.json.sample │ ├── sample_logging.conf │ ├── setup.py │ ├── target_s3_csv │ │ ├── __init__.py │ │ ├── s3.py │ │ └── utils.py │ └── tests │ │ ├── __init__.py │ │ ├── integration │ │ ├── __init__.py │ │ ├── resources │ │ │ ├── invalid-json.json │ │ │ ├── invalid-message-order.json │ │ │ └── messages-with-three-streams.json │ │ ├── test_target_s3_csv.py │ │ └── utils.py │ │ └── unit │ │ ├── __init__.py │ │ ├── test_main.py │ │ ├── test_s3.py │ │ └── test_utils.py ├── target-snowflake │ ├── .gitignore │ ├── CHANGELOG.md │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── flow-diagram.jpg │ ├── flow-diagram.svg │ ├── pylintrc │ ├── sample_logging.conf │ ├── setup.py │ ├── target_snowflake │ │ ├── __init__.py │ │ ├── db_sync.py │ │ ├── exceptions.py │ │ ├── file_format.py │ │ ├── file_formats │ │ │ ├── __init__.py │ │ │ ├── csv.py │ │ │ └── parquet.py │ │ ├── flattening.py │ │ ├── stream_utils.py │ │ └── upload_clients │ │ │ ├── __init__.py │ │ │ ├── base_upload_client.py │ │ │ ├── s3_upload_client.py │ │ │ └── snowflake_upload_client.py │ └── tests │ │ ├── __init__.py │ │ ├── integration │ │ ├── .env.sample │ │ ├── __init__.py │ │ ├── resources │ │ │ ├── invalid-json.json │ │ │ ├── invalid-message-order.json │ │ │ ├── messages-pg-logical-streams-no-records.json │ │ │ ├── messages-pg-logical-streams.json │ │ │ ├── messages-pg-with-invalid-records.json │ │ │ ├── messages-pg-with-valid-records.json │ │ │ ├── messages-simple-table.json │ │ │ ├── messages-with-binary-columns.json │ │ │ ├── messages-with-changing-pk.json │ │ │ ├── messages-with-falsy-pk-values.json │ │ │ ├── messages-with-invalid-records.json │ │ │ ├── messages-with-multi-schemas.json │ │ │ ├── messages-with-nested-schema.json │ │ │ ├── messages-with-new-pk.json │ │ │ ├── messages-with-non-db-friendly-columns.json │ │ │ ├── messages-with-null-pk.json │ │ │ ├── messages-with-reserved-name-as-table-name.json │ │ │ ├── messages-with-space-in-table-name.json │ │ │ ├── messages-with-three-streams-modified-column.json │ │ │ ├── messages-with-three-streams.json │ │ │ ├── messages-with-unexpected-types.json │ │ │ └── messages-with-unicode-characters.json │ │ ├── test_target_snowflake.py │ │ └── utils.py │ │ └── unit │ │ ├── __init__.py │ │ ├── file_formats │ │ ├── __init__.py │ │ ├── test_csv.py │ │ └── test_parquet.py │ │ ├── resources │ │ ├── logical-streams.json │ │ ├── messages-simple-table.json │ │ ├── same-schemas-multiple-times.json │ │ └── streams_only_state.json │ │ ├── test_db_sync.py │ │ ├── test_file_format.py │ │ ├── test_flattening.py │ │ ├── test_stream_utils.py │ │ └── test_target_snowflake.py └── transform-field │ ├── .gitignore │ ├── .pylintrc │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── sample_config.json │ ├── sample_logging.conf │ ├── setup.py │ ├── tests │ ├── __init__.py │ ├── integration │ │ ├── __init__.py │ │ ├── resources │ │ │ ├── catalog.json │ │ │ ├── invalid_config.json │ │ │ ├── invalid_messages.json │ │ │ ├── messages.json │ │ │ ├── streams_with_changing_schema.json │ │ │ ├── streams_with_object.json │ │ │ └── valid_config.json │ │ └── test_integrations.py │ └── unit │ │ ├── __init__.py │ │ ├── test_init.py │ │ ├── test_transform.py │ │ └── test_utils.py │ └── transform_field │ ├── __init__.py │ ├── errors.py │ ├── timings.py │ ├── transform.py │ └── utils.py └── tests ├── __init__.py ├── db ├── mongodb_data │ ├── all_datatypes.bson.gz │ ├── listings.csv │ └── my_collection.bson.gz ├── tap_mongodb.sh ├── tap_mysql_data.sql ├── tap_mysql_db.sh ├── tap_postgres_data.sql ├── tap_postgres_data_logical.sql ├── tap_postgres_db.sh ├── target_postgres.sh └── target_postgres_data.sql ├── end_to_end ├── __init__.py ├── helpers │ ├── __init__.py │ ├── assertions.py │ ├── db.py │ ├── env.py │ └── tasks.py ├── target_snowflake │ ├── __init__.py │ ├── tap_mariadb │ │ ├── __init__.py │ │ ├── test_defined_partial_sync_mariadb_to_sf.py │ │ ├── test_partial_sync_mariadb_to_sf.py │ │ ├── test_replicate_mariadb_replica_to_sf.py │ │ ├── test_replicate_mariadb_to_sf.py │ │ ├── test_replicate_mariadb_to_sf_with_custom_buffer_size.py │ │ ├── test_resync_mariadb_to_sf.py │ │ ├── test_resync_mariadb_to_sf_table_size_check.py │ │ └── test_resync_mariadb_to_sf_with_split_large_files.py │ ├── tap_mongodb │ │ ├── __init__.py │ │ └── test_replicate_mongodb_to_sf.py │ ├── tap_postgres │ │ ├── __init__.py │ │ ├── test_defined_partial_sync_pg_to_sf.py │ │ ├── test_partial_sync_pg_to_sf.py │ │ ├── test_replicate_pg_to_sf.py │ │ ├── test_replicate_pg_to_sf_with_archive_load_files.py │ │ ├── test_resync_pg_to_sf_table_size_check.py │ │ └── test_resync_pg_to_sf_with_split_large_files.py │ └── tap_s3 │ │ ├── __init__.py │ │ └── test_replicate_s3_to_sf.py ├── test-project │ ├── s3_mock_data │ │ ├── mock_data_1.csv │ │ └── mock_data_2.csv │ ├── tap_mongodb_to_pg.yml.template │ ├── tap_mongodb_to_sf.yml.template │ ├── tap_mysql_replica_to_pg.yml.template │ ├── tap_mysql_replica_to_sf.yml.template │ ├── tap_mysql_to_pg.yml.template │ ├── tap_mysql_to_pg_buffered_stream.yml.template │ ├── tap_mysql_to_sf.yml.template │ ├── tap_mysql_to_sf_buffered_stream.yml.template │ ├── tap_mysql_to_sf_defined_partial_sync.yml.template │ ├── tap_mysql_to_sf_soft_delete.yml.template │ ├── tap_mysql_to_sf_split_large_files.yml.template │ ├── tap_postgres_to_pg.yml.template │ ├── tap_postgres_to_sf.yml.template │ ├── tap_postgres_to_sf_archive_load_files.yml.template │ ├── tap_postgres_to_sf_defined_partial_sync.yml.template │ ├── tap_postgres_to_sf_soft_delete.yml.template │ ├── tap_postgres_to_sf_split_large_files.yml.template │ ├── tap_s3_csv_to_pg.yml.template │ ├── tap_s3_csv_to_sf.yml.template │ ├── target_postgres.yml.template │ └── target_snowflake.yml.template └── test_target_postgres.py └── units ├── __init__.py ├── cli ├── __init__.py ├── cli_args.py ├── resources │ ├── example-with-full-vault.yml │ ├── example-with-invalid-vault.yml │ ├── example-with-jinja-env-var.yml │ ├── example-with-vault.yml │ ├── example.json │ ├── example.yml │ ├── invalid.json │ ├── invalid.yml │ ├── sample_json_config │ │ ├── config.json │ │ ├── target_one │ │ │ ├── config.json │ │ │ ├── tap_one │ │ │ │ ├── config.json │ │ │ │ ├── inheritable_config.json │ │ │ │ ├── properties.json │ │ │ │ ├── properties_updated.json │ │ │ │ ├── selection.json │ │ │ │ ├── state.json.bak │ │ │ │ └── transformation.json │ │ │ └── tap_two │ │ │ │ ├── config.json │ │ │ │ ├── inheritable_config.json │ │ │ │ ├── properties.json │ │ │ │ ├── selection.json │ │ │ │ └── transformation.json │ │ └── target_two │ │ │ └── tap_three │ │ │ └── placeholder │ ├── sample_json_config_for_specific_slack_channel │ │ ├── config.json │ │ └── target_one │ │ │ └── tap_one │ │ │ └── config.json │ ├── sample_log_files │ │ ├── tap-run-errors.log │ │ ├── tap-run-lot-of-errors.log │ │ └── tap-run-no-errors.log │ ├── tap-github.yml │ ├── tap-inheritable-config.json │ ├── tap-invalid.yml │ ├── tap-valid-kafka.yml │ ├── tap-valid-mysql.yml │ ├── tap-valid-snowflake.yml │ ├── target-config.json │ ├── target-invalid-s3-csv.yml │ ├── target-valid-s3-csv.yml │ ├── test_import_command │ │ ├── tap_one.yml │ │ ├── tap_three.yml │ │ ├── tap_two.yml │ │ └── target_test.yml │ ├── test_invalid_tap_mongo_yaml_config │ │ ├── tap_test_invalid_replication_method.yml │ │ └── target_test.yml │ ├── test_invalid_yaml_config │ │ ├── tap_test_unknown_target.yml │ │ └── target_test.yml │ ├── test_invalid_yaml_config_with_duplicate_targets │ │ ├── tap_test_unknown_target.yml │ │ ├── target_test.yaml │ │ └── target_test.yml │ ├── test_partial_sync │ │ ├── config.json │ │ ├── target_not_supported │ │ │ ├── config.json │ │ │ └── tap_mysql │ │ │ │ └── config.json │ │ └── target_snowflake │ │ │ ├── config.json │ │ │ ├── tap_mysql │ │ │ ├── config.json │ │ │ ├── inheritable_config.json │ │ │ ├── properties.json │ │ │ ├── state.json │ │ │ └── transformation.json │ │ │ ├── tap_mysql_disabled │ │ │ └── config.json │ │ │ ├── tap_no_state │ │ │ ├── config.json │ │ │ ├── inheritable_config.json │ │ │ ├── properties.json │ │ │ └── transformation.json │ │ │ └── tap_not_support │ │ │ └── config.json │ ├── test_post_import_checks │ │ ├── tap_config_pk_not_defined.json │ │ ├── tap_config_pk_not_required.json │ │ ├── tap_config_pk_required.json │ │ ├── tap_config_with_transformations.json │ │ ├── tap_properties_with_no_pk_full_table.json │ │ ├── tap_properties_with_no_pk_incremental.json │ │ ├── tap_properties_with_no_pk_log_based.json │ │ ├── tap_properties_with_no_pk_not_selected.json │ │ ├── tap_properties_with_pk.json │ │ └── transformations.json │ ├── test_reset_state │ │ ├── config.json │ │ └── target_foo │ │ │ ├── tap_bar │ │ │ └── state.json │ │ │ ├── tap_mysql │ │ │ ├── config.json │ │ │ └── state.json │ │ │ └── tap_pg │ │ │ └── state.json │ ├── test_stop_tap │ │ ├── pipelinewise-mock.sh │ │ ├── scheduler-mock.sh │ │ ├── tap-mock.sh │ │ └── target-mock.sh │ ├── test_tap_target_names │ │ ├── tap2_test.yml │ │ ├── tap_2test.yml │ │ ├── tap_test.yml │ │ ├── tap_valid.yaml │ │ └── target_test.yml │ ├── test_validate_command │ │ ├── invalid_target │ │ │ ├── tap_test.yml │ │ │ └── target_test.yml │ │ ├── json_transformation_in_fastsync │ │ │ ├── tap_test.yml │ │ │ └── target_test.yml │ │ ├── missing_replication_key │ │ │ ├── tap_missing_replication_key.yml │ │ │ └── target_test.yml │ │ ├── missing_replication_key_incremental │ │ │ ├── tap_missing_replication_key.yml │ │ │ └── target_test.yml │ │ └── test_yaml_config_two_targets │ │ │ ├── tap_test.yml │ │ │ ├── target_test.yaml │ │ │ └── target_test.yml │ ├── test_yaml_config │ │ ├── config.yml │ │ ├── tap_test.yml │ │ └── target_test.yml │ ├── test_yaml_config_with_slack_channel │ │ ├── config.yml │ │ ├── tap_1_test.yml │ │ ├── tap_2_test.yml │ │ ├── tap_3_test.yml │ │ └── target_test.yml │ ├── transform-config-empty.json │ ├── transform-config.json │ └── vault-secret.txt ├── test_alert_sender.py ├── test_cli.py ├── test_cli_2.py ├── test_cli_utils.py ├── test_cli_utils_tap_github.py ├── test_commands.py ├── test_config.py ├── test_config_validation.py ├── test_partial_sync.py └── test_reset_state.py ├── fastsync ├── __init__.py ├── assertions.py ├── commons │ ├── __init__.py │ ├── resources │ │ ├── dummy_data.csv.gz │ │ ├── properties_mysql.json │ │ ├── properties_postgres.json │ │ └── properties_s3_csv.json │ ├── test_fastsync_tap_mongodb.py │ ├── test_fastsync_tap_mysql.py │ ├── test_fastsync_tap_postgres.py │ ├── test_fastsync_target_postgres.py │ ├── test_fastsync_target_snowflake.py │ ├── test_fastsync_utils.py │ ├── test_split_gzip.py │ └── test_transform_utils.py ├── test_mongodb_to_postgres.py ├── test_mongodb_to_snowflake.py ├── test_mysql_to_postgres.py ├── test_mysql_to_snowflake.py ├── test_postgres_to_postgres.py └── test_postgres_to_snowflake.py ├── logging_custom.conf ├── partialsync ├── __init__.py ├── resources │ ├── test_partial_sync │ │ ├── target_snowflake │ │ │ ├── tap_mysql │ │ │ │ ├── config.json │ │ │ │ ├── properties.json │ │ │ │ └── transformation.json │ │ │ └── tap_postgres │ │ │ │ └── config.json │ │ └── tmp │ │ │ └── target_config_tmp.json │ └── test_partial_sync_utils │ │ ├── __init__.py │ │ └── sample_sf_columns.py ├── test_mysql_to_snowflake.py ├── test_partial_sync_utils.py ├── test_postgres_to_snowflake.py └── utils.py ├── test_logger.py └── test_utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch=True 3 | source = pipelinewise 4 | 5 | [report] 6 | show_missing=False 7 | fail_under=20 8 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.git 2 | **/.virtualenvs 3 | **/.github 4 | **/.circleci 5 | **/.pytest_cache 6 | **/__pycache__ 7 | *.egg-info 8 | *.egg/ 9 | *.rpm 10 | **/venv 11 | **/.venv 12 | **/.coverage 13 | 14 | bin 15 | dev-project 16 | docs 17 | scripts 18 | tests 19 | test-reports 20 | 21 | .coveragerc 22 | .pre-commit-config.yaml 23 | .gitignore 24 | .style.yapf 25 | .yapfignore 26 | CONTRIBUTING.md 27 | CHANGELOG.md 28 | pylintrc 29 | pytest.ini 30 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @transferwise/analytics-platform 2 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Context 2 | 3 | _Describe the problem your PR is trying to solve_ 4 | 5 | 6 | ## Types of changes 7 | 8 | What types of changes does your code introduce to PipelineWise? 9 | _Put an `x` in the boxes that apply_ 10 | 11 | - [ ] Bugfix (non-breaking change which fixes an issue) [optional] 12 | - [ ] New feature (non-breaking change which adds functionality) [optional] 13 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) [optional] 14 | - [ ] Documentation Update (if none of the other choices apply) [optional] 15 | 16 | 17 | ## Checklist 18 | 19 | - [ ] I have read the [CONTRIBUTING](../CONTRIBUTING.md) doc 20 | - [ ] Description above provides context of the change 21 | - [ ] I have added tests that prove my fix is effective or that my feature works 22 | - [ ] Unit tests for changes (not needed for documentation changes) 23 | - [ ] CI checks pass with my changes 24 | - [ ] Branch name starts with `AP-NNN` (if applicable. AP-NNN = JIRA ID) 25 | - [ ] Relevant documentation is updated including usage instructions 26 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # This is an automatically generated base configuration 2 | # For further configuration options and tuning: 3 | # https://docs.github.com/en/free-pro-team@latest/github/administering-a-repository/configuration-options-for-dependency-updates 4 | 5 | version: 2 6 | updates: 7 | - package-ecosystem: "pip" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" 11 | -------------------------------------------------------------------------------- /.github/tw-rules.yaml: -------------------------------------------------------------------------------- 1 | runChecks: true 2 | 3 | actions: 4 | repo-settings: 5 | deleteBranchOnMerge: true 6 | branch-protection-settings: 7 | branches: 8 | - name: master 9 | dismissStaleReviews: true 10 | numRequiredReviews: 1 11 | requireLinearHistory: true 12 | requireConversationResolution: true 13 | requireBranchUpToDate: true 14 | restrictMerge: 15 | teams: 16 | - name: analytics-platform 17 | - name: machine-users 18 | checks: 19 | - name: lint_and_test 20 | type: tests 21 | - name: test_install_connectors 22 | type: tests 23 | - name: publish 24 | type: tests 25 | - name: e2e_tests_target_pg 26 | type: tests 27 | - name: e2e_tests_mariadb_to_sf 28 | type: tests 29 | - name: e2e_tests_pg_to_sf 30 | type: tests 31 | - name: e2e_tests_mg_to_sf 32 | type: tests 33 | - name: e2e_tests_s3_to_sf 34 | type: tests 35 | - name: e2e_tests_target_pg 36 | type: tests 37 | 38 | sync-codeowners: 39 | extraWriters: 40 | - full-time-technical-staff 41 | - machine-users 42 | -------------------------------------------------------------------------------- /.github/workflows/connectors.yml: -------------------------------------------------------------------------------- 1 | # Workflow to check if all singer connectors are installable 2 | name: Singer connectors 3 | 4 | on: 5 | pull_request: 6 | branches: [master] 7 | 8 | workflow_dispatch: 9 | 10 | concurrency: 11 | group: singer-connectors-${{ github.head_ref }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | test_install_connectors: 16 | runs-on: ubuntu-22.04 17 | 18 | steps: 19 | - name: Checking out repo 20 | uses: actions/checkout@v4.1.7 21 | 22 | - name: Check if python changes are present 23 | id: check 24 | env: 25 | GITHUB_REPO: ${{ github.repository }} 26 | PR_NUMBER: ${{ github.event.pull_request.number }} 27 | continue-on-error: true 28 | run: ./scripts/ci_check_no_file_changes.sh python 29 | 30 | - name: Set up Python 3.10 31 | uses: actions/setup-python@v5.1.0 32 | with: 33 | python-version: '3.10' 34 | 35 | - name: Check PipelineWise and all connectors are installable 36 | run: | 37 | make pipelinewise_no_test_extras all_connectors 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .vscode 3 | .idea/* 4 | *.iml 5 | 6 | # Python 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | .virtualenvs 11 | *.egg-info/ 12 | .pytest_cache 13 | .coverage 14 | .coverage.* 15 | coverage.log 16 | coverage_html 17 | test-reports/ 18 | 19 | # Singer Tap state 20 | .state.json 21 | 22 | # Generated test yamls 23 | tests/end_to_end/test-project/*.yml 24 | 25 | # Dev project 26 | dev-project/.pipelinewise 27 | dev-project/.env 28 | dev-project/pipelinewise-config/*_test.yml* 29 | 30 | # Docker 31 | #.env 32 | 33 | *.db 34 | .DS_Store 35 | venv 36 | env 37 | blog_old.md 38 | node_modules 39 | *.pyc 40 | tmp 41 | 42 | # Docs 43 | docs/_build/ 44 | docs/_templates/ 45 | /build/ 46 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/mirrors-yapf 3 | rev: v0.29.0 4 | hooks: 5 | - id: yapf 6 | - repo: https://github.com/pre-commit/pre-commit-hooks 7 | rev: v2.4.0 8 | hooks: 9 | - id: double-quote-string-fixer 10 | - repo: https://github.com/pre-commit/mirrors-pylint 11 | rev: v2.4.4 12 | hooks: 13 | - id: pylint 14 | -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = google 3 | column_limit = 120 -------------------------------------------------------------------------------- /.yapfignore: -------------------------------------------------------------------------------- 1 | setup.py -------------------------------------------------------------------------------- /Dockerfile.barebone: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim-buster 2 | 3 | RUN apt-get -qq update \ 4 | && apt-get -qqy --no-install-recommends install \ 5 | apt-utils \ 6 | alien \ 7 | gnupg \ 8 | libaio1 \ 9 | wget \ 10 | && rm -rf /var/lib/apt/lists/* \ 11 | && pip install -U --no-cache-dir pip 12 | 13 | COPY . /app 14 | 15 | RUN echo "setup pipelinewise" \ 16 | && cd /app \ 17 | && make pipelinewise_no_test_extras -e pw_acceptlicenses=y \ 18 | && ln -s /root/.pipelinewise /app/.pipelinewise 19 | 20 | ENTRYPOINT ["/app/entrypoint.sh"] 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | 2 | include pipelinewise/cli/schemas/*.json 3 | include pipelinewise/cli/samples/README.md 4 | include pipelinewise/cli/samples/*.yml.sample 5 | -------------------------------------------------------------------------------- /dev-project/mongo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mongo:5.0.26-focal 2 | 3 | COPY --chown=mongodb:root --chmod=400 rootCA.pem /etc/ssl/rootCA.pem 4 | COPY --chown=mongodb:root --chmod=400 mongodb.pem /etc/ssl/mongodb.pem 5 | COPY --chown=mongodb:root --chmod=400 replica.key /etc/ssl/replica.key 6 | 7 | COPY create-pipelinewise-user.sh /docker-entrypoint-initdb.d/create-pipelinewise-user.sh 8 | -------------------------------------------------------------------------------- /dev-project/mongo/create-pipelinewise-user.sh: -------------------------------------------------------------------------------- 1 | echo 'CREATE MONGODB PIPELINEWISE USER' 2 | 3 | mongo --tls --tlsAllowInvalidCertificates --tlsAllowInvalidHostnames -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin admin <NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/project/about.rst: -------------------------------------------------------------------------------- 1 | 2 | ******* 3 | Project 4 | ******* 5 | 6 | History 7 | ------- 8 | 9 | PipelineWise was started in August 2018 by Peter Kosztolanyi at Wise 10 | and the project was open sourced at the end of August 2019. 11 | 12 | Committers 13 | ---------- 14 | 15 | - `@koszti `_ (Peter Kosztolanyi) 16 | - `@kasparg `_ (Kaspar Gering) 17 | - `@louis-pie `_ (Louis Pieterse) 18 | - `@samira-el `_ (Samira El Aabidi) 19 | 20 | For the full list of contributors, take a look at `PipelineWise's Github 21 | Contributor page 22 | `_ 23 | 24 | 25 | Contribution 26 | ------------ 27 | 28 | We are happy to receive bug fixes and new features to PipelineWise. 29 | If you want to contribute read further the :ref:`contribution` section. 30 | 31 | Singer Community 32 | ---------------- 33 | 34 | Singer community to discuss issues an features about taps and targets is available on `Slack `_ 35 | 36 | Resources & links 37 | ----------------- 38 | 39 | * `PipelineWise's official documentation `_ 40 | * `Issues on Github `_ 41 | 42 | -------------------------------------------------------------------------------- /docs/snowflake.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/docs/snowflake.png -------------------------------------------------------------------------------- /docs/user_guide/yaml_config.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _yaml_configuration: 3 | 4 | YAML configuration 5 | ------------------ 6 | 7 | .. warning:: 8 | 9 | This section of the documentation is work in progress. 10 | 11 | Pipelines using YAML format for configurations. You can find some information 12 | and usage of these YAML files in the following sections: 13 | 14 | * :ref:`creating_pipelines` 15 | * :ref:`alerts` 16 | * :ref:`tap-jira` 17 | * :ref:`tap-kafka` 18 | * :ref:`tap-mongodb` 19 | * :ref:`tap-mysql` 20 | * :ref:`tap-postgres` 21 | * :ref:`tap-s3-csv` 22 | * :ref:`tap-snowflake` 23 | * :ref:`tap-zendesk` 24 | * :ref:`tap-github` 25 | * :ref:`tap-slack` 26 | * :ref:`tap-mixpanel` 27 | * :ref:`target-postgres` 28 | * :ref:`target-snowflake` 29 | * :ref:`target-s3-csv` 30 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Activate virtual environment 5 | source /app/.virtualenvs/pipelinewise/bin/activate 6 | export PIPELINEWISE_HOME=/app 7 | 8 | WORK_DIR=/app/wrk 9 | mkdir -p ${WORK_DIR} 10 | cd ${WORK_DIR} 11 | 12 | exec pipelinewise \ 13 | "$@" 14 | -------------------------------------------------------------------------------- /motd: -------------------------------------------------------------------------------- 1 | ,_ 2 | :`. .--._ 3 | `.`-. / ',-""""' 4 | `. ``~-._.'_."/ PipelineWise 5 | `~-._ .` `~; 6 | ;. / (C) Wise - $CURRENT_YEAR 7 | / / 8 | jgs ,_.-';_,.'` 9 | `"-;`/ 10 | ,' 11 | 12 | -------------------------------------------------------------------------------- /pipelinewise/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/pipelinewise/__init__.py -------------------------------------------------------------------------------- /pipelinewise/cli/alert_handlers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/pipelinewise/cli/alert_handlers/__init__.py -------------------------------------------------------------------------------- /pipelinewise/cli/alert_handlers/base_alert_handler.py: -------------------------------------------------------------------------------- 1 | """ 2 | PipelineWise CLI - Base class of alert handlers 3 | """ 4 | from abc import ABC, abstractmethod 5 | 6 | 7 | # pylint: disable=too-few-public-methods 8 | class BaseAlertHandler(ABC): 9 | """ 10 | Abstract base class for alert handlers 11 | """ 12 | 13 | LOG = 'log' 14 | INFO = 'info' 15 | WARNING = 'warning' 16 | ERROR = 'error' 17 | 18 | @abstractmethod 19 | # pylint: disable=unnecessary-pass 20 | def send(self, message: str, level: str = ERROR, exc: Exception = None, **kwargs) -> None: 21 | """ 22 | Send alert 23 | 24 | Args: 25 | message: the alert message 26 | level: alert level 27 | exc: optional exception that triggered the alert 28 | 29 | Returns: 30 | Initialised alert handler object 31 | """ 32 | pass 33 | -------------------------------------------------------------------------------- /pipelinewise/cli/alert_handlers/errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | PipelineWise CLI - Alert handler exceptions 3 | """ 4 | 5 | 6 | class NotImplementedAlertHandlerException(Exception): 7 | """ 8 | Exception to raise when attempted to use a not implemented alert handler class 9 | """ 10 | 11 | def __init__(self, *args, **kwargs): 12 | super().__init__(self, *args, **kwargs) 13 | 14 | 15 | class NotConfiguredAlertHandlerException(Exception): 16 | """ 17 | Exception to raise when attempted to use a not configured alert handler 18 | """ 19 | 20 | def __init__(self, *args, **kwargs): 21 | super().__init__(self, *args, **kwargs) 22 | 23 | 24 | class InvalidAlertHandlerException(Exception): 25 | """ 26 | Exception to raise when alert handler not configured correctly 27 | """ 28 | 29 | def __init__(self, *args, **kwargs): 30 | super().__init__(self, *args, **kwargs) 31 | -------------------------------------------------------------------------------- /pipelinewise/cli/constants.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | 4 | class ConnectorType(enum.Enum): 5 | """ 6 | Enums for various Singer connector type names 7 | """ 8 | 9 | TAP_GITHUB = 'tap-github' 10 | TAP_GOOGLE_ANALYTICS = 'tap-google-analytics' 11 | TAP_JIRA = 'tap-jira' 12 | TAP_KAFKA = 'tap-kafka' 13 | TAP_MIXPANEL = 'tap-mixpanel' 14 | TAP_MONGODB = 'tap-mongodb' 15 | TAP_MYSQL = 'tap-mysql' 16 | TAP_ORACLE = 'tap-oracle' 17 | TAP_POSTGRES = 'tap-postgres' 18 | TAP_S3_CSV = 'tap-s3-csv' 19 | TAP_SALESFORCE = 'tap-salesforce' 20 | TAP_SHOPIFY = 'tap-shopify' 21 | TAP_SLACK = 'tap-slack' 22 | TAP_SNOWFLAKE = 'tap-snowflake' 23 | TAP_TWILIO = 'tap-twilio' 24 | TAP_ZENDESK = 'tap-zendesk' 25 | TAP_ZUORA = 'tap-zuora' 26 | 27 | TARGET_BIGQUERY = 'target-bigquery' 28 | TARGET_POSTGRES = 'target-postgres' 29 | TARGET_SNOWFLAKE = 'target-snowflake' 30 | TARGET_REDSHIFT = 'target-redshift' 31 | TARGET_S3_CSV = 'target-s3-csv' 32 | 33 | TRANSFORM_FIELD = 'transform-field' 34 | -------------------------------------------------------------------------------- /pipelinewise/cli/multiprocess.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import traceback 3 | 4 | 5 | class Process(multiprocessing.Process): 6 | """ 7 | This is an extension of Process to let catching raised exceptions inside the 8 | process. 9 | """ 10 | def __init__(self, *args, **kwargs): 11 | multiprocessing.Process.__init__(self, *args, **kwargs) 12 | self._pconn, self._cconn = multiprocessing.Pipe() 13 | self._exception = None 14 | 15 | def run(self): 16 | try: 17 | multiprocessing.Process.run(self) 18 | self._cconn.send(None) 19 | except Exception as exp: 20 | t_b = traceback.format_exc() 21 | self._cconn.send((exp, t_b)) 22 | 23 | @property 24 | def exception(self): 25 | """ 26 | Returning exception of the process 27 | """ 28 | if self._pconn.poll(): 29 | self._exception = self._pconn.recv() 30 | return self._exception 31 | -------------------------------------------------------------------------------- /pipelinewise/cli/samples/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Alert handlers send alerts on run failures 4 | # Configure the certain alert handler if you want to send alerts 5 | # on failures otherwise comment it 6 | alert_handlers: 7 | # slack: 8 | # token: "slack-token" 9 | # channel: "#slack-channel" 10 | 11 | # victorops: 12 | # base_url: "https://alert.victorops.com/integrations/generic/.../alert/.../..." 13 | # routing_key: "victorops-routing-key" 14 | 15 | -------------------------------------------------------------------------------- /pipelinewise/cli/samples/target_postgres.yml.sample: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # ------------------------------------------------------------------------------ 4 | # General Properties 5 | # ------------------------------------------------------------------------------ 6 | id: "postgres_dwh" # Unique identifier of the target 7 | name: "Postgres Data Warehouse" # Name of the target 8 | type: "target-postgres" # !! THIS SHOULD NOT CHANGE !! 9 | 10 | 11 | # ------------------------------------------------------------------------------ 12 | # Target - Data Warehouse connection details 13 | # ------------------------------------------------------------------------------ 14 | db_conn: 15 | host: "" # Postgres host 16 | port: 5432 # Postgres port 17 | user: "" # Postgres user 18 | password: "" # Plain string or vault encrypted 19 | dbname: "" # Postgres database name 20 | #ssl: "true" # Optional: Using SSL via postgres sslmode 'require' option. 21 | # If the server does not accept SSL connections or the client 22 | # certificate is not recognized the connection will fail 23 | 24 | -------------------------------------------------------------------------------- /pipelinewise/fastsync/README.md: -------------------------------------------------------------------------------- 1 | # FastSync 2 | 3 | ## Description 4 | 5 | Fast Sync is one of the Replication Methods that is functionally identical to Full Table 6 | replication but Fast Sync is bypassing the Singer Specification for optimised performance. 7 | Primary use case of FastSync is initial sync or to resyncing large tables with hundreds of 8 | millions of rows where singer component would usually be running for long hours or sometimes 9 | for days. 10 | 11 | PipelineWise detects automatically when Fast Sync gives better performance than the singer 12 | components and uses it automatically whenever it’s possible. 13 | 14 | ## Supported tap-target routes 15 | 16 | 17 | | Source | Destination | 18 | |---------------|-------------------------------------------------------------| 19 | | MySQL/MariaDB | * BigQuery
* Snowflake
* Postgres
* Redshift | 20 | | Postgres | * BigQuery
* Snowflake
* Postgres
* Redshift | 21 | | MongoDB | * BigQuery
* Snowflake
* Postgres
| 22 | -------------------------------------------------------------------------------- /pipelinewise/fastsync/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/pipelinewise/fastsync/__init__.py -------------------------------------------------------------------------------- /pipelinewise/fastsync/commons/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/pipelinewise/fastsync/commons/__init__.py -------------------------------------------------------------------------------- /pipelinewise/fastsync/commons/errors.py: -------------------------------------------------------------------------------- 1 | class ExportError(Exception): 2 | """Raised when export fails""" 3 | 4 | 5 | class TableNotFoundError(Exception): 6 | """Raised when configured table doesn't exist in source""" 7 | 8 | 9 | class MongoDBInvalidDatetimeError(Exception): 10 | """Raised when a bson datetime is invalid and cannot be serialized""" 11 | 12 | 13 | class UnsupportedKeyTypeException(Exception): 14 | """Raised if key type is unsupported""" 15 | -------------------------------------------------------------------------------- /pipelinewise/fastsync/partialsync/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/pipelinewise/fastsync/partialsync/__init__.py -------------------------------------------------------------------------------- /pipelinewise/logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root,pipelinewise 3 | 4 | [handlers] 5 | keys=stderr,stdout 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [logger_pipelinewise] 17 | level=INFO 18 | handlers=stdout 19 | formatter=child 20 | propagate=0 21 | qualname=pipelinewise 22 | 23 | [handler_stderr] 24 | level=INFO 25 | class=StreamHandler 26 | formatter=child 27 | args=(sys.stderr,) 28 | 29 | [handler_stdout] 30 | level=INFO 31 | class=StreamHandler 32 | formatter=child 33 | args=(sys.stdout,) 34 | 35 | [formatter_child] 36 | class=logging.Formatter 37 | format=time=%(asctime)s logger_name=%(name)s log_level=%(levelname)s message=%(message)s 38 | datefmt=%Y-%m-%d %H:%M:%S 39 | 40 | -------------------------------------------------------------------------------- /pipelinewise/logging_debug.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root,pipelinewise 3 | 4 | [handlers] 5 | keys=stderr,stdout 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=DEBUG 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [logger_pipelinewise] 17 | level=DEBUG 18 | handlers=stdout 19 | formatter=child 20 | propagate=0 21 | qualname=pipelinewise 22 | 23 | [handler_stderr] 24 | level=DEBUG 25 | class=StreamHandler 26 | formatter=child 27 | args=(sys.stderr,) 28 | 29 | [handler_stdout] 30 | level=DEBUG 31 | class=StreamHandler 32 | formatter=child 33 | args=(sys.stdout,) 34 | 35 | [formatter_child] 36 | class=logging.Formatter 37 | format=time=%(asctime)s logger_name=%(name)s file=%(pathname)s:%(lineno)d log_level=%(levelname)s message=%(message)s 38 | datefmt=%Y-%m-%d %H:%M:%S 39 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | # https://github.com/boto/boto3/issues/1968 4 | ignore::DeprecationWarning:botocore 5 | ignore::DeprecationWarning:ansible 6 | ignore::DeprecationWarning:tabulate 7 | ignore::DeprecationWarning:_yaml 8 | ignore::DeprecationWarning:messytables 9 | ignore::DeprecationWarning:setuptools 10 | ignore::DeprecationWarning:pandas 11 | -------------------------------------------------------------------------------- /scripts/check_any_file_changed.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | 5 | import requests 6 | 7 | 8 | def main(): 9 | with open(os.environ["GITHUB_EVENT_PATH"], mode="r", encoding="utf-8") as f: 10 | gh_event_data = json.load(f) 11 | PR_URL = gh_event_data["pull_request"]["url"] 12 | 13 | with requests.get( 14 | f"{PR_URL}/files", 15 | headers={"Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}"}, 16 | ) as resp: 17 | files_changed = [f["filename"] for f in resp.json()] 18 | 19 | for f in sys.argv[1:]: 20 | if f in files_changed: 21 | sys.exit(0) 22 | 23 | sys.exit(1) 24 | 25 | 26 | if __name__ == "__main__": 27 | main() 28 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 1.1.0 (2023-09-13) 2 | ------------------ 3 | - Update GHA 4 | - add additional field merge_commit_hash 5 | 6 | 1.0.3 (2021-12-10) 7 | ------------------ 8 | - Fixed pagination issue while fetching comments. 9 | - Lock versions required by test packages. 10 | 11 | 1.0.2 (2021-08-17) 12 | ------------------ 13 | - Skipping repos with size 0 on list all repos 14 | - Fixing wrong order of include/exclude params 15 | - Improving rate throttling logs 16 | - Added 1 minute more to wait on rate limit reset time when rate limit exceed 17 | 18 | 1.0.1 (2021-08-09) 19 | ------------------ 20 | - Do not do rate throtting during discovery 21 | 22 | 1.0.0 (2021-07-19) 23 | ------------------ 24 | 25 | - This is a fork of https://github.com/singer-io/tap-github v1.10.0. 26 | - Add `organization`, `repos_include`, `repos_exclude`, `include_archived` and `include_disabled` options. 27 | - Add `max_rate_limit_wait_seconds` option to wait if you hit the github api limit. 28 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include tap_github/schemas/*.json 3 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/config.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "access_token": "abcdefghijklmnopqrstuvwxyz1234567890ABCD", 3 | "repository": "singer-io/target-stitch", 4 | "start_date": "2021-01-01T00:00:00Z" 5 | } 6 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | setup(name='pipelinewise-tap-github', 9 | version='1.1.1', 10 | description='Singer.io tap for extracting data from the GitHub API', 11 | long_description=long_description, 12 | long_description_content_type='text/markdown', 13 | author='Wise', 14 | url='https://github.com/transferwise/pipelinewise-tap-github', 15 | classifiers=[ 16 | 'License :: OSI Approved :: GNU Affero General Public License v3', 17 | 'Programming Language :: Python :: 3 :: Only' 18 | ], 19 | py_modules=['tap_github'], 20 | install_requires=[ 21 | 'pipelinewise-singer-python==1.*', 22 | 'requests==2.32.2' 23 | ], 24 | extras_require={ 25 | 'test': [ 26 | 'pylint==2.10.2', 27 | 'pytest==6.2.4' 28 | ] 29 | }, 30 | entry_points=''' 31 | [console_scripts] 32 | tap-github=tap_github:main 33 | ''', 34 | packages=['tap_github'], 35 | package_data={ 36 | 'tap_github': ['tap_github/schemas/*.json'] 37 | }, 38 | include_package_data=True 39 | ) 40 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/tap_github/schemas/assignees.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": ["null", "object"], 3 | "additionalProperties": false, 4 | "properties": { 5 | "login": { 6 | "type": ["null", "string"] 7 | }, 8 | "id": { 9 | "type": ["null", "integer"] 10 | }, 11 | "url": { 12 | "type": ["null", "string"] 13 | }, 14 | "type": { 15 | "type": ["null", "string"] 16 | }, 17 | "_sdc_repository": { 18 | "type": ["string"] 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/tap_github/schemas/collaborators.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": ["null", "object"], 3 | "additionalProperties": false, 4 | "properties": { 5 | "login": { 6 | "type": ["null", "string"] 7 | }, 8 | "id": { 9 | "type": ["null", "integer"] 10 | }, 11 | "url": { 12 | "type": ["null", "string"] 13 | }, 14 | "type": { 15 | "type": ["null", "string"] 16 | }, 17 | "_sdc_repository": { 18 | "type": ["string"] 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/tap_github/schemas/issue_labels.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": [ 3 | "null", 4 | "object" 5 | ], 6 | "properties": { 7 | "_sdc_repository": { 8 | "type": [ 9 | "null", 10 | "string" 11 | ] 12 | }, 13 | "id": { 14 | "type": [ 15 | "null", 16 | "number" 17 | ] 18 | }, 19 | "node_id": { 20 | "type": [ 21 | "null", 22 | "string" 23 | ] 24 | }, 25 | "url": { 26 | "type": [ 27 | "null", 28 | "string" 29 | ] 30 | }, 31 | "name": { 32 | "type": [ 33 | "null", 34 | "string" 35 | ] 36 | }, 37 | "description": { 38 | "type": [ 39 | "null", 40 | "string" 41 | ] 42 | }, 43 | "color": { 44 | "type": [ 45 | "null", 46 | "string" 47 | ] 48 | }, 49 | "default": { 50 | "type": [ 51 | "null", 52 | "boolean" 53 | ] 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /singer-connectors/tap-github/tap_github/schemas/project_columns.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": [ 3 | "null", 4 | "object" 5 | ], 6 | "properties": { 7 | "_sdc_repository": { 8 | "type": [ 9 | "null", 10 | "string" 11 | ] 12 | }, 13 | "url": { 14 | "type": [ 15 | "null", 16 | "string" 17 | ] 18 | }, 19 | "project_url": { 20 | "type": [ 21 | "null", 22 | "string" 23 | ] 24 | }, 25 | "cards_url": { 26 | "type": [ 27 | "null", 28 | "string" 29 | ] 30 | }, 31 | "id": { 32 | "type": [ 33 | "null", 34 | "number" 35 | ] 36 | }, 37 | "node_id": { 38 | "type": [ 39 | "null", 40 | "string" 41 | ] 42 | }, 43 | "name": { 44 | "type": [ 45 | "null", 46 | "string" 47 | ] 48 | }, 49 | "created_at": { 50 | "type": [ 51 | "null", 52 | "string" 53 | ], 54 | "format": "date-time" 55 | }, 56 | "updated_at": { 57 | "type": [ 58 | "null", 59 | "string" 60 | ], 61 | "format": "date-time" 62 | } 63 | } 64 | } -------------------------------------------------------------------------------- /singer-connectors/tap-github/tap_github/schemas/releases.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": ["null", "object"], 3 | "additionalProperties": false, 4 | "properties": { 5 | "_sdc_repository": { 6 | "type": ["string"] 7 | }, 8 | "id": { 9 | "type": ["null", "string"] 10 | }, 11 | "url": { 12 | "type": ["null", "string"] 13 | }, 14 | "html_url": { 15 | "type": ["null", "string"] 16 | }, 17 | "target_commitish": { 18 | "type": ["null", "string"] 19 | }, 20 | "tag_name": { 21 | "type": ["null", "string"] 22 | }, 23 | "name": { 24 | "type": ["null", "string"] 25 | }, 26 | "body": { 27 | "type": ["null", "string"] 28 | }, 29 | "draft": { 30 | "type": ["null", "boolean"] 31 | }, 32 | "prerelease": { 33 | "type": ["null", "boolean"] 34 | }, 35 | "author": { 36 | "type": ["null", "object"], 37 | "additionalProperties": false, 38 | "properties": { 39 | "login": { 40 | "type": ["null", "string"] 41 | }, 42 | "id": { 43 | "type": ["null", "integer"] 44 | } 45 | } 46 | }, 47 | "created_at": { 48 | "type": ["null", "string"], 49 | "format": "date-time" 50 | }, 51 | "published_at": { 52 | "type": ["null", "string"], 53 | "format": "date-time" 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /singer-connectors/tap-github/tap_github/schemas/reviews.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": ["null", "object"], 3 | "additionalProperties": false, 4 | "properties": { 5 | "_sdc_repository": { 6 | "type": ["string"] 7 | }, 8 | "id": { 9 | "type": ["null", "integer"] 10 | }, 11 | "user": { 12 | "type": ["null", "object"], 13 | "additionalProperties": false, 14 | "properties": { 15 | "login": { 16 | "type": ["null", "string"] 17 | }, 18 | "id": { 19 | "type": ["null", "integer"] 20 | } 21 | } 22 | }, 23 | "body": { 24 | "type": ["null", "string"] 25 | }, 26 | "state": { 27 | "type": ["null", "string"] 28 | }, 29 | "commit_id": { 30 | "type": ["null", "string"] 31 | }, 32 | "html_url": { 33 | "type": ["null", "string"] 34 | }, 35 | "pull_request_url": { 36 | "type": ["null", "string"] 37 | }, 38 | "submitted_at": { 39 | "type": ["null", "string"], 40 | "format": "date-time" 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/tap_github/schemas/stargazers.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": ["null", "object"], 3 | "additionalProperties": false, 4 | "properties": { 5 | "_sdc_repository": { 6 | "type": ["string"] 7 | }, 8 | "user": { 9 | "type": ["null", "object"], 10 | "additionalProperties": false, 11 | "properties": { 12 | "id": { 13 | "type": ["null", "integer"] 14 | } 15 | } 16 | }, 17 | "starred_at": { 18 | "type": ["null", "string"], 19 | "format": "date-time" 20 | }, 21 | "user_id": { 22 | "type": ["null", "integer"] 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/tap_github/schemas/team_memberships.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": [ 3 | "null", 4 | "object" 5 | ], 6 | "properties": { 7 | "url": { 8 | "type": [ 9 | "null", 10 | "string" 11 | ] 12 | }, 13 | "role": { 14 | "type": [ 15 | "null", 16 | "string" 17 | ] 18 | }, 19 | "state": { 20 | "type": [ 21 | "null", 22 | "string" 23 | ] 24 | }, 25 | "_sdc_repository": { 26 | "type": [ 27 | "null", 28 | "string" 29 | ] 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /singer-connectors/tap-github/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-github/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-github/tests/test_github_sync.py: -------------------------------------------------------------------------------- 1 | from tap_tester import connections 2 | from base import TestGithubBase 3 | 4 | class TestGithubSync(TestGithubBase): 5 | """Test tap sync mode and metadata conforms to standards.""" 6 | 7 | @staticmethod 8 | def name(): 9 | return "tap_tester_github_sync_test" 10 | 11 | def test_run(self): 12 | """ 13 | Testing that sync creates the appropriate catalog with valid metadata. 14 | Verify that all fields and all streams have selected set to True in the metadata 15 | """ 16 | conn_id = connections.ensure_connection(self) 17 | 18 | found_catalogs = self.run_and_verify_check_mode(conn_id) 19 | 20 | self.perform_and_verify_table_and_field_selection(conn_id,found_catalogs) 21 | 22 | record_count_by_stream = self.run_and_verify_sync(conn_id) 23 | 24 | self.assertGreater(sum(record_count_by_stream.values()), 0) 25 | -------------------------------------------------------------------------------- /singer-connectors/tap-github/tests/unittests/test_stargazers_full_table.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | import tap_github.__init__ as tap_github 4 | 5 | @mock.patch("tap_github.__init__.authed_get_all_pages") 6 | class TestStargazersFullTable(unittest.TestCase): 7 | 8 | def test_stargazers_without_query_params(self, mocked_request): 9 | 10 | schemas = {"stargazers": "None"} 11 | 12 | tap_github.get_all_stargazers(schemas, "tap-github", {}, {}, "") 13 | 14 | mocked_request.assert_called_with(mock.ANY, "https://api.github.com/repos/tap-github/stargazers", mock.ANY) 15 | -------------------------------------------------------------------------------- /singer-connectors/tap-jira/requirements.txt: -------------------------------------------------------------------------------- 1 | tap-jira==2.2.0 2 | -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .vscode 3 | .idea/* 4 | 5 | 6 | # Python 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | .virtualenvs 11 | *.egg-info/ 12 | *__pycache__/ 13 | *~ 14 | build/ 15 | dist/ 16 | test_dir/ 17 | tap-kafka-proto-classes/ 18 | .coverage 19 | 20 | # Singer JSON files 21 | properties.json 22 | catalog.json 23 | config.json 24 | state.json 25 | 26 | *.db 27 | .DS_Store 28 | venv 29 | env 30 | blog_old.md 31 | node_modules 32 | *.pyc 33 | tmp 34 | 35 | # Docs 36 | docs/_build/ 37 | docs/_templates/ 38 | -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | setup(name='pipelinewise-tap-kafka', 9 | version='8.2.1', 10 | description='Singer.io tap for extracting data from Kafka topic - PipelineWise compatible', 11 | long_description=long_description, 12 | long_description_content_type='text/markdown', 13 | author='TransferWise', 14 | url='https://github.com/transferwise/pipelinewise-tap-kafka', 15 | classifiers=[ 16 | 'License :: OSI Approved :: GNU Affero General Public License v3', 17 | 'Programming Language :: Python :: 3 :: Only' 18 | ], 19 | install_requires=[ 20 | 'pipelinewise-singer-python==2.*', 21 | 'dpath==2.1.*', 22 | 'confluent-kafka[protobuf]==2.3.*', 23 | 'grpcio-tools==1.57.*' 24 | ], 25 | extras_require={ 26 | 'test': [ 27 | 'pytest==7.4.*', 28 | 'pylint==2.17.*', 29 | 'pytest-cov==4.0.*' 30 | ] 31 | }, 32 | entry_points=''' 33 | [console_scripts] 34 | tap-kafka=tap_kafka:main 35 | ''', 36 | packages=['tap_kafka', 'tap_kafka.serialization'] 37 | ) 38 | -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tap_kafka/errors.py: -------------------------------------------------------------------------------- 1 | class InvalidConfigException(Exception): 2 | """ 3 | Exception to raise when the config is not valid 4 | """ 5 | pass 6 | 7 | 8 | class InvalidBookmarkException(Exception): 9 | """ 10 | Exception to raise when bookmark is not valid 11 | """ 12 | pass 13 | 14 | 15 | class DiscoveryException(Exception): 16 | """ 17 | Exception to raise when discovery failed 18 | """ 19 | pass 20 | 21 | 22 | class InvalidTimestampException(Exception): 23 | """ 24 | Exception to raise when a kafka timestamp tuple is invalid 25 | """ 26 | pass 27 | 28 | 29 | class TimestampNotAvailableException(Exception): 30 | """ 31 | Exception to raise when timestamp not available in a kafka message 32 | """ 33 | pass 34 | 35 | 36 | class ProtobufCompilerException(Exception): 37 | """ 38 | Exception to raise when protobuf compiler fails 39 | """ 40 | pass 41 | 42 | 43 | class AllBrokersDownException(Exception): 44 | """ 45 | Exception to raise when kafka broker is not available 46 | """ 47 | pass 48 | 49 | 50 | class PrimaryKeyNotFoundException(Exception): 51 | """ 52 | Exception to raise if either the custom primary or message key not found in the message 53 | """ 54 | pass 55 | -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tap_kafka/serialization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-kafka/tap_kafka/serialization/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-kafka/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-kafka/tests/integration/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/integration/resources/json_messages_to_produce.json: -------------------------------------------------------------------------------- 1 | {"message_key": "1", "message": {"id":1, "value":"initial id 1"}} 2 | {"message_key": "1", "message": {"id":1, "value":"updated id 1"}} 3 | {"message_key": "2", "message": {"id":2, "value":"initial id 2"}} 4 | {"message_key": "2", "message": {"id":2, "value":"updated id 2"}} 5 | {"message_key": "3", "message": {"id":3, "value":"initial id 3"}} -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/integration/resources/json_messages_to_produce_2.json: -------------------------------------------------------------------------------- 1 | {"message_key": "3", "message": {"id":3, "value":"updated id 3"}} 2 | {"message_key": "4", "message": {"id":4, "value":"initial id 4"}} 3 | {"message_key": "4", "message": {"id":4, "value":"updated id 4"}} -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/integration/resources/protobuf/messages_to_produce.json: -------------------------------------------------------------------------------- 1 | {"message_key": "1", "message": {"name": "Sir Lancelot of Camelot", "favourite_number": 3, "favourite_color": "Blue"}} 2 | {"message_key": "2", "message": {"name": "Sir Galahad of Camelot", "favourite_number": 8, "favourite_color": "Yellow"}} 3 | {"message_key": "3", "message": {"name": "Arthur, King of the Britons", "favourite_number": 12, "favourite_color": null}} -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/integration/resources/protobuf/user.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message User { 4 | string name = 1; 5 | int64 favourite_number = 2; 6 | string favourite_color = 3; 7 | } -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-kafka/tests/unit/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/unit/helper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-kafka/tests/unit/helper/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/unit/resources/kafka-messages-from-multiple-partitions.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "topic": "dummy_topic", 4 | "partition": 1, 5 | "offset": 1, 6 | "timestamp": [1, 1575895711187], 7 | "value": {"result": "SUCCESS", "details": {"id": "1001", "type": "TYPE_1", "profileId": 1234}} 8 | }, 9 | { 10 | "topic": "dummy_topic", 11 | "partition": 2, 12 | "offset": 2, 13 | "timestamp": [1, 1575895711188], 14 | "value": {"result": "SUCCESS", "details": {"id": "1002", "type": "TYPE_2", "profileId": 1234}} 15 | }, 16 | { 17 | "topic": "dummy_topic", 18 | "partition": 2, 19 | "offset": 3, 20 | "timestamp": [1, 1575895711189], 21 | "value": {"result": "SUCCESS", "details": {"id": "1003", "type": "TYPE_3", "profileId": 1234}} 22 | } 23 | ] -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/unit/resources/state-with-bookmark-with-version.json: -------------------------------------------------------------------------------- 1 | {"bookmarks": {"dummy_topic": {"partition_1": {"partition": 1, "offset": 1, "timestamp": 1575895711187}, "version": 9999}}} -------------------------------------------------------------------------------- /singer-connectors/tap-kafka/tests/unit/resources/state-with-bookmark.json: -------------------------------------------------------------------------------- 1 | {"bookmarks": {"dummy_topic": {"partition_1": {"partition": 1, "offset": 1, "timestamp": 1575895711187}}}} -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/.gitignore: -------------------------------------------------------------------------------- 1 | # emacs generated files 2 | \#*\# 3 | .\#* 4 | *~ 5 | 6 | /.vscode/* 7 | /.ipynb_checkpoints/* 8 | config/ 9 | virtualenvs/ 10 | *catalog*.json 11 | *config*.json 12 | *state*.json 13 | target*.json 14 | *.sublime-* 15 | .python-version 16 | singer-check-tap-data 17 | *.pyc 18 | *.egg-info 19 | dist/ 20 | __pycache__/ 21 | venv/ 22 | build/ 23 | tap_mixpanel/.vscode/settings.json 24 | *.ipynb 25 | .DS_Store 26 | tap_target_commands.sh 27 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 1.2.0 (2020-10-29) 2 | ------------------- 3 | 4 | - Add optional `denest_properties` optional param 5 | - Fix key properties in the export stream 6 | 7 | 1.1.0 (2020-10-29) 8 | ------------------- 9 | 10 | - Add key properties to export stream 11 | - Add `mp_reserved_insert_id` to export stream 12 | 13 | 1.0.1 (2020-10-27) 14 | ------------------- 15 | 16 | - Republish to PyPI to have long description 17 | 18 | 1.0.0 (2020-10-27) 19 | ------------------- 20 | 21 | - Add `export_events` optional param 22 | - Remove `page_size` param when checking access 23 | - Few more info log 24 | - Initial fork of https://github.com/singer-io/tap-mixpanel 1.1.0 25 | 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include tap_mixpanel/schemas/*.json 3 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | pylint: 8 | . venv/bin/activate ;\ 9 | pylint tap_mixpanel -d C,W,unexpected-keyword-arg,duplicate-code,too-many-arguments,too-many-locals,too-many-nested-blocks,too-many-statements,too-many-branches,no-else-return,inconsistent-return-statements,no-else-raise,useless-object-inheritance,no-self-argument,raising-non-exception,no-member 10 | 11 | unit_test: 12 | . venv/bin/activate ;\ 13 | pytest tests/unittests 14 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/config.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "api_secret": "YOUR_API_SECRET", 3 | "date_window_size": "30", 4 | "attribution_window": "5", 5 | "project_timezone": "US/Pacific", 6 | "select_properties_by_default": "true", 7 | "start_date": "2019-01-01T00:00:00Z", 8 | "user_agent": "tap-mixpanel " 9 | } 10 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | 4 | [tool:pytest] 5 | addopts = -ra -q 6 | testpaths = 7 | tests/unittests 8 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/state.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "currently_syncing": "engage", 3 | "bookmarks": { 4 | "export": "2020-03-01T18:27:19.000000Z", 5 | "funnels": "2020-03-01T00:00:00.000000Z", 6 | "revenue": "2020-03-01T00:00:00.000000Z" 7 | } 8 | } -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tap_mixpanel/discover.py: -------------------------------------------------------------------------------- 1 | from singer.catalog import Catalog, CatalogEntry, Schema 2 | from tap_mixpanel.schema import get_schemas, STREAMS 3 | 4 | def discover(client, properties_flag, denest_properties): 5 | schemas, field_metadata = get_schemas(client, properties_flag, denest_properties) 6 | catalog = Catalog([]) 7 | 8 | for stream_name, schema_dict in schemas.items(): 9 | schema = Schema.from_dict(schema_dict) 10 | mdata = field_metadata[stream_name] 11 | 12 | catalog.streams.append(CatalogEntry( 13 | stream=stream_name, 14 | tap_stream_id=stream_name, 15 | key_properties=STREAMS[stream_name]['key_properties'], 16 | schema=schema, 17 | metadata=mdata 18 | )) 19 | 20 | return catalog 21 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tap_mixpanel/schemas/annotations.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "date": { 6 | "type": ["null", "string"], 7 | "format": "date-time" 8 | }, 9 | "project_id": { 10 | "type": ["null", "integer"] 11 | }, 12 | "id": { 13 | "type": ["null", "integer"] 14 | }, 15 | "description": { 16 | "type": ["null", "string"] 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tap_mixpanel/schemas/cohort_members.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "cohort_id": { 6 | "type": ["null", "integer"] 7 | }, 8 | "distinct_id": { 9 | "type": ["null", "string"] 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tap_mixpanel/schemas/cohorts.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "id": { 6 | "type": ["null", "integer"] 7 | }, 8 | "name": { 9 | "type": ["null", "string"] 10 | }, 11 | "description": { 12 | "type": ["null", "string"] 13 | }, 14 | "created": { 15 | "type": ["null", "string"], 16 | "format": "date-time" 17 | }, 18 | "count": { 19 | "type": ["null", "integer"] 20 | }, 21 | "is_visible": { 22 | "type": ["null", "integer"] 23 | }, 24 | "project_id": { 25 | "type": ["null", "integer"] 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tap_mixpanel/schemas/engage.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "distinct_id": { 6 | "type": ["null", "string"] 7 | }, 8 | "properties": { 9 | "type": ["null", "object"] 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tap_mixpanel/schemas/export.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "mp_reserved_insert_id": { 6 | "type": ["null", "string"] 7 | }, 8 | "event": { 9 | "type": ["null", "string"] 10 | }, 11 | "distinct_id": { 12 | "type": ["null", "string"] 13 | }, 14 | "time": { 15 | "type": ["null", "string"], 16 | "format": "date-time" 17 | }, 18 | "properties": { 19 | "type": ["null", "object"] 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tap_mixpanel/schemas/revenue.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "date": { 6 | "type": ["null", "string"], 7 | "format": "date" 8 | }, 9 | "datetime": { 10 | "type": ["null", "string"], 11 | "format": "date-time" 12 | }, 13 | "count": { 14 | "type": ["null", "integer"] 15 | }, 16 | "paid_count": { 17 | "type": ["null", "integer"] 18 | }, 19 | "amount": { 20 | "type": ["null", "number"], 21 | "multipleOf": 1e-20 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-mixpanel/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tests/configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-mixpanel/tests/configuration/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tests/configuration/fixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from tap_mixpanel.client import MixpanelClient 3 | 4 | 5 | @pytest.fixture 6 | def mixpanel_client(): 7 | mixpanel_client = MixpanelClient('API_SECRET') 8 | mixpanel_client._MixpanelClient__verified = True 9 | return mixpanel_client 10 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tests/tap_tester/__init__.py: -------------------------------------------------------------------------------- 1 | import tap_tester.imports 2 | 3 | """ Dynamically loads subtests based on this file's location and module name. """ 4 | tap_tester.imports.import_subtests(__file__, __name__) 5 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tests/tap_tester/test_configuration.py: -------------------------------------------------------------------------------- 1 | config = { 2 | "test_name": "tap_mixpanel_combined_test", 3 | "tap_name": "tap-mixpanel", 4 | "type": "platform.mixpanel", 5 | "credentials": { 6 | "api_secret": "TAP_MIXPANEL_API_SECRET" 7 | }, 8 | "streams" : { 9 | "annotations": {"date"}, 10 | "cohort_members": {"cohort_id", "distinct_id"}, 11 | "cohorts": {"id"}, 12 | "engage": {"distinct_id"}, 13 | "export": {"event", "time", "distinct_id"}, 14 | "revenue": {"date"} 15 | }, 16 | "exclude_streams": [ 17 | "annotations", 18 | "cohort_members", 19 | "cohorts", 20 | "engage", 21 | "funnels" 22 | ] 23 | } 24 | 25 | -------------------------------------------------------------------------------- /singer-connectors/tap-mixpanel/tests/unittests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-mixpanel/tests/unittests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-mongodb/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Catchall 2 | * @transferwise/analytics-platform -------------------------------------------------------------------------------- /singer-connectors/tap-mongodb/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := setup 2 | 3 | create_venv: 4 | rm -Rf venv 5 | python3 -m venv venv 6 | 7 | setup_local_db: 8 | chmod u+x bin/setup_local_db.sh 9 | ./bin/setup_local_db.sh 10 | 11 | upgrade_pip: 12 | . venv/bin/activate; \ 13 | python3 -m pip install -U pip setuptools 14 | 15 | populate_db: 16 | venv/bin/python3 ./bin/populate_test_database.py etl secret@1 17 | 18 | install_dep: 19 | . venv/bin/activate; \ 20 | python3 -m pip install -e .[test,dev] 21 | 22 | check_dep: 23 | . venv/bin/activate; \ 24 | python3 -m pip check && echo "No conflicts" || exit 1 25 | 26 | setup: create_venv setup_local_db upgrade_pip install_dep check_dep 27 | echo "Setup is finished" 28 | 29 | pylint: 30 | pylint tap_mongodb tap_mongodb/sync_strategies --rcfile=pylintrc 31 | 32 | test: 33 | pytest tests -v 34 | 35 | test_cov: 36 | pytest --cov=tap_mongodb tests -v --cov-fail-under=42 -------------------------------------------------------------------------------- /singer-connectors/tap-mongodb/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | container_name: "tap_mongodb_dev" 4 | image: "mongo:5.0.26-focal" 5 | 6 | ports: 7 | - 27017:27017 8 | environment: 9 | MONGO_INITDB_ROOT_USERNAME: mongoAdmin # These are dummy credentials for dev and test only 10 | MONGO_INITDB_ROOT_PASSWORD: Password1 11 | command: [mongod, --replSet, rs0] 12 | networks: 13 | - default 14 | 15 | networks: 16 | default: 17 | ipam: 18 | driver: default 19 | config: 20 | - subnet: 192.169.57.0/24 21 | -------------------------------------------------------------------------------- /singer-connectors/tap-mongodb/sample_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "password": "", 3 | "user": "", 4 | "host": "", 5 | "port": "", 6 | "auth_database": "", 7 | "database": "", 8 | "replica_set": "", 9 | "verify_mode": "", 10 | "ssl": "", 11 | "update_buffer_size": "", 12 | "await_time_ms": "" 13 | } -------------------------------------------------------------------------------- /singer-connectors/tap-mongodb/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-mysql/.noserc: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | # enable coverage 3 | with-coverage=1 4 | 5 | # cover only the main package tap_mysql 6 | cover-package=tap_mysql 7 | 8 | # product html coverage report 9 | cover-html=1 10 | 11 | # folder where to produce the html coverage report 12 | cover-html-dir=coverage_report 13 | 14 | # Erase previously collected coverage statistics before run 15 | cover-erase=1 -------------------------------------------------------------------------------- /singer-connectors/tap-mysql/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | pylint: 8 | . ./venv/bin/activate ;\ 9 | pylint --rcfile .pylintrc tap_mysql/ 10 | 11 | unit_test: 12 | . ./venv/bin/activate ;\ 13 | nosetests -c .noserc --cover-min-percentage=47 tests/unit $(extra_args) 14 | 15 | integration_test: 16 | . ./venv/bin/activate ;\ 17 | nosetests -c .noserc tests/integration $(extra_args) 18 | -------------------------------------------------------------------------------- /singer-connectors/tap-mysql/config.json.sample: -------------------------------------------------------------------------------- 1 | { 2 | "host": "", 3 | "port": "", 4 | "user": "", 5 | "password": "", 6 | "filter_db": "db1,db2", 7 | "engine": "", 8 | "use_gtid": , 9 | "cursorclass": "" 10 | "server_id": "", 11 | "database": "", 12 | "ssl": "", 13 | "ssl_ca": "", 14 | "ssl_cert": "", 15 | "ssl_key": "", 16 | "internal_hostname": "", 17 | "session_sqls": [ 18 | "SET @@session.time_zone=\"+0:00\"", 19 | "SET @@session.wait_timeout=28800", 20 | "SET @@session.net_read_timeout=3600", 21 | "SET @@session.innodb_lock_wait_timeout=3600" 22 | ] 23 | } -------------------------------------------------------------------------------- /singer-connectors/tap-mysql/dev/init.sql: -------------------------------------------------------------------------------- 1 | # configure replication user 2 | grant replication client on *.* to 'replication_user'@'%'; 3 | grant replication slave on *.* to 'replication_user'@'%'; 4 | flush privileges; 5 | 6 | use tap_mysql_test; 7 | 8 | # create objects 9 | create table r1 ( 10 | i1 int auto_increment primary key, 11 | c1 varchar(100), 12 | d1 datetime default current_timestamp() 13 | ); 14 | 15 | select * from r1; 16 | 17 | insert into r1 (c1) values ('#1'),('#2'),('#3'),('#4'),('#5'),('#6'),('#7'); 18 | insert into r1 (c1) values ('#8'),('#9'),('#10'),('#11'),('#12'),('#13'),('#14'); 19 | insert into r1 (c1) values ('#15'),('#16'),('#17'),('#18'); 20 | 21 | update r1 set c1=concat(c1, '- updated 1') where i1 < 10; 22 | 23 | create table r2 ( 24 | i2 int primary key, 25 | d2 datetime 26 | ) ; 27 | insert into r2 (i2, d2) values (1, now()), (2, now()), (3, now()), (4, now()); 28 | 29 | update r1 set c1=concat(c1, '- update 2') where i1 >= 10; 30 | 31 | select * from r2; 32 | 33 | delete from r1 where i1 < 4; 34 | 35 | drop table r2; 36 | 37 | alter table r1 add column b1 bool default False; 38 | insert into r1 (c1, b1) values ('#8', True); 39 | -------------------------------------------------------------------------------- /singer-connectors/tap-mysql/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-mysql/tap_mysql/stream_utils.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring,too-many-locals 2 | 3 | import singer 4 | 5 | from singer import metadata 6 | 7 | 8 | def write_schema_message(catalog_entry, bookmark_properties=None): 9 | if bookmark_properties is None: 10 | bookmark_properties = [] 11 | 12 | key_properties = get_key_properties(catalog_entry) 13 | 14 | singer.write_message(singer.SchemaMessage( 15 | stream=catalog_entry.stream, 16 | schema=catalog_entry.schema.to_dict(), 17 | key_properties=key_properties, 18 | bookmark_properties=bookmark_properties 19 | )) 20 | 21 | 22 | def get_key_properties(catalog_entry): 23 | catalog_metadata = metadata.to_map(catalog_entry.metadata) 24 | stream_metadata = catalog_metadata.get((), {}) 25 | 26 | is_view = get_is_view(catalog_entry) 27 | 28 | if is_view: 29 | key_properties = stream_metadata.get('view-key-properties', []) 30 | else: 31 | key_properties = stream_metadata.get('table-key-properties', []) 32 | 33 | return key_properties 34 | 35 | 36 | def get_is_view(catalog_entry): 37 | md_map = metadata.to_map(catalog_entry.metadata) 38 | 39 | return md_map.get((), {}).get('is-view') 40 | -------------------------------------------------------------------------------- /singer-connectors/tap-mysql/tap_mysql/sync_strategies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-mysql/tap_mysql/sync_strategies/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-mysql/tests/unit/sync_strategies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-mysql/tests/unit/sync_strategies/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/bitnami/postgresql:12 2 | 3 | USER root 4 | 5 | RUN apt-get update \ 6 | && apt-get -y install git build-essential \ 7 | && git clone --depth 1 --branch wal2json_2_3 https://github.com/eulerto/wal2json.git \ 8 | && cd /wal2json \ 9 | && make && make install \ 10 | && cd / \ 11 | && rm -rf wal2json \ 12 | && rm -r /var/lib/apt/lists /var/cache/apt/archives 13 | 14 | USER 1001 15 | -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/Makefile: -------------------------------------------------------------------------------- 1 | COV_OPTIONS ?= 2 | 3 | venv: 4 | python3 -m venv venv ;\ 5 | . ./venv/bin/activate ;\ 6 | pip install --upgrade pip setuptools wheel;\ 7 | pip install -e .[test] 8 | 9 | pylint: 10 | . ./venv/bin/activate ;\ 11 | pylint --rcfile .pylintrc tap_postgres/ 12 | 13 | start_db: 14 | docker-compose up -d 15 | 16 | unit_test: 17 | . ./venv/bin/activate ;\ 18 | coverage run --data-file=.coverage.unit --source=tap_postgres -m pytest -v tests/unit ;\ 19 | 20 | unit_test_cov: unit_test 21 | . ./venv/bin/activate ;\ 22 | coverage report --data-file=.coverage.unit --fail-under=58 23 | 24 | integration_test: 25 | . ./venv/bin/activate ;\ 26 | . ./tests/integration/env ;\ 27 | coverage run --data-file=.coverage.integration --source=tap_postgres -m pytest -v tests/integration ;\ 28 | 29 | integration_test_cov: integration_test 30 | . ./venv/bin/activate ;\ 31 | coverage report --data-file=.coverage.integration --fail-under=63 32 | 33 | total_cov: 34 | . ./venv/bin/activate ;\ 35 | coverage combine ;\ 36 | coverage report --fail-under=85 37 | -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/db_setup/config/postgresql.conf: -------------------------------------------------------------------------------- 1 | wal_level = 'logical' 2 | 3 | listen_addresses = '*' 4 | port = '5432' 5 | max_wal_size = '400MB' 6 | max_wal_senders = '16' 7 | wal_keep_segments = '12' 8 | hot_standby = 'on' 9 | fsync = 'on' 10 | shared_preload_libraries = 'pgaudit' 11 | client_min_messages = 'error' -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | 3 | services: 4 | db_primary: 5 | container_name: "primary" 6 | build: 7 | context: . 8 | dockerfile: Dockerfile 9 | ports: 10 | - "5432:5432" 11 | environment: 12 | - POSTGRESQL_REPLICATION_MODE=master 13 | - POSTGRESQL_REPLICATION_USER=repl_user 14 | - POSTGRESQL_REPLICATION_PASSWORD=repl_password 15 | - POSTGRES_USER=test_user 16 | - POSTGRES_PASSWORD=my-secret-passwd 17 | - POSTGRESQL_POSTGRES_PASSWORD=my-secret-passwd 18 | - POSTGRESQL_DATABASE=tap_postgres_test 19 | - ALLOW_EMPTY_PASSWORD=yes 20 | volumes: 21 | - ./db_setup/config:/bitnami/postgresql/conf 22 | 23 | db_replica: 24 | image: "docker.io/bitnami/postgresql:12" 25 | container_name: replica 26 | ports: 27 | - "5433:5432" 28 | depends_on: 29 | - db_primary 30 | environment: 31 | - POSTGRESQL_REPLICATION_MODE=slave 32 | - POSTGRESQL_REPLICATION_USER=repl_user 33 | - POSTGRESQL_REPLICATION_PASSWORD=repl_password 34 | - POSTGRESQL_MASTER_HOST=db_primary 35 | - POSTGRESQL_MASTER_PORT_NUMBER=5432 36 | - ALLOW_EMPTY_PASSWORD=yes 37 | -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | with open('README.md') as f: 6 | long_description = f.read() 7 | 8 | setup(name='pipelinewise-tap-postgres', 9 | version='2.1.0', 10 | description='Singer.io tap for extracting data from PostgresSQL - PipelineWise compatible', 11 | long_description=long_description, 12 | long_description_content_type='text/markdown', 13 | author='Wise', 14 | url='https://github.com/transferwise/pipelinewise-tap-postgres', 15 | classifiers=[ 16 | 'License :: OSI Approved :: GNU Affero General Public License v3', 17 | 'Programming Language :: Python :: 3 :: Only' 18 | ], 19 | python_requires=">=3.10", 20 | install_requires=[ 21 | 'pipelinewise-singer-python==1.*', 22 | 'psycopg2-binary==2.9.5', 23 | 'strict-rfc3339==0.7', 24 | ], 25 | extras_require={ 26 | "test": [ 27 | 'pytest==7.2.2', 28 | 'pylint==2.12.*', 29 | 'pytest-cov==4.0.0' 30 | ] 31 | }, 32 | entry_points=''' 33 | [console_scripts] 34 | tap-postgres=tap_postgres:main 35 | ''', 36 | packages=['tap_postgres', 'tap_postgres.sync_strategies'] 37 | ) 38 | -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/tap_postgres/sync_strategies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-postgres/tap_postgres/sync_strategies/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-postgres/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-postgres/tests/integration/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-postgres/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-postgres/tests/unit/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.egg-info/ 3 | *__pycache__/ 4 | *~ 5 | dist/ 6 | 7 | .vscode 8 | .venv 9 | .idea 10 | venv 11 | virtualenvs 12 | config.json 13 | -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/Makefile: -------------------------------------------------------------------------------- 1 | .EXPORT_ALL_VARIABLES: 2 | 3 | TAP_S3_CSV_ENDPOINT=http://0.0.0.0:9000 4 | TAP_S3_CSV_ACCESS_KEY_ID=ACCESS_KEY 5 | TAP_S3_CSV_SECRET_ACCESS_KEY=SECRET_ACCESS_KEY 6 | TAP_S3_CSV_BUCKET=awesome_bucket 7 | 8 | 9 | venv: 10 | python3 -m venv venv ;\ 11 | . ./venv/bin/activate ;\ 12 | pip install --upgrade pip setuptools wheel;\ 13 | pip install -e .[test] 14 | 15 | pylint: 16 | . ./venv/bin/activate ;\ 17 | pylint --rcfile .pylintrc tap_s3_csv/ 18 | 19 | unit_tests: 20 | . ./venv/bin/activate ;\ 21 | pytest tests/unit --cov=tap_s3_csv --cov-fail-under=30 22 | 23 | integration_tests: 24 | . ./venv/bin/activate ;\ 25 | pytest tests/integration --cov=tap_s3_csv --cov-fail-under=84 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/config.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "aws_endpoint_url": "http://0.0.0.0:9000", 3 | "aws_access_key_id": "ACCESS_KEY", 4 | "aws_secret_access_key": "SECRET_ACCESS_KEY", 5 | "start_date": "2000-01-01T00:00:00Z", 6 | "bucket": "awesome_bucket", 7 | "tables": [{ 8 | "search_prefix": "feeds", 9 | "search_pattern": ".csv", 10 | "table_name": "my_table", 11 | "key_properties": ["id"], 12 | "delimiter": "," 13 | }] 14 | } 15 | -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | 3 | services: 4 | minio_server: 5 | image: "minio/minio" 6 | container_name: "minio_sever" 7 | ports: 8 | - "9000:9000" 9 | - "9090:9090" 10 | user: "${UID}:${GID}" 11 | environment: 12 | MINIO_ROOT_USER: ACCESS_KEY 13 | MINIO_ROOT_PASSWORD: SECRET_ACCESS_KEY 14 | volumes: 15 | - ./minio/data:/data 16 | command: server /data --console-address :9090 17 | -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/tap_s3_csv/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tap configuration related stuff 3 | """ 4 | from voluptuous import Schema, Required, Optional 5 | 6 | CONFIG_CONTRACT = Schema([{ 7 | Required('table_name'): str, 8 | Required('search_pattern'): str, 9 | Optional('key_properties'): [str], 10 | Optional('search_prefix'): str, 11 | Optional('date_overrides'): [str], 12 | Optional('delimiter'): str 13 | }]) 14 | -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-s3-csv/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-s3-csv/tests/integration/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/tests/integration/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def get_config(): 5 | # -------------------------------------------------------------------------- 6 | # Default configuration settings for integration tests. 7 | # -------------------------------------------------------------------------- 8 | # The following values needs to be defined in environment variables with 9 | # valid details to an S3 bucket 10 | # -------------------------------------------------------------------------- 11 | # S3 bucket 12 | config = { 13 | 'aws_endpoint_url': os.environ.get('TAP_S3_CSV_ENDPOINT'), 14 | 'aws_access_key_id': os.environ.get('TAP_S3_CSV_ACCESS_KEY_ID'), 15 | 'aws_secret_access_key': os.environ.get('TAP_S3_CSV_SECRET_ACCESS_KEY'), 16 | 'bucket': os.environ.get('TAP_S3_CSV_BUCKET'), 17 | 'start_date': '2000-01-01', 18 | 'tables': None 19 | } 20 | 21 | return config 22 | 23 | 24 | def get_test_config(): 25 | return get_config() 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-s3-csv/tests/unit/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/tests/unit/test__init.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import io 3 | import unittest 4 | from unittest.mock import patch 5 | 6 | from tap_s3_csv import do_discover 7 | 8 | 9 | class InitTestCase(unittest.TestCase): 10 | 11 | @patch('tap_s3_csv.discover_streams') 12 | def test_do_discover(self, discover_streams): 13 | discover_streams.return_value = [{'stream': '1'}, {'stream': '2'}] 14 | 15 | f = io.StringIO() 16 | with contextlib.redirect_stdout(f): 17 | do_discover({}) 18 | 19 | self.assertEqual( 20 | """{ 21 | "streams": [ 22 | { 23 | "stream": "1" 24 | }, 25 | { 26 | "stream": "2" 27 | } 28 | ] 29 | }""", f.getvalue()) 30 | -------------------------------------------------------------------------------- /singer-connectors/tap-s3-csv/tests/unit/test_s3.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from tap_s3_csv.s3 import generate_schema 4 | 5 | 6 | class TestS3(unittest.TestCase): 7 | def test_generate_schema(self): 8 | samples = [ 9 | dict(id='1', name='productA', added_at='2017/05/18 10:40:22', price='22.99', sold='true', 10 | sold_at='2019-11-29'), 11 | dict(id='4', name='productB', added_at='2017/05/18 10:40:22', price='18', sold='false'), 12 | dict(id='6', name='productC', added_at='2017/05/18 10:40:22', price='14.6', sold='true', 13 | sold_at='2019-12-11'), 14 | ] 15 | 16 | table_specs = { 17 | 'date_overrides': ['added_at'] 18 | } 19 | 20 | schema = generate_schema(samples, table_specs) 21 | 22 | self.assertDictEqual({ 23 | 'id': { 24 | 'type': ['null', 'string'] 25 | }, 26 | 'name': { 27 | 'type': ['null', 'string'] 28 | }, 29 | 'added_at': {'type': ['null', 'string'], 'format': 'date-time'}, 30 | 'price': { 31 | 'type': ['null', 'string'] 32 | }, 33 | 'sold': { 34 | 'type': ['null', 'string'] 35 | }, 36 | 'sold_at': {'type': ['null', 'string']} 37 | }, schema) 38 | -------------------------------------------------------------------------------- /singer-connectors/tap-salesforce/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | pylint: 8 | . ./venv/bin/activate ;\ 9 | pylint tap_salesforce -d missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,no-else-return,chained-comparison,broad-except 10 | -------------------------------------------------------------------------------- /singer-connectors/tap-salesforce/tap_salesforce/salesforce/exceptions.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=super-init-not-called 2 | 3 | class TapSalesforceException(Exception): 4 | pass 5 | 6 | 7 | class TapSalesforceQuotaExceededException(TapSalesforceException): 8 | pass 9 | -------------------------------------------------------------------------------- /singer-connectors/tap-slack/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 1.1.1 (2023-02-28) 2 | ------------------ 3 | - Use `slack-sdk` instead of deprecated `slackclient` library. 4 | - Increase page size when extracting user profiles from `users.list` API endpoint to circumvent rate limiting. 5 | 6 | 1.1.0 (2020-10-20) 7 | ------------------ 8 | 9 | - Extract user profiles from `users.list` API endpoint 10 | - Extract message attachments from `conversations.history` API endpoint 11 | - Fixed an issue when incremental bookmarks were not sent correctly in the `STATE` messages 12 | 13 | 1.0.1 (2020-10-02) 14 | ------------------ 15 | 16 | - Fixed an issue when `thread_ts` values were not populated correctly in `messages` and `threads` streams 17 | 18 | 1.0.0 (2020-09-30) 19 | ------------------- 20 | 21 | - Initial release and fork of https://github.com/singer-io/tap-slack 1.0.0 22 | -------------------------------------------------------------------------------- /singer-connectors/tap-slack/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | pylint: 8 | . ./venv/bin/activate ;\ 9 | pylint tap_slack -d C,W,unexpected-keyword-arg,duplicate-code,too-many-arguments,too-many-locals,too-many-nested-blocks,useless-object-inheritance,no-self-argument,raising-non-exception,no-member 10 | 11 | unit_test: 12 | . ./venv/bin/activate ;\ 13 | pytest tests --cov tap_slack 14 | -------------------------------------------------------------------------------- /singer-connectors/tap-slack/tap_slack/catalog.py: -------------------------------------------------------------------------------- 1 | import singer 2 | 3 | def generate_catalog(streams): 4 | 5 | catalog = {} 6 | catalog['streams'] = [] 7 | for stream in streams: 8 | schema = stream.load_schema() 9 | catalog_entry = { 10 | 'stream': stream.name, 11 | 'tap_stream_id': stream.name, 12 | 'schema': schema, 13 | 'metadata': singer.metadata.get_standard_metadata( 14 | schema=schema, 15 | key_properties=stream.key_properties, 16 | valid_replication_keys=stream.valid_replication_keys, 17 | replication_method=stream.replication_method) 18 | } 19 | catalog['streams'].append(catalog_entry) 20 | 21 | return catalog 22 | -------------------------------------------------------------------------------- /singer-connectors/tap-slack/tap_slack/schemas/channel_members.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": ["null", "object"], 3 | "additionalProperties": false, 4 | "properties": { 5 | "channel_id": { 6 | "type": ["null", "string"] 7 | }, 8 | "user_id": { 9 | "type": ["null", "string"] 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /singer-connectors/tap-slack/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-slack/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-slack/tests/test_schemas.py: -------------------------------------------------------------------------------- 1 | import os 2 | import singer 3 | import jsonschema 4 | 5 | class TestSchemas: 6 | """ Test schemas """ 7 | schemas_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'tap_slack', 'schemas')) 8 | 9 | def get_schemas(self): 10 | """Function to get all json schema paths""" 11 | schemas = [] 12 | for f in os.listdir(self.schemas_dir): 13 | if os.path.isfile(os.path.join(self.schemas_dir, f)): 14 | if '.json' in f: 15 | schemas.append(os.path.join(self.schemas_dir, f)) 16 | 17 | return schemas 18 | 19 | def test_schemas(self): 20 | """Check if every schema is a valid JSON Schema""" 21 | for schema in self.get_schemas(): 22 | s = singer.utils.load_json(schema) 23 | assert jsonschema.Draft7Validator(s).check_schema(s) is None 24 | -------------------------------------------------------------------------------- /singer-connectors/tap-snowflake/.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .vscode 3 | .idea/* 4 | 5 | 6 | # Python 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | .virtualenvs 11 | *.egg-info/ 12 | *__pycache__/ 13 | *~ 14 | dist/ 15 | .coverage 16 | 17 | # Singer JSON files 18 | properties.json 19 | config.json 20 | state.json 21 | 22 | *.db 23 | .DS_Store 24 | venv 25 | env 26 | blog_old.md 27 | node_modules 28 | *.pyc 29 | tmp 30 | 31 | # Docs 32 | docs/_build/ 33 | docs/_templates/ 34 | -------------------------------------------------------------------------------- /singer-connectors/tap-snowflake/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 3.0.0 (2022-03-04) 2 | ------------------ 3 | 4 | - Added private key authentication 5 | - Bumped `snowflake-connector-python[pandas]` from `2.4` to `2.7` 6 | - Dropped python3.6 support 7 | 8 | 2.0.3 (2021-01-11) 9 | ------------------- 10 | 11 | - Stop using `LAST_QUERY_ID()` Snowflake function 12 | - Bumping dependencies 13 | 14 | 2.0.2 (2020-12-04) 15 | ------------------- 16 | 17 | - Bump `snowflake-connector-python` to 2.3.6 18 | 19 | 2.0.1 (2020-07-27) 20 | ------------------- 21 | 22 | - Add optional `pandas` and `pyarrow` packages to avoid runtime warning messages 23 | 24 | 2.0.0 (2020-04-08) 25 | ------------------- 26 | 27 | - Discover only the required tables 28 | 29 | 1.1.2 (2020-03-19) 30 | ------------------- 31 | 32 | - Delete redundant library `pytz` package 33 | 34 | 1.1.1 (2020-03-18) 35 | ------------------- 36 | 37 | - Use `SHOW SCHEMAS|TABLES|COLUMNS` instead of `INFORMATION_SCHEMA` 38 | - Bump `snowflake-connector-python` to 2.2.2 39 | 40 | 1.1.0 (2020-02-20) 41 | ------------------- 42 | 43 | - Make logging customizable 44 | 45 | 1.0.0 (2019-06-02) 46 | ------------------- 47 | 48 | - Initial release 49 | -------------------------------------------------------------------------------- /singer-connectors/tap-snowflake/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | format: 8 | . ./venv/bin/activate ;\ 9 | find tap_snowflake tests -type f -name '*.py' | xargs unify --check-only 10 | 11 | pylint: 12 | . ./venv/bin/activate ;\ 13 | pylint --rcfile pylintrc tap_snowflake/ 14 | 15 | unit_test: 16 | . ./venv/bin/activate ;\ 17 | pytest tests/unit 18 | 19 | integration_test: 20 | . ./venv/bin/activate ;\ 21 | pytest tests/integration/ -vvx --cov tap_snowflake 22 | -------------------------------------------------------------------------------- /singer-connectors/tap-snowflake/config_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "xxxxx.eu-central-1", 3 | "dbname": "database_name", 4 | "user": "my_user", 5 | "role": "my_role", 6 | "password": "password", 7 | "private_key_path": "/path/pk", 8 | "private_key_passphrase": "strong passphrase", 9 | "warehouse": "my_virtual_warehouse", 10 | "tables": "db.schema.table1,db.schema.table2" 11 | } -------------------------------------------------------------------------------- /singer-connectors/tap-snowflake/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-snowflake/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | with open('README.md') as f: 6 | long_description = f.read() 7 | 8 | setup(name='pipelinewise-tap-snowflake', 9 | version='3.1.0', 10 | description='Singer.io tap for extracting data from Snowflake - PipelineWise compatible', 11 | long_description=long_description, 12 | long_description_content_type='text/markdown', 13 | author="TransferWise", 14 | url='https://github.com/transferwise/pipelinewise-tap-snowflake', 15 | classifiers=[ 16 | 'License :: OSI Approved :: Apache Software License', 17 | 'Programming Language :: Python :: 3 :: Only' 18 | ], 19 | py_modules=['tap_snowflake'], 20 | install_requires=[ 21 | 'pipelinewise-singer-python==1.*', 22 | 'snowflake-connector-python[pandas]==3.15.*', 23 | 'pendulum==1.2.0' 24 | ], 25 | extras_require={ 26 | 'test': [ 27 | 'pylint==2.8.*', 28 | 'pytest==6.2.*', 29 | 'pytest-cov==2.12.*', 30 | 'unify==0.5' 31 | ] 32 | }, 33 | entry_points=''' 34 | [console_scripts] 35 | tap-snowflake=tap_snowflake:main 36 | ''', 37 | packages=['tap_snowflake', 'tap_snowflake.sync_strategies'], 38 | ) 39 | -------------------------------------------------------------------------------- /singer-connectors/tap-snowflake/tap_snowflake/sync_strategies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-snowflake/tap_snowflake/sync_strategies/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-snowflake/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-snowflake/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/.gitignore: -------------------------------------------------------------------------------- 1 | /.vscode/* 2 | /.ipynb_checkpoints/* 3 | config/ 4 | virtualenvs/ 5 | *catalog*.json 6 | *config*.json 7 | previous-state.json 8 | target*.json 9 | *.sublime-* 10 | .python-version 11 | singer-check-tap-data 12 | *.pyc 13 | *.egg-info 14 | dist/ 15 | __pycache__/ 16 | venv/ 17 | build/ 18 | tap_twilio/.vscode/settings.json 19 | *.ipynb 20 | .DS_Store 21 | test_configuration.py 22 | tap_target_commands.sh 23 | .idea/ 24 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 1.1.2 (2021-03-16) 2 | ------------------- 3 | - Fix missing elements for streams without ordered response 4 | 5 | 1.1.1 (2021-02-23) 6 | ------------------- 7 | - Fix wrong JSON parsing of state.json 8 | 9 | 1.1.0 (2021-02-22) 10 | ------------------- 11 | - Transforms stringified JSONs to JSON objects 12 | - Enable streams to by synced since the last bookmark or start_date with query parameter 13 | - Get more fields for events stream 14 | 15 | 1.0.2 (2021-01-21) 16 | ------------------- 17 | - Fix publishing to PyPI 18 | 19 | 1.0.1 (2021-01-19) 20 | ------------------- 21 | - Fix UsageRecords and UsageTriggers schemas 22 | - Fix pagination for Account subresources 23 | 24 | 1.0.0 (2021-01-12) 25 | ------------------- 26 | 27 | - Add programmable chat schemas and streams 28 | - Add taskrouter streams 29 | 30 | 0.0.1 (2020-10-23) 31 | ------------------- 32 | 33 | - Initial fork of singer-io/tap-twilio 0.0.1 34 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include tap_twilio/schemas/*.json -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | pylint: 8 | . venv/bin/activate ;\ 9 | pylint tap_twilio --disable 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports' 10 | 11 | test: 12 | . venv/bin/activate ;\ 13 | pytest tests/unit -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/config.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "account_sid": "YOUR_ACCOUNT_SID", 3 | "auth_token": "YOUR_AUTH_TOKEN", 4 | "start_date": "2019-01-01T00:00:00Z", 5 | "user_agent": "tap-twilio ", 6 | } 7 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | setup(name='pipelinewise-tap-twilio', 9 | version='1.1.2', 10 | description='Singer.io tap for extracting data from the Twilio API - PipelineWise compatible', 11 | long_description=long_description, 12 | long_description_content_type='text/markdown', 13 | author='Wise', 14 | url='https://github.com/transferwise/pipelinewise-tap-twilio', 15 | classifiers=[ 16 | 'License :: OSI Approved :: GNU Affero General Public License v3', 17 | 'Programming Language :: Python :: 3 :: Only' 18 | ], 19 | py_modules=['tap_twilio'], 20 | install_requires=[ 21 | 'requests==2.25.*', 22 | 'pipelinewise-singer-python==1.*' 23 | ], 24 | extras_require={ 25 | 'test': [ 26 | 'pylint==2.9.*', 27 | 'pytest==6.2.*' 28 | ] 29 | }, 30 | python_requires='>=3.6', 31 | entry_points=''' 32 | [console_scripts] 33 | tap-twilio=tap_twilio:main 34 | ''', 35 | packages=find_packages(), 36 | package_data={ 37 | 'tap_twilio': [ 38 | 'schemas/*.json' 39 | ] 40 | }) 41 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/state.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "currently_syncing": "accounts", 3 | "bookmarks": { 4 | "accounts": "2020-03-23T10:31:14.000000Z" 5 | } 6 | } -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/discover.py: -------------------------------------------------------------------------------- 1 | from singer.catalog import Catalog, CatalogEntry, Schema 2 | from tap_twilio.schema import get_schemas 3 | from tap_twilio.streams import flatten_streams 4 | 5 | def discover(): 6 | schemas, field_metadata = get_schemas() 7 | catalog = Catalog([]) 8 | 9 | flat_streams = flatten_streams() 10 | for stream_name, schema_dict in schemas.items(): 11 | schema = Schema.from_dict(schema_dict) 12 | mdata = field_metadata[stream_name] 13 | 14 | catalog.streams.append(CatalogEntry( 15 | stream=stream_name, 16 | tap_stream_id=stream_name, 17 | key_properties=flat_streams.get(stream_name, {}).get('key_properties', None), 18 | schema=schema, 19 | metadata=mdata 20 | )) 21 | 22 | return catalog 23 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/account_balance.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "currency": { 6 | "type": ["null", "string"] 7 | }, 8 | "balance": { 9 | "type": ["null", "number"], 10 | "multipleOf": 1e-8 11 | }, 12 | "account_sid": { 13 | "type": ["null", "string"] 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/activities.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "account_sid": { 6 | "type": [ 7 | "null", 8 | "string" 9 | ] 10 | }, 11 | "available": { 12 | "type": [ 13 | "null", 14 | "boolean" 15 | ] 16 | }, 17 | "date_created": { 18 | "type": [ 19 | "null", 20 | "string" 21 | ], 22 | "format": "date-time" 23 | }, 24 | "date_updated": { 25 | "type": [ 26 | "null", 27 | "string" 28 | ], 29 | "format": "date-time" 30 | }, 31 | "friendly_name": { 32 | "type": [ 33 | "null", 34 | "string" 35 | ] 36 | }, 37 | "sid": { 38 | "type": [ 39 | "null", 40 | "string" 41 | ] 42 | }, 43 | "url": { 44 | "type": [ 45 | "null", 46 | "string" 47 | ] 48 | }, 49 | "workspace_sid": { 50 | "type": [ 51 | "null", 52 | "string" 53 | ] 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/addresses.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "account_sid": { 6 | "type": ["null", "string"] 7 | }, 8 | "city": { 9 | "type": ["null", "string"] 10 | }, 11 | "customer_name": { 12 | "type": ["null", "string"] 13 | }, 14 | "date_created": { 15 | "format": "date-time", 16 | "type": ["null", "string"] 17 | }, 18 | "date_updated": { 19 | "format": "date-time", 20 | "type": ["null", "string"] 21 | }, 22 | "emergency_enabled": { 23 | "type": ["null", "boolean"] 24 | }, 25 | "friendly_name": { 26 | "type": ["null", "string"] 27 | }, 28 | "iso_country": { 29 | "type": ["null", "string"] 30 | }, 31 | "postal_code": { 32 | "type": ["null", "string"] 33 | }, 34 | "region": { 35 | "type": ["null", "string"] 36 | }, 37 | "sid": { 38 | "type": ["null", "string"] 39 | }, 40 | "street": { 41 | "type": ["null", "string"] 42 | }, 43 | "validated": { 44 | "type": ["null", "boolean"] 45 | }, 46 | "verified": { 47 | "type": ["null", "boolean"] 48 | }, 49 | "uri": { 50 | "type": ["null", "string"] 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/alerts.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "log_level": { 6 | "type": ["null", "string"] 7 | }, 8 | "resource_sid": { 9 | "type": ["null", "string"] 10 | }, 11 | "date_updated": { 12 | "type": ["null", "string"], 13 | "format": "date-time" 14 | }, 15 | "account_sid": { 16 | "type": ["null", "string"] 17 | }, 18 | "url": { 19 | "type": ["null", "string"] 20 | }, 21 | "request_method": { 22 | "type": ["null", "string"] 23 | }, 24 | "date_generated": { 25 | "type": ["null", "string"], 26 | "format": "date-time" 27 | }, 28 | "alert_text": { 29 | "type": ["null", "string"] 30 | }, 31 | "sid": { 32 | "type": ["null", "string"] 33 | }, 34 | "date_created": { 35 | "type": ["null", "string"], 36 | "format": "date-time" 37 | }, 38 | "request_url": { 39 | "type": ["null", "string"] 40 | }, 41 | "service_sid": { 42 | "type": ["null", "string"] 43 | }, 44 | "error_code": { 45 | "type": ["null", "string"] 46 | }, 47 | "api_version": { 48 | "type": ["null", "string"] 49 | }, 50 | "more_info": { 51 | "type": ["null", "string"] 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/conference_participants.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "account_sid": { 6 | "type": ["null", "string"] 7 | }, 8 | "call_sid": { 9 | "type": ["null", "string"] 10 | }, 11 | "conference_sid": { 12 | "type": ["null", "string"] 13 | }, 14 | "date_created": { 15 | "type": ["null", "string"], 16 | "format": "date-time" 17 | }, 18 | "date_updated": { 19 | "type": ["null", "string"], 20 | "format": "date-time" 21 | }, 22 | "end_conference_on_exit": { 23 | "type": ["null", "boolean"] 24 | }, 25 | "muted": { 26 | "type": ["null", "boolean"] 27 | }, 28 | "hold": { 29 | "type": ["null", "boolean"] 30 | }, 31 | "status": { 32 | "type": ["null", "string"] 33 | }, 34 | "start_conference_on_enter": { 35 | "type": ["null", "boolean"] 36 | }, 37 | "coaching": { 38 | "type": ["null", "boolean"] 39 | }, 40 | "call_sid_to_coach": { 41 | "type": ["null", "string"] 42 | }, 43 | "uri": { 44 | "type": ["null", "string"] 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/keys.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "account_sid": { 6 | "type": ["null", "string"] 7 | }, 8 | "date_created": { 9 | "format": "date-time", 10 | "type": ["null", "string"] 11 | }, 12 | "date_updated": { 13 | "format": "date-time", 14 | "type": ["null", "string"] 15 | }, 16 | "friendly_name": { 17 | "type": ["null", "string"] 18 | }, 19 | "sid": { 20 | "type": ["null", "string"] 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/message_media.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "sid": { 6 | "type": ["null", "string"] 7 | }, 8 | "account_sid": { 9 | "type": ["null", "string"] 10 | }, 11 | "parent_sid": { 12 | "type": ["null", "string"] 13 | }, 14 | "content_type": { 15 | "type": ["null", "string"] 16 | }, 17 | "date_created": { 18 | "type": ["null", "string"], 19 | "format": "date-time" 20 | }, 21 | "date_updated": { 22 | "type": ["null", "string"], 23 | "format": "date-time" 24 | }, 25 | "uri": { 26 | "type": ["null", "string"] 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/outgoing_caller_ids.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "account_sid": { 6 | "type": ["null", "string"] 7 | }, 8 | "date_created": { 9 | "type": ["null", "string"], 10 | "format": "date-time" 11 | }, 12 | "date_updated": { 13 | "type": ["null", "string"], 14 | "format": "date-time" 15 | }, 16 | "friendly_name": { 17 | "type": ["null", "string"] 18 | }, 19 | "phone_number": { 20 | "type": ["null", "string"] 21 | }, 22 | "sid": { 23 | "type": ["null", "string"] 24 | }, 25 | "uri": { 26 | "type": ["null", "string"] 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/roles.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "sid": { 6 | "type": [ 7 | "null", 8 | "string" 9 | ] 10 | }, 11 | "account_sid": { 12 | "type": [ 13 | "null", 14 | "string" 15 | ] 16 | }, 17 | "service_sid": { 18 | "type": [ 19 | "null", 20 | "string" 21 | ] 22 | }, 23 | "friendly_name": { 24 | "type": [ 25 | "null", 26 | "string" 27 | ] 28 | }, 29 | "permissions": { 30 | "type": [ 31 | "null", 32 | "array" 33 | ], 34 | "items": { 35 | "additionalProperties": false, 36 | "type": "string" 37 | } 38 | }, 39 | "date_created": { 40 | "type": [ 41 | "null", 42 | "string" 43 | ], 44 | "format": "date-time" 45 | }, 46 | "date_updated": { 47 | "type": [ 48 | "null", 49 | "string" 50 | ], 51 | "format": "date-time" 52 | }, 53 | "url": { 54 | "type": [ 55 | "null", 56 | "string" 57 | ] 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tap_twilio/schemas/transcriptions.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "additionalProperties": false, 4 | "properties": { 5 | "account_sid": { 6 | "type": ["null", "string"] 7 | }, 8 | "api_version": { 9 | "type": ["null", "string"] 10 | }, 11 | "date_created": { 12 | "type": ["null", "string"], 13 | "format": "date-time" 14 | }, 15 | "date_updated": { 16 | "type": ["null", "string"], 17 | "format": "date-time" 18 | }, 19 | "duration": { 20 | "type": ["null", "integer"] 21 | }, 22 | "price": { 23 | "type": ["null", "number"], 24 | "multipleOf": 1e-8 25 | }, 26 | "price_unit": { 27 | "type": ["null", "string"] 28 | }, 29 | "recording_sid": { 30 | "type": ["null", "string"] 31 | }, 32 | "sid": { 33 | "type": ["null", "string"] 34 | }, 35 | "status": { 36 | "type": ["null", "string"] 37 | }, 38 | "transcription_text": { 39 | "type": ["null", "string"] 40 | }, 41 | "type": { 42 | "type": ["null", "string"] 43 | }, 44 | "uri": { 45 | "type": ["null", "string"] 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /singer-connectors/tap-twilio/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-twilio/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 1.2.1 (2020-07-23) 4 | ------------------ 5 | 6 | - Use `start_time` query parameter for satisfaction_ratings stream 7 | 8 | 1.2.0 (2020-07-20) 9 | ------------------ 10 | 11 | - Make `rate_limit`, `max_workers` and `batch_size` options configurable 12 | 13 | 1.1.0 (2020-02-19) 14 | ------------------ 15 | 16 | - Make logging customizable 17 | 18 | 1.0.0 (2019-11-14) 19 | ------------------ 20 | 21 | - This is a fork of https://github.com/singer-io/tap-zendesk v1.4.6. 22 | - Parallelize zendesk requests 23 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include tap_zendesk/*.json 3 | include tap_zendesk/schemas/*.json 4 | include tap_zendesk/schemas/shared/*.json -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | with open('README.md', 'r') as fh: 6 | long_description = fh.read() 7 | 8 | setup(name='pipelinewise-tap-zendesk', 9 | version='1.2.1', 10 | description='Singer.io tap for extracting data from the Zendesk API', 11 | long_description=long_description, 12 | long_description_content_type='text/markdown', 13 | author='TransferWise', 14 | url='https://github.com/transferwise/pipelinewise-tap-zendesk', 15 | classifiers=['Programming Language :: Python :: 3 :: Only'], 16 | py_modules=['tap_zendesk'], 17 | install_requires=[ 18 | 'pipelinewise-singer-python==1.*', 19 | 'zenpy==2.0.52', 20 | ], 21 | extras_require={ 22 | 'test': [ 23 | 'ipdb', 24 | 'pylint', 25 | 'nose', 26 | 'nose-watch', 27 | ] 28 | }, 29 | entry_points=''' 30 | [console_scripts] 31 | tap-zendesk=tap_zendesk:main 32 | ''', 33 | packages=['tap_zendesk'], 34 | include_package_data=True, 35 | ) 36 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/tap_zendesk/default_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "rate_limit": 1000, 3 | "max_workers": 10, 4 | "batch_size": 50 5 | } -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/tap_zendesk/discover.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=invalid-name,missing-function-docstring 2 | import os 3 | import json 4 | import singer 5 | from tap_zendesk.streams import STREAMS 6 | 7 | def get_abs_path(path): 8 | return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) 9 | 10 | def load_shared_schema_refs(): 11 | ref_sub_path = 'shared' 12 | shared_schemas_path = get_abs_path('schemas/' + ref_sub_path) 13 | 14 | shared_file_names = [f for f in os.listdir(shared_schemas_path) 15 | if os.path.isfile(os.path.join(shared_schemas_path, f))] 16 | 17 | shared_schema_refs = {} 18 | for shared_file in shared_file_names: 19 | with open(os.path.join(shared_schemas_path, shared_file)) as data_file: 20 | shared_schema_refs[ref_sub_path + '/' + shared_file] = json.load(data_file) 21 | 22 | return shared_schema_refs 23 | 24 | def discover_streams(client): 25 | streams = [] 26 | refs = load_shared_schema_refs() 27 | 28 | for s in STREAMS.values(): 29 | s = s(client) 30 | schema = singer.resolve_schema_references(s.load_schema(), refs) 31 | streams.append({'stream': s.name, 'tap_stream_id': s.name, 'schema': schema, 'metadata': s.load_metadata()}) 32 | return streams 33 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/tap_zendesk/schemas/group_memberships.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "default": { 4 | "type": [ 5 | "null", 6 | "boolean" 7 | ] 8 | }, 9 | "url": { 10 | "type": [ 11 | "null", 12 | "string" 13 | ] 14 | }, 15 | "user_id": { 16 | "type": [ 17 | "null", 18 | "integer" 19 | ] 20 | }, 21 | "updated_at": { 22 | "type": [ 23 | "null", 24 | "string" 25 | ], 26 | "format": "date-time" 27 | }, 28 | "group_id": { 29 | "type": [ 30 | "null", 31 | "integer" 32 | ] 33 | }, 34 | "created_at": { 35 | "type": [ 36 | "null", 37 | "string" 38 | ], 39 | "format": "date-time" 40 | }, 41 | "id": { 42 | "type": [ 43 | "null", 44 | "integer" 45 | ] 46 | } 47 | }, 48 | "type": [ 49 | "null", 50 | "object" 51 | ] 52 | } 53 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/tap_zendesk/schemas/groups.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": [ 3 | "null", 4 | "object" 5 | ], 6 | "properties": { 7 | "name": { 8 | "type": [ 9 | "null", 10 | "string" 11 | ] 12 | }, 13 | "created_at": { 14 | "type": [ 15 | "null", 16 | "string" 17 | ], 18 | "format": "date-time" 19 | }, 20 | "url": { 21 | "type": [ 22 | "null", 23 | "string" 24 | ] 25 | }, 26 | "updated_at": { 27 | "type": [ 28 | "null", 29 | "string" 30 | ], 31 | "format": "date-time" 32 | }, 33 | "deleted": { 34 | "type": [ 35 | "null", 36 | "boolean" 37 | ] 38 | }, 39 | "id": { 40 | "type": [ 41 | "null", 42 | "integer" 43 | ] 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/tap_zendesk/schemas/satisfaction_ratings.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": [ 3 | "null", 4 | "object", 5 | "string" 6 | ], 7 | "properties": 8 | { 9 | "id": { 10 | "type": ["null", "integer"] 11 | }, 12 | "assignee_id": { 13 | "type": ["null", "integer"] 14 | }, 15 | "group_id": { 16 | "type": ["null", "integer"] 17 | }, 18 | "reason_id": { 19 | "type": ["null", "integer"] 20 | }, 21 | "requester_id": { 22 | "type": ["null", "integer"] 23 | }, 24 | "ticket_id": { 25 | "type": ["null", "integer"] 26 | }, 27 | "updated_at": { 28 | "type": ["null", "string"], 29 | "format": "date-time" 30 | }, 31 | "created_at": { 32 | "type": ["null", "string"], 33 | "format": "date-time" 34 | }, 35 | "url": { 36 | "type": ["null", "string"] 37 | }, 38 | "score": { 39 | "type": ["null", "string"] 40 | }, 41 | "reason": { 42 | "type": ["null", "string"] 43 | }, 44 | "comment": { 45 | "type": ["null", "string"] 46 | } 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/tap_zendesk/schemas/tags.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "count": { 4 | "type": [ 5 | "null", 6 | "integer" 7 | ] 8 | }, 9 | "name": { 10 | "type": [ 11 | "null", 12 | "string" 13 | ] 14 | } 15 | }, 16 | "type": [ 17 | "null", 18 | "object" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/tap-zendesk/test/__init__.py -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/test/test_init.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from tap_zendesk import get_session 3 | 4 | class TestGetSession(unittest.TestCase): 5 | """ 6 | Confirm that partner information is added to session headers when 7 | present in config. 8 | """ 9 | def test_no_partner_info_returns_none(self): 10 | test_session = get_session({}) 11 | self.assertEqual(test_session, None) 12 | 13 | def test_incomplete_partner_info_returns_none(self): 14 | test_session = get_session({"marketplace_name": "Hithere"}) 15 | self.assertEqual(test_session, None) 16 | 17 | def test_adds_headers_when_all_present_in_config(self): 18 | test_session = get_session({"marketplace_name": "Hithere", 19 | "marketplace_organization_id": 1234, 20 | "marketplace_app_id": 12345}) 21 | self.assertEqual("Hithere", test_session.headers.get("X-Zendesk-Marketplace-Name")) 22 | self.assertEqual("1234", test_session.headers.get("X-Zendesk-Marketplace-Organization-Id")) 23 | self.assertEqual("12345", test_session.headers.get("X-Zendesk-Marketplace-App-Id")) 24 | -------------------------------------------------------------------------------- /singer-connectors/tap-zendesk/test/test_state.json: -------------------------------------------------------------------------------- 1 | {"bookmarks": {"tickets": {"generated_timestamp": "2019-11-12T20:04:02.000000Z"}, "ticket_fields": {"updated_at": "2019-11-08T14:01:27Z"}, "macros": {"updated_at": "2019-11-11T12:22:56Z"}, "satisfaction_ratings": {"updated_at": "2019-05-23T07:48:12Z"}}} 2 | -------------------------------------------------------------------------------- /singer-connectors/target-postgres/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | pylint: 8 | . ./venv/bin/activate ;\ 9 | pylint --rcfile .pylintrc target_postgres/ 10 | 11 | unit_test: 12 | . ./venv/bin/activate ;\ 13 | pytest --cov=target_postgres --cov-fail-under=44 tests/unit -v 14 | 15 | env: 16 | export TARGET_POSTGRES_PORT=5432 17 | export TARGET_POSTGRES_DBNAME=target_db 18 | export TARGET_POSTGRES_USER=my_user 19 | export TARGET_POSTGRES_PASSWORD=secret 20 | export TARGET_POSTGRES_HOST=localhost 21 | export TARGET_POSTGRES_SCHEMA=public 22 | 23 | integration_test: env 24 | . ./venv/bin/activate ;\ 25 | pytest tests/integration --cov=target_postgres --cov-fail-under=87 -v 26 | -------------------------------------------------------------------------------- /singer-connectors/target-postgres/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | db: 5 | image: postgres:12-alpine 6 | environment: 7 | POSTGRES_DB: "target_db" 8 | POSTGRES_USER: "my_user" 9 | POSTGRES_PASSWORD: "secret" 10 | ports: 11 | - 5432:5432 12 | -------------------------------------------------------------------------------- /singer-connectors/target-postgres/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/target-postgres/tests/integration/resources/invalid-json.json: -------------------------------------------------------------------------------- 1 | {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_one"}} 2 | {"type": "SCHEMA", "stream": "tap_mysql_test-test_table_one", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}}, "type": "object"}, "key_properties": ["c_pk"]} 3 | THIS IS A TEST INPUT FROM A TAP WITH A LINE WITH INVALID JSON 4 | {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_one", "version": 1} 5 | -------------------------------------------------------------------------------- /singer-connectors/target-postgres/tests/integration/resources/invalid-message-order.json: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "tap_mysql_test-test_table_one", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}}, "type": "object"}, "key_properties": ["c_pk"]} 2 | {"type": "RECORD", "stream": "tap_mysql_test-test_table_one", "record": {"c_pk": 1, "c_varchar": "1", "c_int": 1}, "version": 1, "time_extracted": "2019-01-31T15:51:47.465408Z"} 3 | {"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-10 02:00:00"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .vscode 3 | .idea/* 4 | 5 | 6 | # Python 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | .virtualenvs 11 | *.egg-info/ 12 | *__pycache__/ 13 | *~ 14 | dist/ 15 | 16 | # Singer JSON files 17 | properties.json 18 | config.json 19 | state.json 20 | 21 | *.db 22 | .DS_Store 23 | venv 24 | env 25 | blog_old.md 26 | node_modules 27 | *.pyc 28 | tmp 29 | 30 | # Docs 31 | docs/_build/ 32 | docs/_templates/ 33 | -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | pylint: 8 | . ./venv/bin/activate ;\ 9 | pylint target_s3_csv -d C,W 10 | 11 | unit_test: 12 | . ./venv/bin/activate ;\ 13 | pytest tests/unit --cov target_s3_csv --cov-fail-under=75 14 | 15 | integration_test: 16 | . ./venv/bin/activate ;\ 17 | pytest tests/integration --cov target_s3_csv --cov-fail-under=72 18 | -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/config.json.sample: -------------------------------------------------------------------------------- 1 | { 2 | "aws_access_key_id": "ACCESS-KEY", 3 | "aws_secret_access_key": "SECRET", 4 | "s3_bucket": "BUCKET", 5 | "s3_key_prefix": "SOME-PREFIX/", 6 | "delimiter": "," 7 | } 8 | -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-s3-csv/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-s3-csv/tests/integration/__init__.py -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/tests/integration/resources/invalid-json.json: -------------------------------------------------------------------------------- 1 | {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_one"}} 2 | {"type": "SCHEMA", "stream": "tap_mysql_test-test_table_one", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}}, "type": "object"}, "key_properties": ["c_pk"]} 3 | THIS IS A TEST INPUT FROM A TAP WITH A LINE WITH INVALID JSON 4 | {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_one", "version": 1} 5 | -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/tests/integration/resources/invalid-message-order.json: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "tap_mysql_test-test_table_one", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}}, "type": "object"}, "key_properties": ["c_pk"]} 2 | {"type": "RECORD", "stream": "tap_mysql_test-test_table_one", "record": {"c_pk": 1, "c_varchar": "1", "c_int": 1}, "version": 1, "time_extracted": "2019-01-31T15:51:47.465408Z"} 3 | {"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-10 02:00:00"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} -------------------------------------------------------------------------------- /singer-connectors/target-s3-csv/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-s3-csv/tests/unit/__init__.py -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .vscode 3 | .idea/* 4 | *.iml 5 | 6 | # Python 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | .virtualenvs 11 | .venv 12 | *.egg-info/ 13 | *__pycache__/ 14 | *~ 15 | dist/ 16 | .coverage 17 | 18 | # Singer JSON files 19 | properties.json 20 | config.json 21 | state.json 22 | 23 | *.db 24 | .DS_Store 25 | venv 26 | env 27 | blog_old.md 28 | node_modules 29 | *.pyc 30 | tmp 31 | 32 | # Docs 33 | docs/_build/ 34 | docs/_templates/ 35 | 36 | # Environment 37 | .env 38 | -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv ;\ 3 | . ./venv/bin/activate ;\ 4 | pip install --upgrade pip setuptools wheel ;\ 5 | pip install -e .[test] 6 | 7 | pylint: 8 | . ./venv/bin/activate ;\ 9 | pylint --rcfile pylintrc target_snowflake/ 10 | 11 | unit_test: 12 | . ./venv/bin/activate ;\ 13 | pytest tests/unit -vv --cov target_snowflake --cov-fail-under=67 14 | 15 | integration_test: 16 | . ./venv/bin/activate ;\ 17 | pytest tests/integration/ -vvx --cov target_snowflake --cov-fail-under=86 18 | -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/flow-diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-snowflake/flow-diagram.jpg -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/target_snowflake/exceptions.py: -------------------------------------------------------------------------------- 1 | """Exceptions used by pipelinewise-target-snowflake""" 2 | 3 | 4 | class RecordValidationException(Exception): 5 | """Exception to raise when record validation failed""" 6 | 7 | 8 | class UnexpectedValueTypeException(Exception): 9 | """Exception to raise when record value type doesn't match the expected schema type""" 10 | 11 | 12 | class InvalidValidationOperationException(Exception): 13 | """Exception to raise when internal JSON schema validation process failed""" 14 | 15 | 16 | class TooManyRecordsException(Exception): 17 | """Exception to raise when query returns more records than max_records""" 18 | 19 | 20 | class FileFormatNotFoundException(Exception): 21 | """Exception to raise when name file format not found""" 22 | 23 | 24 | class InvalidFileFormatException(Exception): 25 | """Exception to raise when name file format is not compatible""" 26 | 27 | 28 | class UnexpectedMessageTypeException(Exception): 29 | """Exception to raise when provided message doesn't match the expected type""" 30 | 31 | 32 | class PrimaryKeyNotFoundException(Exception): 33 | """Exception to raise when primary key not found in the record message""" 34 | -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/target_snowflake/file_formats/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-snowflake/target_snowflake/file_formats/__init__.py -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/target_snowflake/upload_clients/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-snowflake/target_snowflake/upload_clients/__init__.py -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/target_snowflake/upload_clients/base_upload_client.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base class for upload clients 3 | """ 4 | from abc import ABC, abstractmethod 5 | from singer import get_logger 6 | 7 | 8 | class BaseUploadClient(ABC): 9 | """ 10 | Abstract class for upload clients 11 | """ 12 | def __init__(self, connection_config): 13 | self.connection_config = connection_config 14 | self.logger = get_logger('target_snowflake') 15 | 16 | @abstractmethod 17 | def upload_file(self, file: str, stream: str, temp_dir: str = None) -> None: 18 | """ 19 | Upload file 20 | """ 21 | 22 | @abstractmethod 23 | def delete_object(self, stream: str, key: str) -> None: 24 | """ 25 | Delete object 26 | """ 27 | 28 | @abstractmethod 29 | def copy_object(self, copy_source: str, target_bucket: str, target_key: str, target_metadata: dict) -> None: 30 | """ 31 | Copy object 32 | """ 33 | -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-snowflake/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/tests/integration/.env.sample: -------------------------------------------------------------------------------- 1 | TARGET_SNOWFLAKE_ACCOUNT= 2 | TARGET_SNOWFLAKE_DBNAME= 3 | TARGET_SNOWFLAKE_USER= 4 | TARGET_SNOWFLAKE_PASSWORD= 5 | TARGET_SNOWFLAKE_WAREHOUSE= 6 | TARGET_SNOWFLAKE_SCHEMA= 7 | TARGET_SNOWFLAKE_AWS_ACCESS_KEY= 8 | TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY= 9 | TARGET_SNOWFLAKE_S3_ACL= 10 | TARGET_SNOWFLAKE_S3_BUCKET= 11 | TARGET_SNOWFLAKE_S3_KEY_PREFIX= 12 | TARGET_SNOWFLAKE_STAGE= 13 | TARGET_SNOWFLAKE_FILE_FORMAT_CSV= 14 | TARGET_SNOWFLAKE_FILE_FORMAT_PARQUET= 15 | CLIENT_SIDE_ENCRYPTION_MASTER_KEY= 16 | -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-snowflake/tests/integration/__init__.py -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/tests/integration/resources/invalid-json.json: -------------------------------------------------------------------------------- 1 | {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_one"}} 2 | {"type": "SCHEMA", "stream": "tap_mysql_test-test_table_one", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}}, "type": "object"}, "key_properties": ["c_pk"]} 3 | THIS IS A TEST INPUT FROM A TAP WITH A LINE WITH INVALID JSON 4 | {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_one", "version": 1} 5 | -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/tests/integration/resources/invalid-message-order.json: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "tap_mysql_test-test_table_one", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}}, "type": "object"}, "key_properties": ["c_pk"]} 2 | {"type": "RECORD", "stream": "tap_mysql_test-test_table_one", "record": {"c_pk": 1, "c_varchar": "1", "c_int": 1}, "version": 1, "time_extracted": "2019-01-31T15:51:47.465408Z"} 3 | {"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-10 02:00:00"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/tests/integration/resources/messages-with-unexpected-types.json: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream": "db-table_1", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_time": {"format": "time", "inclusion": "available", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["c_pk"]} 2 | {"type": "ACTIVATE_VERSION", "stream": "db-table_1", "version": 2} 3 | {"type": "RECORD", "stream": "db-table_1", "record": {"c_pk": 1, "c_time": "04:00:00"}, "version": 2, "time_extracted": "2019-01-31T15:51:50.215998Z"} 4 | {"type": "RECORD", "stream": "db-table_1", "record": {"c_pk": 2, "c_time": "07:15:00"}, "version": 2, "time_extracted": "2019-01-31T15:51:50.215998Z"} 5 | {"type": "RECORD", "stream": "db-table_1", "record": {"c_pk": 3, "c_time": 235667, "_sdc_deleted_at": "2019-02-10T15:51:50.215998Z"}, "version": 2, "time_extracted": "2019-01-31T15:51:50.215998Z"} 6 | {"type": "STATE", "value": {"currently_syncing": "db-table_1", "bookmarks": {}}} 7 | {"type": "ACTIVATE_VERSION", "stream": "db-table_1", "version": 2} 8 | {"type": "STATE", "value": {"currently_syncing": null, "bookmarks": {"db-table_1": {"initial_full_table_complete": true}}}} 9 | -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-snowflake/tests/unit/__init__.py -------------------------------------------------------------------------------- /singer-connectors/target-snowflake/tests/unit/file_formats/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/target-snowflake/tests/unit/file_formats/__init__.py -------------------------------------------------------------------------------- /singer-connectors/transform-field/.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .vscode 3 | .idea/* 4 | 5 | 6 | # Python 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | .virtualenvs 11 | *.egg-info/ 12 | *~ 13 | dist/ 14 | 15 | # Singer JSON files 16 | properties.json 17 | config.json 18 | state.json 19 | 20 | *.db 21 | .DS_Store 22 | venv 23 | env 24 | blog_old.md 25 | node_modules 26 | *.pyc 27 | tmp 28 | 29 | # Docs 30 | docs/_build/ 31 | docs/_templates/ 32 | -------------------------------------------------------------------------------- /singer-connectors/transform-field/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | ## 2.3.0 (2021-12-16) 4 | ### Added 5 | - Transformation of specific fields in object/array type properties in `RECORD` by using XPath syntax. 6 | - Conditions on specific fields in object/array type properties in `RECORD`. 7 | 8 | ## 2.2.0 (2021-09-17) 9 | ### Added 10 | - New transformation MASK-STRING-SKIP-ENDS-n. The transformation masks the string except start and end n-characters. 11 | 12 | ## 2.1.0 (2021-03-11) 13 | ### Addedd 14 | - `--validate` flag to do one-off validatation of the transformation config using a given catalog file. 15 | 16 | ### Changed 17 | - Validation of the transformation during runtime whenever a new `SCHEMA` type message has been received. 18 | 19 | 20 | ## 2.0.0 (2020-03-17) 21 | 22 | ### Changed 23 | - Stop trimming transformed values 24 | -------------------------------------------------------------------------------- /singer-connectors/transform-field/sample_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations": [ 3 | { 4 | "field_id": "password_hash", 5 | "tap_stream_name": "stream-id-sent-by-the-tap", 6 | "type": "MASK-HIDDEN" 7 | }, 8 | { 9 | "field_id": "salt", 10 | "tap_stream_name": "stream-id-sent-by-the-tap", 11 | "type": "HASH" 12 | }, 13 | { 14 | "field_id": "value", 15 | "tap_stream_name": "stream-id-sent-by-the-tap", 16 | "type": "SET-NULL", 17 | "when": [ 18 | {"column": "string_column_1", "equals": "Property" }, 19 | {"column": "numeric_column", "equals": 200 }, 20 | {"column": "string_column_2", "regex_match": "sensitive.*PII" }, 21 | {"column": "json_column", "field_path": "metadata/comment", "regex_match": "sensitive" } 22 | ] 23 | }, 24 | { 25 | "field_id": "metadata", 26 | "tap_stream_name": "stream-id-sent-by-the-tap", 27 | "type": "MASK-HIDDEN", 28 | "field_paths": ["user/address", "user/zip_code"] 29 | } 30 | ] 31 | } -------------------------------------------------------------------------------- /singer-connectors/transform-field/sample_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=stderr 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [handler_stderr] 17 | level=INFO 18 | class=StreamHandler 19 | formatter=child 20 | args=(sys.stderr,) 21 | 22 | [formatter_child] 23 | class=logging.Formatter 24 | format=time=%(asctime)s name=%(name)s level=%(levelname)s message=%(message)s 25 | datefmt=%Y-%m-%d %H:%M:%S 26 | -------------------------------------------------------------------------------- /singer-connectors/transform-field/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/transform-field/tests/__init__.py -------------------------------------------------------------------------------- /singer-connectors/transform-field/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/transform-field/tests/integration/__init__.py -------------------------------------------------------------------------------- /singer-connectors/transform-field/tests/integration/resources/invalid_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations":[ 3 | { 4 | "tap_stream_name":"dummy_stream", 5 | "field_id":"column_1", 6 | "type":"SET-NULL" 7 | }, 8 | { 9 | "tap_stream_name":"dummy_stream", 10 | "field_id":"column_2", 11 | "type":"HASH" 12 | }, 13 | { 14 | "tap_stream_name": "dummy_stream", 15 | "field_id": "column_5", 16 | "type": "MASK-DATE" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /singer-connectors/transform-field/tests/integration/resources/invalid_messages.json: -------------------------------------------------------------------------------- 1 | {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_one"}} 2 | {"type": "SCHEMA", "stream": "tap_mysql_test-test_table_one", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}}, "type": "object"}, "key_properties": ["c_pk"]} 3 | THIS IS A TEST INPUT FROM A TAP WITH A LINE WITH INVALID JSON 4 | {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_one", "version": 1} 5 | -------------------------------------------------------------------------------- /singer-connectors/transform-field/tests/integration/resources/streams_with_changing_schema.json: -------------------------------------------------------------------------------- 1 | {"type": "SCHEMA", "stream":"dummy_stream", "schema": {"properties": {"column_2": {"type": ["null", "integer"]}}}, "key_properties": []} 2 | {"type": "RECORD", "stream":"dummy_stream", "record": {"column_2": 1}} 3 | {"type": "RECORD", "stream":"dummy_stream", "record": {"column_2": 2}} 4 | {"type": "RECORD", "stream":"dummy_stream", "record": {"column_2": 3}} 5 | {"type": "SCHEMA", "stream":"dummy_stream", "schema": {"properties": {"column_2": {"type": ["null", "string"]}}}, "key_properties": []} 6 | {"type": "RECORD", "stream":"dummy_stream", "record": {"column_2": "ABC"}} -------------------------------------------------------------------------------- /singer-connectors/transform-field/tests/integration/resources/valid_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations":[ 3 | { 4 | "tap_stream_name":"dummy_stream", 5 | "field_id":"column_1", 6 | "type":"SET-NULL" 7 | }, 8 | { 9 | "tap_stream_name":"dummy_stream", 10 | "field_id":"column_2", 11 | "type":"MASK-NUMBER" 12 | }, 13 | { 14 | "tap_stream_name": "dummy_stream", 15 | "field_id": "column_5", 16 | "type": "MASK-DATE" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /singer-connectors/transform-field/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/singer-connectors/transform-field/tests/unit/__init__.py -------------------------------------------------------------------------------- /singer-connectors/transform-field/transform_field/errors.py: -------------------------------------------------------------------------------- 1 | class CatalogRequiredException(Exception): 2 | """Raised when catalog needs to be provided but it has not been""" 3 | 4 | 5 | class StreamNotFoundException(Exception): 6 | """Raised when catalog doesn't have a given selected stream""" 7 | 8 | def __init__(self, stream): 9 | message = f'Catalog doesn\'t have the selected stream `{stream}`!' 10 | 11 | super().__init__(message) 12 | 13 | 14 | class NoStreamSchemaException(Exception): 15 | """Raised when stream has an empty schema""" 16 | 17 | def __init__(self, stream): 18 | message = f'Stream `{stream}` has an empty schema!' 19 | 20 | super().__init__(message) 21 | 22 | 23 | class InvalidTransformationException(Exception): 24 | """Raised when the given transformation is invalid""" 25 | 26 | 27 | class UnsupportedTransformationTypeException(Exception): 28 | """Raised when the given transformation type is not supported""" 29 | 30 | def __init__(self, trans_type): 31 | message = f'Transformation `{trans_type}` is not supported!' 32 | 33 | super().__init__(message) 34 | -------------------------------------------------------------------------------- /singer-connectors/transform-field/transform_field/timings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import time 4 | 5 | from contextlib import contextmanager 6 | 7 | 8 | class Timings: 9 | """Gathers timing information for the three main steps of the Transformer.""" 10 | 11 | def __init__(self, logger): 12 | self.logger = logger 13 | self.last_time = time.time() 14 | self.timings = { 15 | 'validating': 0.0, 16 | 'transforming': 0.0, 17 | None: 0.0 18 | } 19 | 20 | @contextmanager 21 | def mode(self, mode): 22 | """We wrap the big steps of the Tap in this context manager to accumulate 23 | timing info.""" 24 | 25 | start = time.time() 26 | yield 27 | end = time.time() 28 | self.timings[None] += start - self.last_time 29 | self.timings[mode] += end - start 30 | self.last_time = end 31 | 32 | def log_timings(self): 33 | """We call this with every flush to print out the accumulated timings""" 34 | self.logger.debug('Timings: unspecified: %.3f; validating: %.3f; transforming: %.3f;', 35 | self.timings[None], 36 | self.timings['validating'], 37 | self.timings['transforming']) 38 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/__init__.py -------------------------------------------------------------------------------- /tests/db/mongodb_data/all_datatypes.bson.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/db/mongodb_data/all_datatypes.bson.gz -------------------------------------------------------------------------------- /tests/db/mongodb_data/my_collection.bson.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/db/mongodb_data/my_collection.bson.gz -------------------------------------------------------------------------------- /tests/db/target_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Building a test PostgreSQL target database for integration testing of tap-postgres 4 | PWD="$(dirname "$0")" 5 | 6 | echo "Building test PostgreSQL target database..." 7 | 8 | # To run this script some environment variables must be set. 9 | # Normally it's defined in .circleci/config.yml 10 | if [[ -z "${TARGET_POSTGRES_HOST}" || -z "${TARGET_POSTGRES_PORT}" || -z "${TARGET_POSTGRES_USER}" || -z "${TARGET_POSTGRES_PASSWORD}" || -z "${TARGET_POSTGRES_DB}" ]]; then 11 | echo "ERROR: One or more required environment variable is not defined:" 12 | echo " - TARGET_POSTGRES_HOST" 13 | echo " - TARGET_POSTGRES_PORT" 14 | echo " - TARGET_POSTGRES_USER" 15 | echo " - TARGET_POSTGRES_PASSWORD" 16 | echo " - TARGET_POSTGRES_DB" 17 | exit 1 18 | fi 19 | 20 | # Create a postgres password file for non-interaction connection 21 | PGPASSFILE=~/.pgpass 22 | echo ${TARGET_POSTGRES_HOST}:${TARGET_POSTGRES_PORT}:${TARGET_POSTGRES_DB}:${TARGET_POSTGRES_USER}:${TARGET_POSTGRES_PASSWORD} > ${PGPASSFILE} 23 | chmod 0600 ${PGPASSFILE} 24 | 25 | # Build the test Databases 26 | TEST_DB_SQL=${PWD}/target_postgres_data.sql 27 | psql -U ${TARGET_POSTGRES_USER} -h ${TARGET_POSTGRES_HOST} -f ${TEST_DB_SQL} -d ${TARGET_POSTGRES_DB} 28 | -------------------------------------------------------------------------------- /tests/db/target_postgres_data.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Postgres as a target data store requires the pgcrypto extension to apply transformation 3 | -- and obfuscation rules. The required DIGEST function available in the pgcrypto extension 4 | -- 5 | CREATE EXTENSION IF NOT EXISTS pgcrypto; 6 | -------------------------------------------------------------------------------- /tests/end_to_end/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/end_to_end/__init__.py -------------------------------------------------------------------------------- /tests/end_to_end/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | # we want to have pytest assert introspection in the assertions helper 4 | pytest.register_assert_rewrite('tests.end_to_end.helpers.assertions') 5 | -------------------------------------------------------------------------------- /tests/end_to_end/target_snowflake/tap_mariadb/__init__.py: -------------------------------------------------------------------------------- 1 | from tests.end_to_end.target_snowflake import TargetSnowflake 2 | 3 | 4 | class TapMariaDB(TargetSnowflake): 5 | """ 6 | Base class for E2E tests for tap mysql -> target snowflake 7 | """ 8 | 9 | # pylint: disable=arguments-differ 10 | def setUp(self, tap_id: str, target_id: str): 11 | super().setUp(tap_id=tap_id, target_id=target_id, tap_type='TAP_MYSQL') 12 | self.e2e_env.setup_tap_mysql() 13 | -------------------------------------------------------------------------------- /tests/end_to_end/target_snowflake/tap_mongodb/__init__.py: -------------------------------------------------------------------------------- 1 | from tests.end_to_end.target_snowflake import TargetSnowflake 2 | 3 | 4 | class TapMongoDB(TargetSnowflake): 5 | """ 6 | Base class for E2E tests for tap mongodb -> target snowflake 7 | """ 8 | 9 | # pylint: disable=arguments-differ 10 | def setUp(self, tap_id: str, target_id: str): 11 | super().setUp(tap_id=tap_id, target_id=target_id, tap_type='TAP_MONGODB') 12 | self.e2e_env.setup_tap_mongodb() 13 | -------------------------------------------------------------------------------- /tests/end_to_end/target_snowflake/tap_postgres/__init__.py: -------------------------------------------------------------------------------- 1 | from tests.end_to_end.target_snowflake import TargetSnowflake 2 | 3 | 4 | class TapPostgres(TargetSnowflake): 5 | """ 6 | Base class for E2E tests for tap postgres -> target snowflake 7 | """ 8 | 9 | # pylint: disable=arguments-differ 10 | def setUp(self, tap_id: str, target_id: str): 11 | super().setUp(tap_id=tap_id, target_id=target_id, tap_type='TAP_POSTGRES') 12 | self.e2e_env.clean_up_temp_dir() 13 | self.e2e_env.setup_tap_postgres() 14 | -------------------------------------------------------------------------------- /tests/end_to_end/target_snowflake/tap_s3/__init__.py: -------------------------------------------------------------------------------- 1 | from tests.end_to_end.target_snowflake import TargetSnowflake 2 | 3 | 4 | class TapS3(TargetSnowflake): 5 | """ 6 | Base class for E2E tests for tap S3 -> target snowflake 7 | """ 8 | 9 | # pylint: disable=arguments-differ 10 | def setUp(self, tap_id: str, target_id: str): 11 | super().setUp(tap_id=tap_id, target_id=target_id, tap_type='TAP_S3_CSV') 12 | self.e2e_env.setup_tap_s3_csv() 13 | -------------------------------------------------------------------------------- /tests/end_to_end/test-project/target_postgres.yml.template: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # ------------------------------------------------------------------------------ 4 | # General Properties 5 | # ------------------------------------------------------------------------------ 6 | id: "postgres_dwh" # Unique identifier of the target 7 | name: "Postgres Data Warehouse" # Name of the target 8 | type: "target-postgres" # !! THIS SHOULD NOT CHANGE !! 9 | 10 | 11 | # ------------------------------------------------------------------------------ 12 | # Target - Data Warehouse connection details 13 | # ------------------------------------------------------------------------------ 14 | db_conn: 15 | host: "${TARGET_POSTGRES_HOST}" # PostgreSQL host 16 | port: ${TARGET_POSTGRES_PORT} # PostgreSQL port 17 | user: "${TARGET_POSTGRES_USER}" # PostgreSQL user 18 | password: "${TARGET_POSTGRES_PASSWORD}" # Plain string or vault encrypted 19 | dbname: "${TARGET_POSTGRES_DB}" # PostgreSQL database name 20 | -------------------------------------------------------------------------------- /tests/units/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/__init__.py -------------------------------------------------------------------------------- /tests/units/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/cli/__init__.py -------------------------------------------------------------------------------- /tests/units/cli/resources/example-with-full-vault.yml: -------------------------------------------------------------------------------- 1 | !vault | 2 | $ANSIBLE_VAULT;1.1;AES256 3 | 36356635646633303836363135373037333962623066396231616465333263373062316332376366 4 | 3463646538653435313239643861616263323063386238390a366236316435333465373031643765 5 | 66356162356363383166353165623735336438396161636364623237373237356635643461376433 6 | 6665626464363935630a316666343339353765386535316261623639383862646464336433303533 7 | 3734 -------------------------------------------------------------------------------- /tests/units/cli/resources/example-with-invalid-vault.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # A list of tasty fruits 3 | - Apple 4 | - Orange 5 | - Strawberry 6 | - Mango 7 | - !vault | 8 | $ANSIBLE_VAULT;1.1;AES256 9 | 123456789012345678901234567890123456789012345678901234567890 -------------------------------------------------------------------------------- /tests/units/cli/resources/example-with-jinja-env-var.yml: -------------------------------------------------------------------------------- 1 | --- 2 | app: "my-app" 3 | secret: "{{ env_var['APP_SECRET'] }}" 4 | environment: "{{ env_var['APP_ENVIRONMENT'] }}" 5 | -------------------------------------------------------------------------------- /tests/units/cli/resources/example-with-vault.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # A list of tasty fruits 3 | - Apple 4 | - Orange 5 | - Strawberry 6 | - Mango 7 | - !vault | 8 | $ANSIBLE_VAULT;1.1;AES256 9 | 39353938316661653737376632333765636234643638613464303564393431626133313238353334 10 | 3666316661373931363839346630626662393664633731340a613366333563386536316363313432 11 | 37636531343162656430396630383736323564363637303730613133316139633964303466306364 12 | 6231336535353465360a636262316639336134366337373033303265383235393638383035366633 13 | 37643039346438666439616332626430323531363933663430633662323936633630 -------------------------------------------------------------------------------- /tests/units/cli/resources/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "glossary": { 3 | "title": "example glossary", 4 | "GlossDiv": { 5 | "title": "S", 6 | "GlossList": { 7 | "GlossEntry": { 8 | "ID": "SGML", 9 | "SortAs": "SGML", 10 | "GlossTerm": "Standard Generalized Markup Language", 11 | "Acronym": "SGML", 12 | "Abbrev": "ISO 8879:1986", 13 | "GlossDef": { 14 | "para": "A meta-markup language, used to create markup languages such as DocBook.", 15 | "GlossSeeAlso": ["GML", "XML"] 16 | }, 17 | "GlossSee": "markup" 18 | } 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # A list of tasty fruits 3 | - Apple 4 | - Orange 5 | - Strawberry 6 | - Mango -------------------------------------------------------------------------------- /tests/units/cli/resources/invalid.json: -------------------------------------------------------------------------------- 1 | {THIS IS AN INVALID JSON SYNTAX -- [] --} -------------------------------------------------------------------------------- /tests/units/cli/resources/invalid.yml: -------------------------------------------------------------------------------- 1 | THIS IS AN INVALID: YAML: SYNTAX: -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "foo", 3 | "aws_access_key_id": "secret", 4 | "aws_secret_access_key": "secret/", 5 | "client_side_encryption_master_key": "secret=", 6 | "dbname": "my_db", 7 | "file_format": "my_file_format", 8 | "password": "secret", 9 | "s3_bucket": "foo", 10 | "s3_key_prefix": "foo/", 11 | "stage": "my_stage", 12 | "user": "user", 13 | "warehouse": "MY_WAREHOUSE" 14 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_one/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "db_test_mysql", 3 | "filter_dbs": "db_test_mysql", 4 | "host": "localhost", 5 | "password": "password", 6 | "port": 3306, 7 | "user": "user" 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_one/inheritable_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size_rows": 20000, 3 | "data_flattening_max_level": 0, 4 | "hard_delete": true, 5 | "primary_key_required": true, 6 | "schema_mapping": { 7 | "db_test_mysql": { 8 | "target_schema": "db_test_mysql", 9 | "target_schema_select_permissions": [ 10 | "grp_stats" 11 | ] 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_one/selection.json: -------------------------------------------------------------------------------- 1 | { 2 | "selection": [ 3 | { 4 | "replication_method": "LOG_BASED", 5 | "tap_stream_id": "db_test_mysql-table_one", 6 | "sync_start_from": { 7 | "column": "id", 8 | "value": "5" 9 | } 10 | }, 11 | { 12 | "replication_method": "INCREMENTAL", 13 | "replication_key": "id", 14 | "tap_stream_id": "db_test_mysql-table_two" 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_one/state.json.bak: -------------------------------------------------------------------------------- 1 | ORIGINAL FILE DID NOT EXIST! -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_one/transformation.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations": [ 3 | { 4 | "field_id": "email", 5 | "tap_stream_name": "db_test_mysql-table_one", 6 | "type": "HASH", 7 | "when": null 8 | } 9 | ] 10 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_two/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "db_test_postgres", 3 | "filter_dbs": "db_test_postgres", 4 | "host": "localhost", 5 | "password": "password", 6 | "port": 5432, 7 | "user": "user" 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_two/inheritable_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size_rows": 20000, 3 | "data_flattening_max_level": 0, 4 | "hard_delete": true, 5 | "primary_key_required": true, 6 | "schema_mapping": { 7 | "public": { 8 | "target_schema": "db_test_postgres", 9 | "target_schema_select_permissions": null 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_two/selection.json: -------------------------------------------------------------------------------- 1 | { 2 | "selection": [ 3 | { 4 | "replication_method": "LOG_BASED", 5 | "tap_stream_id": "db_test_postgres-public-table_two" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_one/tap_two/transformation.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations": [] 3 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config/target_two/tap_three/placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/cli/resources/sample_json_config/target_two/tap_three/placeholder -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config_for_specific_slack_channel/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "alert_handlers": { 3 | "slack": { 4 | "channel": "#test-channel", 5 | "token": "foo" 6 | } 7 | }, 8 | "targets": [ 9 | { 10 | "id": "target_one", 11 | "name": "Target One", 12 | "type": "target-s3-csv", 13 | "status": "ready", 14 | "taps": [ 15 | { 16 | "enabled": true, 17 | "id": "tap_one", 18 | "name": "Source Three", 19 | "owner": "somebody@transferwise.com", 20 | "type": "tap-mysql", 21 | "send_alert": true, 22 | "slack_alert_channel": "#test-channel-tap-one" 23 | } 24 | ] 25 | } 26 | ] 27 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_json_config_for_specific_slack_channel/target_one/tap_one/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "db_test_mysql", 3 | "filter_dbs": "db_test_mysql", 4 | "host": "localhost", 5 | "password": "password", 6 | "port": 3306, 7 | "user": "user" 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/sample_log_files/tap-run-no-errors.log: -------------------------------------------------------------------------------- 1 | time=2020-07-17 06:20:18 logger_name=tap_mysql log_level=INFO message=Server Parameters: version: x.x.x-MariaDB-log, wait_timeout: 28800, innodb_lock_wait_timeout: 3600, max_allowed_packet: 33554432, interactive_timeout: 28800 2 | time=2020-07-17 06:20:18 logger_name=tap_mysql log_level=INFO message=Server SSL Parameters(blank means SSL is not active): [ssl_version: ], [ssl_cipher: ] 3 | time=2020-07-17 06:20:19 logger_name=tap_mysql log_level=WARNING message=Columns {'column_one'} were selected but are not supported. Skipping them. 4 | time=2020-07-17 06:20:19 logger_name=tap_mysql log_level=WARNING message=Columns {'column_two'} were selected but are not supported. Skipping them. 5 | time=2020-07-17 06:20:19 logger_name=tap_mysql log_level=WARNING message=Columns {'column_three'} were selected but are not supported. Skipping them. 6 | time=2020-07-17 06:20:23 logger_name=target_snowflake log_level=INFO message=Table 'my_schema."TABLE_ONE"' exists 7 | time=2020-07-17 06:20:23 logger_name=target_snowflake log_level=INFO message=Table 'my_schema."TABLE_TWO"' exists 8 | time=2020-07-17 06:20:23 logger_name=target_snowflake log_level=INFO message=Table 'my_schema."TABLE_THREE"' exists -------------------------------------------------------------------------------- /tests/units/cli/resources/tap-github.yml: -------------------------------------------------------------------------------- 1 | id: "github" 2 | name: "Github" 3 | type: "tap-github" 4 | owner: "somebody@foo.com" 5 | sync_period: "*/90 * * * *" 6 | 7 | db_conn: 8 | access_token: "" 9 | start_date: "2021-07-14T00:00:00Z" 10 | organization: "gnome" 11 | repos_include: "gnome* polari" 12 | repos_exclude: "*tests* api-docs" 13 | repository: "gnome/gnome-software" 14 | include_archived: false 15 | include_disabled: false 16 | max_rate_limit_wait_seconds: 600 17 | 18 | target: "snowflake" 19 | batch_size_rows: 20000 20 | 21 | schemas: 22 | - source_schema: "my_db" 23 | target_schema: "repl_my_db" 24 | target_schema_select_permissions: 25 | - grp_stats 26 | tables: 27 | - table_name: "table_one" 28 | replication_method: "INCREMENTAL" 29 | replication_key: "last_update" 30 | - table_name: "table_two" 31 | replication_method: "LOG_BASED" 32 | -------------------------------------------------------------------------------- /tests/units/cli/resources/tap-inheritable-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size_rows": 5000, 3 | "data_flattening_max_level": 0, 4 | "default_target_schema": "jira_clear", 5 | "default_target_schema_select_permissions": [ 6 | "grp_power" 7 | ], 8 | "hard_delete": true, 9 | "primary_key_required": true, 10 | "schema_mapping": { 11 | "jira": { 12 | "target_schema": "jira_clear", 13 | "target_schema_select_permissions": [ 14 | "grp_power" 15 | ] 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/target-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "foo", 3 | "aws_access_key_id": "secret", 4 | "aws_secret_access_key": "secret/", 5 | "client_side_encryption_master_key": "secret=", 6 | "dbname": "my_db", 7 | "file_format": "my_file_format", 8 | "password": "secret", 9 | "s3_bucket": "foo", 10 | "s3_key_prefix": "foo/", 11 | "stage": "my_stage", 12 | "user": "user", 13 | "warehouse": "MY_WAREHOUSE" 14 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/target-invalid-s3-csv.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # ------------------------------------------------------------------------------ 4 | # General Properties 5 | # ------------------------------------------------------------------------------ 6 | id: "s3" # Unique identifier of the target 7 | name: "S3 Target connector" # Name of the target 8 | type: "target-s3-csv" # !! THIS SHOULD NOT CHANGE !! 9 | 10 | 11 | # ------------------------------------------------------------------------------ 12 | # Target - S3 details 13 | # ------------------------------------------------------------------------------ 14 | db_conn: 15 | # Profile based authentication 16 | aws_profile: "" 17 | 18 | # Non-profile based authentication 19 | #aws_access_key_id: "" 20 | #aws_secret_access_key: "" 22 | 23 | s3_bucket_is_required: "" 24 | 25 | s3_key_prefix: "pipelinewise-exports/" 26 | delimiter: "," 27 | quotechar: "\"" -------------------------------------------------------------------------------- /tests/units/cli/resources/target-valid-s3-csv.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # ------------------------------------------------------------------------------ 4 | # General Properties 5 | # ------------------------------------------------------------------------------ 6 | id: "s3" # Unique identifier of the target 7 | name: "S3 Target connector" # Name of the target 8 | type: "target-s3-csv" # !! THIS SHOULD NOT CHANGE !! 9 | 10 | 11 | # ------------------------------------------------------------------------------ 12 | # Target - S3 details 13 | # ------------------------------------------------------------------------------ 14 | db_conn: 15 | # Profile based authentication 16 | aws_profile: "" 17 | 18 | # Non-profile based authentication 19 | #aws_access_key_id: "" 20 | #aws_secret_access_key: "" 22 | 23 | s3_bucket: "" 24 | 25 | s3_key_prefix: "pipelinewise-exports/" 26 | delimiter: "," 27 | quotechar: "\"" -------------------------------------------------------------------------------- /tests/units/cli/resources/test_import_command/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "test_snowflake_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_invalid_tap_mongo_yaml_config/tap_test_invalid_replication_method.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | id: "mysql_sample" 4 | name: "Sample MySQL Database" 5 | type: "tap-mongodb" 6 | owner: "somebody@foo.com" 7 | 8 | 9 | db_conn: 10 | host: "" 11 | port: 3306 12 | user: "" 13 | password: "" 14 | dbname: "" 15 | 16 | target: "test_snowflake_target" 17 | batch_size_rows: 20000 18 | 19 | schemas: 20 | - source_schema: "my_db" 21 | target_schema: "repl_my_db" 22 | target_schema_select_permissions: 23 | - grp_stats 24 | 25 | tables: 26 | - table_name: "table_one" 27 | replication_method: "INCREMENTAL" 28 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_invalid_tap_mongo_yaml_config/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "test_snowflake_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_invalid_yaml_config/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "test_snowflake_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_invalid_yaml_config_with_duplicate_targets/target_test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "my_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_invalid_yaml_config_with_duplicate_targets/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "my_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_not_supported/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "foo", 3 | "aws_access_key_id": "secret", 4 | "aws_secret_access_key": "secret/", 5 | "client_side_encryption_master_key": "secret=", 6 | "dbname": "my_db", 7 | "file_format": "my_file_format", 8 | "password": "secret", 9 | "s3_bucket": "foo", 10 | "s3_key_prefix": "foo/", 11 | "stage": "my_stage", 12 | "user": "user", 13 | "warehouse": "MY_WAREHOUSE" 14 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_not_supported/tap_mysql/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "db_test_mysql", 3 | "filter_dbs": "db_test_mysql", 4 | "host": "localhost", 5 | "password": "password", 6 | "port": 3306, 7 | "user": "user" 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "foo", 3 | "aws_access_key_id": "secret", 4 | "aws_secret_access_key": "secret/", 5 | "client_side_encryption_master_key": "secret=", 6 | "dbname": "my_db", 7 | "file_format": "my_file_format", 8 | "password": "secret", 9 | "s3_bucket": "foo", 10 | "s3_key_prefix": "foo/", 11 | "stage": "my_stage", 12 | "user": "user", 13 | "warehouse": "MY_WAREHOUSE" 14 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_mysql/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "mysql_source_db", 3 | "filter_dbs": "db_test_mysql", 4 | "host": "localhost", 5 | "password": "secret", 6 | "port": 13306, 7 | "user": "pipelinewise" 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_mysql/inheritable_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size_rows": 20000, 3 | "data_flattening_max_level": 0, 4 | "hard_delete": true, 5 | "primary_key_required": true, 6 | "schema_mapping": { 7 | "mysql_source_db": { 8 | "target_schema": "db_test_mysql", 9 | "target_schema_select_permissions": [ 10 | "grp_stats" 11 | ] 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_mysql/state.json: -------------------------------------------------------------------------------- 1 | { 2 | "currently_syncing": null, 3 | "bookmarks": { 4 | "mysql_source_db-full": { 5 | "replication_key": "begin", 6 | "replication_key_value": 7, 7 | "version": 1 8 | }, 9 | "mysql_source_db-table_with_binary": { 10 | "log_file": "mysql-bin.000005", 11 | "log_pos": 286544, 12 | "version": 1 13 | }, 14 | "mysql_source_db-no_pk_table": {}, 15 | "mysql_source_db-weight_unit": { 16 | "log_file": "mysql-bin.000005", 17 | "log_pos": 286544, 18 | "version": 1 19 | }, 20 | "mysql_source_db-edgydata": { 21 | "log_file": "mysql-bin.000005", 22 | "log_pos": 286544, 23 | "version": 1 24 | }, 25 | "mysql_source_db-all_datatypes": { 26 | "log_file": "mysql-bin.000005", 27 | "log_pos": 286544, 28 | "version": 1 29 | }, 30 | "mysql_source_db-table_with_space and UPPERCase": { 31 | "log_file": "mysql-bin.000005", 32 | "log_pos": 286544, 33 | "version": 1 34 | }, 35 | "mysql_source_db-order": {}, 36 | "mysql_source_db-address": { 37 | "replication_key": "date_updated", 38 | "replication_key_value": "2022-03-22T11:47:17+00:00", 39 | "version": 1 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_mysql/transformation.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations": [ 3 | { 4 | "field_id": "email", 5 | "tap_stream_name": "db_test_mysql-table one", 6 | "type": "HASH", 7 | "when": null 8 | } 9 | ] 10 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_mysql_disabled/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "db_test_mysql", 3 | "filter_dbs": "db_test_mysql", 4 | "host": "localhost", 5 | "password": "password", 6 | "port": 3306, 7 | "user": "user" 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_no_state/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "mysql_source_db", 3 | "filter_dbs": "db_test_mysql", 4 | "host": "localhost", 5 | "password": "secret", 6 | "port": 13306, 7 | "user": "pipelinewise" 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_no_state/inheritable_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size_rows": 20000, 3 | "data_flattening_max_level": 0, 4 | "hard_delete": true, 5 | "primary_key_required": true, 6 | "schema_mapping": { 7 | "mysql_source_db": { 8 | "target_schema": "db_test_mysql", 9 | "target_schema_select_permissions": [ 10 | "grp_stats" 11 | ] 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_no_state/transformation.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations": [ 3 | { 4 | "field_id": "email", 5 | "tap_stream_name": "db_test_mysql-table_one", 6 | "type": "HASH", 7 | "when": null 8 | } 9 | ] 10 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_partial_sync/target_snowflake/tap_not_support/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "bootstrap_servers": "127.0.0.2:11000", 3 | "consumer_timeout_ms": 600000, 4 | "encoding": "utf-8", 5 | "group_id": "test_group_id", 6 | "heartbeat_interval_ms": 10000, 7 | "local_store_dir": "/dummy_path", 8 | "max_poll_interval_ms": 720000, 9 | "max_runtime_ms": 900000, 10 | "primary_keys": { 11 | "id": "/id" 12 | }, 13 | "session_timeout_ms": 120000, 14 | "topic": "Test.Topic" 15 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_post_import_checks/tap_config_pk_not_defined.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "my_tap" 3 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_post_import_checks/tap_config_pk_not_required.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "my_tap", 3 | "primary_key_required": false 4 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_post_import_checks/tap_config_pk_required.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "my_tap", 3 | "primary_key_required": true 4 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_post_import_checks/tap_config_with_transformations.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "my_tap", 3 | "primary_key_required": true, 4 | "files": { 5 | "transformation": "./transformations.json" 6 | } 7 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_post_import_checks/transformations.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations" : [ 3 | { 4 | "type": "HASH", 5 | "field_id": "col_1" 6 | } 7 | ] 8 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_reset_state/config.json: -------------------------------------------------------------------------------- 1 | { "switch_over_data_file": "/tmp/switch_over_test.json", 2 | "targets": [ 3 | { 4 | "id": "target_foo", 5 | "type": "target-snowflake", 6 | "taps": [ 7 | { 8 | "id": "tap_pg", 9 | "type": "tap-postgres" 10 | }, 11 | { 12 | "id": "tap_bar", 13 | "type": "tap-bar" 14 | }, 15 | { 16 | "id": "tap_mysql", 17 | "type": "tap-mysql" 18 | } 19 | ] 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_reset_state/target_foo/tap_bar/state.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/cli/resources/test_reset_state/target_foo/tap_bar/state.json -------------------------------------------------------------------------------- /tests/units/cli/resources/test_reset_state/target_foo/tap_mysql/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "new_database_url" 3 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_reset_state/target_foo/tap_mysql/state.json: -------------------------------------------------------------------------------- 1 | {"bookmarks": {"foo_table": {"log_file": "mysql-bin.000001", "log_pos": 1, "version": 1}, "bar_table": {"foo": "bar"}}} -------------------------------------------------------------------------------- /tests/units/cli/resources/test_reset_state/target_foo/tap_pg/state.json: -------------------------------------------------------------------------------- 1 | { 2 | "bookmarks": { 3 | "foo_table": { 4 | "lsn": 456321 5 | }, 6 | "bar_table": { 7 | "foo": "bar" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/test_stop_tap/pipelinewise-mock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This is a mock of main PPW executable that is running 3 | # linux piped tap and target dummy connectors 4 | 5 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 6 | 7 | echo "this is a mock of main PPW executable" 8 | $DIR/tap-mock.sh | $DIR/target-mock.sh 9 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_stop_tap/scheduler-mock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This is a mock of scheduler that starts PPW executable 3 | 4 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 5 | 6 | echo "this is a mock of scheduled that starts PPW executable" 7 | $DIR/pipelinewise-mock.sh 8 | 9 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_stop_tap/tap-mock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This is a mock tap that sends a dummy message to STDOUT in every second 3 | 4 | for i in {1..30} 5 | do 6 | echo "Message generated by tap-mock" 7 | sleep 1 8 | done 9 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_stop_tap/target-mock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This is a mock target that read messages from a tap on STDIN 3 | # and prints the received message to STDOUT 4 | 5 | # Reading message from STDIN 6 | while read line 7 | do 8 | echo "Message received: $line" 9 | done < "${1:-/dev/stdin}" 10 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_tap_target_names/tap2_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/cli/resources/test_tap_target_names/tap2_test.yml -------------------------------------------------------------------------------- /tests/units/cli/resources/test_tap_target_names/tap_2test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/cli/resources/test_tap_target_names/tap_2test.yml -------------------------------------------------------------------------------- /tests/units/cli/resources/test_tap_target_names/tap_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/cli/resources/test_tap_target_names/tap_test.yml -------------------------------------------------------------------------------- /tests/units/cli/resources/test_tap_target_names/tap_valid.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/cli/resources/test_tap_target_names/tap_valid.yaml -------------------------------------------------------------------------------- /tests/units/cli/resources/test_tap_target_names/target_test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/cli/resources/test_tap_target_names/target_test.yml -------------------------------------------------------------------------------- /tests/units/cli/resources/test_validate_command/invalid_target/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "invalid_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "sf_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_validate_command/missing_replication_key/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_validate_command/missing_replication_key_incremental/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_validate_command/test_yaml_config_two_targets/target_test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "test_snowflake_target" 4 | name: "Test Target Connector #2" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_validate_command/test_yaml_config_two_targets/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "test_snowflake_target" 4 | name: "Test Target Connector #1" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_yaml_config/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | alert_handlers: 4 | slack: 5 | token: !vault | 6 | $ANSIBLE_VAULT;1.1;AES256 7 | 39353938316661653737376632333765636234643638613464303564393431626133313238353334 8 | 3666316661373931363839346630626662393664633731340a613366333563386536316363313432 9 | 37636531343162656430396630383736323564363637303730613133316139633964303466306364 10 | 6231336535353465360a636262316639336134366337373033303265383235393638383035366633 11 | 37643039346438666439616332626430323531363933663430633662323936633630 12 | channel: "#slack-channel" 13 | 14 | # victorops: 15 | # base_url: "https://alert.victorops.com/integrations/generic/.../alert/.../..." 16 | # routing_key: "victorops-routing-key" -------------------------------------------------------------------------------- /tests/units/cli/resources/test_yaml_config/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "test_snowflake_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/test_yaml_config_with_slack_channel/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | alert_handlers: 4 | slack: 5 | token: foo 6 | channel: "#slack-channel" 7 | 8 | # victorops: 9 | # base_url: "https://alert.victorops.com/integrations/generic/.../alert/.../..." 10 | # routing_key: "victorops-routing-key" -------------------------------------------------------------------------------- /tests/units/cli/resources/test_yaml_config_with_slack_channel/target_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is a minimalistic target configuration that used only for testing purposes 3 | id: "test_snowflake_target" 4 | name: "Test Target Connector" 5 | type: "target-snowflake" 6 | db_conn: 7 | account: "account" 8 | dbname: "foo_db" 9 | user: "user" 10 | password: "secret" 11 | warehouse: "MY_WAREHOUSE" 12 | s3_bucket: "s3_bucket" 13 | s3_key_prefix: "s3_prefix/" 14 | aws_access_key_id: "access_key_id" 15 | stage: "foo_stage" 16 | file_format: "foo_file_format" 17 | aws_secret_access_key: "secret_access_key" 18 | client_side_encryption_master_key: "master_key" 19 | -------------------------------------------------------------------------------- /tests/units/cli/resources/transform-config-empty.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations": [] 3 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/transform-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations": [ 3 | { 4 | "field_id": "device_name", 5 | "safe_field_id": "\"DEVICE_NAME\"", 6 | "tap_stream_name": "public-payment_token", 7 | "type": "SET-NULL", 8 | "when": null 9 | }, 10 | { 11 | "field_id": "token_pan", 12 | "safe_field_id": "\"TOKEN_PAN\"", 13 | "tap_stream_name": "public-payment_token", 14 | "type": "SET-NULL", 15 | "when": null 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /tests/units/cli/resources/vault-secret.txt: -------------------------------------------------------------------------------- 1 | vault-secret-password -------------------------------------------------------------------------------- /tests/units/fastsync/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/fastsync/__init__.py -------------------------------------------------------------------------------- /tests/units/fastsync/commons/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/fastsync/commons/__init__.py -------------------------------------------------------------------------------- /tests/units/fastsync/commons/resources/dummy_data.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/fastsync/commons/resources/dummy_data.csv.gz -------------------------------------------------------------------------------- /tests/units/logging_custom.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root,pipelinewise 3 | 4 | [handlers] 5 | keys=stderr,stdout 6 | 7 | [formatters] 8 | keys=child 9 | 10 | [logger_root] 11 | level=DEBUG 12 | handlers=stderr 13 | formatter=child 14 | propagate=0 15 | 16 | [logger_pipelinewise] 17 | level=DEBUG 18 | handlers=stdout 19 | formatter=child 20 | propagate=0 21 | qualname=pipelinewise 22 | 23 | [handler_stderr] 24 | level=DEBUG 25 | class=StreamHandler 26 | formatter=child 27 | args=(sys.stderr,) 28 | 29 | [handler_stdout] 30 | level=DEBUG 31 | class=StreamHandler 32 | formatter=child 33 | args=(sys.stdout,) 34 | 35 | [formatter_child] 36 | class=logging.Formatter 37 | format=time=%(asctime)s logger_name=%(name)s file=%(pathname)s:%(lineno)d log_level=%(levelname)s message=%(message)s 38 | datefmt=%Y-%m-%d %H:%M:%S 39 | -------------------------------------------------------------------------------- /tests/units/partialsync/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/partialsync/__init__.py -------------------------------------------------------------------------------- /tests/units/partialsync/resources/test_partial_sync/target_snowflake/tap_mysql/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "mysql_source_db", 3 | "filter_dbs": "db_test_mysql", 4 | "host": "localhost", 5 | "password": "secret", 6 | "port": 3306, 7 | "user": "db_user" 8 | } -------------------------------------------------------------------------------- /tests/units/partialsync/resources/test_partial_sync/target_snowflake/tap_mysql/transformation.json: -------------------------------------------------------------------------------- 1 | { 2 | "transformations": [ 3 | { 4 | "field_id": "email", 5 | "tap_stream_name": "db_test_mysql-table_one", 6 | "type": "HASH", 7 | "when": null 8 | } 9 | ] 10 | } -------------------------------------------------------------------------------- /tests/units/partialsync/resources/test_partial_sync/target_snowflake/tap_postgres/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbname": "postgres_source_db", 3 | "host": "db_postgres_source", 4 | "password": "secret", 5 | "port": 5432, 6 | "tap_id": "test_postgres", 7 | "user": "pipelinewise" 8 | } -------------------------------------------------------------------------------- /tests/units/partialsync/resources/test_partial_sync_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/transferwise/pipelinewise/e4ba417e2a5cc35ce2803e1ac36e4125c26cfbd8/tests/units/partialsync/resources/test_partial_sync_utils/__init__.py --------------------------------------------------------------------------------