├── tests ├── __init__.py ├── fakes │ ├── mock_source.yml │ ├── mock_destination.yml │ └── test_backup.yml ├── test_airbyte_response.py ├── test_config_validator.py ├── test_airbyte_config_model.py ├── test_airbyte_dto_factory.py └── test_fixtures.py ├── requirements.txt ├── utils.py ├── .codecov.yml ├── examples ├── example_secrets.yml └── config.yml ├── .cirrus.yml ├── .gitignore ├── config_validator.py ├── airbyte_config_model.py ├── topiary.py ├── airbyte_client.py ├── README.md ├── airbyte_dto_factory.py └── controller.py /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest>=3.0.7 2 | requests>=2.25.1 3 | pyyaml==5.4.1 -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | def is_yaml(name): 2 | if name.strip().split('.')[-1] == 'yml' or name.strip().split('.')[-1] == 'yaml': 3 | return True 4 | else: 5 | return False -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | target: auto 6 | # adjust accordingly based on how flaky your tests are 7 | # this allows a 10% drop from the previous base commit coverage 8 | threshold: 10% -------------------------------------------------------------------------------- /tests/fakes/mock_source.yml: -------------------------------------------------------------------------------- 1 | - connectionConfiguration: 2 | access_token: '**********' 3 | repository: apache/superset 4 | name: apache/superset 5 | sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e 6 | sourceId: 7d95ec85-47c6-42d4-a7a2-8e5c22c810d2 7 | sourceName: GitHub 8 | workspaceId: f3b9e848-790c-4cdd-a475-5c6bb156dc10 -------------------------------------------------------------------------------- /examples/example_secrets.yml: -------------------------------------------------------------------------------- 1 | sources: 2 | GitHub: # both the source name (GitHub) 3 | access_token: 'GITHUB_ACCESS_TOKEN' # and the name of the secret (access_token) need to match what Airbyte expects 4 | Slack: 5 | token: 'SECRET_SLACK_TOKEN' 6 | 7 | destinations: 8 | Postgres: 9 | password: 'postgres_password' 10 | BigQuery: 11 | credentials_json: '{ 12 | "type": "service_account", 13 | # etc 14 | }' -------------------------------------------------------------------------------- /tests/fakes/mock_destination.yml: -------------------------------------------------------------------------------- 1 | - connectionConfiguration: 2 | database: postgres 3 | host: devrel-rds.cxiotftzsypc.us-west-2.rds.amazonaws.com 4 | password: '**********' 5 | port: 5432 6 | schema: demo 7 | username: devrel_master 8 | destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503 9 | destinationId: a41cb2f8-fcce-4c91-adfe-37c4586609f5 10 | destinationName: Postgres 11 | name: devrel-rds 12 | workspaceId: f3b9e848-790c-4cdd-a475-5c6bb156dc10 -------------------------------------------------------------------------------- /.cirrus.yml: -------------------------------------------------------------------------------- 1 | task: 2 | matrix: 3 | - name: Unit Tests (python:3.7-slim) 4 | container: {image: "python:3.7-slim"} 5 | populate_script: python3 -m pip install --verbose -r requirements.txt 6 | test_script: python3 -m pytest -v 7 | - name: Unit Tests (python:3.8-slim) 8 | container: { image: "python:3.8-slim" } 9 | populate_script: python3 -m pip install --verbose -r requirements.txt 10 | test_script: python3 -m pytest -v 11 | - name: Unit Tests (python:3.9-slim) 12 | container: {image: "python:3.9-slim"} 13 | populate_script: python3 -m pip install --verbose -r requirements.txt 14 | test_script: python3 -m pytest -v -------------------------------------------------------------------------------- /tests/test_airbyte_response.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from requests import HTTPError 3 | from airbyte_client import AirbyteResponse 4 | from airbyte_client import RESPONSE_CODES 5 | 6 | 7 | def json(): 8 | return { 9 | 'attribute': 'value' 10 | } 11 | 12 | 13 | def raise_error(): 14 | raise HTTPError 15 | 16 | 17 | def test_recognized_response_code(monkeypatch): 18 | response = requests.Response() 19 | response.status_code = 200 20 | response.json = json 21 | response.raise_for_status = lambda: True 22 | airbyte_response = AirbyteResponse(response) 23 | 24 | assert airbyte_response.message == RESPONSE_CODES[200] 25 | assert airbyte_response.payload['attribute'] == 'value' 26 | assert airbyte_response.ok 27 | 28 | 29 | def test_unrecognized_response_code(): 30 | response = requests.Response() 31 | response.status_code = 500 32 | response.json = json 33 | response.raise_for_status = raise_error 34 | airbyte_response = AirbyteResponse(response) 35 | 36 | assert airbyte_response.message == "Error: Unrecognized response code" 37 | assert airbyte_response.payload['attribute'] == 'value' 38 | assert not airbyte_response.ok 39 | -------------------------------------------------------------------------------- /examples/config.yml: -------------------------------------------------------------------------------- 1 | # This config file creates (or modifies) two new sources (one Slack and one GitHub), a destination, and creates a 2 | # connection with default settings between the GitHub source and the BigQuery destination 3 | sources: 4 | - name: superset-slack 5 | sourceName: Slack 6 | connectionConfiguration: 7 | start_date: '2015-01-01T00:00:00Z' 8 | api_token: '**********' 9 | lookback_window: 7 10 | join_channels: True 11 | - name: apache/superset 12 | sourceName: GitHub 13 | connectionConfiguration: 14 | access_token: '**********' 15 | repository: apache/superset 16 | start_date: '2017-01-01T00:00:00Z' 17 | 18 | destinations: 19 | - connectionConfiguration: 20 | big_query_client_buffer_size_mb: 15 21 | credentials_json: '**********' 22 | dataset_id: test 23 | dataset_location: US 24 | project_id: bigquery-project-id 25 | destinationName: BigQuery 26 | name: community-data-bq 27 | 28 | connections: 29 | - sourceName: apache/superset 30 | destinationName: community-data-bq 31 | prefix: 'github_superset_' 32 | namespaceDefinition: destination 33 | schedule: 34 | units: 24 35 | timeUnit: hours 36 | status: 'inactive' 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #project files 2 | secrets.yml 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | 104 | # Jetbrains 105 | .idea 106 | -------------------------------------------------------------------------------- /tests/fakes/test_backup.yml: -------------------------------------------------------------------------------- 1 | connections: [] 2 | destinations: 3 | - connectionConfiguration: 4 | basic_normalization: true 5 | database: postgres 6 | host: devrel-rds.cxiotftzsypc.us-west-2.rds.amazonaws.com 7 | password: '**********' 8 | port: 5432 9 | schema: demo 10 | username: devrel_master 11 | destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503 12 | destinationId: a41cb2f8-fcce-4c91-adfe-37c4586609f5 13 | destinationName: Postgres 14 | name: devrel-rds 15 | workspaceId: 5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6 16 | sources: 17 | - connectionConfiguration: 18 | access_token: '**********' 19 | repository: apache/superset 20 | name: apache/superset 21 | sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e 22 | sourceId: def25ce1-b7dd-476f-8051-1908b8a6da32 23 | sourceName: GitHub 24 | workspaceId: 5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6 25 | - connectionConfiguration: 26 | access_token: '**********' 27 | repository: preset-io/superset 28 | name: preset-io/superset 29 | sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e 30 | sourceId: 7d95ec85-47c6-42d4-a7a2-8e5c22c810d2 31 | sourceName: GitHub 32 | workspaceId: 5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6 33 | - connectionConfiguration: 34 | start_date: '2015-01-01T00:00:00Z' 35 | token: '**********' 36 | name: apache superset slack 37 | sourceDefinitionId: 41375467-61ae-4204-8e38-e2b8b7365f23 38 | sourceId: b63274c3-6da3-4584-b6f7-6b74cc30bd94 39 | sourceName: Slack 40 | workspaceId: 5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6 41 | 42 | @pytest.fixture 43 | def dummy_source_dto(): 44 | """ 45 | Creates a dummy SourceDto 46 | """ 47 | source = SourceDto() 48 | source.source_definition_id = 'ef69ef6e-aa7f-4af1-a01d-ef775033524e' 49 | source.source_id = '7d95ec85-47c6-42d4-a7a2-8e5c22c810d2' 50 | source.workspace_id = 'f3b9e848-790c-4cdd-a475-5c6bb156dc10' 51 | source.connection_configuration = {} 52 | source.name = 'apache/superset' 53 | source.source_name = 'GitHub' 54 | source.tag = 'tag1' 55 | return source -------------------------------------------------------------------------------- /tests/test_config_validator.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from tests.test_fixtures import * 3 | 4 | import config_validator 5 | 6 | 7 | def test_check_source(dummy_source_dict, dummy_config_loader): 8 | assert dummy_config_loader.check_source(dummy_source_dict) is True 9 | dummy_source_dict.pop('sourceName') # pop a required field 10 | assert dummy_config_loader.check_source(dummy_source_dict) is False 11 | 12 | 13 | def test_check_destination(dummy_destination_dict, dummy_config_loader): 14 | assert dummy_config_loader.check_destination(dummy_destination_dict) is True 15 | dummy_destination_dict.pop('destinationName') # pop a required field 16 | assert dummy_config_loader.check_destination(dummy_destination_dict) is False 17 | 18 | 19 | def test_check_required(dummy_config_loader, dummy_source_dict, dummy_destination_dict, dummy_new_connection_dict): 20 | required_source_fields = {x: config_validator.SOURCE_FIELDS[x] for x in config_validator.SOURCE_FIELDS if 21 | config_validator.SOURCE_FIELDS[x] is True} 22 | required_destination_fields = {x: config_validator.DESTINATION_FIELDS[x] for x in config_validator.DESTINATION_FIELDS if 23 | config_validator.DESTINATION_FIELDS[x] is True} 24 | required_connection_fields = {x: config_validator.CONNECTION_FIELDS[x] for x in config_validator.CONNECTION_FIELDS if 25 | config_validator.CONNECTION_FIELDS[x] is True} 26 | assert dummy_config_loader.check_required(dummy_source_dict, required_source_fields) is True 27 | assert dummy_config_loader.check_required(dummy_destination_dict, required_destination_fields) is True 28 | assert dummy_config_loader.check_required(dummy_new_connection_dict, required_connection_fields) is True 29 | dummy_source_dict.pop("name") 30 | dummy_destination_dict.pop("name") 31 | dummy_new_connection_dict.pop("sourceName") 32 | assert dummy_config_loader.check_required(dummy_source_dict, required_source_fields) is False 33 | assert dummy_config_loader.check_required(dummy_destination_dict, required_destination_fields) is False 34 | assert dummy_config_loader.check_required(dummy_new_connection_dict, required_connection_fields) is False -------------------------------------------------------------------------------- /config_validator.py: -------------------------------------------------------------------------------- 1 | SOURCE_FIELDS = { # {fieldName: is_required} 2 | "sourceId": False, 3 | "name": True, 4 | "sourceName": True, 5 | "connectionConfiguration": True, 6 | "syncCatalog": False 7 | } 8 | 9 | DESTINATION_FIELDS = { # {fieldName: is_required} 10 | "destinationId": False, 11 | "name": True, 12 | "destinationName": True, 13 | "connectionConfiguration": True, 14 | } 15 | 16 | CONNECTION_FIELDS = { # {fieldName: is_required} 17 | "sourceName": True, 18 | "destinationName": True, 19 | "prefix": False, 20 | "namespaceDefinition": False, 21 | "schedule": True, 22 | "status": True, 23 | "syncCatalog": False 24 | } 25 | 26 | 27 | class ConfigValidator: 28 | 29 | def __init__(self): 30 | self.config_is_good = True 31 | 32 | def validate_config(self, config: dict) -> bool: 33 | """Validates that a config read from yaml is valid. Validity here means: 34 | - all sources have required information specified (see above or Airbyte API docs) 35 | - all destinations have required information specified 36 | - all connections refer to a valid source and a valid destination 37 | Not to be confused with the unrelated --validate command line argument 38 | """ 39 | 40 | if 'sources' in config: 41 | for source in config['sources']: 42 | if not self.check_source(source): # if source is not valid 43 | print("Error: invalid source") 44 | print(repr(source)) 45 | self.config_is_good = False 46 | if 'destinations' in config: 47 | for destination in config['destinations']: 48 | if not self.check_destination(destination): 49 | print("Error: invalid destination") 50 | print(repr(destination)) 51 | self.config_is_good = False 52 | if 'connections' in config and not ('sources' in config and 'destinations' in config): 53 | print("Error: Connections can't be defined without at least one valid source and one valid destination") 54 | self.config_is_good = False 55 | return self.config_is_good 56 | 57 | def check_secrets(self, secrets): # TODO: On hold for secrets v2 58 | pass 59 | 60 | def check_source(self, source_dict) -> bool: 61 | return self.check_required(source_dict, SOURCE_FIELDS) 62 | 63 | def check_destination(self, destination_dict) -> bool: 64 | return self.check_required(destination_dict, DESTINATION_FIELDS) 65 | 66 | def check_connection(self, connection_dict) -> bool: 67 | return self.check_required(connection_dict, CONNECTION_FIELDS) 68 | 69 | def check_required(self, input_dict: dict, parameters: dict) -> bool: 70 | """ 71 | Checks that required parameters are a subset of provided parameters for the given source or destination 72 | """ 73 | 74 | test_a = set([x for x in parameters.keys() if parameters[x] is True]) 75 | test_b = set(input_dict.keys()) 76 | result = test_a.issubset(test_b) 77 | return result 78 | -------------------------------------------------------------------------------- /tests/test_airbyte_config_model.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from tests.test_fixtures import * 3 | import yaml 4 | import utils 5 | import copy 6 | 7 | 8 | def test_airbyte_config_model__write_yaml(dummy_airbyte_config_model, dummy_source_dict, 9 | dummy_destination_dict): 10 | config_model = dummy_airbyte_config_model 11 | config_model.write_yaml("test_airbyte_config_model__write_yaml.yml") 12 | t = yaml.safe_load(open("test_airbyte_config_model__write_yaml.yml", 'r')) 13 | assert len(t['sources']) == 1 14 | assert len(t['destinations']) == 1 15 | dummy_source_dict.pop('tags') # tags will never be part of a config model in a workflow where write_yaml is invoked 16 | dummy_destination_dict.pop('tags') 17 | assert t['sources'][0] == dummy_source_dict 18 | assert t['destinations'][0] == dummy_destination_dict 19 | 20 | 21 | def test_has(dummy_airbyte_config_model, dummy_source_dto, dummy_destination_dto, dummy_connection_dto): 22 | dummy_source_dto = copy.copy(dummy_source_dto) 23 | dummy_destination_dto = copy.copy(dummy_destination_dto) 24 | dummy_connection_dto = copy.copy(dummy_connection_dto) 25 | assert dummy_airbyte_config_model.has(dummy_source_dto) is True 26 | assert dummy_airbyte_config_model.has(dummy_destination_dto) is True 27 | assert dummy_airbyte_config_model.has(dummy_connection_dto) is True 28 | # modify ids and check false 29 | dummy_source_dto.source_id += 'mod' 30 | dummy_destination_dto.destination_id += 'mod' 31 | dummy_connection_dto.connection_id += 'mod' 32 | assert dummy_airbyte_config_model.has(dummy_source_dto) is True 33 | assert dummy_airbyte_config_model.has(dummy_destination_dto) is True 34 | assert dummy_airbyte_config_model.has(dummy_connection_dto) is True 35 | # remove ids, causing .has to check against name. check true 36 | dummy_source_dto.source_id = None 37 | dummy_destination_dto.destination_id = None 38 | dummy_connection_dto.connection_id = None 39 | assert dummy_airbyte_config_model.has(dummy_source_dto) is True 40 | assert dummy_airbyte_config_model.has(dummy_destination_dto) is True 41 | assert dummy_airbyte_config_model.has(dummy_connection_dto) is True 42 | # modify names and check false 43 | dummy_source_dto.name += 'mod' 44 | dummy_destination_dto.name += 'mod' 45 | dummy_connection_dto.name += 'mod' 46 | assert dummy_airbyte_config_model.has(dummy_source_dto) is False 47 | assert dummy_airbyte_config_model.has(dummy_destination_dto) is False 48 | assert dummy_airbyte_config_model.has(dummy_connection_dto) is False 49 | 50 | 51 | def test_name_to_id(dummy_airbyte_config_model, dummy_source_dto, dummy_destination_dto, dummy_connection_dto): 52 | dummy_source_dto = copy.copy(dummy_source_dto) 53 | dummy_destination_dto = copy.copy(dummy_destination_dto) 54 | dummy_connection_dto = copy.copy(dummy_connection_dto) 55 | # test sources 56 | assert dummy_airbyte_config_model.name_to_id(dummy_source_dto.name) \ 57 | == dummy_source_dto.source_id 58 | dummy_source_dto.name += 'mod' 59 | assert dummy_airbyte_config_model.name_to_id(dummy_source_dto.name) is None 60 | # test destinations 61 | assert dummy_airbyte_config_model.name_to_id(dummy_destination_dto.name) \ 62 | == dummy_destination_dto.destination_id 63 | dummy_destination_dto.name += 'mod' 64 | assert dummy_airbyte_config_model.name_to_id(dummy_destination_dto.name) is None 65 | # test connections 66 | assert dummy_airbyte_config_model.name_to_id(dummy_connection_dto.name) \ 67 | == dummy_connection_dto.connection_id 68 | dummy_connection_dto.name += 'mod' 69 | assert dummy_airbyte_config_model.name_to_id(dummy_connection_dto.name) is None 70 | -------------------------------------------------------------------------------- /airbyte_config_model.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | 4 | class AirbyteConfigModel: 5 | def __init__(self): 6 | self.sources = {} 7 | self.destinations = {} 8 | self.connections = {} 9 | self.workspaces = {} 10 | self.global_config = {} 11 | 12 | def has(self, dto): 13 | """ 14 | Determines if the AirbyteConfigModel, which should always match the airbyte deployment, contains a given dto. 15 | The match is first attempted on id, and if no match is found, is attempted on name. 16 | """ 17 | dto_id, name = dto.get_identity() 18 | if dto_id and dto_id in {**self.sources, **self.destinations, **self.connections}: 19 | return True 20 | else: # search on name 21 | for x in {**self.sources, **self.destinations, **self.connections}.values(): 22 | if x.get_identity()[1] == name: 23 | return True 24 | return False 25 | 26 | def name_to_id(self, dto_name): 27 | """ 28 | Uses the name of a DTO object to return the associated uuid, or None if not found 29 | """ 30 | 31 | for x in {**self.sources, **self.destinations, **self.connections}.values(): 32 | if x.get_identity()[1] == dto_name: 33 | return x.get_identity()[0] 34 | return None 35 | 36 | def write_yaml(self, filename): 37 | payload = {'sources': [source.to_payload() for source in self.sources.values()], 38 | 'destinations': [destination.to_payload() for destination in self.destinations.values()], 39 | 'connections': [connection.to_payload() for connection in self.connections.values()]} 40 | with open(filename, 'w') as yaml_file: 41 | yaml.safe_dump(payload, yaml_file) 42 | 43 | def wipe_sources(self, client): 44 | """Removes all sources in self.sources from the deployment and the model""" 45 | 46 | # TODO: delete_sources would ideally return an AirbyteResponse and not a bool. 47 | removed = [] 48 | for source in self.sources.values(): 49 | if client.delete_source(source): 50 | removed.append(source.source_id) 51 | else: 52 | print("AirbyteConfigModel.wipe_sources : Unable to delete source: " + repr(source)) 53 | for source_id in removed: 54 | self.sources.pop(source_id) 55 | 56 | def wipe_destinations(self, client): 57 | """Removes all destinations in self.destinations from deployment and the model""" 58 | 59 | removed = [] 60 | for destination in self.destinations.values(): 61 | if client.delete_destination(destination): 62 | removed.append(destination.destination_id) 63 | else: 64 | print("AirbyteConfigModel.wipe_destinations : Unable to delete destination: " + repr(destination)) 65 | for destination_id in removed: 66 | self.destinations.pop(destination_id) 67 | 68 | def wipe_connections(self, client): 69 | """Removes all connections in self.connections from deployment and the model""" 70 | 71 | removed = [] 72 | for connection in self.connections.values(): 73 | if client.delete_connection(connection): 74 | removed.append(connection.connection_id) 75 | else: 76 | print("AirbyteConfigModel.wipe_connections : Unable to delete connection: " + repr(connection)) 77 | for connection_id in removed: 78 | self.connections.pop(connection_id) 79 | 80 | def validate(self, client): 81 | """this function validates the model and all included connectors""" 82 | 83 | print("Validating connectors...") 84 | for source in self.sources.values(): 85 | response = client.check_source_connection(source).payload 86 | print("Source is valid: " + source.source_id + ' ' + repr(response['jobInfo']['succeeded'])) 87 | for destination in self.destinations.values(): 88 | response = client.check_destination_connection(destination).payload 89 | print("Destination is valid: " + destination.destination_id + ' ' + repr(response['jobInfo']['succeeded'])) 90 | for connection in self.connections.values(): 91 | pass # TODO: implement connection validation 92 | 93 | def validate_sources(self, client): 94 | for source in self.sources.values(): 95 | response = client.check_source_connection(source).payload 96 | print("Source is valid: " + source.source_id + ' ' + repr(response['jobInfo']['succeeded'])) 97 | 98 | def validate_destinations(self, client): 99 | for destination in self.destinations.values(): 100 | response = client.check_destination_connection(destination).payload 101 | print("Destination is valid: " + destination.destination_id + ' ' + repr(response['jobInfo']['succeeded'])) 102 | 103 | def validate_connections(self, client): 104 | for connection in self.connections.values(): 105 | response = client.check_connection_connection(connection).payload 106 | print("connection is valid: " + connection.connection_id + ' ' + repr(response['jobInfo']['succeeded'])) 107 | -------------------------------------------------------------------------------- /topiary.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | This is a simple, open-source tool designed to help manage Airbyte deployments through via the API. 4 | """ 5 | 6 | __author__ = "Robert Stolz" 7 | __version__ = "0.1.0" 8 | __license__ = "MIT" 9 | 10 | import argparse 11 | import utils 12 | from airbyte_client import AirbyteClient 13 | from airbyte_config_model import AirbyteConfigModel 14 | from controller import Controller 15 | from config_validator import ConfigValidator 16 | 17 | VALID_MODES = ['wipe', 'validate', 'sync'] 18 | 19 | 20 | def main(args): 21 | """Handles arguments and setup tasks. Invokes controller methods to carry out the specified workflow""" 22 | controller: Controller = Controller() 23 | config_validator: ConfigValidator = ConfigValidator() 24 | client: AirbyteClient = controller.instantiate_client(args) 25 | definitions: dict = controller.get_definitions(client) 26 | controller.instantiate_dto_factory(definitions['source_definitions'], definitions['destination_definitions']) 27 | workspace: str = controller.get_workspace(args, client) 28 | airbyte_model: AirbyteConfigModel = controller.get_airbyte_configuration(client, workspace) 29 | 30 | # sync workflow 31 | if args.mode == 'sync': 32 | if utils.is_yaml(args.target): # deployment to yaml sync workflow 33 | airbyte_model.write_yaml(args.target) 34 | print("Output written to: " + args.target) 35 | else: # yaml to deployment sync workflow 36 | yaml_config, secrets = controller.read_yaml_config(args) 37 | if not config_validator.validate_config(yaml_config): 38 | print("Error: Invalid config provided as yaml. Exiting...") 39 | exit(2) 40 | dtos_from_config = controller.build_dtos_from_yaml_config(yaml_config, secrets) 41 | if args.backup_file: 42 | airbyte_model.write_yaml(args.backup_file) 43 | if args.wipe: 44 | controller.wipe_all(airbyte_model, client) 45 | print("Applying changes to deployment: " + client.airbyte_url) 46 | if args.sources or args.all: 47 | controller.sync_sources_to_deployment(airbyte_model, client, workspace, dtos_from_config) 48 | if args.validate: 49 | controller.validate_sources(airbyte_model, client) 50 | if args.destinations or args.all: 51 | controller.sync_destinations_to_deployment(airbyte_model, client, workspace, dtos_from_config) 52 | if args.validate: 53 | controller.validate_destinations(airbyte_model, client) 54 | if args.connections or args.all: 55 | controller.sync_connections_to_deployment(airbyte_model, client, dtos_from_config) 56 | if args.validate: 57 | controller.validate_connections(airbyte_model, client) 58 | 59 | # wipe workflow 60 | elif args.mode == 'wipe': 61 | if args.sources or args.all: 62 | controller.wipe_sources(airbyte_model, client) 63 | if args.destinations or args.all: 64 | controller.wipe_destinations(airbyte_model, client) 65 | if args.connections or args.all: 66 | controller.wipe_connections(airbyte_model, client) 67 | 68 | # validate workflow 69 | elif args.mode == 'validate': 70 | if args.sources or args.all: 71 | controller.validate_sources(airbyte_model, client) 72 | if args.destinations or args.all: 73 | controller.validate_destinations(airbyte_model, client) 74 | if args.connections or args.all: 75 | # Q: what does it mean to validate a connection beyond validating its source and destination? 76 | pass # TODO: Implement Controller.validate_connections ? 77 | else: 78 | print("main: unrecognized mode " + args.mode) 79 | 80 | 81 | if __name__ == "__main__": 82 | """ This is executed when run from the command line """ 83 | parser = argparse.ArgumentParser() 84 | 85 | # Required positional argument 86 | #parser.add_argument("arg", help="Required positional argument") 87 | parser.add_argument("mode", help="Operating mode. Choices are sync, validate, wipe") 88 | parser.add_argument("origin", help="location of the source Airbyte deployment or yaml file") 89 | 90 | # Optional argument flag which defaults to False 91 | parser.add_argument("-s", "--sources", action="store_true", default=False, 92 | help="sync sources") 93 | parser.add_argument("-d", "--destinations", action="store_true", default=False, 94 | help="syncs destinations") 95 | parser.add_argument("-c", "--connections", action="store_true", default=False, 96 | help="syncs connections") 97 | parser.add_argument("-a", "--all", action="store_true", default=False, 98 | help="syncs sources, destinations, and connections") 99 | parser.add_argument("-w", "--wipe", action="store_true", default=False, 100 | help="deletes all connectors on the target") 101 | parser.add_argument("-v", "--validate", action="store_true", default=False, 102 | help="validates all connectors on the destination after applying changes") 103 | 104 | # Optional argument which requires a parameter (eg. -d test) 105 | parser.add_argument("--target", action="store", dest="target", 106 | help="specifies the airbyte deployment or yaml file to modify") 107 | parser.add_argument("--backup", action="store", dest="backup_file", 108 | help="specifies a .yaml file to backup the configuration of the target before syncing") 109 | parser.add_argument("--secrets", action="store", dest="secrets", 110 | help="specifies a .yaml file containing the secrets for each source and destination type") 111 | parser.add_argument("--workspace", action="store", dest="workspace_slug", 112 | help="species the workspace name (slug). Allows use of a non-default workspace") 113 | # Specify output of "--version" 114 | parser.add_argument( 115 | "--version", 116 | action="version", 117 | version="%(prog)s (version {version})".format(version=__version__)) 118 | 119 | args = parser.parse_args() 120 | main(args) 121 | -------------------------------------------------------------------------------- /airbyte_client.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | RESPONSE_CODES = { 4 | 200: "Operation successful", 5 | 204: "The resource was deleted successfully", 6 | 404: "Resource not found", 7 | 422: "Invalid input", 8 | } 9 | 10 | 11 | class AirbyteResponse: 12 | def __init__(self, response): 13 | self.status_code = response.status_code 14 | if self.status_code in RESPONSE_CODES: 15 | self.message = RESPONSE_CODES[response.status_code] 16 | else: 17 | self.message = "Error: Unrecognized response code" 18 | self.payload = response.json() 19 | self.ok = response.ok 20 | # TODO: include the full response object 21 | 22 | 23 | class AirbyteClient: 24 | """ 25 | Handles interactions with the Airbyte API 26 | """ 27 | 28 | def __init__(self, url): 29 | self.airbyte_url = url.strip('/') + '/' 30 | print('API connection is healthy: ' + repr(self.health_check().ok)) 31 | 32 | def get(self, relative_url) -> AirbyteResponse: 33 | route = self.airbyte_url + relative_url 34 | try: 35 | r = requests.get(route) 36 | except: 37 | print('Error: Unable to connect to the Airbyte API at: ' + self.airbyte_url + ' using route ' + route) 38 | exit(2) 39 | return AirbyteResponse(r) 40 | 41 | def post(self, relative_url, payload) -> AirbyteResponse: 42 | route = self.airbyte_url + relative_url 43 | r = requests.post(route, payload) 44 | return AirbyteResponse(r) 45 | 46 | def health_check(self): 47 | """Route: GET /v1/openapi""" 48 | return self.get('api/v1/health') 49 | 50 | def get_workspace_by_slug(self, slug='default'): 51 | """Route: POST /v1/workspaces/get_by_slug""" 52 | return self.post('api/v1/workspaces/get_by_slug', dict(slug=slug)) 53 | 54 | def get_workspace_by_id(self, workspace_uuid): 55 | """Route: POST /v1/workspaces/get""" 56 | uuid = None 57 | route = self.airbyte_url + 'api/v1/workspaces/get' 58 | r = requests.post(route, json={"workspace_id": uuid}) 59 | return AirbyteResponse(r) 60 | 61 | def list_workspaces(self): 62 | """Route: /v1/workspaces/list""" 63 | return self.post('api/v1/workspaces/list', payload={}) 64 | 65 | def get_source_definitions(self): 66 | """Route: /v1/source_definitions/list""" 67 | route = self.airbyte_url + 'api/v1/source_definitions/list' 68 | r = requests.post(route) 69 | return AirbyteResponse(r) 70 | 71 | def get_source_definition_connection_spec(self, source_definition_id): 72 | """Route: /v1/source_definition_specifications/get""" 73 | route = self.airbyte_url + 'api/v1/source_definition_specifications/get' 74 | r = requests.post(route, json={'sourceDefinitionId': source_definition_id}) 75 | return AirbyteResponse(r) 76 | 77 | def get_destination_definitions(self): 78 | """Route: /v1/destination_definitions/list""" 79 | route = self.airbyte_url + 'api/v1/destination_definitions/list' 80 | r = requests.post(route) 81 | return AirbyteResponse(r) 82 | 83 | def check_source_connection(self, source_dto): 84 | """Route: POST /v1/sources/check_connection""" 85 | route = self.airbyte_url + 'api/v1/sources/check_connection' 86 | payload = {'sourceId': source_dto.source_id} 87 | r = requests.post(route, json=payload) 88 | if r.status_code == '404': 89 | print(source_dto.source_id + ': Unable to validate, source not found') 90 | return AirbyteResponse(r) 91 | 92 | def create_source(self, source_dto, workspace) -> AirbyteResponse: 93 | """ Route: POST /v1/sources/create""" 94 | route = self.airbyte_url + 'api/v1/sources/create' 95 | connection_spec = self.get_source_definition_connection_spec(source_dto.source_definition_id) 96 | payload = {'sourceDefinitionId': source_dto.source_definition_id, 97 | 'workspaceId': workspace['workspaceId'], 98 | 'connectionConfiguration': source_dto.connection_configuration, 99 | 'name': source_dto.name} 100 | r = requests.post(route, json=payload) 101 | return AirbyteResponse(r) 102 | 103 | def delete_source(self, source_dto): 104 | """Route: POST /v1/sources/delete""" 105 | route = self.airbyte_url + 'api/v1/sources/delete' 106 | payload = {'sourceId': source_dto.source_id} 107 | print("Deleting source: " + source_dto.source_id) 108 | r = requests.post(route, json=payload) 109 | return r.ok # TODO: should return an AirbyteResponse, but Airbyte API returns a different type for this route 110 | 111 | def get_configured_sources(self, workspace): 112 | """Route: POST /v1/sources/list""" 113 | route = self.airbyte_url + 'api/v1/sources/list' 114 | r = requests.post(route, json={'workspaceId': workspace['workspaceId']}) 115 | return AirbyteResponse(r) 116 | 117 | def update_source(self, source_dto): 118 | """Route: POST /v1/sources/update""" 119 | route = self.airbyte_url + 'api/v1/sources/update' 120 | payload = {'sourceId': source_dto.source_id, 121 | 'connectionConfiguration': source_dto.connection_configuration, 122 | 'name': source_dto.name} 123 | r = requests.post(route, json=payload) 124 | return AirbyteResponse(r) 125 | 126 | def discover_source_schema(self, source_dto): 127 | """Route: POST /v1/sources/discover_schema""" 128 | route = self.airbyte_url + 'api/v1/sources/discover_schema' 129 | payload = {'sourceId': source_dto.source_id} 130 | r = requests.post(route, json=payload) 131 | return AirbyteResponse(r) 132 | 133 | def check_destination_connection(self, destination_dto): 134 | """Route: POST /v1/destinations/check_connection""" 135 | route = self.airbyte_url + 'api/v1/destinations/check_connection' 136 | payload = {'destinationId': destination_dto.destination_id} 137 | r = requests.post(route, json=payload) 138 | if r.status_code == '404': 139 | print(destination_dto.destination_id + ': Unable to validate, destination not found') 140 | return AirbyteResponse(r) 141 | 142 | def create_destination(self, destination_dto, workspace): 143 | """ Route: POST /v1/destinations/create""" 144 | route = self.airbyte_url + 'api/v1/destinations/create' 145 | payload = {'destinationDefinitionId': destination_dto.destination_definition_id, 146 | 'workspaceId': workspace['workspaceId'], 147 | 'connectionConfiguration': destination_dto.connection_configuration, 148 | 'name': destination_dto.name} 149 | r = requests.post(route, json=payload) 150 | return AirbyteResponse(r) 151 | 152 | def delete_destination(self, destination_dto): 153 | """Route: POST /v1/destinations/delete""" 154 | route = self.airbyte_url + 'api/v1/destinations/delete' 155 | payload = {'destinationId': destination_dto.destination_id} 156 | print("Deleting destination: " + destination_dto.destination_id) 157 | r = requests.post(route, json=payload) 158 | return r.ok # TODO: should return an AirbyteResponse, but Airbyte API returns a different type for this route 159 | 160 | def list_destinations(self): 161 | """Route: POST /v1/destinations/list""" 162 | pass 163 | 164 | def get_configured_destinations(self, workspace): 165 | """Route: POST /v1/destinations/list""" 166 | route = self.airbyte_url + 'api/v1/destinations/list' 167 | r = requests.post(route, json={'workspaceId': workspace['workspaceId']}) 168 | return AirbyteResponse(r) 169 | 170 | def update_destination(self, destination_dto): 171 | """Route: POST /v1/destinations/update""" 172 | route = self.airbyte_url + 'api/v1/destinations/update' 173 | payload = {'destinationId': destination_dto.destination_id, 174 | 'connectionConfiguration': destination_dto.connection_configuration, 175 | 'name': destination_dto.name} 176 | r = requests.post(route, json=payload) 177 | return AirbyteResponse(r) 178 | 179 | def create_connection(self, connection_dto, source_dto): 180 | """Route: POST /v1/connections/create""" 181 | route = self.airbyte_url + 'api/v1/connections/create' 182 | if not connection_dto.sync_catalog: 183 | source_schema = self.discover_source_schema(source_dto).payload 184 | connection_dto.sync_catalog = source_schema['catalog'] 185 | payload = { 186 | 'name': connection_dto.name, 187 | 'prefix': connection_dto.prefix, 188 | 'sourceId': connection_dto.source_id, 189 | 'destinationId': connection_dto.destination_id, 190 | 'status': connection_dto.status, 191 | 'syncCatalog': connection_dto.sync_catalog, 192 | 'schedule': connection_dto.schedule 193 | } 194 | r = requests.post(route, json=payload) 195 | return AirbyteResponse(r) 196 | 197 | def delete_connection(self, connection_dto): 198 | """Route: POST /v1/connections/delete""" 199 | route = self.airbyte_url + 'api/v1/connections/delete' 200 | payload = {'connectionId': connection_dto.connection_id} 201 | print("Deleting connection: " + connection_dto.connection_id) 202 | r = requests.post(route, json=payload) 203 | return r.ok 204 | 205 | def reset_conection(self): 206 | """Route: POST /v1/connections/reset""" 207 | pass 208 | 209 | def sync_connection(self): 210 | """Route: POST /v1/connections/sync""" 211 | pass 212 | 213 | def update_connection(self, connection_dto): 214 | """Route: POST /v1/connections/update""" 215 | route = self.airbyte_url + 'api/v1/connections/update' 216 | payload = { 217 | 'connectionId': connection_dto.connection_id, 218 | 'prefix': connection_dto.prefix, 219 | 'status': connection_dto.status, 220 | 'syncCatalog': connection_dto.sync_catalog, 221 | 'schedule': connection_dto.schedule 222 | } 223 | r = requests.post(route, json=payload) 224 | return AirbyteResponse(r) 225 | 226 | def get_connection_state(self): 227 | """Route: POST /v1/state/get""" 228 | pass 229 | 230 | def get_configured_connections(self, workspace): 231 | """Route: POST /v1/connections/list""" 232 | route = self.airbyte_url + 'api/v1/connections/list' 233 | r = requests.post(route, json={'workspaceId': workspace['workspaceId']}) 234 | return AirbyteResponse(r) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Airbyte Topiary 2 | Airbyte Topiary is an open-source configuration and deployment management tool for [Airbyte](https://github.com/airbytehq/airbyte). **As this tool is early in development, I *highly* recommend reading below before using the tool to avoid irreversible (potentially unexpected) changes to your Airbyte deployment.** 3 | 4 | # Releases 5 | No releases yet. 6 | 7 | # Setup 8 | 1. Clone this repo to your working environment 9 | `git clone github.com/garden-of-delete/airbyte-topiary` 10 | 2. Create a new Python3 virtual environment. For example, to use venv to create a virtual environment in your current working directory: 11 | `python3 -m venv .venv` 12 | Activate the environment with `. .venv/bin/activate` 13 | 3. Install all requirements: 14 | `pip3 install requirements.txt` 15 | 4. Run Topiary. `python topiary.py` will display help and usage. 16 | 17 | # Configuration as YAML 18 | Airbyte Topiary allows configuration for an airbyte deployment to be moved to and from yaml files through interaction with the Airbyte API. Provided .yml configuration is first validated, but care should be taken to ensure all the details are correct. Check the `examples/` directory to see some example configurations. 19 | 20 | ### Sources 21 | Sources require the following: 22 | `name`: a name given to the source. **should be unique across the whole Airbyte deployment** 23 | `sourceName`: the name associated with the Airbyte connector. e.g. GitHub, Slack. Used to choose the right connector type when creating a new source. 24 | `connectionConfiguration`: specific to each source. Check the documentation for that source to get a list. Will include things like: 25 | `access_token` / `api_token` / `some_other_secret` 26 | `repository` 27 | `start_date` (provided as a standard timestamp YYYY-MM-DDThr:mm:ssZ) 28 | 29 | Optionally, a `sourceId` (uuid) can be provided to bypass using `name` to check if the source already exists in the Airbyte deployment during a `sync` operation. 30 | 31 | ### Destinations 32 | Same as Sources, but will probably have more destination specific details in the `connectionConfiguration` section. For example, the BigQuery destination requires something like: 33 | ``` 34 | big_query_client_buffer_size_mb: 15 35 | credentials_json: '**********' 36 | dataset_id: somedataset 37 | dataset_location: US 38 | project_id: some-project 39 | ``` 40 | Optionally, a `destinationId` (uuid) can be provided to bypass using `name` to check if the source already exists in the Airbyte deployment during a `sync` operation. 41 | 42 | ### Connections 43 | Connections require the following: 44 | `sourceName` or `sourceId`: used to identify the source. Id will be tried first, then name. 45 | `destinationName` or `destinationId`: used to identify the destination. Id will be tried first, then name. 46 | `connectionName` or `connectionId`: used to provide a name for a new connection (not visible in Airbyte's GUI), or to target an existing connection for changes 47 | `prefix`: prefixes the tables produced by the connection. For example `github_superset_` 48 | `namespaceDefinition`: tells the connection to use the namespace configuration (schema / dataset information, other details) of the source, destination, or custom. I personally leave the namespace configuration up to the destination (`destination`). 49 | `schedule`: 50 | `units`: number of units of time as an integer 51 | `timeUnit`: units of time used (`hours`, `days`, etc) 52 | `status`: active or inactive. Note: an "active" connector with a schedule will start a sync attempt in Airbyte immediately upon creation. 53 | 54 | Optionally, a `syncCatalog` can also be specified. This monstrosity is specific to each source and contains the configuration for each of the streams in the connection. Since the `syncCatalog` as expected by the Airbyte API is not particularly human readable, Topiary provides some options here: 55 | - If a `syncCatalog` is not provided, Topiary will retrieve the default sync catalog from the source and use that. Note, the default syncCatalog has all available streams selected with the default sync mode (usually "Full Refresh - Overwrite/Append") 56 | - To modify the `syncCatalog` for an existing connection, I would recommend first syncing the connection to yaml before making changes and applying back to Airbyte. Read the "**The Sync Workflow**" section below to see how to do this. 57 | 58 | A connection connecting a GitHub source to a BigQuery destination might look something like this (no SyncCatalog provided, so defaults will be used): 59 | ``` 60 | - sourceName: apache/superset 61 | destinationName: community-data-bq 62 | prefix: 'github_superset_' 63 | namespaceDefinition: destination 64 | schedule: 65 | units: 24 66 | timeUnit: hours 67 | status: 'inactive' 68 | ``` 69 | 70 | # Workflows 71 | Airbyte Topiary supports a number of workflows designed to make managing Airbyte deployments at scale easier. These are: 72 | - **sync**: applies configuration provided as yml to an Airbyte deployment, OR retrieves the configuration of an Airbyte deployment and writes it to .yml 73 | - **wipe**: deletes the specified connectors (sources, destinations) and associated connections / configuration 74 | - **validate**: validates all sources and destinations 75 | 76 | ## The Sync Workflow 77 | All the sync workflows described below are accessed through the **sync** master mode like so: 78 | 79 | `python topiary.py sync ...` 80 | 81 | In all cases, a configuration origin, which follows the `sync` command, and a `--target` are required. 82 | 83 | Topiary will use the .yaml or .yml file extensions following the `source` and `--target` arguments to choose the right sync workflow. 84 | 85 | During setup, Airbyte creates a default workspace called 'default'. Topiary allows the user to specify an alternative existing workspace by name using the optional `--workspace` argument, followed by the name of the workspace. 86 | 87 | ### Sync yaml to Airbyte 88 | 89 | The yaml to deployment workflow takes a .yaml file as the origin and applies the configuration contained within to a destination Airbyte deployment. 90 | 91 | These optional arguments can be used in combination to define what to apply the sync operation to: 92 | - `--sources` 93 | - `--destinations` 94 | - `--connections` 95 | - `--all` (same as `--sources --destinations --connections`) 96 | 97 | **Note:** if none of the four optional arguments above are given, no changes will be made to the `--target`. 98 | 99 | Basic usage could be something like: 100 | 101 | `python topiary.py sync config.yml --target http://123.456.789.0:8081 --all` 102 | 103 | Almost all sources and destinations will have associated secrets. topiary ignores any secrets specified in the source config.yml. Secrets are specified separately using the `--secrets` argument, followed by a .yaml file. For example: 104 | 105 | `python topiary.py sync config.yml --target http://123.456.789.0:8081 --secrets secrets.yml --all` 106 | 107 | There are a number of additional optional parameters that modify how a sync operation is carried out: 108 | - `--wipe` removes all sources, destinations, and connectors **before** applying config.yml 109 | - `--backup` followed by a filename. Dumps the full configuration of airbyte to the specified file **before** applying `--wipe` and config.yml 110 | - `--validate` validates the sources, destinations, and connections on the destination Airbyte deployment **after** applying changes. 111 | 112 | Used together, a realistic invocation of topiary might look something like: 113 | `python topiary.py sync config.yml --target http://123.456.789.0:8081 --secrets secrets.yml --all --validate --backup backup_config.yml` 114 | 115 | ### Modifying existing sources and destinations 116 | If the yaml file specifies connectors with valid `sourceId`, `destinationId`, or `connectionId` matching matching Airbyte deployment, or failing that, valid `name`s, then topiary will attempt to modify the existing source/destination/connection instead of creating a new one. 117 | 118 | ### Sync deployment to yaml 119 | An existing Airbyte deployment can be written to a .yaml by following the `--target` argument with a filename having the .yaml or .yml extension. For example: 120 | 121 | `python topiary.py sync http://123.456.789.0:8081 --target my_deployment.yml` 122 | 123 | will write the configuration of all sources, destinations, and connections to `my_deployment.yml`. 124 | 125 | Note in this case, no `--secrets` file is specified, since it has no meaning in this workflow. Secrets can't be extracted from the Airbyte API. 126 | 127 | ## Wipe a deployment 128 | The `wipe` mode deletes sources, destinations, connections or any combination in an existing Airbyte deployment. 129 | 130 | `python topiary.py wipe http://123.456.789.0:8081 --all` 131 | 132 | As with `sync`ing a .yaml file to a deployment, these optional arguments can be used in combination to define what to apply the wipe operation to: 133 | - `--sources` 134 | - `--destinations` 135 | - `--connections` 136 | - `--all` (same as `--sources --destinations --connections`) 137 | 138 | **Note**: the `--wipe` argument when used in the `sync` workflow will wipe **ALL** sources/destinations/connections, not just those specified. 139 | 140 | ## Validate a deployment 141 | The `validate` mode validates sources, destinations, connections or any combination in an existing Airbyte deployment. 142 | 143 | `python topiary.py validate http://123.456.789.0:8081 --all` 144 | 145 | As with `wipe` mode, these optional arguments can be used in combination to define what to apply the `validate` operation to: 146 | - `--sources` 147 | - `--destinations` 148 | - `--connections` 149 | - `--all` (same as `--sources --destinations --connections`) 150 | 151 | # Contributing 152 | This is a small project I've been building in my free time, so there isn't much structure needed around contributing (for now). Check the issue list, open an issue for your change if needed, fork the project, modify it, then open a PR :) 153 | 154 | # Acknowledgements 155 | Thanks to Abhi and the Airbyte team for being responsive to questions and feedback during the development process. Also big thanks to the team at Preset.io for supporting the concept and my use of Airbyte while employed there. 156 | 157 | # License 158 | Copyright 2021 Robert Stolz 159 | 160 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 161 | 162 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 163 | 164 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /airbyte_dto_factory.py: -------------------------------------------------------------------------------- 1 | class SourceDto: 2 | """ 3 | Data transfer object class for Source-type Airbyte abstractions 4 | """ 5 | 6 | def __init__(self): 7 | self.source_definition_id = None 8 | self.source_id = None 9 | self.workspace_id = None 10 | self.connection_configuration = {} 11 | self.name = None 12 | self.source_name = None 13 | self.tags = [] 14 | 15 | def get_identity(self): 16 | return self.source_id, self.name 17 | 18 | def to_payload(self): 19 | """ 20 | sends this dto object to a dict formatted as a payload 21 | """ 22 | 23 | r = {} 24 | r['sourceDefinitionId'] = self.source_definition_id 25 | r['sourceId'] = self.source_id 26 | r['workspaceId'] = self.workspace_id 27 | r['connectionConfiguration'] = self.connection_configuration 28 | r['name'] = self.name 29 | r['sourceName'] = self.source_name 30 | return r 31 | 32 | 33 | class DestinationDto: 34 | """ 35 | Data transfer object class for Destination-type Airbyte abstractions 36 | """ 37 | 38 | def __init__(self): 39 | self.destination_definition_id = None 40 | self.destination_id = None 41 | self.workspace_id = None 42 | self.connection_configuration = {} 43 | self.name = None 44 | self.destination_name = None 45 | self.tags = [] 46 | 47 | def get_identity(self): 48 | return self.destination_id, self.name 49 | 50 | def to_payload(self): 51 | """ 52 | Sends this dto object to a dict formatted as a payload 53 | """ 54 | 55 | r = {} 56 | r['destinationDefinitionId'] = self.destination_definition_id 57 | r['destinationId'] = self.destination_id 58 | r['workspaceId'] = self.workspace_id 59 | r['connectionConfiguration'] = self.connection_configuration 60 | r['name'] = self.name 61 | r['destinationName'] = self.destination_name 62 | return r 63 | 64 | 65 | class ConnectionDto: 66 | """ 67 | Data transfer object class for Connection-type Airbyte abstractions 68 | """ 69 | 70 | def __init__(self): 71 | self.connection_id = None 72 | self.name = 'default' 73 | self.prefix = '' 74 | self.source_id = None 75 | self.source_name = None 76 | self.destination_id = None 77 | self.destination_name = None 78 | self.sync_catalog = {} # sync_catalog['streams'] is a list of dicts {stream:, config:} 79 | self.schedule = {} 80 | self.namespace_definition = None 81 | self.status = 'active' 82 | 83 | def get_identity(self): 84 | return self.connection_id, self.name 85 | 86 | def to_payload(self): 87 | r = {} 88 | r['connectionId'] = self.connection_id 89 | r['sourceId'] = self.source_id 90 | r['destinationId'] = self.destination_id 91 | r['name'] = self.name 92 | r['prefix'] = self.prefix 93 | r['schedule'] = self.schedule 94 | r['status'] = self.status 95 | r['syncCatalog'] = self.sync_catalog 96 | r['namespaceDefinition'] = self.namespace_definition 97 | return r 98 | 99 | 100 | class ConnectionGroupDto: 101 | """Data transfer object class for connection groups, each one representing a set of connections 102 | Note, Airbyte does not have this abstraction internally. 103 | 104 | ConnectionGroupDto also does not have a to_payload method, as it will never need to be written to .yml, 105 | or interact directly with the client, only read. Instead, to_incomplete_connection_dict sends the 106 | relevant info for making a new ConnectionDto to a dict. 107 | """ 108 | 109 | def __init__(self): 110 | self.group_name = None 111 | self.prefix = '' 112 | self.source_tags = None 113 | self.destination_tags = None 114 | self.sync_catalog = {} # sync_catalog['streams'] is a list of dicts {stream:, config:} 115 | self.schedule = {} 116 | self.status = 'active' 117 | 118 | def to_incomplete_connection_dict(self): 119 | """ 120 | This function returns what AirbyteDtoFactory.build_connection_dto craves 121 | """ 122 | 123 | r = { 124 | 'name': self.group_name, 125 | 'prefix': self.prefix, 126 | 'syncCatalog': self.sync_catalog, 127 | 'schedule': self.schedule, 128 | 'status': self.status 129 | } 130 | return r 131 | 132 | 133 | class StreamDto: 134 | """ 135 | Data transfer object class for the stream, belongs to the connection abstraction 136 | """ 137 | 138 | def __init__(self): 139 | self.name = None 140 | self.json_schema = {} 141 | self.supported_sync_modes = [] 142 | self.source_defined_cursor = None 143 | self.default_cursor_field = [] 144 | self.source_defined_primary_key = [] 145 | self.namespace = None 146 | 147 | 148 | class StreamConfigDto: 149 | """ 150 | Data transfer object class for the stream configuration, belongs to the connection abstraction 151 | """ 152 | 153 | def __init__(self): 154 | self.sync_mode = None 155 | self.cursor_field = [] 156 | self.destination_sync_mode = None 157 | self.primary_key = [] 158 | self.alias_name = None 159 | self.selected = None 160 | 161 | 162 | class WorkspaceDto: 163 | """ 164 | Data transfer object class for Workspace-type Airbyte abstractions 165 | """ 166 | 167 | 168 | class AirbyteDtoFactory: 169 | """ 170 | Builds data transfer objects, each modeling an abstraction inside Airbyte 171 | """ 172 | 173 | def __init__(self, source_definitions, destination_definitions): 174 | self.source_definitions = source_definitions 175 | self.destination_definitions = destination_definitions 176 | 177 | def populate_secrets(self, secrets, new_dtos): 178 | # TODO: Find a better way to deal with unpredictable naming in secrets v2 179 | if 'sources' in new_dtos: 180 | for source in new_dtos['sources']: 181 | if source.source_name in secrets['sources']: 182 | if 'access_token' in source.connection_configuration: 183 | source.connection_configuration['access_token'] = secrets['sources'][source.source_name]['access_token'] 184 | elif 'token' in source.connection_configuration: 185 | source.connection_configuration['token'] = secrets['sources'][source.source_name]['token'] 186 | if 'destinations' in new_dtos: 187 | for destination in new_dtos['destinations']: 188 | if destination.destination_name in secrets['destinations']: 189 | if 'password' in destination.connection_configuration: 190 | destination.connection_configuration['password'] = secrets['destinations'][destination.destination_name]['password'] 191 | elif 'credentials_json' in destination.connection_configuration: 192 | destination.connection_configuration['credentials_json'] = \ 193 | secrets['destinations'][destination.destination_name]['credentials_json'] 194 | 195 | def build_source_dto(self, source: dict) -> SourceDto: 196 | """ 197 | Builds a SourceDto object from a dict representing a source 198 | """ 199 | 200 | r = SourceDto() 201 | if 'connectionConfiguration' in source: 202 | r.connection_configuration = source['connectionConfiguration'] 203 | r.name = source['name'] 204 | r.source_name = source['sourceName'] 205 | if 'sourceDefinitionId' in source: 206 | r.source_definition_id = source['sourceDefinitionId'] 207 | else: 208 | for definition in self.source_definitions['sourceDefinitions']: 209 | if r.source_name == definition['name']: 210 | r.source_definition_id = definition['sourceDefinitionId'] 211 | # TODO: handle exception where no sourceDefinitionId matches the provided source name 212 | if 'sourceId' in source: 213 | r.source_id = source['sourceId'] 214 | if 'workspaceId' in source: 215 | r.workspace_id = source['workspaceId'] 216 | if 'tags' in source: 217 | r.tags = source['tags'] 218 | return r 219 | 220 | def build_destination_dto(self, destination): 221 | """ 222 | Builds a DestinationDto object from a dict representing a source 223 | """ 224 | 225 | r = DestinationDto() 226 | r.connection_configuration = destination['connectionConfiguration'] 227 | r.destination_name = destination['destinationName'] 228 | r.name = destination['name'] 229 | if 'destinationDefinitionId' in destination: 230 | r.destination_definition_id = destination['destinationDefinitionId'] 231 | else: 232 | for definition in self.destination_definitions['destinationDefinitions']: 233 | if r.destination_name == definition['name']: 234 | r.destination_definition_id = definition['destinationDefinitionId'] 235 | if 'destinationId' in destination: 236 | r.destination_id = destination['destinationId'] 237 | if 'workspaceId' in destination: 238 | r.workspace_id = destination['workspaceId'] 239 | if 'tags' in destination: 240 | r.tags = destination['tags'] 241 | return r 242 | 243 | def build_connection_dto(self, connection): 244 | """ 245 | Builds a ConnectionDto from a dict representing a connection 246 | """ 247 | 248 | r = ConnectionDto() 249 | if 'prefix' in connection: 250 | r.prefix = connection['prefix'] 251 | if 'connectionId' in connection: # => connection is already defined in an Airbyte deployment 252 | r.connection_id = connection['connectionId'] 253 | if 'sourceId' in connection: 254 | r.source_id = connection['sourceId'] 255 | if 'sourceName' in connection: 256 | r.source_name = connection['sourceName'] 257 | if 'destinationId' in connection: 258 | r.destination_id = connection['destinationId'] 259 | if 'destinationName' in connection: 260 | r.destination_name = connection['destinationName'] 261 | if 'name' in connection: 262 | r.name = connection['name'] 263 | if 'syncCatalog' in connection: 264 | r.sync_catalog = connection['syncCatalog'] 265 | if 'status' in connection: 266 | r.status = connection['status'] 267 | if 'namespaceDefinition' in connection: 268 | r.status = connection['namespaceDefinition'] 269 | r.schedule = connection['schedule'] 270 | r.status = connection['status'] 271 | return r 272 | 273 | def build_connection_group_dto(self, connection_group): 274 | """ 275 | Builds a ConnectionGroupDto from a dict representing a connection_group 276 | Note: unlike the other DTO classes, ConnectionGroupDto doesn't represent an abstraction inside Airbyte 277 | """ 278 | 279 | r = ConnectionGroupDto() 280 | r.group_name = connection_group['groupName'] 281 | if 'syncCatalog' in connection_group: 282 | r.sync_catalog = connection_group['syncCatalog'] 283 | r.schedule = connection_group['schedule'] 284 | r.status = connection_group['status'] 285 | r.source_tags = connection_group['sourceTags'] 286 | r.destination_tags = connection_group['destinationTags'] 287 | r.prefix = connection_group['prefix'] 288 | return r 289 | -------------------------------------------------------------------------------- /controller.py: -------------------------------------------------------------------------------- 1 | from airbyte_config_model import AirbyteConfigModel 2 | from airbyte_client import AirbyteClient 3 | from airbyte_dto_factory import AirbyteDtoFactory 4 | import utils 5 | import yaml 6 | 7 | 8 | class Controller: 9 | """Communicates with the user. Provides methods to execute the tasks for each workflow.""" 10 | 11 | def __init__(self): 12 | self.dto_factory = None 13 | 14 | def instantiate_dto_factory(self, source_definitions, destination_definitions): 15 | self.dto_factory = AirbyteDtoFactory(source_definitions,destination_definitions) 16 | 17 | def instantiate_client(self, args) -> AirbyteClient: 18 | # if origin is a deployment and target is not specified 19 | if not utils.is_yaml(args.origin) and args.target is None: 20 | client = AirbyteClient(args.origin) 21 | # if in sync mode and source is a yaml file 22 | elif utils.is_yaml(args.origin): 23 | if utils.is_yaml(args.target): 24 | print("Fatal error: --target must be followed by a valid " 25 | "Airbyte deployment url when the origin is a .yaml file") 26 | exit(2) 27 | client = AirbyteClient(args.target) 28 | elif utils.is_yaml(args.target): 29 | if utils.is_yaml(args.origin): 30 | print("Fatal error: --target must be followed by a valid " 31 | "Airbyte deployment url when the origin is a .yaml file") 32 | exit(2) 33 | client = AirbyteClient(args.origin) 34 | else: 35 | print("Fatal error: the origin or --target must be a valid .yaml configuration file") 36 | exit(2) 37 | return client 38 | 39 | def read_yaml_config(self, args): 40 | """get config from config.yml""" 41 | 42 | secrets = None 43 | if utils.is_yaml(args.origin): 44 | yaml_config = yaml.safe_load(open(args.origin, 'r')) 45 | else: 46 | yaml_config = yaml.safe_load(open(args.target, 'r')) 47 | if args.secrets: 48 | secrets = yaml.safe_load(open(args.secrets, 'r')) # TODO: if no --secrets specified, skip 49 | else: 50 | print("Warning: Reading yaml config but --secrets not specified. Is this intentional?") 51 | return yaml_config, secrets 52 | 53 | def get_definitions(self, client): 54 | """Retrieves source and destination definitions for configured sources""" 55 | 56 | print("Retrieving source and destination definitions from: " + client.airbyte_url) 57 | available_sources = client.get_source_definitions().payload 58 | available_destinations = client.get_destination_definitions().payload 59 | return {'source_definitions': available_sources, 'destination_definitions': available_destinations} 60 | 61 | def get_airbyte_configuration(self, client, workspace): 62 | """Retrieves the configuration from an airbyte deployment and returns an AirbyteConfigModel representing it""" 63 | 64 | print("Retrieving Airbyte configuration from: " + client.airbyte_url) 65 | configured_sources = client.get_configured_sources(workspace).payload['sources'] 66 | configured_destinations = client.get_configured_destinations(workspace).payload['destinations'] 67 | configured_connections = client.get_configured_connections(workspace).payload['connections'] 68 | airbyte_model = AirbyteConfigModel() 69 | for source in configured_sources: 70 | source_dto = self.dto_factory.build_source_dto(source) 71 | airbyte_model.sources[source_dto.source_id] = source_dto 72 | for destination in configured_destinations: 73 | destination_dto = self.dto_factory.build_destination_dto(destination) 74 | airbyte_model.destinations[destination_dto.destination_id] = destination_dto 75 | for connection in configured_connections: 76 | connection_dto = self.dto_factory.build_connection_dto(connection) 77 | airbyte_model.connections[connection_dto.connection_id] = connection_dto 78 | return airbyte_model 79 | 80 | def get_workspace(self, args, client) -> str: 81 | """Retrieves workspace specified by the --workspace argument, or the default workspace otherwise""" 82 | 83 | if args.workspace_slug: 84 | workspace = client.get_workspace_by_slug(args.workspace_slug).payload 85 | else: 86 | workspace = client.list_workspaces().payload['workspaces'][0] # TODO: support for multiple workspaces 87 | return workspace 88 | 89 | def build_dtos_from_yaml_config(self, yaml_config, secrets): 90 | """Creates a dict of new DTOs by parsing the user specified .yml config file""" 91 | 92 | new_dtos = {} 93 | if 'global' in yaml_config.keys(): 94 | for item in yaml_config['global']: 95 | pass # TODO: placeholder for global configuration 96 | if 'workspaces' in yaml_config.keys(): 97 | for item in yaml_config['workspaces']: 98 | pass # TODO: placeholder for multi-workspace support 99 | if 'sources' in yaml_config.keys(): 100 | new_sources = [] 101 | for item in yaml_config['sources']: 102 | new_sources.append(self.dto_factory.build_source_dto(item)) 103 | new_dtos['sources'] = new_sources 104 | if 'destinations' in yaml_config.keys(): 105 | new_destinations = [] 106 | for item in yaml_config['destinations']: 107 | new_destinations.append(self.dto_factory.build_destination_dto(item)) 108 | new_dtos['destinations'] = new_destinations 109 | if 'connections' in yaml_config.keys(): 110 | new_connections = [] 111 | new_connection_groups = [] 112 | for connection_entity in yaml_config['connections']: 113 | if 'groupName' in connection_entity: # if entity is a group defined with in shorthand with tags 114 | new_connection_groups.append(self.dto_factory.build_connection_group_dto(connection_entity)) 115 | else: # else connection_entity is a connection, not a connection group 116 | new_connections.append(self.dto_factory.build_connection_dto(connection_entity)) 117 | if len(new_connections) > 0: 118 | new_dtos['connections'] = new_connections 119 | if len(new_connection_groups) > 0: 120 | new_dtos['connectionGroups'] = new_connection_groups 121 | self.dto_factory.populate_secrets(secrets, new_dtos) 122 | return new_dtos 123 | 124 | def sync_sources_to_deployment(self, 125 | airbyte_model: AirbyteConfigModel, 126 | client: AirbyteClient, 127 | workspace: str, 128 | dtos_from_config: dict): 129 | """Applies destinations defined by the provided dict of dtos to a proved airbyte deployment in the specified 130 | workspace using the provided client 131 | """ 132 | if 'sources' in dtos_from_config: 133 | for new_source in dtos_from_config['sources']: 134 | if airbyte_model.has(new_source): # source already exists by name or id in the deployment 135 | if new_source.source_id is None: # if no id on the provided source 136 | new_source.source_id = airbyte_model.name_to_id(new_source.name) 137 | response = client.update_source(new_source) 138 | if response.ok: 139 | source_dto = self.dto_factory.build_source_dto(response.payload) 140 | print("Updated source: " + source_dto.source_id) 141 | airbyte_model.sources[source_dto.source_id] = source_dto 142 | else: 143 | print("Error: unable to modify source: " + new_source.source_id) 144 | print('Response code: ' + repr(response.status_code) + ' ' + response.message) 145 | else: # source does not exist 146 | response = client.create_source(new_source, workspace) 147 | if response.ok: 148 | source_dto = self.dto_factory.build_source_dto(response.payload) 149 | print("Updated source: " + source_dto.source_id) 150 | airbyte_model.sources[source_dto.source_id] = source_dto 151 | else: 152 | print("Error: unable to modify source: " + new_source.source_id) 153 | print('Response code: ' + repr(response.status_code) + ' ' + response.message) 154 | else: 155 | print('Warning: --sources option used, but no sources found in provided config.yml') 156 | 157 | def sync_destinations_to_deployment(self, 158 | airbyte_model: AirbyteConfigModel, 159 | client: AirbyteClient, 160 | workspace: str, 161 | dtos_from_config: dict): 162 | """Applies destinations defined by the provided dict of dtos to a proved airbyte deployment in the specified 163 | workspace using the provided client 164 | """ 165 | 166 | if 'destinations' in dtos_from_config: 167 | for new_destination in dtos_from_config['destinations']: 168 | if airbyte_model.has(new_destination): # destination already exists by name or id in the deployment 169 | if new_destination.destination_id is None: # if no id on the provided destination 170 | new_destination.destination_id = airbyte_model.name_to_id(new_destination.name) 171 | response = client.update_destination(new_destination) 172 | if response.ok: 173 | destination_dto = self.dto_factory.build_destination_dto(response.payload) 174 | print("Updated destination: " + destination_dto.destination_id) 175 | airbyte_model.destinations[destination_dto.destination_id] = destination_dto 176 | else: 177 | print("Error: unable to modify destination: " + new_destination.destination_id) 178 | print('Response code: ' + repr(response.status_code) + ' ' + response.message) 179 | else: # destination does not exist 180 | response = client.create_destination(new_destination, workspace) 181 | if response.ok: 182 | destination_dto = self.dto_factory.build_destination_dto(response.payload) 183 | print("Updated destination: " + destination_dto.destination_id) 184 | airbyte_model.destinations[destination_dto.destination_id] = destination_dto 185 | else: 186 | print("Error: unable to modify destination: " + new_destination.destination_id) 187 | print('Response code: ' + repr(response.status_code) + ' ' + response.message) 188 | else: 189 | print('Warning: --destinations option used, but no destinations found in provided config.yml') 190 | 191 | def sync_connections_to_deployment(self, 192 | airbyte_model: AirbyteConfigModel, 193 | client: AirbyteClient, 194 | dtos_from_config: dict): 195 | """ 196 | Applies a collection of connectionDtos and/or connectionGroupDtos (experimental), to an airbyte deployment 197 | """ 198 | # create or modify each connection defined in yml 199 | if 'connections' in dtos_from_config: 200 | for new_connection in dtos_from_config['connections']: 201 | # verify the new_connection has a valid source_id and destination_id before proceeding 202 | if new_connection.source_id is None: 203 | new_connection.source_id = airbyte_model.name_to_id(new_connection.source_name) 204 | if new_connection.destination_id is None: 205 | new_connection.destination_id = airbyte_model.name_to_id(new_connection.destination_name) 206 | if airbyte_model.has(new_connection): # connection already exists by name or id in the deployment 207 | if new_connection.connection_id is None: # if no id on the provided connection 208 | new_connection.connection_id = airbyte_model.name_to_id(new_connection.name) 209 | if new_connection.source_id is None or new_connection.destination_id is None: 210 | print("Error: Failed to create or update a connection : sourceId or destinationId unresolved") 211 | return 212 | if new_connection.connection_id is None: # create new connection 213 | response = client.create_connection(new_connection, airbyte_model.sources[new_connection.source_id]) 214 | if response.ok: 215 | connection_dto = self.dto_factory.build_connection_dto(response.payload) 216 | print("Created connection: " + connection_dto.connection_id) 217 | airbyte_model.connections[connection_dto.connection_id] = connection_dto 218 | else: 219 | print("Error: unable to create connection: " + new_connection.name + ' ' 220 | + new_connection.connection_id) 221 | print('Response code: ' + repr(response.status_code) + ' ' + response.message) 222 | else: # modify existing connection 223 | if not new_connection.sync_catalog: 224 | new_connection.sync_catalog = airbyte_model.connections[new_connection.connection_id]\ 225 | .sync_catalog 226 | response = client.update_connection(new_connection) 227 | if response.ok: 228 | connection_dto = self.dto_factory.build_connection_dto(response.payload) 229 | airbyte_model.connections[connection_dto.connection_id] = connection_dto 230 | print("Updated connection: " + connection_dto.connection_id) 231 | else: 232 | print("Error: unable to modify connection: " + new_connection.name + ' ' 233 | + new_connection.connection_id) 234 | print('Response code: ' + repr(response.status_code) + ' ' + response.message) 235 | else: 236 | print('Warning: --connections option used, but no connections found in provided config.yml') 237 | 238 | 239 | def wipe_sources(self, airbyte_model, client): 240 | """Wrapper for AirbyteConfigModel.wipe_sources""" 241 | print("Wiping sources on " + client.airbyte_url) 242 | airbyte_model.wipe_sources(client) 243 | 244 | def wipe_destinations(self, airbyte_model, client): 245 | """Wrapper for AirbyteConfigModel.wipe_destinations""" 246 | print("Wiping destinations on " + client.airbyte_url) 247 | airbyte_model.wipe_destinations(client) 248 | 249 | def wipe_connections(self, airbyte_model, client): 250 | """Wrapper for AirbyteConfigModel.wipe_connections""" 251 | print("Wiping connections on " + client.airbyte_url) 252 | airbyte_model.wipe_connections(client) 253 | 254 | def wipe_all(self, airbyte_model, client): 255 | """Wipes all sources, destinations, and connections in the specified airbyte deployment""" 256 | print("Wiping deployment: " + client.airbyte_url) 257 | self.wipe_sources(airbyte_model, client) 258 | self.wipe_destinations(airbyte_model, client) 259 | self.wipe_connections(airbyte_model,client) 260 | 261 | def validate_sources(self, airbyte_model, client): 262 | """Wrapper for AirbyteConfigModel.validate_sources""" 263 | print("Validating sources...") 264 | airbyte_model.validate_sources(client) 265 | 266 | def validate_destinations(self, airbyte_model, client): 267 | """Wrapper for AirbyteConfigModel.validate_destinations""" 268 | print("Validating destinations...") 269 | airbyte_model.validate_destinations(client) 270 | 271 | def validate_connections(self, airbyte_mopdel, client): 272 | """Wrapper for AirbyteConfigModel.validate_connections""" 273 | pass # TODO: implement Controller.validate_connections 274 | 275 | def validate_all(self, airbyte_model, client): 276 | """Validates all sources, destinations, and connections in the specified AirbyteConfigModel""" 277 | self.validate_sources(airbyte_model, client) 278 | self.validate_destinations(airbyte_model, client) 279 | #self.validate_connections(airbyte_model, client) # TODO: turn on -------------------------------------------------------------------------------- /tests/test_airbyte_dto_factory.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from tests.test_fixtures import * 3 | 4 | 5 | def test_source_dto__to_payload(dummy_source_dto): 6 | """ 7 | Test SourceDto.to_payload 8 | Verifies the data is not mutated by to_payload 9 | """ 10 | payload = dummy_source_dto.to_payload() 11 | assert payload['sourceDefinitionId'] == 'ef69ef6e-aa7f-4af1-a01d-ef775033524e' 12 | assert payload['sourceId'] == '7d95ec85-47c6-42d4-a7a2-8e5c22c810d2' 13 | assert payload['workspaceId'] == 'f3b9e848-790c-4cdd-a475-5c6bb156dc10' 14 | assert payload['connectionConfiguration'] == {'access_token': '**********'} 15 | assert payload['name'] == 'apache/superset' 16 | assert payload['sourceName'] == 'GitHub' 17 | 18 | 19 | def test_destination_dto__to_payload(dummy_destination_dto): 20 | """ 21 | Test DestinationDto.to_payload 22 | Verifies the data is not mutated by to_payload 23 | """ 24 | payload = dummy_destination_dto.to_payload() 25 | assert payload['destinationDefinitionId'] == '25c5221d-dce2-4163-ade9-739ef790f503' 26 | assert payload['destinationId'] == 'a41cb2f8-fcce-4c91-adfe-37c4586609f5' 27 | assert payload['workspaceId'] == 'f3b9e848-790c-4cdd-a475-5c6bb156dc10' 28 | assert payload['connectionConfiguration']['database'] == 'postgres' 29 | assert payload['connectionConfiguration']['host'] == 'hostname.com' 30 | assert payload['connectionConfiguration']['schema'] == 'demo' 31 | assert payload['connectionConfiguration']['username'] == 'devrel_master' 32 | assert payload['name'] == 'devrel-rds' 33 | assert payload['destinationName'] == 'Postgres' 34 | 35 | 36 | def test_connection_dto__to_payload(dummy_connection_dto): 37 | """ 38 | Test ConnectionDto.to_payload 39 | Verifies the data is not mutated by to_payload 40 | """ 41 | payload = dummy_connection_dto.to_payload() 42 | assert payload['connectionId'] == '10290824-9305-47cc-8966-6dd032abd3c0' 43 | assert payload['sourceId'] == '7d95ec85-47c6-42d4-a7a2-8e5c22c810d2' 44 | assert payload['destinationId'] == 'a41cb2f8-fcce-4c91-adfe-37c4586609f5' 45 | assert payload['name'] == 'superset-to-postgres' 46 | assert payload['prefix'] == 'github_superset_' 47 | assert payload['schedule'] == {'units': 24, 'timeUnit': 'hours'} 48 | assert payload['status'] == 'active' 49 | assert payload['syncCatalog'] == {'streams': [{'stream': {'name': 'assignees', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'assignees', 'selected': True}}, {'stream': {'name': 'branches', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'name': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'protected': {'type': ['null', 'boolean']}, 'protection': {'type': ['null', 'object'], 'properties': {'required_status_checks': {'type': ['null', 'object'], 'properties': {'contexts': {'type': ['null', 'array'], 'items': [{'type': ['null', 'string']}, {'type': ['null', 'string']}]}, 'enforcement_level': {'type': ['null', 'string']}}}}}, 'repository': {'type': ['string']}, 'protection_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [], 'aliasName': 'branches', 'selected': True}}, {'stream': {'name': 'collaborators', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'permissions': {'type': ['null', 'object'], 'properties': {'pull': {'type': ['null', 'boolean']}, 'push': {'type': ['null', 'boolean']}, 'admin': {'type': ['null', 'boolean']}}}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'collaborators', 'selected': True}}, {'stream': {'name': 'comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'user_id': {'type': ['null', 'integer']}, 'html_url': {'type': ['null', 'string']}, 'issue_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'comments', 'selected': True}}, {'stream': {'name': 'commit_comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'line': {'type': ['null', 'integer']}, 'path': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'position': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'commit_comments', 'selected': True}}, {'stream': {'name': 'commits', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'url': {'type': ['null', 'string']}, 'tree': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'author': {'type': ['null', 'object'], 'properties': {'date': {'type': ['null', 'string'], 'format': 'date-time'}, 'name': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}}}, 'message': {'type': ['null', 'string']}, 'committer': {'type': ['null', 'object'], 'properties': {'date': {'type': ['null', 'string'], 'format': 'date-time'}, 'name': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}}}, 'verification': {'type': ['null', 'object'], 'properties': {'reason': {'type': ['null', 'string']}, 'payload': {'type': ['null', 'string']}, 'verified': {'type': ['null', 'boolean']}, 'signature': {'type': ['null', 'string']}}}, 'comment_count': {'type': ['null', 'integer']}}}, 'node_id': {'type': ['null', 'string']}, 'parents': {'type': ['null', 'array'], 'items': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}}}}, 'html_url': {'type': ['null', 'string']}, 'author_id': {'type': ['null', 'integer']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'comments_url': {'type': ['null', 'string']}, 'committer_id': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['sha']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['sha']], 'aliasName': 'commits', 'selected': True}}, {'stream': {'name': 'events', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'org_id': {'type': ['null', 'integer']}, 'public': {'type': ['null', 'boolean']}, 'payload': {'type': ['null', 'object'], 'properties': {}}, 'repo_id': {'type': ['null', 'integer']}, 'actor_id': {'type': ['null', 'integer']}, 'created_at': {'type': ['null', 'string']}, 'repository': {'type': ['string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'events', 'selected': True}}, {'stream': {'name': 'issue_events', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'event': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'actor_id': {'type': ['null', 'integer']}, 'issue_id': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'commit_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_events', 'selected': True}}, {'stream': {'name': 'issue_labels', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'color': {'type': ['null', 'string']}, 'default': {'type': ['null', 'boolean']}, 'node_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'description': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_labels', 'selected': True}}, {'stream': {'name': 'issue_milestones', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, 'due_on': {'type': ['null', 'string'], 'format': 'date-time'}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'creator_id': {'type': ['null', 'integer']}, 'labels_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'description': {'type': ['null', 'string']}, 'open_issues': {'type': ['null', 'integer']}, 'closed_issues': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_milestones', 'selected': True}}, {'stream': {'name': 'issues', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, 'labels': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'locked': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'user_id': {'type': ['null', 'integer']}, 'comments': {'type': ['null', 'integer']}, 'html_url': {'type': ['null', 'string']}, 'assignees': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'events_url': {'type': ['null', 'string']}, 'labels_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'assignee_id': {'type': ['null', 'integer']}, 'comments_url': {'type': ['null', 'string']}, 'milestone_id': {'type': ['null', 'integer']}, 'pull_request': {'type': ['null', 'object'], 'properties': {'url': {'type': ['null', 'string']}, 'diff_url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'patch_url': {'type': ['null', 'string']}}}, 'repository_url': {'type': ['null', 'string']}, 'active_lock_reason': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issues', 'selected': True}}, {'stream': {'name': 'organizations', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'blog': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'plan': {'type': ['null', 'object'], 'properties': {'name': {'type': ['null', 'string']}, 'seats': {'type': ['null', 'integer']}, 'space': {'type': ['null', 'integer']}, 'filled_seats': {'type': ['null', 'integer']}, 'private_repos': {'type': ['null', 'integer']}}}, 'type': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'company': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'location': {'type': ['null', 'string']}, 'followers': {'type': ['null', 'integer']}, 'following': {'type': ['null', 'integer']}, 'hooks_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'disk_usage': {'type': ['null', 'integer']}, 'events_url': {'type': ['null', 'string']}, 'issues_url': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'description': {'type': ['null', 'string']}, 'is_verified': {'type': ['null', 'boolean']}, 'members_url': {'type': ['null', 'string']}, 'public_gists': {'type': ['null', 'integer']}, 'public_repos': {'type': ['null', 'integer']}, 'billing_email': {'type': ['null', 'string']}, 'collaborators': {'type': ['null', 'integer']}, 'private_gists': {'type': ['null', 'integer']}, 'twitter_username': {'type': ['null', 'string']}, 'public_members_url': {'type': ['null', 'string']}, 'owned_private_repos': {'type': ['null', 'integer']}, 'total_private_repos': {'type': ['null', 'integer']}, 'has_repository_projects': {'type': ['null', 'boolean']}, 'members_can_create_pages': {'type': ['null', 'boolean']}, 'has_organization_projects': {'type': ['null', 'boolean']}, 'default_repository_permission': {'type': ['null', 'string']}, 'two_factor_requirement_enabled': {'type': ['null', 'boolean']}, 'members_can_create_public_pages': {'type': ['null', 'boolean']}, 'members_can_create_repositories': {'type': ['null', 'boolean']}, 'members_can_create_private_pages': {'type': ['null', 'boolean']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'organizations', 'selected': True}}, {'stream': {'name': 'projects', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'owner_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'creator_id': {'type': ['null', 'integer']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'columns_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'projects', 'selected': True}}, {'stream': {'name': 'pull_request_stats', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'merged': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'commits': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'comments': {'type': ['null', 'integer']}, 'additions': {'type': ['null', 'integer']}, 'deletions': {'type': ['null', 'integer']}, 'mergeable': {'type': ['null', 'boolean']}, 'merged_by': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'rebaseable': {'type': ['null', 'boolean']}, 'repository': {'type': ['string']}, 'changed_files': {'type': ['null', 'integer']}, 'mergeable_state': {'type': ['null', 'string']}, 'review_comments': {'type': ['null', 'integer']}, 'maintainer_can_modify': {'type': ['null', 'boolean']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'pull_request_stats', 'selected': True}}, {'stream': {'name': 'pull_requests', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'base': {'type': ['null', 'object'], 'properties': {'ref': {'type': ['null', 'string']}, 'sha': {'type': ['null', 'string']}, 'label': {'type': ['null', 'string']}, 'repo_id': {'type': ['null', 'integer']}, 'user_id': {'type': ['null', 'integer']}}}, 'body': {'type': ['null', 'string']}, 'head': {'type': ['null', 'object'], 'properties': {'ref': {'type': ['null', 'string']}, 'sha': {'type': ['null', 'string']}, 'label': {'type': ['null', 'string']}, 'repo_id': {'type': ['null', 'integer']}, 'user_id': {'type': ['null', 'integer']}}}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'draft': {'type': ['null', 'boolean']}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'self': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'issue': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'commits': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'comments': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'statuses': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'review_comment': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'review_comments': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'labels': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'locked': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'diff_url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'assignees': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'issue_url': {'type': ['null', 'string']}, 'merged_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'patch_url': {'type': ['null', 'string']}, 'auto_merge': {'type': ['null', 'boolean']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'assignee_id': {'type': ['null', 'integer']}, 'commits_url': {'type': ['null', 'string']}, 'comments_url': {'type': ['null', 'string']}, 'milestone_id': {'type': ['null', 'integer']}, 'statuses_url': {'type': ['null', 'string']}, 'requested_teams': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'merge_commit_sha': {'type': ['null', 'string']}, 'active_lock_reason': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}, 'review_comment_url': {'type': ['null', 'string']}, 'requested_reviewers': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'review_comments_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'pull_requests', 'selected': True}}, {'stream': {'name': 'releases', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'draft': {'type': ['null', 'boolean']}, 'assets': {'type': ['null', 'array'], 'items': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'size': {'type': ['null', 'integer']}, 'label': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'uploader_id': {'type': ['null', 'integer']}, 'content_type': {'type': ['null', 'string']}, 'download_count': {'type': ['null', 'integer']}, 'browser_download_url': {'type': ['null', 'string']}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'tag_name': {'type': ['null', 'string']}, 'author_id': {'type': ['null', 'integer']}, 'assets_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'prerelease': {'type': ['null', 'boolean']}, 'repository': {'type': ['string']}, 'upload_url': {'type': ['null', 'string']}, 'tarball_url': {'type': ['null', 'string']}, 'zipball_url': {'type': ['null', 'string']}, 'published_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'target_commitish': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'releases', 'selected': True}}, {'stream': {'name': 'repositories', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'fork': {'type': ['null', 'boolean']}, 'name': {'type': ['null', 'string']}, 'size': {'type': ['null', 'integer']}, 'owner': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'topics': {'type': ['null', 'array'], 'items': {'type': ['null', 'string']}}, 'git_url': {'type': ['null', 'string']}, 'license': {'type': ['null', 'object'], 'properties': {'key': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'spdx_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'private': {'type': ['null', 'boolean']}, 'ssh_url': {'type': ['null', 'string']}, 'svn_url': {'type': ['null', 'string']}, 'archived': {'type': ['null', 'boolean']}, 'disabled': {'type': ['null', 'boolean']}, 'has_wiki': {'type': ['null', 'boolean']}, 'homepage': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'keys_url': {'type': ['null', 'string']}, 'language': {'type': ['null', 'string']}, 'tags_url': {'type': ['null', 'string']}, 'blobs_url': {'type': ['null', 'string']}, 'clone_url': {'type': ['null', 'string']}, 'forks_url': {'type': ['null', 'string']}, 'full_name': {'type': ['null', 'string']}, 'has_pages': {'type': ['null', 'boolean']}, 'hooks_url': {'type': ['null', 'string']}, 'pulls_url': {'type': ['null', 'string']}, 'pushed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'teams_url': {'type': ['null', 'string']}, 'trees_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'events_url': {'type': ['null', 'string']}, 'has_issues': {'type': ['null', 'boolean']}, 'issues_url': {'type': ['null', 'string']}, 'labels_url': {'type': ['null', 'string']}, 'merges_url': {'type': ['null', 'string']}, 'mirror_url': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'visibility': {'type': ['null', 'string']}, 'archive_url': {'type': ['null', 'string']}, 'commits_url': {'type': ['null', 'string']}, 'compare_url': {'type': ['null', 'string']}, 'description': {'type': ['null', 'string']}, 'forks_count': {'type': ['null', 'integer']}, 'is_template': {'type': ['null', 'boolean']}, 'permissions': {'type': ['null', 'object'], 'properties': {'pull': {'type': ['null', 'boolean']}, 'push': {'type': ['null', 'boolean']}, 'admin': {'type': ['null', 'boolean']}}}, 'branches_url': {'type': ['null', 'string']}, 'comments_url': {'type': ['null', 'string']}, 'contents_url': {'type': ['null', 'string']}, 'git_refs_url': {'type': ['null', 'string']}, 'git_tags_url': {'type': ['null', 'string']}, 'has_projects': {'type': ['null', 'boolean']}, 'releases_url': {'type': ['null', 'string']}, 'statuses_url': {'type': ['null', 'string']}, 'assignees_url': {'type': ['null', 'string']}, 'downloads_url': {'type': ['null', 'string']}, 'has_downloads': {'type': ['null', 'boolean']}, 'languages_url': {'type': ['null', 'string']}, 'default_branch': {'type': ['null', 'string']}, 'milestones_url': {'type': ['null', 'string']}, 'stargazers_url': {'type': ['null', 'string']}, 'watchers_count': {'type': ['null', 'integer']}, 'deployments_url': {'type': ['null', 'string']}, 'git_commits_url': {'type': ['null', 'string']}, 'subscribers_url': {'type': ['null', 'string']}, 'contributors_url': {'type': ['null', 'string']}, 'issue_events_url': {'type': ['null', 'string']}, 'stargazers_count': {'type': ['null', 'integer']}, 'subscription_url': {'type': ['null', 'string']}, 'collaborators_url': {'type': ['null', 'string']}, 'issue_comment_url': {'type': ['null', 'string']}, 'notifications_url': {'type': ['null', 'string']}, 'open_issues_count': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'repositories', 'selected': True}}, {'stream': {'name': 'review_comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'line': {'type': ['null', 'integer']}, 'path': {'type': ['null', 'string']}, 'side': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'self': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'pull_request': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'position': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'diff_hunk': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'start_line': {'type': ['null', 'integer']}, 'start_side': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'original_line': {'type': ['null', 'integer']}, 'in_reply_to_id': {'type': ['null', 'integer']}, 'pull_request_url': {'type': ['null', 'string']}, 'original_position': {'type': ['null', 'integer']}, 'author_association': {'type': ['null', 'string']}, 'original_commit_id': {'type': ['null', 'string']}, 'original_start_line': {'type': ['null', 'integer']}, 'pull_request_review_id': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'review_comments', 'selected': True}}, {'stream': {'name': 'reviews', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'state': {'type': ['null', 'string']}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'pull_request': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'commit_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'submitted_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'pull_request_url': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'reviews', 'selected': True}}, {'stream': {'name': 'stargazers', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'user_id': {'type': ['null', 'integer']}, 'repository': {'type': ['string']}, 'starred_at': {'type': ['null', 'string'], 'format': 'date-time'}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['starred_at'], 'sourceDefinedPrimaryKey': [['user_id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['starred_at'], 'destinationSyncMode': 'append', 'primaryKey': [['user_id']], 'aliasName': 'stargazers', 'selected': True}}, {'stream': {'name': 'tags', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'name': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'tarball_url': {'type': ['null', 'string']}, 'zipball_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [], 'aliasName': 'tags', 'selected': True}}, {'stream': {'name': 'teams', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'slug': {'type': ['null', 'string']}, 'parent': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'privacy': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'permission': {'type': ['null', 'string']}, 'repository': {'type': ['null', 'string']}, 'description': {'type': ['null', 'string']}, 'members_url': {'type': ['null', 'string']}, 'organization': {'type': ['null', 'string']}, 'repositories_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'teams', 'selected': True}}, {'stream': {'name': 'users', 'jsonSchema': {'type': ['null', 'object'], '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'organization': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'users', 'selected': True}}]} 50 | 51 | 52 | def test_dto__get_identity(dummy_source_dto, dummy_destination_dto, dummy_connection_dto): 53 | assert dummy_source_dto.get_identity()[0] == dummy_source_dto.source_id 54 | assert dummy_source_dto.get_identity()[1] == dummy_source_dto.name 55 | assert dummy_destination_dto.get_identity()[0] == dummy_destination_dto.destination_id 56 | assert dummy_destination_dto.get_identity()[1] == dummy_destination_dto.name 57 | assert dummy_connection_dto.get_identity()[0] == dummy_connection_dto.connection_id 58 | assert dummy_connection_dto.get_identity()[1] == dummy_connection_dto.name 59 | dummy_source_dto.source_id = None 60 | dummy_destination_dto.destination_id = None 61 | dummy_connection_dto.connection_id = None 62 | assert dummy_source_dto.get_identity()[0] is None 63 | assert dummy_destination_dto.get_identity()[0] is None 64 | assert dummy_connection_dto.get_identity()[0] is None 65 | 66 | 67 | def test_dto_factory__build_source_dto(dummy_airbyte_dto_factory, dummy_source_dict, dummy_source_dto): 68 | """ 69 | Test AirbyteDtoFactory.build_source_dto 70 | """ 71 | t = dummy_airbyte_dto_factory.build_source_dto(dummy_source_dict) 72 | assert t.source_definition_id == dummy_source_dto.source_definition_id 73 | assert t.source_id == dummy_source_dto.source_id 74 | assert t.workspace_id == dummy_source_dto.workspace_id 75 | assert t.connection_configuration == dummy_source_dto.connection_configuration 76 | assert t.source_name == dummy_source_dto.source_name 77 | assert t.name == dummy_source_dto.name 78 | assert t.tags == dummy_source_dto.tags 79 | 80 | 81 | def test_dto_factory__build_destination_dto(dummy_airbyte_dto_factory, dummy_destination_dict, dummy_destination_dto): 82 | """ 83 | Test AirbyteDtoFactory.build_destination_dto 84 | """ 85 | t = dummy_airbyte_dto_factory.build_destination_dto(dummy_destination_dict) 86 | assert t.destination_definition_id == dummy_destination_dto.destination_definition_id 87 | assert t.destination_id == dummy_destination_dto.destination_id 88 | assert t.workspace_id == dummy_destination_dto.workspace_id 89 | assert t.connection_configuration == dummy_destination_dto.connection_configuration 90 | assert t.destination_name == dummy_destination_dto.destination_name 91 | assert t.name == dummy_destination_dto.name 92 | assert t.tags == dummy_destination_dto.tags 93 | 94 | 95 | def test_dto_factory__build_connection_dto__existing_connection(dummy_airbyte_dto_factory, dummy_existing_connection_dict, 96 | dummy_connection_dto): 97 | """ 98 | Test AirbyteDtoFactory.build_connection_dto using a dict for an existing connection 99 | Ensures that the ConnectionDto being built by the AirbyteDtoFactory is faithful to the input dict 100 | """ 101 | t = dummy_airbyte_dto_factory.build_connection_dto(dummy_existing_connection_dict) 102 | assert t.connection_id == dummy_connection_dto.connection_id 103 | assert t.source_id == dummy_connection_dto.source_id 104 | assert t.destination_id == dummy_connection_dto.destination_id 105 | assert t.name == dummy_connection_dto.name 106 | assert t.prefix == dummy_connection_dto.prefix 107 | assert t.schedule == dummy_connection_dto.schedule 108 | assert t.status == dummy_connection_dto.status 109 | assert t.sync_catalog == dummy_connection_dto.sync_catalog 110 | 111 | 112 | def test_dto_factory__build_connection_dto__new_connection(dummy_airbyte_dto_factory, dummy_new_connection_dict, 113 | dummy_connection_dto): 114 | """ 115 | Test AirbyteDtoFactory.build_connection_dto using a dict for an existing connection 116 | Ensures that the ConnectionDto being built by the AirbyteDtoFactory is faithful to the input dict 117 | """ 118 | t = dummy_airbyte_dto_factory.build_connection_dto(dummy_new_connection_dict) 119 | assert t.source_name == dummy_connection_dto.source_name 120 | assert t.destination_name == dummy_connection_dto.destination_name 121 | assert t.name == dummy_connection_dto.name 122 | assert t.prefix == dummy_connection_dto.prefix 123 | assert t.schedule == dummy_connection_dto.schedule 124 | assert t.status == dummy_connection_dto.status 125 | 126 | def test_dto_factory__build_connection_group_dto(dummy_airbyte_dto_factory, dummy_connection_group_dict, 127 | dummy_connection_group_dto): 128 | """ 129 | Test AirbyteDtoFactory.build_connection_group_dto 130 | Experimental 131 | """ 132 | t = dummy_airbyte_dto_factory.build_connection_group_dto(dummy_connection_group_dict) 133 | assert t.group_name == dummy_connection_group_dto.group_name 134 | assert t.source_tags == dummy_connection_group_dto.source_tags 135 | assert t.destination_tags == dummy_connection_group_dto.destination_tags 136 | assert t.prefix == dummy_connection_group_dto.prefix 137 | assert t.schedule == dummy_connection_group_dto.schedule 138 | assert t.status == dummy_connection_group_dto.status 139 | assert t.sync_catalog == dummy_connection_group_dto.sync_catalog 140 | 141 | 142 | def test_dto_factory__populate_secrets(dummy_airbyte_dto_factory, dummy_secrets_dict, dummy_source_dto, 143 | dummy_destination_dto): 144 | """ 145 | Test AirbyteDtoFactory.populate_secrets 146 | Verifies placeholder secrets in the DTOs are being correctly overridden 147 | """ 148 | new_dtos = {'sources': [dummy_source_dto], 'destinations': [dummy_destination_dto]} 149 | dummy_airbyte_dto_factory.populate_secrets(dummy_secrets_dict, new_dtos) 150 | assert new_dtos['sources'][0].connection_configuration['access_token'] == 'ghp_SECRET_TOKEN' 151 | assert new_dtos['destinations'][0].connection_configuration['password'] == 'SECRET_POSTGRES_PASSWORD' -------------------------------------------------------------------------------- /tests/test_fixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from airbyte_dto_factory import * 3 | from airbyte_config_model import * 4 | from config_validator import ConfigValidator 5 | 6 | @pytest.fixture 7 | def dummy_source_dto(): 8 | """ 9 | Creates a dummy SourceDto 10 | """ 11 | source = SourceDto() 12 | source.source_definition_id = 'ef69ef6e-aa7f-4af1-a01d-ef775033524e' 13 | source.source_id = '7d95ec85-47c6-42d4-a7a2-8e5c22c810d2' 14 | source.workspace_id = 'f3b9e848-790c-4cdd-a475-5c6bb156dc10' 15 | source.connection_configuration = { 16 | 'access_token': '**********' 17 | } 18 | source.name = 'apache/superset' 19 | source.source_name = 'GitHub' 20 | source.tags = ['tag1'] 21 | return source 22 | 23 | 24 | @pytest.fixture 25 | def dummy_destination_dto(): 26 | """ 27 | Creates a dummy DestinationDto 28 | """ 29 | destination = DestinationDto() 30 | destination.destination_definition_id = '25c5221d-dce2-4163-ade9-739ef790f503' 31 | destination.destination_id = 'a41cb2f8-fcce-4c91-adfe-37c4586609f5' 32 | destination.workspace_id = 'f3b9e848-790c-4cdd-a475-5c6bb156dc10' 33 | destination.connection_configuration = { 34 | 'database': 'postgres', 35 | 'host': 'hostname.com', 36 | 'port': '5432', 37 | 'schema': 'demo', 38 | 'username': 'devrel_master', 39 | 'password': '**********' 40 | } 41 | destination.name = 'devrel-rds' 42 | destination.destination_name = 'Postgres' 43 | destination.tags = ['tag2'] 44 | return destination 45 | 46 | 47 | @pytest.fixture 48 | def dummy_connection_dto(): 49 | connection = ConnectionDto() 50 | connection.connection_id = '10290824-9305-47cc-8966-6dd032abd3c0' 51 | connection.source_id = '7d95ec85-47c6-42d4-a7a2-8e5c22c810d2' 52 | connection.source_name = 'apache/superset' 53 | connection.destination_id = 'a41cb2f8-fcce-4c91-adfe-37c4586609f5' 54 | connection.destination_name = 'postgres' 55 | connection.name = 'superset-to-postgres' 56 | connection.prefix = 'github_superset_' 57 | connection.schedule = {'units': 24, 'timeUnit': 'hours'} 58 | connection.status = 'active' 59 | connection.sync_catalog = {'streams': [{'stream': {'name': 'assignees', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'assignees', 'selected': True}}, {'stream': {'name': 'branches', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'name': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'protected': {'type': ['null', 'boolean']}, 'protection': {'type': ['null', 'object'], 'properties': {'required_status_checks': {'type': ['null', 'object'], 'properties': {'contexts': {'type': ['null', 'array'], 'items': [{'type': ['null', 'string']}, {'type': ['null', 'string']}]}, 'enforcement_level': {'type': ['null', 'string']}}}}}, 'repository': {'type': ['string']}, 'protection_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [], 'aliasName': 'branches', 'selected': True}}, {'stream': {'name': 'collaborators', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'permissions': {'type': ['null', 'object'], 'properties': {'pull': {'type': ['null', 'boolean']}, 'push': {'type': ['null', 'boolean']}, 'admin': {'type': ['null', 'boolean']}}}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'collaborators', 'selected': True}}, {'stream': {'name': 'comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'user_id': {'type': ['null', 'integer']}, 'html_url': {'type': ['null', 'string']}, 'issue_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'comments', 'selected': True}}, {'stream': {'name': 'commit_comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'line': {'type': ['null', 'integer']}, 'path': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'position': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'commit_comments', 'selected': True}}, {'stream': {'name': 'commits', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'url': {'type': ['null', 'string']}, 'tree': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'author': {'type': ['null', 'object'], 'properties': {'date': {'type': ['null', 'string'], 'format': 'date-time'}, 'name': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}}}, 'message': {'type': ['null', 'string']}, 'committer': {'type': ['null', 'object'], 'properties': {'date': {'type': ['null', 'string'], 'format': 'date-time'}, 'name': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}}}, 'verification': {'type': ['null', 'object'], 'properties': {'reason': {'type': ['null', 'string']}, 'payload': {'type': ['null', 'string']}, 'verified': {'type': ['null', 'boolean']}, 'signature': {'type': ['null', 'string']}}}, 'comment_count': {'type': ['null', 'integer']}}}, 'node_id': {'type': ['null', 'string']}, 'parents': {'type': ['null', 'array'], 'items': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}}}}, 'html_url': {'type': ['null', 'string']}, 'author_id': {'type': ['null', 'integer']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'comments_url': {'type': ['null', 'string']}, 'committer_id': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['sha']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['sha']], 'aliasName': 'commits', 'selected': True}}, {'stream': {'name': 'events', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'org_id': {'type': ['null', 'integer']}, 'public': {'type': ['null', 'boolean']}, 'payload': {'type': ['null', 'object'], 'properties': {}}, 'repo_id': {'type': ['null', 'integer']}, 'actor_id': {'type': ['null', 'integer']}, 'created_at': {'type': ['null', 'string']}, 'repository': {'type': ['string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'events', 'selected': True}}, {'stream': {'name': 'issue_events', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'event': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'actor_id': {'type': ['null', 'integer']}, 'issue_id': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'commit_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_events', 'selected': True}}, {'stream': {'name': 'issue_labels', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'color': {'type': ['null', 'string']}, 'default': {'type': ['null', 'boolean']}, 'node_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'description': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_labels', 'selected': True}}, {'stream': {'name': 'issue_milestones', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, 'due_on': {'type': ['null', 'string'], 'format': 'date-time'}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'creator_id': {'type': ['null', 'integer']}, 'labels_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'description': {'type': ['null', 'string']}, 'open_issues': {'type': ['null', 'integer']}, 'closed_issues': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_milestones', 'selected': True}}, {'stream': {'name': 'issues', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, 'labels': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'locked': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'user_id': {'type': ['null', 'integer']}, 'comments': {'type': ['null', 'integer']}, 'html_url': {'type': ['null', 'string']}, 'assignees': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'events_url': {'type': ['null', 'string']}, 'labels_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'assignee_id': {'type': ['null', 'integer']}, 'comments_url': {'type': ['null', 'string']}, 'milestone_id': {'type': ['null', 'integer']}, 'pull_request': {'type': ['null', 'object'], 'properties': {'url': {'type': ['null', 'string']}, 'diff_url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'patch_url': {'type': ['null', 'string']}}}, 'repository_url': {'type': ['null', 'string']}, 'active_lock_reason': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issues', 'selected': True}}, {'stream': {'name': 'organizations', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'blog': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'plan': {'type': ['null', 'object'], 'properties': {'name': {'type': ['null', 'string']}, 'seats': {'type': ['null', 'integer']}, 'space': {'type': ['null', 'integer']}, 'filled_seats': {'type': ['null', 'integer']}, 'private_repos': {'type': ['null', 'integer']}}}, 'type': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'company': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'location': {'type': ['null', 'string']}, 'followers': {'type': ['null', 'integer']}, 'following': {'type': ['null', 'integer']}, 'hooks_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'disk_usage': {'type': ['null', 'integer']}, 'events_url': {'type': ['null', 'string']}, 'issues_url': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'description': {'type': ['null', 'string']}, 'is_verified': {'type': ['null', 'boolean']}, 'members_url': {'type': ['null', 'string']}, 'public_gists': {'type': ['null', 'integer']}, 'public_repos': {'type': ['null', 'integer']}, 'billing_email': {'type': ['null', 'string']}, 'collaborators': {'type': ['null', 'integer']}, 'private_gists': {'type': ['null', 'integer']}, 'twitter_username': {'type': ['null', 'string']}, 'public_members_url': {'type': ['null', 'string']}, 'owned_private_repos': {'type': ['null', 'integer']}, 'total_private_repos': {'type': ['null', 'integer']}, 'has_repository_projects': {'type': ['null', 'boolean']}, 'members_can_create_pages': {'type': ['null', 'boolean']}, 'has_organization_projects': {'type': ['null', 'boolean']}, 'default_repository_permission': {'type': ['null', 'string']}, 'two_factor_requirement_enabled': {'type': ['null', 'boolean']}, 'members_can_create_public_pages': {'type': ['null', 'boolean']}, 'members_can_create_repositories': {'type': ['null', 'boolean']}, 'members_can_create_private_pages': {'type': ['null', 'boolean']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'organizations', 'selected': True}}, {'stream': {'name': 'projects', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'owner_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'creator_id': {'type': ['null', 'integer']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'columns_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'projects', 'selected': True}}, {'stream': {'name': 'pull_request_stats', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'merged': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'commits': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'comments': {'type': ['null', 'integer']}, 'additions': {'type': ['null', 'integer']}, 'deletions': {'type': ['null', 'integer']}, 'mergeable': {'type': ['null', 'boolean']}, 'merged_by': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'rebaseable': {'type': ['null', 'boolean']}, 'repository': {'type': ['string']}, 'changed_files': {'type': ['null', 'integer']}, 'mergeable_state': {'type': ['null', 'string']}, 'review_comments': {'type': ['null', 'integer']}, 'maintainer_can_modify': {'type': ['null', 'boolean']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'pull_request_stats', 'selected': True}}, {'stream': {'name': 'pull_requests', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'base': {'type': ['null', 'object'], 'properties': {'ref': {'type': ['null', 'string']}, 'sha': {'type': ['null', 'string']}, 'label': {'type': ['null', 'string']}, 'repo_id': {'type': ['null', 'integer']}, 'user_id': {'type': ['null', 'integer']}}}, 'body': {'type': ['null', 'string']}, 'head': {'type': ['null', 'object'], 'properties': {'ref': {'type': ['null', 'string']}, 'sha': {'type': ['null', 'string']}, 'label': {'type': ['null', 'string']}, 'repo_id': {'type': ['null', 'integer']}, 'user_id': {'type': ['null', 'integer']}}}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'draft': {'type': ['null', 'boolean']}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'self': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'issue': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'commits': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'comments': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'statuses': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'review_comment': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'review_comments': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'labels': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'locked': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'diff_url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'assignees': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'issue_url': {'type': ['null', 'string']}, 'merged_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'patch_url': {'type': ['null', 'string']}, 'auto_merge': {'type': ['null', 'boolean']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'assignee_id': {'type': ['null', 'integer']}, 'commits_url': {'type': ['null', 'string']}, 'comments_url': {'type': ['null', 'string']}, 'milestone_id': {'type': ['null', 'integer']}, 'statuses_url': {'type': ['null', 'string']}, 'requested_teams': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'merge_commit_sha': {'type': ['null', 'string']}, 'active_lock_reason': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}, 'review_comment_url': {'type': ['null', 'string']}, 'requested_reviewers': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'review_comments_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'pull_requests', 'selected': True}}, {'stream': {'name': 'releases', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'draft': {'type': ['null', 'boolean']}, 'assets': {'type': ['null', 'array'], 'items': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'size': {'type': ['null', 'integer']}, 'label': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'uploader_id': {'type': ['null', 'integer']}, 'content_type': {'type': ['null', 'string']}, 'download_count': {'type': ['null', 'integer']}, 'browser_download_url': {'type': ['null', 'string']}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'tag_name': {'type': ['null', 'string']}, 'author_id': {'type': ['null', 'integer']}, 'assets_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'prerelease': {'type': ['null', 'boolean']}, 'repository': {'type': ['string']}, 'upload_url': {'type': ['null', 'string']}, 'tarball_url': {'type': ['null', 'string']}, 'zipball_url': {'type': ['null', 'string']}, 'published_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'target_commitish': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'releases', 'selected': True}}, {'stream': {'name': 'repositories', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'fork': {'type': ['null', 'boolean']}, 'name': {'type': ['null', 'string']}, 'size': {'type': ['null', 'integer']}, 'owner': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'topics': {'type': ['null', 'array'], 'items': {'type': ['null', 'string']}}, 'git_url': {'type': ['null', 'string']}, 'license': {'type': ['null', 'object'], 'properties': {'key': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'spdx_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'private': {'type': ['null', 'boolean']}, 'ssh_url': {'type': ['null', 'string']}, 'svn_url': {'type': ['null', 'string']}, 'archived': {'type': ['null', 'boolean']}, 'disabled': {'type': ['null', 'boolean']}, 'has_wiki': {'type': ['null', 'boolean']}, 'homepage': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'keys_url': {'type': ['null', 'string']}, 'language': {'type': ['null', 'string']}, 'tags_url': {'type': ['null', 'string']}, 'blobs_url': {'type': ['null', 'string']}, 'clone_url': {'type': ['null', 'string']}, 'forks_url': {'type': ['null', 'string']}, 'full_name': {'type': ['null', 'string']}, 'has_pages': {'type': ['null', 'boolean']}, 'hooks_url': {'type': ['null', 'string']}, 'pulls_url': {'type': ['null', 'string']}, 'pushed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'teams_url': {'type': ['null', 'string']}, 'trees_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'events_url': {'type': ['null', 'string']}, 'has_issues': {'type': ['null', 'boolean']}, 'issues_url': {'type': ['null', 'string']}, 'labels_url': {'type': ['null', 'string']}, 'merges_url': {'type': ['null', 'string']}, 'mirror_url': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'visibility': {'type': ['null', 'string']}, 'archive_url': {'type': ['null', 'string']}, 'commits_url': {'type': ['null', 'string']}, 'compare_url': {'type': ['null', 'string']}, 'description': {'type': ['null', 'string']}, 'forks_count': {'type': ['null', 'integer']}, 'is_template': {'type': ['null', 'boolean']}, 'permissions': {'type': ['null', 'object'], 'properties': {'pull': {'type': ['null', 'boolean']}, 'push': {'type': ['null', 'boolean']}, 'admin': {'type': ['null', 'boolean']}}}, 'branches_url': {'type': ['null', 'string']}, 'comments_url': {'type': ['null', 'string']}, 'contents_url': {'type': ['null', 'string']}, 'git_refs_url': {'type': ['null', 'string']}, 'git_tags_url': {'type': ['null', 'string']}, 'has_projects': {'type': ['null', 'boolean']}, 'releases_url': {'type': ['null', 'string']}, 'statuses_url': {'type': ['null', 'string']}, 'assignees_url': {'type': ['null', 'string']}, 'downloads_url': {'type': ['null', 'string']}, 'has_downloads': {'type': ['null', 'boolean']}, 'languages_url': {'type': ['null', 'string']}, 'default_branch': {'type': ['null', 'string']}, 'milestones_url': {'type': ['null', 'string']}, 'stargazers_url': {'type': ['null', 'string']}, 'watchers_count': {'type': ['null', 'integer']}, 'deployments_url': {'type': ['null', 'string']}, 'git_commits_url': {'type': ['null', 'string']}, 'subscribers_url': {'type': ['null', 'string']}, 'contributors_url': {'type': ['null', 'string']}, 'issue_events_url': {'type': ['null', 'string']}, 'stargazers_count': {'type': ['null', 'integer']}, 'subscription_url': {'type': ['null', 'string']}, 'collaborators_url': {'type': ['null', 'string']}, 'issue_comment_url': {'type': ['null', 'string']}, 'notifications_url': {'type': ['null', 'string']}, 'open_issues_count': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'repositories', 'selected': True}}, {'stream': {'name': 'review_comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'line': {'type': ['null', 'integer']}, 'path': {'type': ['null', 'string']}, 'side': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'self': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'pull_request': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'position': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'diff_hunk': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'start_line': {'type': ['null', 'integer']}, 'start_side': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'original_line': {'type': ['null', 'integer']}, 'in_reply_to_id': {'type': ['null', 'integer']}, 'pull_request_url': {'type': ['null', 'string']}, 'original_position': {'type': ['null', 'integer']}, 'author_association': {'type': ['null', 'string']}, 'original_commit_id': {'type': ['null', 'string']}, 'original_start_line': {'type': ['null', 'integer']}, 'pull_request_review_id': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'review_comments', 'selected': True}}, {'stream': {'name': 'reviews', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'state': {'type': ['null', 'string']}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'pull_request': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'commit_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'submitted_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'pull_request_url': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'reviews', 'selected': True}}, {'stream': {'name': 'stargazers', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'user_id': {'type': ['null', 'integer']}, 'repository': {'type': ['string']}, 'starred_at': {'type': ['null', 'string'], 'format': 'date-time'}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['starred_at'], 'sourceDefinedPrimaryKey': [['user_id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['starred_at'], 'destinationSyncMode': 'append', 'primaryKey': [['user_id']], 'aliasName': 'stargazers', 'selected': True}}, {'stream': {'name': 'tags', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'name': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'tarball_url': {'type': ['null', 'string']}, 'zipball_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [], 'aliasName': 'tags', 'selected': True}}, {'stream': {'name': 'teams', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'slug': {'type': ['null', 'string']}, 'parent': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'privacy': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'permission': {'type': ['null', 'string']}, 'repository': {'type': ['null', 'string']}, 'description': {'type': ['null', 'string']}, 'members_url': {'type': ['null', 'string']}, 'organization': {'type': ['null', 'string']}, 'repositories_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'teams', 'selected': True}}, {'stream': {'name': 'users', 'jsonSchema': {'type': ['null', 'object'], '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'organization': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'users', 'selected': True}}]} 60 | return connection 61 | 62 | 63 | @pytest.fixture 64 | def dummy_connection_group_dto(): 65 | connection_group = ConnectionGroupDto() 66 | connection_group.group_name = 'github-to-postgres' 67 | connection_group.source_tags = ['github', 'red'] 68 | connection_group.destination_tags = ['postgres', 'blue'] 69 | connection_group.prefix = 'github_' 70 | connection_group.schedule = {'timeUnit': 'hours', 'units': 24} 71 | connection_group.status = 'active' 72 | connection_group.sync_catalog = None 73 | return connection_group 74 | 75 | 76 | @pytest.fixture 77 | def dummy_source_definitions(): 78 | """ 79 | Create a dummy source definition (as dict) 80 | """ 81 | source_definitions = [{'sourceDefinitionId': 'c2281cee-86f9-4a86-bb48-d23286b4c7bd', 82 | 'name': 'Slack', 'dockerRepository': 'airbyte/source-slack', 83 | 'dockerImageTag': '0.1.9', 84 | 'documentationUrl': 'https://docs.airbyte.io/integrations/sources/slack', 85 | 'icon': 'icon.png'}, 86 | {'sourceDefinitionId': 'ef69ef6e-aa7f-4af1-a01d-ef775033524e', 87 | 'name': 'GitHub', 'dockerRepository': 'airbyte/source-github-singer', 88 | 'dockerImageTag': '0.1.7', 'documentationUrl': 'https://hub.docker.com/r/airbyte/source-github-singer', 89 | 'icon': None}] 90 | return source_definitions 91 | 92 | 93 | @pytest.fixture 94 | def dummy_destination_definitions(): 95 | """ 96 | c=Create a dummy destination definition (as dict) 97 | """ 98 | destination_definitions = [{'destinationDefinitionId': '22f6c74f-5699-40ff-833c-4a879ea40133', 99 | 'name': 'BigQuery', 100 | 'dockerRepository': 'airbyte/destination-bigquery', 101 | 'dockerImageTag': '0.3.12', 102 | 'documentationUrl': 'https://docs.airbyte.io/integrations/destinations/bigquery', 103 | 'icon': None}, 104 | {'destinationDefinitionId': '25c5221d-dce2-4163-ade9-739ef790f503', 105 | 'name': 'Postgres', 106 | 'dockerRepository': 'airbyte/destination-postgres', 107 | 'dockerImageTag': '0.3.5', 108 | 'documentationUrl': 'https://docs.airbyte.io/integrations/destinations/postgres', 109 | 'icon': None}] 110 | return destination_definitions 111 | 112 | 113 | @pytest.fixture 114 | def dummy_source_dict(): 115 | """ 116 | Creates a dummy source dict 117 | """ 118 | source_dict = { 119 | 'sourceDefinitionId': 'ef69ef6e-aa7f-4af1-a01d-ef775033524e', 120 | 'sourceId': '7d95ec85-47c6-42d4-a7a2-8e5c22c810d2', 121 | 'workspaceId': 'f3b9e848-790c-4cdd-a475-5c6bb156dc10', 122 | 'connectionConfiguration': {'access_token': '**********'}, 123 | 'name': 'apache/superset', 124 | 'sourceName': 'GitHub', 125 | 'tags': ['tag1'] 126 | } 127 | return source_dict 128 | 129 | 130 | @pytest.fixture 131 | def dummy_destination_dict(): 132 | """ 133 | Creates a dummy destination dict 134 | """ 135 | destination_dict = { 136 | 'destinationDefinitionId': '25c5221d-dce2-4163-ade9-739ef790f503', 137 | 'destinationId': 'a41cb2f8-fcce-4c91-adfe-37c4586609f5', 138 | 'workspaceId': 'f3b9e848-790c-4cdd-a475-5c6bb156dc10', 139 | 'connectionConfiguration': { 140 | 'database': 'postgres', 141 | 'host': 'hostname.com', 142 | 'port': '5432', 143 | 'schema': 'demo', 144 | 'username': 'devrel_master', 145 | 'password': '**********' 146 | }, 147 | 'name': 'devrel-rds', 148 | 'destinationName': 'Postgres', 149 | 'tags': ['tag2'] 150 | } 151 | return destination_dict 152 | 153 | 154 | @pytest.fixture 155 | def dummy_existing_connection_dict(): 156 | """ 157 | Represents an existing connection pulled from an existing Airbyte deployment as a payload or config.yml 158 | """ 159 | connection_dict = { 160 | 'connectionId': '10290824-9305-47cc-8966-6dd032abd3c0', 161 | 'sourceId': '7d95ec85-47c6-42d4-a7a2-8e5c22c810d2', 162 | 'destinationId': 'a41cb2f8-fcce-4c91-adfe-37c4586609f5', 163 | 'name': 'superset-to-postgres', 164 | 'prefix': 'github_superset_', 165 | 'schedule': {'units': 24, 'timeUnit': 'hours'}, 166 | 'status': 'active', 167 | 'syncCatalog': {'streams': [{'stream': {'name': 'assignees', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'assignees', 'selected': True}}, {'stream': {'name': 'branches', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'name': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'protected': {'type': ['null', 'boolean']}, 'protection': {'type': ['null', 'object'], 'properties': {'required_status_checks': {'type': ['null', 'object'], 'properties': {'contexts': {'type': ['null', 'array'], 'items': [{'type': ['null', 'string']}, {'type': ['null', 'string']}]}, 'enforcement_level': {'type': ['null', 'string']}}}}}, 'repository': {'type': ['string']}, 'protection_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [], 'aliasName': 'branches', 'selected': True}}, {'stream': {'name': 'collaborators', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'permissions': {'type': ['null', 'object'], 'properties': {'pull': {'type': ['null', 'boolean']}, 'push': {'type': ['null', 'boolean']}, 'admin': {'type': ['null', 'boolean']}}}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'collaborators', 'selected': True}}, {'stream': {'name': 'comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'user_id': {'type': ['null', 'integer']}, 'html_url': {'type': ['null', 'string']}, 'issue_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'comments', 'selected': True}}, {'stream': {'name': 'commit_comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'line': {'type': ['null', 'integer']}, 'path': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'position': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'commit_comments', 'selected': True}}, {'stream': {'name': 'commits', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'url': {'type': ['null', 'string']}, 'tree': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'author': {'type': ['null', 'object'], 'properties': {'date': {'type': ['null', 'string'], 'format': 'date-time'}, 'name': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}}}, 'message': {'type': ['null', 'string']}, 'committer': {'type': ['null', 'object'], 'properties': {'date': {'type': ['null', 'string'], 'format': 'date-time'}, 'name': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}}}, 'verification': {'type': ['null', 'object'], 'properties': {'reason': {'type': ['null', 'string']}, 'payload': {'type': ['null', 'string']}, 'verified': {'type': ['null', 'boolean']}, 'signature': {'type': ['null', 'string']}}}, 'comment_count': {'type': ['null', 'integer']}}}, 'node_id': {'type': ['null', 'string']}, 'parents': {'type': ['null', 'array'], 'items': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}}}}, 'html_url': {'type': ['null', 'string']}, 'author_id': {'type': ['null', 'integer']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'comments_url': {'type': ['null', 'string']}, 'committer_id': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['sha']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['sha']], 'aliasName': 'commits', 'selected': True}}, {'stream': {'name': 'events', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'org_id': {'type': ['null', 'integer']}, 'public': {'type': ['null', 'boolean']}, 'payload': {'type': ['null', 'object'], 'properties': {}}, 'repo_id': {'type': ['null', 'integer']}, 'actor_id': {'type': ['null', 'integer']}, 'created_at': {'type': ['null', 'string']}, 'repository': {'type': ['string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'events', 'selected': True}}, {'stream': {'name': 'issue_events', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'event': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'actor_id': {'type': ['null', 'integer']}, 'issue_id': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'commit_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_events', 'selected': True}}, {'stream': {'name': 'issue_labels', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'color': {'type': ['null', 'string']}, 'default': {'type': ['null', 'boolean']}, 'node_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'description': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_labels', 'selected': True}}, {'stream': {'name': 'issue_milestones', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, 'due_on': {'type': ['null', 'string'], 'format': 'date-time'}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'creator_id': {'type': ['null', 'integer']}, 'labels_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'description': {'type': ['null', 'string']}, 'open_issues': {'type': ['null', 'integer']}, 'closed_issues': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issue_milestones', 'selected': True}}, {'stream': {'name': 'issues', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, 'labels': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'locked': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'user_id': {'type': ['null', 'integer']}, 'comments': {'type': ['null', 'integer']}, 'html_url': {'type': ['null', 'string']}, 'assignees': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'events_url': {'type': ['null', 'string']}, 'labels_url': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'assignee_id': {'type': ['null', 'integer']}, 'comments_url': {'type': ['null', 'string']}, 'milestone_id': {'type': ['null', 'integer']}, 'pull_request': {'type': ['null', 'object'], 'properties': {'url': {'type': ['null', 'string']}, 'diff_url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'patch_url': {'type': ['null', 'string']}}}, 'repository_url': {'type': ['null', 'string']}, 'active_lock_reason': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'issues', 'selected': True}}, {'stream': {'name': 'organizations', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'blog': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'plan': {'type': ['null', 'object'], 'properties': {'name': {'type': ['null', 'string']}, 'seats': {'type': ['null', 'integer']}, 'space': {'type': ['null', 'integer']}, 'filled_seats': {'type': ['null', 'integer']}, 'private_repos': {'type': ['null', 'integer']}}}, 'type': {'type': ['null', 'string']}, 'email': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'company': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'location': {'type': ['null', 'string']}, 'followers': {'type': ['null', 'integer']}, 'following': {'type': ['null', 'integer']}, 'hooks_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'disk_usage': {'type': ['null', 'integer']}, 'events_url': {'type': ['null', 'string']}, 'issues_url': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'description': {'type': ['null', 'string']}, 'is_verified': {'type': ['null', 'boolean']}, 'members_url': {'type': ['null', 'string']}, 'public_gists': {'type': ['null', 'integer']}, 'public_repos': {'type': ['null', 'integer']}, 'billing_email': {'type': ['null', 'string']}, 'collaborators': {'type': ['null', 'integer']}, 'private_gists': {'type': ['null', 'integer']}, 'twitter_username': {'type': ['null', 'string']}, 'public_members_url': {'type': ['null', 'string']}, 'owned_private_repos': {'type': ['null', 'integer']}, 'total_private_repos': {'type': ['null', 'integer']}, 'has_repository_projects': {'type': ['null', 'boolean']}, 'members_can_create_pages': {'type': ['null', 'boolean']}, 'has_organization_projects': {'type': ['null', 'boolean']}, 'default_repository_permission': {'type': ['null', 'string']}, 'two_factor_requirement_enabled': {'type': ['null', 'boolean']}, 'members_can_create_public_pages': {'type': ['null', 'boolean']}, 'members_can_create_repositories': {'type': ['null', 'boolean']}, 'members_can_create_private_pages': {'type': ['null', 'boolean']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'organizations', 'selected': True}}, {'stream': {'name': 'projects', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'owner_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'creator_id': {'type': ['null', 'integer']}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'columns_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'projects', 'selected': True}}, {'stream': {'name': 'pull_request_stats', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'merged': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'commits': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'comments': {'type': ['null', 'integer']}, 'additions': {'type': ['null', 'integer']}, 'deletions': {'type': ['null', 'integer']}, 'mergeable': {'type': ['null', 'boolean']}, 'merged_by': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'rebaseable': {'type': ['null', 'boolean']}, 'repository': {'type': ['string']}, 'changed_files': {'type': ['null', 'integer']}, 'mergeable_state': {'type': ['null', 'string']}, 'review_comments': {'type': ['null', 'integer']}, 'maintainer_can_modify': {'type': ['null', 'boolean']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'pull_request_stats', 'selected': True}}, {'stream': {'name': 'pull_requests', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'base': {'type': ['null', 'object'], 'properties': {'ref': {'type': ['null', 'string']}, 'sha': {'type': ['null', 'string']}, 'label': {'type': ['null', 'string']}, 'repo_id': {'type': ['null', 'integer']}, 'user_id': {'type': ['null', 'integer']}}}, 'body': {'type': ['null', 'string']}, 'head': {'type': ['null', 'object'], 'properties': {'ref': {'type': ['null', 'string']}, 'sha': {'type': ['null', 'string']}, 'label': {'type': ['null', 'string']}, 'repo_id': {'type': ['null', 'integer']}, 'user_id': {'type': ['null', 'integer']}}}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'draft': {'type': ['null', 'boolean']}, 'state': {'type': ['null', 'string']}, 'title': {'type': ['null', 'string']}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'self': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'issue': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'commits': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'comments': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'statuses': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'review_comment': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'review_comments': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'labels': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'locked': {'type': ['null', 'boolean']}, 'number': {'type': ['null', 'integer']}, 'node_id': {'type': ['null', 'string']}, 'diff_url': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'assignees': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'closed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'issue_url': {'type': ['null', 'string']}, 'merged_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'patch_url': {'type': ['null', 'string']}, 'auto_merge': {'type': ['null', 'boolean']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'assignee_id': {'type': ['null', 'integer']}, 'commits_url': {'type': ['null', 'string']}, 'comments_url': {'type': ['null', 'string']}, 'milestone_id': {'type': ['null', 'integer']}, 'statuses_url': {'type': ['null', 'string']}, 'requested_teams': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'merge_commit_sha': {'type': ['null', 'string']}, 'active_lock_reason': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}, 'review_comment_url': {'type': ['null', 'string']}, 'requested_reviewers': {'type': ['null', 'array'], 'items': {'type': ['null', 'integer']}}, 'review_comments_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'pull_requests', 'selected': True}}, {'stream': {'name': 'releases', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'draft': {'type': ['null', 'boolean']}, 'assets': {'type': ['null', 'array'], 'items': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'size': {'type': ['null', 'integer']}, 'label': {'type': ['null', 'string']}, 'state': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'uploader_id': {'type': ['null', 'integer']}, 'content_type': {'type': ['null', 'string']}, 'download_count': {'type': ['null', 'integer']}, 'browser_download_url': {'type': ['null', 'string']}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'tag_name': {'type': ['null', 'string']}, 'author_id': {'type': ['null', 'integer']}, 'assets_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'prerelease': {'type': ['null', 'boolean']}, 'repository': {'type': ['string']}, 'upload_url': {'type': ['null', 'string']}, 'tarball_url': {'type': ['null', 'string']}, 'zipball_url': {'type': ['null', 'string']}, 'published_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'target_commitish': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['created_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['created_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'releases', 'selected': True}}, {'stream': {'name': 'repositories', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'fork': {'type': ['null', 'boolean']}, 'name': {'type': ['null', 'string']}, 'size': {'type': ['null', 'integer']}, 'owner': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'topics': {'type': ['null', 'array'], 'items': {'type': ['null', 'string']}}, 'git_url': {'type': ['null', 'string']}, 'license': {'type': ['null', 'object'], 'properties': {'key': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'spdx_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'private': {'type': ['null', 'boolean']}, 'ssh_url': {'type': ['null', 'string']}, 'svn_url': {'type': ['null', 'string']}, 'archived': {'type': ['null', 'boolean']}, 'disabled': {'type': ['null', 'boolean']}, 'has_wiki': {'type': ['null', 'boolean']}, 'homepage': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'keys_url': {'type': ['null', 'string']}, 'language': {'type': ['null', 'string']}, 'tags_url': {'type': ['null', 'string']}, 'blobs_url': {'type': ['null', 'string']}, 'clone_url': {'type': ['null', 'string']}, 'forks_url': {'type': ['null', 'string']}, 'full_name': {'type': ['null', 'string']}, 'has_pages': {'type': ['null', 'boolean']}, 'hooks_url': {'type': ['null', 'string']}, 'pulls_url': {'type': ['null', 'string']}, 'pushed_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'teams_url': {'type': ['null', 'string']}, 'trees_url': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'events_url': {'type': ['null', 'string']}, 'has_issues': {'type': ['null', 'boolean']}, 'issues_url': {'type': ['null', 'string']}, 'labels_url': {'type': ['null', 'string']}, 'merges_url': {'type': ['null', 'string']}, 'mirror_url': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'visibility': {'type': ['null', 'string']}, 'archive_url': {'type': ['null', 'string']}, 'commits_url': {'type': ['null', 'string']}, 'compare_url': {'type': ['null', 'string']}, 'description': {'type': ['null', 'string']}, 'forks_count': {'type': ['null', 'integer']}, 'is_template': {'type': ['null', 'boolean']}, 'permissions': {'type': ['null', 'object'], 'properties': {'pull': {'type': ['null', 'boolean']}, 'push': {'type': ['null', 'boolean']}, 'admin': {'type': ['null', 'boolean']}}}, 'branches_url': {'type': ['null', 'string']}, 'comments_url': {'type': ['null', 'string']}, 'contents_url': {'type': ['null', 'string']}, 'git_refs_url': {'type': ['null', 'string']}, 'git_tags_url': {'type': ['null', 'string']}, 'has_projects': {'type': ['null', 'boolean']}, 'releases_url': {'type': ['null', 'string']}, 'statuses_url': {'type': ['null', 'string']}, 'assignees_url': {'type': ['null', 'string']}, 'downloads_url': {'type': ['null', 'string']}, 'has_downloads': {'type': ['null', 'boolean']}, 'languages_url': {'type': ['null', 'string']}, 'default_branch': {'type': ['null', 'string']}, 'milestones_url': {'type': ['null', 'string']}, 'stargazers_url': {'type': ['null', 'string']}, 'watchers_count': {'type': ['null', 'integer']}, 'deployments_url': {'type': ['null', 'string']}, 'git_commits_url': {'type': ['null', 'string']}, 'subscribers_url': {'type': ['null', 'string']}, 'contributors_url': {'type': ['null', 'string']}, 'issue_events_url': {'type': ['null', 'string']}, 'stargazers_count': {'type': ['null', 'integer']}, 'subscription_url': {'type': ['null', 'string']}, 'collaborators_url': {'type': ['null', 'string']}, 'issue_comment_url': {'type': ['null', 'string']}, 'notifications_url': {'type': ['null', 'string']}, 'open_issues_count': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'repositories', 'selected': True}}, {'stream': {'name': 'review_comments', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'body': {'type': ['null', 'string']}, 'line': {'type': ['null', 'integer']}, 'path': {'type': ['null', 'string']}, 'side': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'self': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'pull_request': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'position': {'type': ['null', 'integer']}, 'commit_id': {'type': ['null', 'string']}, 'diff_hunk': {'type': ['null', 'string']}, 'created_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'repository': {'type': ['string']}, 'start_line': {'type': ['null', 'integer']}, 'start_side': {'type': ['null', 'string']}, 'updated_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'original_line': {'type': ['null', 'integer']}, 'in_reply_to_id': {'type': ['null', 'integer']}, 'pull_request_url': {'type': ['null', 'string']}, 'original_position': {'type': ['null', 'integer']}, 'author_association': {'type': ['null', 'string']}, 'original_commit_id': {'type': ['null', 'string']}, 'original_start_line': {'type': ['null', 'integer']}, 'pull_request_review_id': {'type': ['null', 'integer']}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['updated_at'], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['updated_at'], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'review_comments', 'selected': True}}, {'stream': {'name': 'reviews', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'body': {'type': ['null', 'string']}, 'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'state': {'type': ['null', 'string']}, '_links': {'type': ['null', 'object'], 'properties': {'html': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}, 'pull_request': {'type': ['null', 'object'], 'properties': {'href': {'type': ['null', 'string']}}}}}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'commit_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'submitted_at': {'type': ['null', 'string'], 'format': 'date-time'}, 'pull_request_url': {'type': ['null', 'string']}, 'author_association': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'reviews', 'selected': True}}, {'stream': {'name': 'stargazers', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'user': {'type': ['null', 'object'], 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'user_id': {'type': ['null', 'integer']}, 'repository': {'type': ['string']}, 'starred_at': {'type': ['null', 'string'], 'format': 'date-time'}}}, 'supportedSyncModes': ['full_refresh', 'incremental'], 'sourceDefinedCursor': True, 'defaultCursorField': ['starred_at'], 'sourceDefinedPrimaryKey': [['user_id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': ['starred_at'], 'destinationSyncMode': 'append', 'primaryKey': [['user_id']], 'aliasName': 'stargazers', 'selected': True}}, {'stream': {'name': 'tags', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'name': {'type': ['null', 'string']}, 'commit': {'type': ['null', 'object'], 'properties': {'sha': {'type': ['null', 'string']}, 'url': {'type': ['null', 'string']}}}, 'node_id': {'type': ['null', 'string']}, 'repository': {'type': ['string']}, 'tarball_url': {'type': ['null', 'string']}, 'zipball_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [], 'aliasName': 'tags', 'selected': True}}, {'stream': {'name': 'teams', 'jsonSchema': {'type': 'object', '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'name': {'type': ['null', 'string']}, 'slug': {'type': ['null', 'string']}, 'parent': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'privacy': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'permission': {'type': ['null', 'string']}, 'repository': {'type': ['null', 'string']}, 'description': {'type': ['null', 'string']}, 'members_url': {'type': ['null', 'string']}, 'organization': {'type': ['null', 'string']}, 'repositories_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'teams', 'selected': True}}, {'stream': {'name': 'users', 'jsonSchema': {'type': ['null', 'object'], '$schema': 'http://json-schema.org/draft-07/schema#', 'properties': {'id': {'type': ['null', 'integer']}, 'url': {'type': ['null', 'string']}, 'type': {'type': ['null', 'string']}, 'login': {'type': ['null', 'string']}, 'node_id': {'type': ['null', 'string']}, 'html_url': {'type': ['null', 'string']}, 'gists_url': {'type': ['null', 'string']}, 'repos_url': {'type': ['null', 'string']}, 'avatar_url': {'type': ['null', 'string']}, 'events_url': {'type': ['null', 'string']}, 'site_admin': {'type': ['null', 'boolean']}, 'gravatar_id': {'type': ['null', 'string']}, 'starred_url': {'type': ['null', 'string']}, 'organization': {'type': ['null', 'string']}, 'followers_url': {'type': ['null', 'string']}, 'following_url': {'type': ['null', 'string']}, 'organizations_url': {'type': ['null', 'string']}, 'subscriptions_url': {'type': ['null', 'string']}, 'received_events_url': {'type': ['null', 'string']}}}, 'supportedSyncModes': ['full_refresh'], 'sourceDefinedCursor': None, 'defaultCursorField': [], 'sourceDefinedPrimaryKey': [['id']], 'namespace': None}, 'config': {'syncMode': 'full_refresh', 'cursorField': [], 'destinationSyncMode': 'append', 'primaryKey': [['id']], 'aliasName': 'users', 'selected': True}}]} 168 | } 169 | return connection_dict 170 | 171 | 172 | @pytest.fixture 173 | def dummy_new_connection_dict(): 174 | """ 175 | Represents an existing connection pulled from an existing Airbyte deployment as a payload or config.yml 176 | """ 177 | connection_dict = { 178 | 'sourceName': 'apache/superset', 179 | 'destinationName': 'postgres', 180 | 'name': 'superset-to-postgres', 181 | 'prefix': 'github_superset_', 182 | 'schedule': {'units': 24, 'timeUnit': 'hours'}, 183 | 'status': 'active', 184 | } 185 | return connection_dict 186 | 187 | @pytest.fixture 188 | def dummy_connection_group_dict(): 189 | """ 190 | Represents a set of connections defined in shortform using tags in config.yml 191 | """ 192 | connection_dict = { 193 | 'groupName': 'github-to-postgres', 194 | 'prefix': 'github_', 195 | 'schedule': {'units': 24, 'timeUnit': 'hours'}, 196 | 'status': 'active', 197 | 'sourceTags': ['github', 'red'], 198 | 'destinationTags': ['postgres', 'blue'], 199 | 'syncCatalog': None 200 | } 201 | return connection_dict 202 | 203 | 204 | @pytest.fixture 205 | def dummy_airbyte_dto_factory(dummy_source_definitions, dummy_destination_definitions): 206 | """ 207 | Create a dummy AirbyteDtoFactory given a set of dummy source and destination definitions 208 | """ 209 | dto_factory = AirbyteDtoFactory(dummy_source_definitions, dummy_destination_definitions) 210 | return dto_factory 211 | 212 | 213 | @pytest.fixture 214 | def dummy_airbyte_config_model(dummy_source_dto, dummy_destination_dto, dummy_connection_dto): 215 | """ 216 | Create a dummy AirbyteConfigModel given some dummy dtos 217 | """ 218 | config_model = AirbyteConfigModel() 219 | config_model.sources[dummy_source_dto.source_id] = dummy_source_dto 220 | config_model.destinations[dummy_destination_dto.destination_id] = dummy_destination_dto 221 | config_model.connections[dummy_connection_dto.connection_id] = dummy_connection_dto 222 | return config_model 223 | 224 | 225 | @pytest.fixture 226 | def dummy_populated_airbyte_config_model(dummy_source_dto, dummy_destination_dto, dummy_connection_dto): 227 | config_model = AirbyteConfigModel() 228 | config_model.sources[dummy_source_dto.source_id] = dummy_source_dto 229 | config_model.destinations[dummy_destination_dto.destination_id] = dummy_destination_dto 230 | config_model.connections[dummy_connection_dto.connection_id] = dummy_connection_dto 231 | return config_model 232 | 233 | 234 | @pytest.fixture 235 | def dummy_secrets_dict(): 236 | """ 237 | Create dummy secrets for the dummy sources and destinations (as dict) 238 | """ 239 | secrets_dict = { 240 | 'sources': { 241 | 'GitHub': {'access_token': 'ghp_SECRET_TOKEN'}, 242 | 'Slack': {'token': 'SLACK_SECRET_TOKEN'} 243 | }, 244 | 'destinations': { 245 | 'Postgres': {'password': 'SECRET_POSTGRES_PASSWORD'} 246 | } 247 | } 248 | return secrets_dict 249 | 250 | 251 | @pytest.fixture 252 | def dummy_config_loader(): 253 | return ConfigValidator() 254 | --------------------------------------------------------------------------------