├── tap_airbyte ├── __init__.py └── tap.py ├── tests ├── __init__.py ├── test_core.py ├── test_syncs.py └── fixtures │ ├── KPHX.csv │ └── SMEARGLE.singer ├── output └── .gitignore ├── mypy.ini ├── .vscode └── settings.json ├── .secrets └── .gitignore ├── .github └── workflows │ ├── project_add.yml │ └── ci.yml ├── meltano.yml ├── LICENSE ├── tox.ini ├── pyproject.toml ├── .gitignore └── README.md /tap_airbyte/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Test suite for tap-airbyte.""" 2 | -------------------------------------------------------------------------------- /output/.gitignore: -------------------------------------------------------------------------------- 1 | # This directory is used as a target by target-jsonl, so ignore all files 2 | 3 | * 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version = 3.9 3 | warn_unused_configs = True 4 | 5 | [mypy-backoff.*] 6 | ignore_missing_imports = True 7 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.testing.pytestArgs": [ 3 | "tap_airbyte/tests" 4 | ], 5 | "python.testing.unittestEnabled": false, 6 | "python.testing.pytestEnabled": true 7 | } -------------------------------------------------------------------------------- /.secrets/.gitignore: -------------------------------------------------------------------------------- 1 | # IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, 2 | # make sure those are never staged for commit into your git repo. You can store them here or another 3 | # secure location. 4 | # 5 | # Note: This may be redundant with the global .gitignore for, and is provided 6 | # for redundancy. If the `.secrets` folder is not needed, you may delete it 7 | # from the project. 8 | 9 | * 10 | !.gitignore 11 | -------------------------------------------------------------------------------- /.github/workflows/project_add.yml: -------------------------------------------------------------------------------- 1 | # Managed by Pulumi. Any edits to this file will be overwritten. 2 | 3 | name: Add issues and PRs to MeltanoLabs Overview Project 4 | 5 | on: 6 | issues: 7 | types: 8 | - opened 9 | - reopened 10 | - transferred 11 | pull_request: 12 | types: 13 | - opened 14 | - reopened 15 | 16 | jobs: 17 | add-to-project: 18 | name: Add issue to project 19 | runs-on: ubuntu-latest 20 | if: ${{ github.actor != 'dependabot[bot]' }} 21 | steps: 22 | - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 23 | with: 24 | project-url: https://github.com/orgs/MeltanoLabs/projects/3 25 | github-token: ${{ secrets.MELTYBOT_PROJECT_ADD_PAT }} 26 | -------------------------------------------------------------------------------- /meltano.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | send_anonymous_usage_stats: true 3 | project_id: "tap-airbyte" 4 | default_environment: test 5 | environments: 6 | - name: test 7 | plugins: 8 | extractors: 9 | - name: tap-airbyte 10 | namespace: tap_airbyte 11 | pip_url: -e . 12 | capabilities: 13 | - discover 14 | - catalog 15 | - state 16 | - about 17 | - stream-maps 18 | - name: tap-pokeapi 19 | namespace: tap_pokeapi 20 | inherit_from: tap-airbyte 21 | config: 22 | airbyte_spec: 23 | image: "airbyte/source-pokeapi" 24 | tag: "0.1.5" 25 | airbyte_config: 26 | pokemon_name: smeargle 27 | loaders: 28 | - name: target-jsonl 29 | variant: andyh1203 30 | pip_url: target-jsonl 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2022 Alex Butler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize, reopened] 6 | push: 7 | branches: [main] 8 | workflow_dispatch: 9 | inputs: {} 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 13 | cancel-in-progress: true 14 | 15 | env: 16 | FORCE_COLOR: "1" 17 | 18 | permissions: 19 | contents: read 20 | 21 | jobs: 22 | build: 23 | strategy: 24 | matrix: 25 | py_version: ["3.9", "3.10"] 26 | runs-on: ubuntu-latest 27 | steps: 28 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 29 | - name: Setup Python ${{ matrix.py_version }} 30 | uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 31 | with: 32 | python-version: ${{ matrix.py_version }} 33 | - name: Install Tap Airbyte Wrapper 34 | run: | 35 | python -m pip install --upgrade pip poetry 36 | poetry install 37 | - name: Run SDK Tests 38 | run: | 39 | poetry run pytest -k test_core 40 | - name: Run Airbyte Sync Tests 41 | run: | 42 | poetry run pytest -k test_syncs 43 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy 2 | 3 | [tox] 4 | envlist = py38 5 | ; envlist = py37, py38, py39 6 | isolated_build = true 7 | 8 | [testenv] 9 | whitelist_externals = poetry 10 | 11 | commands = 12 | poetry install -v 13 | poetry run pytest 14 | poetry run black --check tap_airbyte/ 15 | poetry run flake8 tap_airbyte 16 | poetry run pydocstyle tap_airbyte 17 | poetry run mypy tap_airbyte --exclude='tap_airbyte/tests' 18 | 19 | [testenv:pytest] 20 | # Run the python tests. 21 | # To execute, run `tox -e pytest` 22 | envlist = py37, py38, py39 23 | commands = 24 | poetry install -v 25 | poetry run pytest 26 | 27 | [testenv:format] 28 | # Attempt to auto-resolve lint errors before they are raised. 29 | # To execute, run `tox -e format` 30 | commands = 31 | poetry install -v 32 | poetry run black tap_airbyte/ 33 | poetry run isort tap_airbyte 34 | 35 | [testenv:lint] 36 | # Raise an error if lint and style standards are not met. 37 | # To execute, run `tox -e lint` 38 | commands = 39 | poetry install -v 40 | poetry run black --check --diff tap_airbyte/ 41 | poetry run isort --check tap_airbyte 42 | poetry run flake8 tap_airbyte 43 | poetry run pydocstyle tap_airbyte 44 | # refer to mypy.ini for specific settings 45 | poetry run mypy tap_airbyte --exclude='tap_airbyte/tests' 46 | 47 | [flake8] 48 | ignore = W503 49 | max-line-length = 88 50 | max-complexity = 10 51 | 52 | [pydocstyle] 53 | ignore = D105,D203,D213 54 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "tap-airbyte" 3 | version = "0.7.0" 4 | description = "Singer tap for Airbyte, built with the Meltano Singer SDK." 5 | readme = "README.md" 6 | authors = ["Alex Butler"] 7 | keywords = [ 8 | "ELT", 9 | "Airbyte", 10 | ] 11 | classifiers = [ 12 | "Intended Audience :: Developers", 13 | "Operating System :: OS Independent", 14 | "Programming Language :: Python :: 3.9", 15 | "Programming Language :: Python :: 3.10", 16 | ] 17 | license = "Apache-2.0" 18 | 19 | [tool.poetry.dependencies] 20 | python = ">=3.9,<3.11" 21 | singer-sdk = { version="~=0.39.0", extras = [] } 22 | fs-s3fs = { version = "~=1.1.1", optional = true } 23 | orjson = "^3.10.6" 24 | virtualenv = "^20.26.3" 25 | 26 | [tool.poetry.group.dev.dependencies] 27 | pytest = ">=8" 28 | singer-sdk = { version="~=0.39.0", extras = ["testing"] } 29 | 30 | [tool.poetry.extras] 31 | s3 = ["fs-s3fs"] 32 | 33 | [tool.mypy] 34 | python_version = "3.10" 35 | warn_unused_configs = true 36 | 37 | [tool.ruff] 38 | src = ["tap_airbyte"] 39 | target-version = "py39" 40 | 41 | [tool.ruff.lint] 42 | ignore = [ 43 | "ANN101", # missing-type-self 44 | "ANN102", # missing-type-cls 45 | "COM812", # missing-trailing-comma 46 | "ISC001", # single-line-implicit-string-concatenation 47 | ] 48 | select = ["ALL"] 49 | 50 | [tool.ruff.lint.flake8-annotations] 51 | allow-star-arg-any = true 52 | 53 | [tool.ruff.lint.isort] 54 | known-first-party = ["tap_airbyte"] 55 | 56 | [tool.black] # https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#configuration-via-a-file 57 | line-length = 100 58 | target-version = ["py310"] 59 | preview = true 60 | 61 | [build-system] 62 | requires = ["poetry-core==1.9.0"] 63 | build-backend = "poetry.core.masonry.api" 64 | 65 | [tool.poetry.scripts] 66 | # CLI declaration 67 | tap-airbyte = 'tap_airbyte.tap:TapAirbyte.cli' 68 | -------------------------------------------------------------------------------- /tests/test_core.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Alex Butler 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | # software and associated documentation files (the "Software"), to deal in the Software 5 | # without restriction, including without limitation the rights to use, copy, modify, merge, 6 | # publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons 7 | # to whom the Software is furnished to do so, subject to the following conditions: 8 | # 9 | # The above copyright notice and this permission notice shall be included in all copies or 10 | # substantial portions of the Software. 11 | 12 | """Tests standard tap features using the built-in SDK tests library""" 13 | 14 | from singer_sdk.testing.legacy import get_standard_tap_tests 15 | 16 | from tap_airbyte.tap import TapAirbyte 17 | 18 | 19 | # Run standard built-in tap tests from the SDK for native images: 20 | def test_standard_tap_tests_native(): 21 | """Run standard tap tests from the SDK.""" 22 | tests = get_standard_tap_tests( 23 | TapAirbyte, 24 | config={ 25 | "airbyte_spec": {"image": "airbyte/source-pokeapi", "tag": "0.2.14"}, 26 | "airbyte_config": { 27 | "pokemon_name": "chansey", 28 | }, 29 | }, 30 | ) 31 | for test in tests: 32 | test() 33 | 34 | 35 | # Run standard built-in tap tests from the SDK for non-native images: 36 | def test_standard_tap_tests_docker(): 37 | """Run standard tap tests from the SDK.""" 38 | tests = get_standard_tap_tests( 39 | TapAirbyte, 40 | config={ 41 | "airbyte_spec": {"image": "airbyte/source-pokeapi", "tag": "0.2.14"}, 42 | "airbyte_config": { 43 | "pokemon_name": "blissey", 44 | }, 45 | "skip_native_check": True 46 | } 47 | ) 48 | for test in tests: 49 | test() 50 | 51 | 52 | if __name__ == "__main__": 53 | test_standard_tap_tests_native() 54 | test_standard_tap_tests_docker() 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Secrets and internal config files 2 | **/.secrets/* 3 | 4 | # Ignore meltano internal cache and sqlite systemdb 5 | 6 | .meltano/ 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | pip-wheel-metadata/ 31 | share/python-wheels/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | MANIFEST 36 | 37 | # PyInstaller 38 | # Usually these files are written by a python script from a template 39 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 40 | *.manifest 41 | *.spec 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | 47 | # Unit test / coverage reports 48 | htmlcov/ 49 | .tox/ 50 | .nox/ 51 | .coverage 52 | .coverage.* 53 | .cache 54 | nosetests.xml 55 | coverage.xml 56 | *.cover 57 | *.py,cover 58 | .hypothesis/ 59 | .pytest_cache/ 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Django stuff: 66 | *.log 67 | local_settings.py 68 | db.sqlite3 69 | db.sqlite3-journal 70 | 71 | # Flask stuff: 72 | instance/ 73 | .webassets-cache 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | 81 | # PyBuilder 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 102 | __pypackages__/ 103 | 104 | # Celery stuff 105 | celerybeat-schedule 106 | celerybeat.pid 107 | 108 | # SageMath parsed files 109 | *.sage.py 110 | 111 | # Environments 112 | .env 113 | .venv 114 | env/ 115 | venv/ 116 | ENV/ 117 | env.bak/ 118 | venv.bak/ 119 | 120 | # Spyder project settings 121 | .spyderproject 122 | .spyproject 123 | 124 | # Rope project settings 125 | .ropeproject 126 | 127 | # mkdocs documentation 128 | /site 129 | 130 | # mypy 131 | .mypy_cache/ 132 | .dmypy.json 133 | dmypy.json 134 | 135 | # Pyre type checker 136 | .pyre/ 137 | 138 | .vscode/* 139 | .venv-*/* 140 | tap_airbyte/.venv-*/* 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

Tap-Airbyte-Wrapper

2 | 3 |

4 | Actions Status 5 | License: MIT 6 | Code style: black 7 |

8 | 9 | `tap-airbyte` is a Singer tap that wraps *all* Airbyte sources implicitly. This adds over 250 immediately usable extractors to the broader Singer ecosystem. This opens up high quality connectors for an expanded audience further democratizing ELT and encourages contributions upstream where system experts using an airbyte source via this wrapper may be inclined to contribute to the connector source, open issues, etc if it is the better option than what's available in the Singer catalog alone. 10 | 11 | Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps. 12 | 13 | ## Configuration 📝 14 | 15 | | Setting | Required | Default | Description | 16 | |:--------------------|:--------:|:-------:|:------------| 17 | | airbyte_spec | True | None | Specification for the Airbyte source connector. This is a JSON object minimally containing the `image` key. The `tag` key is optional and defaults to `latest`. | 18 | | airbyte_config | False | None | Configuration to pass through to the Airbyte source connector, this can be gleaned by running the the tap with the `--about` flag and the `--config` flag pointing to a file containing the `airbyte_spec` configuration. This is a JSON object. | 19 | | docker_mounts | False | None | Docker mounts to mount to the container. Expects a list of maps containing source, target, and type as is documented in the docker --mount [documentation](https://docs.docker.com/storage/bind-mounts/#choose-the--v-or---mount-flag) | 20 | | stream_maps | False | None | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). | 21 | | stream_map_config | False | None | User-defined config values to be used within map expressions. | 22 | | flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | 23 | | flattening_max_depth| False | None | The max depth to flatten schemas. | 24 | 25 | 26 | ### Configure using environment variables ✏️ 27 | 28 | `OCI_RUNTIME` can be set to override the default of `docker`. This lets the tap work with podman, nerdctl, colima, and so on. 29 | 30 | ```sh 31 | OCI_RUNTIME=nerdctl meltano run tap-pokeapi target-jsonl 32 | ``` 33 | 34 | This Singer tap will automatically import any environment variables within the working directory's 35 | `.env` if the `--config=ENV` is provided, such that config values will be considered if a matching 36 | environment variable is set either in the terminal context or in the `.env` file. 37 | 38 | ### Source Authentication and Authorization 👮🏽‍♂️ 39 | 40 | First, configure your tap by creating a configuration json file. In this example we will call it `github.json` since this tap may use many configurations for different sources. 41 | 42 | 43 | > ❗️ Remember the required keys for `airbyte_config` can be dumped to stdout by running `tap-airbyte --about --config /path/to/FILE` where FILE minimally contains just the airbyte_spec.image value 44 | 45 | ```json 46 | { 47 | "airbyte_spec": { 48 | "image": "source-github" 49 | }, 50 | "airbyte_config": { 51 | "credentials": { 52 | "access_token": "..." 53 | }, 54 | "repositories": "z3z1ma/*", 55 | } 56 | } 57 | ``` 58 | 59 | Run the built in Airbyte connection test to validate your configuration like this where `github.json` represents the above config (note the choice of file name is purely for illustration): 60 | 61 | ```bash 62 | tap-airbyte --config ./github.json --test 63 | ``` 64 | 65 | The `--test` flag will **validate your configuration** as being able to access the configured data source! Be sure to use it. With meltano, configuration is implicitly passed based on what's in your meltano.yml configuration which simplifies it to just `meltano invoke tap-airbyte --test` 66 | 67 | See more configuration examples in the [sync tests](tap_airbyte/tests/test_syncs.py) 68 | 69 | ## Usage 👷‍♀️ 70 | 71 | You can easily run `tap-airbyte` by itself or in a pipeline using [Meltano](https://meltano.com/). 72 | 73 | ### Executing the Tap Directly 🔨 74 | 75 | ```bash 76 | tap-airbyte --version 77 | tap-airbyte --help 78 | tap-airbyte --config CONFIG --discover > ./catalog.json 79 | ``` 80 | 81 | ## Developer Resources 👩🏼‍💻 82 | 83 | Follow these instructions to contribute to this project. 84 | 85 | ### Initialize your Development Environment 86 | 87 | ```bash 88 | pipx install poetry 89 | poetry install 90 | ``` 91 | 92 | ### Create and Run Tests 🧪 93 | 94 | Create tests within the `tap_airbyte/tests` subfolder and 95 | then run: 96 | 97 | ```bash 98 | poetry run pytest 99 | ``` 100 | 101 | You can also test the `tap-airbyte` CLI interface directly using `poetry run`: 102 | 103 | ```bash 104 | poetry run tap-airbyte --help 105 | ``` 106 | 107 | ### Testing with [Meltano](https://www.meltano.com) 108 | 109 | _**Note:** This tap will work in any Singer environment and does not require Meltano. 110 | Examples here are for convenience and to streamline end-to-end orchestration scenarios._ 111 | 112 | 113 | Next, install Meltano (if you haven't already) and any needed plugins: 114 | 115 | ```bash 116 | # Install meltano 117 | pipx install meltano 118 | # Initialize meltano within this directory 119 | cd tap-airbyte 120 | meltano install 121 | ``` 122 | 123 | Now you can test and orchestrate using Meltano: 124 | 125 | ```bash 126 | # Test invocation: 127 | meltano invoke tap-airbyte --version 128 | # OR run a test `elt` pipeline: 129 | meltano elt tap-airbyte target-jsonl 130 | ``` 131 | 132 | ### SDK Dev Guide 133 | 134 | See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html) for more instructions on how to use the SDK to 135 | develop your own taps and targets. 136 | -------------------------------------------------------------------------------- /tests/test_syncs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Alex Butler 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | # software and associated documentation files (the "Software"), to deal in the Software 5 | # without restriction, including without limitation the rights to use, copy, modify, merge, 6 | # publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons 7 | # to whom the Software is furnished to do so, subject to the following conditions: 8 | # 9 | # The above copyright notice and this permission notice shall be included in all copies or 10 | # substantial portions of the Software. 11 | 12 | import io 13 | import json 14 | from contextlib import redirect_stderr, redirect_stdout 15 | from pathlib import Path 16 | 17 | import orjson 18 | 19 | from tap_airbyte.tap import TapAirbyte 20 | 21 | 22 | def test_weather_sync(): 23 | """Run a sync and compare the output to a fixture derived from a public dataset. 24 | This test provides a very strong guarantee that the tap is working as expected.""" 25 | 26 | tap = TapAirbyte( 27 | config={ 28 | "airbyte_spec": {"image": "airbyte/source-file", "tag": "0.5.3"}, 29 | "airbyte_config": { 30 | "dataset_name": "test", 31 | "format": "csv", 32 | "url": "https://raw.githubusercontent.com/fivethirtyeight/data/master/us-weather-history/KPHX.csv", 33 | "provider": { 34 | "storage": "HTTPS", 35 | "user_agent": True, 36 | }, 37 | }, 38 | }, 39 | parse_env_config=True, 40 | ) 41 | 42 | tap.ORJSON_OPTS |= orjson.OPT_SORT_KEYS 43 | 44 | FIXTURE = Path(__file__).parent.joinpath("fixtures", "KPHX.singer") 45 | SINGER_DUMP = FIXTURE.read_text() 46 | 47 | stdout = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") 48 | stderr = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") 49 | with redirect_stdout(stdout), redirect_stderr(stderr): 50 | tap.sync_all() 51 | stdout.seek(0), stderr.seek(0) 52 | 53 | inp = stdout.readlines() 54 | dmp = SINGER_DUMP.splitlines() 55 | 56 | assert len(inp) == len(dmp), f"Expected {len(dmp)} stdout lines, got {len(inp)}" 57 | 58 | for no, test_case, baseline in enumerate(zip(stdout.readlines(), SINGER_DUMP.splitlines())): 59 | try: 60 | parsed_test_case, parsed_baseline = ( 61 | json.loads(test_case), 62 | json.loads(baseline), 63 | ) 64 | if parsed_test_case["type"] == "RECORD": 65 | assert ( 66 | parsed_baseline["type"] == "RECORD" 67 | ), f"Parsed message at line {no} is not a record but the test input is" 68 | parsed_baseline.pop("time_extracted", None) 69 | parsed_test_case.pop("time_extracted", None) 70 | assert ( 71 | parsed_baseline == parsed_test_case 72 | ), f"{no}: {parsed_baseline} != {parsed_test_case}" 73 | except json.JSONDecodeError: 74 | pass 75 | 76 | 77 | def test_poke_sync_native(): 78 | """Run a sync and compare the output to a fixture derived from a public dataset. 79 | This test provides a very strong guarantee that the tap is working as expected.""" 80 | 81 | tap = TapAirbyte( 82 | config={ 83 | "airbyte_spec": {"image": "airbyte/source-pokeapi", "tag": "0.2.14"}, 84 | "airbyte_config": { 85 | # sketch -> spore, shell smash, baton pass, focus sash 86 | # if you know, you know. 87 | "pokemon_name": "smeargle", 88 | }, 89 | }, 90 | ) 91 | 92 | tap.ORJSON_OPTS |= orjson.OPT_SORT_KEYS 93 | 94 | FIXTURE = Path(__file__).parent.joinpath("fixtures", "SMEARGLE.singer") 95 | SINGER_DUMP = FIXTURE.read_text() 96 | 97 | stdout = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") 98 | stderr = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") 99 | with redirect_stdout(stdout), redirect_stderr(stderr): 100 | tap.sync_all() 101 | stdout.seek(0), stderr.seek(0) 102 | 103 | inp = stdout.readlines() 104 | dmp = SINGER_DUMP.splitlines() 105 | 106 | assert len(inp) == len(dmp), f"Expected {len(dmp)} stdout lines, got {len(inp)}" 107 | 108 | for no, test_case, baseline in enumerate(zip(stdout.readlines(), SINGER_DUMP.splitlines())): 109 | try: 110 | parsed_test_case, parsed_baseline = ( 111 | json.loads(test_case), 112 | json.loads(baseline), 113 | ) 114 | if parsed_test_case["type"] == "RECORD": 115 | assert ( 116 | parsed_baseline["type"] == "RECORD" 117 | ), f"Parsed message at line {no} is not a record but the test input is" 118 | parsed_baseline.pop("time_extracted", None) 119 | parsed_test_case.pop("time_extracted", None) 120 | assert ( 121 | parsed_baseline == parsed_test_case 122 | ), f"{no}: {parsed_baseline} != {parsed_test_case}" 123 | except json.JSONDecodeError: 124 | pass 125 | 126 | def test_poke_sync_docker(): 127 | """Run a sync and compare the output to a fixture derived from a public dataset. 128 | This test provides a very strong guarantee that the tap is working as expected.""" 129 | 130 | tap = TapAirbyte( 131 | config={ 132 | "airbyte_spec": {"image": "airbyte/source-pokeapi", "tag": "0.2.14"}, 133 | "airbyte_config": { 134 | # sketch -> spore, shell smash, baton pass, focus sash 135 | # if you know, you know. 136 | "pokemon_name": "smeargle", 137 | }, 138 | "skip_native_check": True 139 | }, 140 | ) 141 | 142 | tap.ORJSON_OPTS |= orjson.OPT_SORT_KEYS 143 | 144 | FIXTURE = Path(__file__).parent.joinpath("fixtures", "SMEARGLE.singer") 145 | SINGER_DUMP = FIXTURE.read_text() 146 | 147 | stdout = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") 148 | stderr = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") 149 | with redirect_stdout(stdout), redirect_stderr(stderr): 150 | tap.sync_all() 151 | stdout.seek(0), stderr.seek(0) 152 | 153 | inp = stdout.readlines() 154 | dmp = SINGER_DUMP.splitlines() 155 | 156 | assert len(inp) == len(dmp), f"Expected {len(dmp)} stdout lines, got {len(inp)}" 157 | 158 | for no, test_case, baseline in enumerate(zip(stdout.readlines(), SINGER_DUMP.splitlines())): 159 | try: 160 | parsed_test_case, parsed_baseline = ( 161 | json.loads(test_case), 162 | json.loads(baseline), 163 | ) 164 | if parsed_test_case["type"] == "RECORD": 165 | assert ( 166 | parsed_baseline["type"] == "RECORD" 167 | ), f"Parsed message at line {no} is not a record but the test input is" 168 | parsed_baseline.pop("time_extracted", None) 169 | parsed_test_case.pop("time_extracted", None) 170 | assert ( 171 | parsed_baseline == parsed_test_case 172 | ), f"{no}: {parsed_baseline} != {parsed_test_case}" 173 | except json.JSONDecodeError: 174 | pass 175 | 176 | 177 | def test_docker_mount_sync(): 178 | """This test ensures that the tap can mount a docker volume and read from it.""" 179 | 180 | data = Path(__file__).parent.joinpath("fixtures", "KPHX.csv") 181 | tap = TapAirbyte( 182 | config={ 183 | "airbyte_spec": {"image": "airbyte/source-file", "tag": "0.5.3"}, 184 | "airbyte_config": { 185 | "dataset_name": "test", 186 | "format": "csv", 187 | "url": "/local/KPHX.csv", 188 | "provider": { 189 | "storage": "local", 190 | }, 191 | }, 192 | "skip_native_check": True, 193 | "docker_mounts": [ 194 | { 195 | "source": str(data.parent), 196 | "target": "/local", 197 | "type": "bind", 198 | } 199 | ], 200 | }, 201 | ) 202 | 203 | tap.ORJSON_OPTS |= orjson.OPT_SORT_KEYS 204 | 205 | FIXTURE = Path(__file__).parent.joinpath("fixtures", "KPHX.singer") 206 | SINGER_DUMP = FIXTURE.read_text() 207 | 208 | stdout = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") 209 | stderr = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") 210 | with redirect_stdout(stdout), redirect_stderr(stderr): 211 | tap.sync_all() 212 | stdout.seek(0), stderr.seek(0) 213 | 214 | inp = stdout.readlines() 215 | dmp = SINGER_DUMP.splitlines(keepends=True) 216 | 217 | assert len(inp) == len(dmp), f"Expected {len(dmp)} stdout lines, got {len(inp)}" 218 | 219 | for no, test_case, baseline in enumerate(zip(stdout.readlines(), SINGER_DUMP.splitlines())): 220 | try: 221 | parsed_test_case, parsed_baseline = ( 222 | json.loads(test_case), 223 | json.loads(baseline), 224 | ) 225 | if parsed_test_case["type"] == "RECORD": 226 | assert ( 227 | parsed_baseline["type"] == "RECORD" 228 | ), f"Parsed message at line {no} is not a record but the test input is" 229 | parsed_baseline.pop("time_extracted", None) 230 | parsed_test_case.pop("time_extracted", None) 231 | assert ( 232 | parsed_baseline == parsed_test_case 233 | ), f"{no}: {parsed_baseline} != {parsed_test_case}" 234 | except json.JSONDecodeError: 235 | pass 236 | 237 | 238 | if __name__ == "__main__": 239 | test_weather_sync() 240 | test_poke_sync_docker() 241 | test_poke_sync_native() 242 | test_docker_mount_sync() 243 | -------------------------------------------------------------------------------- /tests/fixtures/KPHX.csv: -------------------------------------------------------------------------------- 1 | date,actual_mean_temp,actual_min_temp,actual_max_temp,average_min_temp,average_max_temp,record_min_temp,record_max_temp,record_min_temp_year,record_max_temp_year,actual_precipitation,average_precipitation,record_precipitation 2 | 2014-7-1,98,86,109,82,107,65,115,1927,1990,0.00,0.02,2.68 3 | 2014-7-2,98,86,109,82,107,65,118,1911,2011,0.00,0.01,2.81 4 | 2014-7-3,94,79,108,82,107,64,117,1916,1907,0.0,0.02,0.22 5 | 2014-7-4,90,81,98,83,107,63,118,1912,1989,0.00,0.02,0.22 6 | 2014-7-5,94,84,103,83,107,63,116,1912,1983,0.01,0.02,0.18 7 | 2014-7-6,95,84,105,83,107,65,116,1902,1942,0.00,0.02,0.60 8 | 2014-7-7,98,88,108,83,107,66,115,1915,1905,0.00,0.02,0.75 9 | 2014-7-8,96,86,105,83,107,67,115,1955,1985,0.0,0.03,0.53 10 | 2014-7-9,92,83,101,83,107,66,116,1926,1958,0.0,0.02,0.44 11 | 2014-7-10,96,87,105,83,107,67,115,1926,1958,0.00,0.03,0.64 12 | 2014-7-11,98,88,107,84,107,68,118,1944,1958,0.00,0.03,0.48 13 | 2014-7-12,99,88,110,84,107,69,115,1944,2005,0.00,0.03,1.02 14 | 2014-7-13,95,80,110,84,107,66,114,1944,1989,0.03,0.03,1.30 15 | 2014-7-14,92,81,103,84,106,68,116,1962,2003,0.02,0.04,1.24 16 | 2014-7-15,95,86,103,84,106,66,117,1944,1998,0.0,0.04,0.72 17 | 2014-7-16,96,84,107,84,106,68,118,1907,1925,0.00,0.04,1.48 18 | 2014-7-17,96,86,105,84,106,65,116,1908,2005,0.00,0.04,1.31 19 | 2014-7-18,94,83,105,84,106,68,115,1946,1989,0.00,0.04,1.04 20 | 2014-7-19,96,88,104,84,106,68,116,1913,1989,0.0,0.04,0.80 21 | 2014-7-20,96,85,106,84,106,67,114,1900,1978,0.00,0.04,1.70 22 | 2014-7-21,96,85,107,84,106,65,118,1924,2006,0.00,0.05,1.54 23 | 2014-7-22,101,89,113,84,106,66,116,1924,2006,0.00,0.04,1.15 24 | 2014-7-23,104,94,114,84,106,69,114,1913,2006,0.00,0.04,0.41 25 | 2014-7-24,105,93,116,84,106,68,116,1913,2014,0.0,0.04,1.66 26 | 2014-7-25,98,87,109,84,106,67,115,1897,1943,0.0,0.05,1.10 27 | 2014-7-26,96,83,108,84,105,65,116,1952,1995,0.0,0.04,1.38 28 | 2014-7-27,94,83,104,84,105,65,118,1913,1995,0.0,0.04,1.23 29 | 2014-7-28,96,85,107,84,105,66,121,1913,1995,0.00,0.05,1.90 30 | 2014-7-29,101,92,110,84,105,68,115,1913,1995,0.00,0.04,2.05 31 | 2014-7-30,102,92,111,84,105,66,115,1913,1934,0.00,0.04,1.16 32 | 2014-7-31,98,87,109,84,105,70,115,1921,1972,0.0,0.04,1.33 33 | 2014-8-1,90,81,99,84,105,68,116,1950,1972,0.0,0.03,1.48 34 | 2014-8-2,90,78,102,84,105,70,113,1946,1918,0.0,0.04,1.09 35 | 2014-8-3,92,80,103,84,105,66,114,1956,1975,0.0,0.04,2.12 36 | 2014-8-4,92,79,105,84,105,68,116,1956,1975,0.00,0.03,0.78 37 | 2014-8-5,94,83,105,84,105,69,114,1965,1969,0.00,0.04,1.16 38 | 2014-8-6,96,84,108,84,105,68,114,1949,1995,0.00,0.03,2.01 39 | 2014-8-7,94,82,105,83,105,66,112,1928,1905,0.00,0.04,0.87 40 | 2014-8-8,95,85,105,83,105,67,116,1903,2012,0.00,0.03,0.58 41 | 2014-8-9,94,84,104,83,105,70,114,1930,2012,0.0,0.03,0.45 42 | 2014-8-10,94,84,104,83,105,68,116,1949,2003,0.00,0.04,0.32 43 | 2014-8-11,98,87,109,83,105,65,113,1949,1962,0.00,0.03,0.54 44 | 2014-8-12,88,77,98,83,105,64,115,1949,2012,0.44,0.03,0.84 45 | 2014-8-13,84,73,95,83,105,69,115,1949,2012,0.11,0.04,0.30 46 | 2014-8-14,93,83,102,83,105,69,113,1925,1988,0.00,0.03,1.20 47 | 2014-8-15,94,83,104,83,105,66,112,1968,1992,0.00,0.03,1.84 48 | 2014-8-16,97,87,106,83,105,64,113,1918,1992,0.00,0.03,1.32 49 | 2014-8-17,99,88,109,83,104,64,114,1949,2013,0.0,0.03,0.89 50 | 2014-8-18,91,83,98,83,104,62,112,1918,2011,0.0,0.03,1.73 51 | 2014-8-19,83,74,91,83,104,63,113,1918,1986,0.43,0.03,0.78 52 | 2014-8-20,87,75,98,83,104,58,112,1917,1982,0.00,0.04,1.59 53 | 2014-8-21,81,71,90,82,104,62,110,1916,2007,0.19,0.03,0.72 54 | 2014-8-22,83,72,93,82,104,65,113,1917,2011,0.00,0.03,0.50 55 | 2014-8-23,87,76,97,82,104,61,114,1968,2011,0.00,0.03,0.80 56 | 2014-8-24,91,79,103,82,104,61,115,1965,1985,0.00,0.04,1.16 57 | 2014-8-25,92,83,101,82,104,65,113,1965,2011,0.00,0.03,0.90 58 | 2014-8-26,90,80,100,82,104,65,117,1928,2011,0.00,0.03,0.65 59 | 2014-8-27,93,83,103,82,104,64,113,1920,1981,0.00,0.03,2.43 60 | 2014-8-28,95,83,106,82,104,64,113,1920,1998,0.00,0.03,0.94 61 | 2014-8-29,96,83,108,82,104,64,113,1920,1981,0.00,0.03,0.68 62 | 2014-8-30,97,83,111,81,104,64,113,1920,2011,0.00,0.03,0.34 63 | 2014-8-31,96,83,109,81,104,62,113,1962,1950,0.00,0.02,0.61 64 | 2014-9-1,95,81,108,81,104,63,116,1962,1950,0.00,0.03,1.21 65 | 2014-9-2,96,82,109,81,103,64,112,1964,1982,0.00,0.02,1.36 66 | 2014-9-3,96,84,108,81,103,64,112,1964,1983,0.00,0.03,0.35 67 | 2014-9-4,95,83,107,80,103,60,112,1921,1948,0.0,0.02,2.91 68 | 2014-9-5,91,80,101,80,103,61,113,1912,1945,0.00,0.03,2.43 69 | 2014-9-6,96,86,105,80,103,63,111,1921,1986,0.00,0.02,0.43 70 | 2014-9-7,94,84,104,80,102,61,111,1895,1979,0.00,0.03,0.82 71 | 2014-9-8,81,71,90,80,102,62,110,1912,1979,3.29,0.02,3.29 72 | 2014-9-9,86,79,92,79,102,61,110,1920,1974,0.0,0.03,0.66 73 | 2014-9-10,88,79,97,79,102,59,111,1912,1990,0.00,0.02,1.01 74 | 2014-9-11,90,79,100,79,102,58,112,1912,1990,0.00,0.02,1.08 75 | 2014-9-12,91,78,104,78,101,58,110,1920,1971,0.00,0.02,1.90 76 | 2014-9-13,92,83,100,78,101,55,109,1952,2000,0.00,0.02,1.25 77 | 2014-9-14,92,81,102,78,101,58,112,1915,2000,0.00,0.02,1.50 78 | 2014-9-15,96,87,105,77,100,53,110,1915,2000,0.00,0.02,0.07 79 | 2014-9-16,86,80,92,77,100,56,109,1906,1928,0.01,0.03,1.33 80 | 2014-9-17,84,78,89,77,100,56,109,1906,1962,0.14,0.01,1.19 81 | 2014-9-18,86,76,96,76,99,58,109,1950,2010,0.00,0.02,1.46 82 | 2014-9-19,90,82,98,76,99,54,111,1965,2010,0.00,0.02,1.06 83 | 2014-9-20,90,81,99,76,99,47,107,1965,2010,0.00,0.02,0.49 84 | 2014-9-21,90,80,99,75,98,47,107,1965,2003,0.00,0.02,0.63 85 | 2014-9-22,91,80,101,75,98,47,109,1895,1989,0.00,0.02,0.41 86 | 2014-9-23,90,77,102,75,98,54,108,1965,1982,0.00,0.02,0.72 87 | 2014-9-24,91,77,104,74,97,55,108,1968,2002,0.00,0.02,0.68 88 | 2014-9-25,92,81,103,74,97,49,108,1913,1979,0.00,0.01,0.42 89 | 2014-9-26,92,82,101,73,96,50,108,1900,1989,0.00,0.02,2.62 90 | 2014-9-27,82,69,95,73,96,51,107,1920,2009,1.64,0.02,1.64 91 | 2014-9-28,79,71,87,72,96,50,108,1923,1992,0.02,0.02,1.31 92 | 2014-9-29,78,67,89,72,95,52,107,1923,2003,0.00,0.02,1.66 93 | 2014-9-30,78,66,89,72,95,49,107,1965,1980,0.00,0.02,0.31 94 | 2014-10-1,79,67,91,71,94,49,107,1927,1980,0.00,0.02,0.73 95 | 2014-10-2,81,69,92,71,94,49,107,1927,1980,0.00,0.02,0.60 96 | 2014-10-3,83,67,99,70,94,49,105,1907,1988,0.00,0.02,1.29 97 | 2014-10-4,84,69,99,70,93,46,103,1908,1993,0.00,0.02,0.91 98 | 2014-10-5,83,69,96,69,93,46,104,1908,1987,0.00,0.02,0.60 99 | 2014-10-6,82,70,93,69,92,46,105,1913,1917,0.00,0.02,1.17 100 | 2014-10-7,81,72,90,69,92,44,104,1913,1991,0.00,0.02,0.30 101 | 2014-10-8,76,69,82,68,92,47,104,1913,1950,0.05,0.02,1.04 102 | 2014-10-9,77,70,84,68,91,48,103,1949,1996,0.0,0.02,0.62 103 | 2014-10-10,79,69,89,67,91,44,105,1961,1991,0.00,0.01,0.56 104 | 2014-10-11,81,70,92,67,91,45,102,1920,1991,0.00,0.02,0.31 105 | 2014-10-12,82,70,94,66,90,44,103,1924,1950,0.00,0.02,0.32 106 | 2014-10-13,78,65,90,66,90,48,101,1969,1989,0.00,0.02,0.16 107 | 2014-10-14,80,65,94,66,89,43,100,1920,1973,0.00,0.02,2.32 108 | 2014-10-15,80,69,90,65,89,41,103,1899,1991,0.00,0.02,0.72 109 | 2014-10-16,80,67,92,65,89,40,101,1899,1991,0.00,0.01,0.32 110 | 2014-10-17,80,71,89,64,88,42,102,1938,2009,0.0,0.02,0.28 111 | 2014-10-18,79,68,90,64,88,43,103,1966,2003,0.00,0.02,0.75 112 | 2014-10-19,82,69,94,64,87,40,101,1917,2003,0.08,0.02,1.69 113 | 2014-10-20,78,66,90,63,87,40,103,1949,2003,0.00,0.02,0.41 114 | 2014-10-21,80,69,91,63,87,39,103,1949,2003,0.00,0.02,0.40 115 | 2014-10-22,80,67,93,62,86,37,102,1906,2003,0.00,0.02,0.38 116 | 2014-10-23,81,67,94,62,86,37,100,1906,2003,0.00,0.01,0.52 117 | 2014-10-24,82,68,96,62,85,41,96,1920,1959,0.00,0.02,0.22 118 | 2014-10-25,83,70,96,61,85,40,96,1956,1990,0.00,0.02,0.36 119 | 2014-10-26,80,71,89,61,85,42,98,1920,2001,0.00,0.02,0.33 120 | 2014-10-27,80,71,88,60,84,39,98,1918,2001,0.00,0.02,1.12 121 | 2014-10-28,77,65,88,60,84,41,97,1908,2007,0.00,0.01,0.92 122 | 2014-10-29,78,64,91,60,83,40,95,1971,2007,0.00,0.02,0.48 123 | 2014-10-30,79,64,94,59,83,34,94,1971,1990,0.00,0.02,1.00 124 | 2014-10-31,81,67,95,59,83,36,96,1900,1988,0.00,0.02,1.05 125 | 2014-11-1,76,69,83,58,82,37,96,1971,1924,0.00,0.02,1.25 126 | 2014-11-2,67,60,73,58,82,34,96,1956,1924,0.00,0.02,0.44 127 | 2014-11-3,64,53,75,58,81,36,96,1956,2009,0.00,0.02,0.31 128 | 2014-11-4,67,53,81,57,81,32,95,1956,2001,0.00,0.02,0.32 129 | 2014-11-5,73,60,85,57,80,32,93,1922,2007,0.00,0.02,0.63 130 | 2014-11-6,73,60,86,56,80,34,94,1922,2007,0.00,0.02,0.50 131 | 2014-11-7,72,59,84,56,79,34,92,1947,2007,0.00,0.02,0.39 132 | 2014-11-8,74,58,90,56,79,32,91,1897,1906,0.00,0.02,0.36 133 | 2014-11-9,74,61,87,55,79,31,88,1897,1973,0.00,0.02,0.55 134 | 2014-11-10,72,59,84,55,78,33,91,1948,2009,0.00,0.02,2.24 135 | 2014-11-11,71,58,84,54,78,35,91,1935,1934,0.00,0.01,1.41 136 | 2014-11-12,70,60,80,54,77,33,93,1898,1999,0.00,0.02,1.02 137 | 2014-11-13,71,61,80,54,77,31,91,1938,1999,0.00,0.02,0.96 138 | 2014-11-14,67,61,73,53,76,28,91,1916,1999,0.00,0.02,1.58 139 | 2014-11-15,67,57,77,53,76,33,90,1925,1999,0.00,0.02,0.58 140 | 2014-11-16,62,53,71,52,75,32,89,1916,1999,0.00,0.02,0.94 141 | 2014-11-17,59,47,71,52,75,32,87,1964,2008,0.00,0.01,0.78 142 | 2014-11-18,61,47,74,52,74,31,88,1958,2008,0.00,0.02,0.65 143 | 2014-11-19,62,50,74,51,74,30,88,1921,1897,0.00,0.02,0.65 144 | 2014-11-20,61,48,74,51,73,31,89,1956,2006,0.00,0.03,1.16 145 | 2014-11-21,63,53,72,51,73,30,88,1948,1924,0.00,0.02,0.60 146 | 2014-11-22,59,46,72,50,72,30,89,1953,1950,0.00,0.02,1.60 147 | 2014-11-23,61,48,74,50,72,27,87,1931,1950,0.00,0.03,0.98 148 | 2014-11-24,57,42,71,49,72,28,88,1931,1950,0.00,0.03,0.54 149 | 2014-11-25,58,46,69,49,71,32,88,1938,1950,0.00,0.03,0.64 150 | 2014-11-26,59,44,74,49,71,32,88,1966,1950,0.00,0.02,1.02 151 | 2014-11-27,67,47,87,48,70,31,87,1966,2014,0.00,0.03,1.29 152 | 2014-11-28,68,52,84,48,70,32,86,1896,1901,0.00,0.03,0.51 153 | 2014-11-29,66,51,80,48,70,29,85,1919,1999,0.00,0.03,0.72 154 | 2014-11-30,67,53,80,47,69,30,84,1911,1949,0.00,0.02,1.23 155 | 2014-12-1,62,51,73,47,69,31,83,1918,1949,0.00,0.03,0.40 156 | 2014-12-2,62,54,70,47,68,30,81,1913,1940,0.03,0.02,0.41 157 | 2014-12-3,68,60,76,47,68,30,81,1934,1940,0.01,0.03,0.69 158 | 2014-12-4,64,60,68,46,68,28,83,1909,1979,0.61,0.03,0.91 159 | 2014-12-5,65,57,72,46,68,25,82,1909,1965,0.00,0.03,0.67 160 | 2014-12-6,64,57,71,46,67,27,83,1960,1939,0.00,0.03,0.50 161 | 2014-12-7,64,51,77,46,67,27,83,1948,1939,0.00,0.02,1.08 162 | 2014-12-8,66,55,76,45,67,24,84,1916,1939,0.00,0.03,1.06 163 | 2014-12-9,66,54,78,45,67,24,84,1916,1981,0.00,0.03,0.77 164 | 2014-12-10,66,54,77,45,66,29,87,1967,1950,0.00,0.03,1.12 165 | 2014-12-11,66,56,76,45,66,26,81,1916,1950,0.00,0.03,0.87 166 | 2014-12-12,68,57,78,45,66,28,79,1971,2010,0.0,0.03,0.72 167 | 2014-12-13,59,51,66,45,66,26,82,1911,2010,0.08,0.03,1.04 168 | 2014-12-14,55,46,63,45,66,24,78,1901,2010,0.00,0.03,1.43 169 | 2014-12-15,55,46,64,44,66,26,79,1916,1952,0.00,0.03,0.86 170 | 2014-12-16,57,49,65,44,65,28,86,1931,1980,0.00,0.03,0.26 171 | 2014-12-17,56,51,61,44,65,28,82,1927,2013,0.07,0.03,1.35 172 | 2014-12-18,56,50,62,44,65,29,79,1933,1950,0.00,0.03,1.02 173 | 2014-12-19,55,45,64,44,65,25,79,1968,1917,0.00,0.03,0.98 174 | 2014-12-20,56,45,66,44,65,24,78,1897,1954,0.00,0.02,0.28 175 | 2014-12-21,56,47,65,44,65,27,77,1897,1985,0.00,0.03,0.52 176 | 2014-12-22,57,47,66,44,65,23,79,1897,1917,0.00,0.03,0.96 177 | 2014-12-23,58,47,68,44,65,26,79,1990,1950,0.00,0.02,0.83 178 | 2014-12-24,55,43,66,44,65,25,81,1953,1955,0.00,0.03,0.93 179 | 2014-12-25,55,45,64,44,65,27,78,1953,1950,0.00,0.03,0.63 180 | 2014-12-26,48,38,57,44,65,22,78,1911,1980,0.00,0.02,0.56 181 | 2014-12-27,48,35,60,44,65,24,82,1911,1980,0.00,0.03,1.05 182 | 2014-12-28,48,36,60,44,65,24,78,1954,1917,0.00,0.03,0.79 183 | 2014-12-29,49,36,61,44,65,26,85,1966,1980,0.00,0.03,0.47 184 | 2014-12-30,50,37,63,44,65,23,81,1895,1980,0.00,0.03,1.50 185 | 2014-12-31,43,36,50,44,66,22,79,1900,1897,0.11,0.03,0.80 186 | 2015-1-1,41,35,46,44,66,24,81,1919,1981,0.00,0.03,0.22 187 | 2015-1-2,43,31,54,45,66,23,81,1919,1981,0.00,0.03,0.99 188 | 2015-1-3,44,34,54,45,66,25,79,1950,1956,0.00,0.03,0.82 189 | 2015-1-4,50,35,65,45,66,23,83,1950,1927,0.00,0.03,0.65 190 | 2015-1-5,57,40,74,45,66,23,80,1950,1927,0.00,0.03,0.31 191 | 2015-1-6,62,44,79,45,66,17,81,1913,2006,0.00,0.03,0.76 192 | 2015-1-7,64,47,81,45,66,16,81,1913,2015,0.00,0.03,0.67 193 | 2015-1-8,63,54,71,45,66,19,81,1913,2002,0.00,0.03,1.22 194 | 2015-1-9,63,51,75,45,66,24,84,1964,1923,0.00,0.03,1.18 195 | 2015-1-10,60,54,65,45,67,25,84,1964,1990,0.0,0.03,1.51 196 | 2015-1-11,63,54,72,45,67,20,83,1964,1953,0.01,0.03,1.74 197 | 2015-1-12,62,56,67,45,67,22,80,1962,1996,0.02,0.03,1.04 198 | 2015-1-13,57,51,63,45,67,20,79,1963,1904,0.11,0.03,0.81 199 | 2015-1-14,57,46,67,45,67,22,81,1963,2000,0.00,0.04,0.77 200 | 2015-1-15,60,47,72,46,67,23,81,1964,2000,0.00,0.03,0.69 201 | 2015-1-16,62,49,74,46,67,22,83,1964,1976,0.00,0.03,1.05 202 | 2015-1-17,60,47,73,46,67,26,83,1964,1976,0.00,0.03,1.07 203 | 2015-1-18,62,48,75,46,68,27,80,1915,1971,0.00,0.03,0.51 204 | 2015-1-19,63,50,75,46,68,27,88,1915,1971,0.00,0.03,0.87 205 | 2015-1-20,62,48,75,46,68,22,84,1963,1950,0.00,0.03,0.77 206 | 2015-1-21,64,53,74,46,68,24,81,1922,2014,0.00,0.02,1.33 207 | 2015-1-22,57,48,65,46,68,21,81,1937,2013,0.00,0.03,0.35 208 | 2015-1-23,54,43,64,46,68,24,81,1937,2013,0.00,0.03,0.25 209 | 2015-1-24,64,50,77,46,68,24,80,1937,1951,0.00,0.02,0.51 210 | 2015-1-25,63,48,78,46,68,23,83,1898,1951,0.00,0.03,0.71 211 | 2015-1-26,62,53,71,46,68,26,81,1950,1987,0.01,0.03,1.18 212 | 2015-1-27,64,53,74,47,68,26,82,1950,2003,0.00,0.03,0.87 213 | 2015-1-28,64,52,75,47,69,30,83,1937,1971,0.00,0.02,0.57 214 | 2015-1-29,65,58,71,47,69,27,83,1948,1935,0.24,0.03,1.45 215 | 2015-1-30,58,55,60,47,69,28,82,1970,1935,0.35,0.03,1.19 216 | 2015-1-31,57,53,60,47,69,27,86,1949,2003,0.07,0.03,0.59 217 | 2015-2-1,60,50,69,47,69,28,83,1985,2003,0.00,0.03,1.13 218 | 2015-2-2,61,48,73,47,69,29,82,1922,1925,0.00,0.03,0.35 219 | 2015-2-3,63,50,76,47,69,28,86,1922,1925,0.00,0.03,1.06 220 | 2015-2-4,65,52,77,47,69,27,85,1955,1963,0.00,0.02,0.89 221 | 2015-2-5,67,52,82,47,69,28,87,1964,1963,0.00,0.03,0.82 222 | 2015-2-6,69,54,84,48,69,28,87,1955,1963,0.00,0.03,0.92 223 | 2015-2-7,67,53,81,48,70,24,89,1899,1963,0.00,0.03,0.74 224 | 2015-2-8,69,55,82,48,70,24,86,1933,1970,0.00,0.03,0.83 225 | 2015-2-9,69,55,83,48,70,28,84,1933,1987,0.00,0.03,0.88 226 | 2015-2-10,70,57,83,48,70,28,87,1939,1951,0.00,0.03,0.81 227 | 2015-2-11,68,54,81,48,70,26,83,1933,1951,0.00,0.03,0.77 228 | 2015-2-12,68,58,78,48,70,28,84,1965,1971,0.00,0.02,0.74 229 | 2015-2-13,68,55,80,48,70,27,88,1948,1957,0.00,0.03,1.42 230 | 2015-2-14,69,54,83,49,71,28,85,1966,1957,0.00,0.04,0.92 231 | 2015-2-15,71,63,79,49,71,26,86,1949,2014,0.00,0.03,0.79 232 | 2015-2-16,67,55,79,49,71,28,84,1964,1977,0.00,0.03,0.68 233 | 2015-2-17,66,55,77,49,71,26,88,1910,2014,0.00,0.04,0.57 234 | 2015-2-18,65,50,79,49,71,29,86,1964,1977,0.00,0.03,0.72 235 | 2015-2-19,66,52,80,49,71,29,88,1955,1977,0.00,0.04,0.93 236 | 2015-2-20,67,55,79,50,72,27,87,1955,1977,0.00,0.04,0.55 237 | 2015-2-21,67,54,79,50,72,29,86,1955,1982,0.00,0.04,0.73 238 | 2015-2-22,64,56,72,50,72,26,87,1955,1982,0.00,0.04,0.45 239 | 2015-2-23,65,59,70,50,72,29,89,1955,1989,0.01,0.03,0.94 240 | 2015-2-24,59,51,67,50,72,29,91,1964,1904,0.0,0.04,1.46 241 | 2015-2-25,60,47,72,50,72,29,92,1960,1921,0.00,0.04,0.81 242 | 2015-2-26,64,51,77,51,73,33,91,1919,1986,0.00,0.04,0.62 243 | 2015-2-27,64,50,78,51,73,28,92,1964,1986,0.00,0.03,0.57 244 | 2015-2-28,65,60,70,51,73,30,89,1962,1986,0.0,0.04,0.75 245 | 2015-3-1,69,61,77,51,73,31,89,1964,1986,0.00,0.04,0.94 246 | 2015-3-2,59,52,66,51,74,30,90,1971,1921,0.23,0.04,0.74 247 | 2015-3-3,56,47,65,51,74,28,91,1966,1921,0.07,0.04,1.98 248 | 2015-3-4,59,47,70,52,74,25,88,1966,1972,0.00,0.04,1.16 249 | 2015-3-5,61,46,75,52,74,31,93,1966,1972,0.00,0.03,0.97 250 | 2015-3-6,67,51,83,52,74,34,92,1967,1972,0.00,0.04,1.53 251 | 2015-3-7,69,55,83,52,75,34,91,1931,1972,0.00,0.04,0.71 252 | 2015-3-8,69,55,83,52,75,32,92,1969,1989,0.00,0.03,0.84 253 | 2015-3-9,70,56,84,52,75,29,94,1964,1972,0.00,0.04,0.98 254 | 2015-3-10,73,58,87,53,75,34,95,1964,1972,0.00,0.04,0.52 255 | 2015-3-11,73,59,87,53,76,33,94,1899,1900,0.00,0.03,1.40 256 | 2015-3-12,77,68,86,53,76,34,94,1909,1900,0.00,0.04,1.10 257 | 2015-3-13,74,60,88,53,76,32,92,1954,1972,0.00,0.04,0.51 258 | 2015-3-14,76,63,88,53,76,33,95,1969,2013,0.00,0.03,1.07 259 | 2015-3-15,76,63,89,53,77,31,92,1962,2007,0.00,0.04,0.79 260 | 2015-3-16,78,65,90,54,77,34,99,1917,2007,0.00,0.04,1.14 261 | 2015-3-17,78,64,91,54,77,31,99,1917,2007,0.00,0.03,0.54 262 | 2015-3-18,72,62,81,54,77,36,94,1954,2007,0.02,0.03,0.25 263 | 2015-3-19,65,60,70,54,78,34,92,1898,2004,0.01,0.04,0.54 264 | 2015-3-20,71,61,81,54,78,35,95,1898,2004,0.00,0.03,0.86 265 | 2015-3-21,73,60,85,54,78,35,97,1955,2004,0.00,0.02,0.27 266 | 2015-3-22,75,60,89,55,78,32,94,1897,1990,0.00,0.03,0.81 267 | 2015-3-23,75,61,88,55,79,31,93,1897,1990,0.00,0.03,0.52 268 | 2015-3-24,73,59,86,55,79,36,94,1949,1990,0.00,0.03,0.40 269 | 2015-3-25,75,62,88,55,79,34,93,1929,1990,0.00,0.02,0.64 270 | 2015-3-26,77,64,89,55,79,35,100,1913,1988,0.00,0.03,1.19 271 | 2015-3-27,79,63,95,55,80,33,98,1898,1986,0.00,0.02,0.65 272 | 2015-3-28,80,64,95,56,80,34,95,1898,1986,0.00,0.02,0.91 273 | 2015-3-29,81,65,97,56,80,36,97,1907,2015,0.00,0.02,0.42 274 | 2015-3-30,83,70,96,56,81,38,97,1898,2004,0.00,0.02,0.14 275 | 2015-3-31,82,68,95,56,81,32,95,1897,2015,0.00,0.02,0.27 276 | 2015-4-1,79,67,91,56,81,37,100,1904,2011,0.00,0.02,0.46 277 | 2015-4-2,76,64,87,57,81,37,98,1917,1943,0.00,0.01,0.68 278 | 2015-4-3,73,60,86,57,82,38,97,1906,1943,0.00,0.02,0.69 279 | 2015-4-4,76,62,89,57,82,39,98,1955,1961,0.00,0.02,0.43 280 | 2015-4-5,75,61,89,57,82,37,98,1921,1989,0.00,0.01,1.30 281 | 2015-4-6,74,61,86,57,82,38,102,1909,1989,0.00,0.02,0.56 282 | 2015-4-7,71,58,84,58,83,38,104,1922,1989,0.00,0.02,0.19 283 | 2015-4-8,70,61,78,58,83,39,104,1929,1989,0.00,0.01,0.99 284 | 2015-4-9,69,54,83,58,83,40,102,1953,1989,0.00,0.02,0.36 285 | 2015-4-10,72,57,87,58,84,35,100,1922,1989,0.00,0.01,0.83 286 | 2015-4-11,73,60,86,59,84,39,98,1979,1989,0.00,0.01,1.05 287 | 2015-4-12,76,65,86,59,84,40,99,1967,1936,0.00,0.01,0.27 288 | 2015-4-13,77,63,90,59,84,39,99,1927,1962,0.00,0.01,0.16 289 | 2015-4-14,81,67,94,60,85,40,103,1945,1925,0.00,0.01,0.38 290 | 2015-4-15,75,65,84,60,85,40,101,1970,1962,0.00,0.00,0.30 291 | 2015-4-16,66,56,75,60,85,40,101,1924,1984,0.00,0.01,0.67 292 | 2015-4-17,69,55,82,61,86,38,100,1924,1987,0.00,0.01,0.45 293 | 2015-4-18,75,59,90,61,86,38,100,1896,1962,0.00,0.00,0.21 294 | 2015-4-19,77,62,92,61,86,37,103,1968,1989,0.00,0.01,0.82 295 | 2015-4-20,79,65,93,61,87,38,105,1933,1989,0.00,0.01,0.41 296 | 2015-4-21,78,65,91,62,87,42,103,1967,1989,0.00,0.00,0.44 297 | 2015-4-22,77,65,89,62,87,39,105,1904,2012,0.00,0.01,0.90 298 | 2015-4-23,73,62,83,62,87,40,103,1923,2012,0.00,0.00,0.57 299 | 2015-4-24,70,62,78,63,88,41,99,1923,1987,0.02,0.00,0.17 300 | 2015-4-25,69,56,81,63,88,41,102,1964,1898,0.02,0.01,0.33 301 | 2015-4-26,65,54,75,63,88,42,101,1964,1992,0.14,0.00,0.20 302 | 2015-4-27,72,57,87,64,89,45,104,1963,1992,0.00,0.01,0.34 303 | 2015-4-28,81,68,93,64,89,45,104,1896,1992,0.00,0.00,0.65 304 | 2015-4-29,84,71,96,64,89,43,105,1970,1992,0.00,0.01,0.22 305 | 2015-4-30,85,70,99,65,90,39,102,1967,1943,0.00,0.00,0.17 306 | 2015-5-1,86,70,101,65,90,42,103,1915,1985,0.00,0.00,0.27 307 | 2015-5-2,85,72,98,65,90,40,107,1967,1947,0.00,0.01,0.47 308 | 2015-5-3,86,75,96,66,91,39,109,1899,1947,0.00,0.00,0.09 309 | 2015-5-4,75,65,85,66,91,45,106,1899,1947,0.24,0.01,0.91 310 | 2015-5-5,77,67,86,66,91,45,105,1964,1989,0.00,0.00,0.46 311 | 2015-5-6,80,69,90,67,92,44,105,1964,1947,0.00,0.00,0.12 312 | 2015-5-7,76,65,86,67,92,45,108,1896,1989,0.00,0.01,0.15 313 | 2015-5-8,68,61,75,67,92,40,110,1965,1989,0.0,0.00,0.12 314 | 2015-5-9,67,57,77,68,93,45,108,1950,1934,0.00,0.00,0.46 315 | 2015-5-10,75,61,89,68,93,46,111,1930,1934,0.00,0.01,0.78 316 | 2015-5-11,82,67,96,68,93,43,110,1905,1934,0.00,0.00,0.17 317 | 2015-5-12,81,70,91,68,94,48,109,1905,1996,0.00,0.01,0.22 318 | 2015-5-13,80,69,91,69,94,48,108,1962,1978,0.00,0.00,0.08 319 | 2015-5-14,79,70,87,69,94,50,107,1907,1927,0.00,0.00,0.21 320 | 2015-5-15,67,57,76,69,95,50,107,1968,1937,0.93,0.01,0.93 321 | 2015-5-16,66,57,75,70,95,49,106,1955,1997,0.0,0.00,0.08 322 | 2015-5-17,75,64,86,70,95,49,108,1943,1970,0.00,0.01,0.09 323 | 2015-5-18,78,65,90,70,95,44,107,1903,1970,0.00,0.00,0.06 324 | 2015-5-19,76,65,87,70,96,48,110,1903,2008,0.00,0.00,0.23 325 | 2015-5-20,80,67,93,71,96,49,108,1902,2008,0.00,0.01,0.73 326 | 2015-5-21,82,72,91,71,96,48,109,1902,2005,0.00,0.00,0.42 327 | 2015-5-22,74,66,82,71,97,48,109,1962,2000,0.00,0.00,0.28 328 | 2015-5-23,74,64,83,71,97,49,109,1965,2001,0.00,0.01,0.12 329 | 2015-5-24,76,66,85,72,97,52,109,1944,1983,0.00,0.00,0.34 330 | 2015-5-25,80,67,92,72,98,48,108,1965,2001,0.00,0.00,0.05 331 | 2015-5-26,84,71,96,72,98,48,112,1916,1951,0.00,0.01,0.14 332 | 2015-5-27,85,73,97,72,98,48,111,1917,1951,0.00,0.00,0.05 333 | 2015-5-28,85,71,98,73,99,53,113,1929,1983,0.00,0.00,0.27 334 | 2015-5-29,88,73,103,73,99,50,112,1918,1910,0.00,0.00,0.08 335 | 2015-5-30,90,74,105,73,99,51,114,1918,1910,0.00,0.01,0.01 336 | 2015-5-31,93,78,107,73,100,54,109,1918,2001,0.00,0.00,0.10 337 | 2015-6-1,92,78,105,74,100,50,111,1917,2012,0.00,0.00,0.02 338 | 2015-6-2,90,76,104,74,100,53,110,1917,1977,0.00,0.00,0.14 339 | 2015-6-3,88,74,102,74,101,51,112,1965,2006,0.00,0.00,0.41 340 | 2015-6-4,85,73,97,75,101,49,113,1908,1990,0.00,0.00,0.16 341 | 2015-6-5,81,72,90,75,101,55,112,1932,1990,0.16,0.00,0.16 342 | 2015-6-6,85,73,97,75,102,58,110,1967,2002,0.03,0.00,0.11 343 | 2015-6-7,86,71,100,75,102,56,115,1963,1985,0.00,0.00,0.06 344 | 2015-6-8,92,78,105,76,102,55,115,1950,1985,0.00,0.00,0.15 345 | 2015-6-9,89,83,95,76,103,54,114,1907,1985,0.0,0.00,0.07 346 | 2015-6-10,88,80,95,76,103,55,111,1965,1978,0.0,0.00,0.26 347 | 2015-6-11,91,82,100,76,103,55,114,1913,1918,0.00,0.00,0.02 348 | 2015-6-12,91,78,104,77,103,59,112,1913,1974,0.00,0.00,0.92 349 | 2015-6-13,93,80,105,77,104,59,114,1917,1936,0.00,0.00,0.04 350 | 2015-6-14,95,83,107,77,104,56,115,1922,1974,0.00,0.00,0.02 351 | 2015-6-15,99,86,112,78,104,54,115,1907,1974,0.00,0.00,0.08 352 | 2015-6-16,99,85,112,78,104,54,115,1907,1974,0.00,0.00,0.01 353 | 2015-6-17,100,86,114,78,105,54,114,1965,1896,0.00,0.00,0.19 354 | 2015-6-18,101,86,115,78,105,54,115,1921,1989,0.00,0.01,0.25 355 | 2015-6-19,100,85,114,79,105,61,115,1965,1968,0.00,0.00,0.19 356 | 2015-6-20,96,80,112,79,105,61,115,1964,1968,0.00,0.00,0.15 357 | 2015-6-21,99,88,109,79,105,57,115,1923,1968,0.00,0.00,0.27 358 | 2015-6-22,97,85,109,80,106,56,116,1965,1988,0.00,0.00,1.37 359 | 2015-6-23,99,87,111,80,106,61,116,1923,1974,0.00,0.00,0.84 360 | 2015-6-24,100,87,112,80,106,62,118,1945,1929,0.00,0.00,0.26 361 | 2015-6-25,100,90,110,80,106,60,120,1965,1990,0.00,0.00,0.34 362 | 2015-6-26,98,89,107,81,106,62,122,1963,1990,0.00,0.00,0.07 363 | 2015-6-27,99,90,108,81,106,55,118,1965,1990,0.01,0.00,0.04 364 | 2015-6-28,99,87,110,81,106,59,118,1965,1990,0.00,0.01,0.26 365 | 2015-6-29,98,86,110,81,107,59,119,1913,2013,0.05,0.00,0.09 366 | 2015-6-30,96,85,107,82,107,64,115,1913,1950,0.00,0.00,0.21 -------------------------------------------------------------------------------- /tap_airbyte/tap.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Alex Butler 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | # software and associated documentation files (the "Software"), to deal in the Software 5 | # without restriction, including without limitation the rights to use, copy, modify, merge, 6 | # publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons 7 | # to whom the Software is furnished to do so, subject to the following conditions: 8 | # 9 | # The above copyright notice and this permission notice shall be included in all copies or 10 | # substantial portions of the Software. 11 | """Airbyte tap class""" 12 | 13 | from __future__ import annotations 14 | 15 | from copy import deepcopy 16 | import errno 17 | import os 18 | import shutil 19 | import subprocess 20 | import sys 21 | import time 22 | import typing as t 23 | from contextlib import contextmanager 24 | from datetime import date, datetime 25 | from decimal import Decimal 26 | from enum import Enum 27 | from functools import lru_cache 28 | from pathlib import Path, PurePath 29 | from queue import Empty, Queue 30 | from tempfile import TemporaryDirectory 31 | from threading import Lock, Thread 32 | from uuid import UUID 33 | 34 | import click 35 | import orjson 36 | import requests 37 | import singer_sdk._singerlib as singer 38 | import virtualenv 39 | from singer_sdk import Stream, Tap 40 | from singer_sdk import typing as th 41 | from singer_sdk.cli import common_options 42 | from singer_sdk.helpers._classproperty import classproperty 43 | 44 | # Sentinel value for broken pipe 45 | PIPE_CLOSED = object() 46 | 47 | 48 | def default(obj): 49 | if isinstance(obj, (datetime, date)): 50 | return obj.isoformat() 51 | elif isinstance(obj, Decimal): 52 | return float(obj) 53 | elif isinstance(obj, UUID): 54 | return str(obj) 55 | elif isinstance(obj, bytes): 56 | return obj.decode("utf-8") 57 | elif isinstance(obj, Enum): 58 | return obj.value 59 | return str(obj) 60 | 61 | 62 | def write_message(message) -> None: 63 | try: 64 | sys.stdout.buffer.write( 65 | orjson.dumps(message.to_dict(), option=TapAirbyte.ORJSON_OPTS, default=default) 66 | ) 67 | sys.stdout.buffer.flush() 68 | except IOError as e: 69 | # Broken pipe 70 | if e.errno == errno.EPIPE and TapAirbyte.pipe_status is not PIPE_CLOSED: 71 | TapAirbyte.logger.info("Received SIGPIPE, stopping sync of stream.") # type: ignore 72 | TapAirbyte.pipe_status = PIPE_CLOSED # type: ignore 73 | # Prevent BrokenPipe writes to closed stdout 74 | os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) 75 | else: 76 | raise 77 | 78 | 79 | STDOUT_LOCK = Lock() 80 | singer.write_message = write_message 81 | 82 | 83 | class AirbyteException(Exception): 84 | pass 85 | 86 | 87 | class AirbyteMessage(str, Enum): 88 | RECORD = "RECORD" 89 | STATE = "STATE" 90 | LOG = "LOG" 91 | TRACE = "TRACE" 92 | CATALOG = "CATALOG" 93 | SPEC = "SPEC" 94 | CONNECTION_STATUS = "CONNECTION_STATUS" 95 | CONTROL = "CONTROL" 96 | 97 | 98 | # These translate between Singer's replication method and Airbyte's sync mode 99 | REPLICATION_METHOD_MAP = { 100 | "FULL_TABLE": "FULL_REFRESH", 101 | "INCREMENTAL": "INCREMENTAL", 102 | "LOG_BASED": "INCREMENTAL", 103 | } 104 | # We are piping to Singer targets, so this field is irrelevant 105 | NOOP_AIRBYTE_SYNC_MODE = "append" 106 | 107 | 108 | class TapAirbyte(Tap): 109 | name = "tap-airbyte" 110 | config_jsonschema = th.PropertiesList( 111 | th.Property( 112 | "airbyte_spec", 113 | th.ObjectType( 114 | th.Property( 115 | "image", 116 | th.StringType, 117 | required=True, 118 | description="Airbyte image to run", 119 | ), 120 | th.Property("tag", th.StringType, required=False, default="latest"), 121 | ), 122 | required=True, 123 | description=( 124 | "Specification for the Airbyte source connector. This is a JSON object minimally" 125 | " containing the `image` key. The `tag` key is optional and defaults to `latest`." 126 | ), 127 | ), 128 | th.Property( 129 | "airbyte_config", 130 | th.ObjectType(), 131 | required=False, 132 | description=( 133 | "Configuration to pass through to the Airbyte source connector, this can be gleaned" 134 | " by running the the tap with the `--about` flag and the `--config` flag pointing" 135 | " to a file containing the `airbyte_spec` configuration. This is a JSON object." 136 | ), 137 | ), 138 | th.Property( 139 | "docker_mounts", 140 | th.ArrayType( 141 | th.ObjectType( 142 | th.Property( 143 | "source", 144 | th.StringType, 145 | required=True, 146 | description="Source path to mount", 147 | ), 148 | th.Property( 149 | "target", 150 | th.StringType, 151 | required=True, 152 | description="Target path to mount", 153 | ), 154 | th.Property( 155 | "type", 156 | th.StringType, 157 | default="bind", 158 | description="Type of mount", 159 | ), 160 | ) 161 | ), 162 | required=False, 163 | description=( 164 | "Docker mounts to make available to the Airbyte container. Expects a list of maps" 165 | " containing source, target, and type as is documented in the docker --mount" 166 | " documentation" 167 | ), 168 | ), 169 | th.Property( 170 | "skip_native_check", 171 | th.BooleanType, 172 | required=False, 173 | default=False, 174 | description="Disables the check for natively executable sources. By default, AirByte sources are checked " 175 | "to see if they are able to be executed natively without using containers. This disables that " 176 | "check and forces them to run in containers.", 177 | ), 178 | th.Property( 179 | "native_source_python", 180 | th.StringType, 181 | required=False, 182 | description="Path to Python executable to use.", 183 | ), 184 | th.Property( 185 | "force_native", 186 | th.BooleanType, 187 | required=False, 188 | default=False, 189 | description="This flag forces the connector to run in native mode without checking.", 190 | ) 191 | 192 | ).to_dict() 193 | airbyte_mount_dir: str = os.getenv("AIRBYTE_MOUNT_DIR", "/tmp") 194 | pipe_status = None 195 | eof_received = None 196 | # Airbyte image to run 197 | _image: t.Optional[str] = None # type: ignore 198 | _tag: t.Optional[str] = None # type: ignore 199 | _docker_mounts: t.Optional[t.List[t.Dict[str, str]]] = None # type: ignore 200 | container_runtime = os.getenv("OCI_RUNTIME", "docker") 201 | 202 | # Airbyte -> Demultiplexer -< Singer Streams 203 | singer_consumers: t.List[Thread] = [] 204 | buffers: t.Dict[str, Queue] = {} 205 | 206 | # State container 207 | airbyte_state: t.Dict[str, t.Any] = {} 208 | 209 | ORJSON_OPTS = orjson.OPT_APPEND_NEWLINE 210 | 211 | @classproperty 212 | def cli(cls) -> t.Callable: 213 | @common_options.PLUGIN_VERSION 214 | @common_options.PLUGIN_ABOUT 215 | @common_options.PLUGIN_ABOUT_FORMAT 216 | @common_options.PLUGIN_CONFIG 217 | @click.option( 218 | "--discover", 219 | is_flag=True, 220 | help="Run the tap in discovery mode.", 221 | ) 222 | @click.option( 223 | "--test", 224 | is_flag=True, 225 | help="Use --test to run the Airbyte connection test.", 226 | ) 227 | @click.option( 228 | "--catalog", 229 | help="Use a Singer catalog file with the tap.", 230 | type=click.Path(), 231 | ) 232 | @click.option( 233 | "--state", 234 | help="Use a bookmarks file for incremental replication.", 235 | type=click.Path(), 236 | ) 237 | @click.command( 238 | help="Execute the Singer tap.", 239 | context_settings={"help_option_names": ["--help"]}, 240 | ) 241 | def cli( 242 | version: bool = False, 243 | about: bool = False, 244 | discover: bool = False, 245 | test: bool = False, 246 | config: tuple[str, ...] = (), 247 | state: t.Optional[str] = None, 248 | catalog: t.Optional[str] = None, 249 | about_format: t.Optional[str] = None, 250 | ) -> None: 251 | if version: 252 | cls.print_version() 253 | return 254 | if not about: 255 | cls.print_version(print_fn=cls.logger.info) 256 | validate_config: bool = True 257 | if discover or about: 258 | validate_config = False 259 | parse_env_config = False 260 | config_files: list[PurePath] = [] 261 | for config_path in config: 262 | if config_path == "ENV": 263 | parse_env_config = True 264 | continue 265 | if not Path(config_path).is_file(): 266 | raise FileNotFoundError( 267 | f"Could not locate config file at '{config_path}'." 268 | "Please check that the file exists." 269 | ) 270 | config_files.append(Path(config_path)) 271 | # Enrich about info with spec if possible 272 | if about: 273 | cls.discover_streams = lambda *_: t.cast(t.List[AirbyteStream], []) 274 | try: 275 | tap: TapAirbyte = cls( # type: ignore 276 | config=config_files or None, 277 | state=state, 278 | catalog=catalog, 279 | parse_env_config=parse_env_config, 280 | validate_config=validate_config, 281 | ) 282 | spec = tap.run_spec()["connectionSpecification"] 283 | except Exception: 284 | cls.logger.info("Tap-Airbyte instantiation failed. Printing basic about info.") 285 | cls.print_about(output_format=about_format) 286 | else: 287 | cls.logger.info( 288 | "Tap-Airbyte instantiation succeeded. Printing spec-enriched about info." 289 | ) 290 | cls.config_jsonschema["properties"]["airbyte_config"] = spec 291 | cls.print_about(output_format=about_format) 292 | cls.print_spec_as_config(spec) 293 | return 294 | # End modification 295 | tap: TapAirbyte = cls( # type: ignore 296 | config=config_files or None, 297 | state=state, 298 | catalog=catalog, 299 | parse_env_config=parse_env_config, 300 | validate_config=validate_config, 301 | ) 302 | if discover: 303 | tap.run_discovery() 304 | if test: 305 | tap.run_connection_test() 306 | elif test: 307 | tap.run_connection_test() 308 | else: 309 | tap.sync_all() 310 | 311 | return cli 312 | 313 | def _ensure_oci(self) -> None: 314 | """Ensure that the OCI runtime is installed and available.""" 315 | self.logger.info("Checking for %s on PATH.", self.container_runtime) 316 | if not shutil.which(self.container_runtime): 317 | self.logger.error( 318 | "Could not find %s on PATH. Please verify that %s is installed and on PATH.", 319 | self.container_runtime, 320 | self.container_runtime, 321 | ) 322 | sys.exit(1) 323 | self.logger.info("Found %s on PATH.", self.container_runtime) 324 | self.logger.info("Checking %s version.", self.container_runtime) 325 | try: 326 | subprocess.check_call([self.container_runtime, "version"], stdout=subprocess.DEVNULL) 327 | except subprocess.CalledProcessError as e: 328 | self.logger.error( 329 | ( 330 | "Failed to execute %s version with exit code %d. Please verify that %s is" 331 | " configured correctly." 332 | ), 333 | self.container_runtime, 334 | e.returncode, 335 | self.container_runtime, 336 | ) 337 | sys.exit(1) 338 | self.logger.info("Successfully executed %s version.", self.container_runtime) 339 | 340 | @property 341 | def native_venv_path(self) -> Path: 342 | """Get the path to the virtual environment for the connector.""" 343 | return Path(__file__).parent.resolve() / f".venv-airbyte-{self.source_name}" 344 | 345 | @property 346 | def native_venv_bin_path(self) -> Path: 347 | """Get the path to the virtual environment for the connector bin.""" 348 | return self.native_venv_path / ("Scripts" if sys.platform == "win32" else "bin") 349 | 350 | def setup_native_connector_venv(self) -> None: 351 | """Creates a virtual environment and installs the source connector via PyPI""" 352 | if self.native_venv_path.exists(): 353 | self.logger.info("Virtual environment for source already exists.") 354 | return 355 | 356 | self.logger.info( 357 | "Creating virtual environment at %s, using %s Python.", 358 | self.native_venv_path, 359 | self.config.get("native_source_python", "default") 360 | ) 361 | 362 | # Construct the arguments list for virtualenv 363 | args = [] 364 | 365 | if self.config.get("native_source_python"): 366 | args.append(["-p", self.config["native_source_python"]]) 367 | args.append(str(self.native_venv_path)) 368 | 369 | # Run the virtualenv command 370 | virtualenv.cli_run(args) 371 | 372 | self.logger.info( 373 | "Installing %s in the virtual environment..", 374 | self._get_requirement_string() 375 | ) 376 | 377 | subprocess.run( 378 | [self.native_venv_bin_path / "pip", "install", 379 | self._get_requirement_string()], 380 | check=True, 381 | stdout=subprocess.PIPE, 382 | stderr=subprocess.STDOUT 383 | ) 384 | 385 | def _run_pip_check(self) -> str: 386 | process = subprocess.run( 387 | [self.native_venv_bin_path / "pip", "check"], 388 | capture_output=True, 389 | text=True 390 | ) 391 | 392 | return process.stdout 393 | 394 | def _get_requirement_string(self) -> str: 395 | """Get the requirement string for the source connector.""" 396 | name = f"airbyte-{self.source_name}" 397 | if self.config["airbyte_spec"]["tag"] != "latest": 398 | name += f"~={self.config['airbyte_spec']['tag']}" 399 | return name 400 | 401 | def _is_native_connector(self) -> bool: 402 | """Check if the connector is available on PyPI and can be managed natively without Docker.""" 403 | # If force_native is set, skip the check and return True 404 | if self.config.get("force_native"): 405 | self.logger.info("Forcing native mode as requested by configuration.") 406 | return True 407 | 408 | is_native = False 409 | try: 410 | response = requests.get( 411 | "https://connectors.airbyte.com/files/registries/v0/oss_registry.json", 412 | timeout=5, 413 | ) 414 | response.raise_for_status() 415 | data = response.json() 416 | sources = data["sources"] 417 | image_name = self.config["airbyte_spec"]["image"] 418 | for source in sources: 419 | if source["dockerRepository"] == image_name: 420 | is_native = source.get("remoteRegistries", {}).get("pypi", {}).get("enabled") 421 | break 422 | except Exception: 423 | pass 424 | return is_native 425 | 426 | @lru_cache(maxsize=None) 427 | def is_native(self) -> bool: 428 | """Check if the connector is available on PyPI and can be managed natively without Docker.""" 429 | if self.config.get("skip_native_check", False): 430 | return False 431 | 432 | is_native = self._is_native_connector() 433 | if is_native: 434 | self.setup_native_connector_venv() 435 | pip_result = self._run_pip_check() 436 | self.logger.info(f"pip check results: {pip_result}") 437 | else: 438 | self._ensure_oci() 439 | return is_native 440 | 441 | def to_command( 442 | self, *airbyte_cmd: str, docker_args: t.Optional[t.List[str]] = None 443 | ) -> t.List[t.Union[str, Path]]: 444 | """Construct the command to run the Airbyte connector.""" 445 | return ( 446 | [self.venv / "bin" / self.source_name, *airbyte_cmd] 447 | if self.is_native() 448 | else [ 449 | "docker", 450 | "run", 451 | *(docker_args or []), 452 | f"{self.image}:{self.tag}", 453 | *airbyte_cmd, 454 | ] 455 | ) 456 | 457 | @property 458 | def venv(self) -> Path: 459 | """Get the path to the virtual environment for the connector.""" 460 | return Path(__file__).parent.resolve() / f".venv-airbyte-{self.source_name}" 461 | 462 | @property 463 | def source_name(self) -> str: 464 | """Get the name of the source connector.""" 465 | return self.config["airbyte_spec"]["image"].split("/")[1] 466 | 467 | def run_help(self) -> None: 468 | """Run the help command for the Airbyte connector.""" 469 | subprocess.run(self.to_command("--help"), check=True) 470 | 471 | def run_spec(self) -> t.Dict[str, t.Any]: 472 | """Run the spec command for the Airbyte connector.""" 473 | proc = subprocess.run( 474 | self.to_command("spec"), 475 | stdout=subprocess.PIPE, 476 | stderr=subprocess.PIPE, 477 | ) 478 | for line in proc.stdout.decode("utf-8").splitlines(): 479 | try: 480 | message = orjson.loads(line) 481 | except orjson.JSONDecodeError: 482 | if line: 483 | self.logger.warning("Could not parse message: %s", line) 484 | continue 485 | if message["type"] in (AirbyteMessage.LOG, AirbyteMessage.TRACE): 486 | self._process_log_message(message) 487 | elif message["type"] == AirbyteMessage.SPEC: 488 | return message["spec"] 489 | else: 490 | self.logger.warning("Unhandled message: %s", message) 491 | if proc.returncode != 0: 492 | raise AirbyteException(f"Could not run spec for {self.image}:{self.tag}: {proc.stderr}") 493 | raise AirbyteException( 494 | "Could not output spec, no spec message received.\n" 495 | f"Stdout: {proc.stdout.decode('utf-8')}\n" 496 | f"Stderr: {proc.stderr.decode('utf-8')}" 497 | ) 498 | 499 | @staticmethod 500 | def print_spec_as_config(spec: t.Dict[str, t.Any]) -> None: 501 | """Print the spec as a config file to stdout.""" 502 | print("\nSetup Instructions:\n") 503 | print("airbyte_config:") 504 | for prop, schema in spec["properties"].items(): 505 | if "description" in schema: 506 | print(f" # {schema['description']}") 507 | print(f" {prop}: {'fixme' if schema['type'] != 'object' else ''}") 508 | if schema["type"] == "object": 509 | if "oneOf" in schema: 510 | for i, one_of in enumerate(schema["oneOf"]): 511 | print(f" # Option {i + 1}") 512 | for inner_prop, inner_schema in one_of["properties"].items(): 513 | if inner_prop == "option_title": 514 | continue 515 | if "description" in inner_schema: 516 | print(f" # {inner_schema['description']}") 517 | print(f" {inner_prop}: fixme") 518 | else: 519 | for inner_prop, inner_schema in schema["properties"].items(): 520 | if "description" in inner_schema: 521 | print(f" # {inner_schema['description']}") 522 | print(f" {inner_prop}: fixme") 523 | 524 | def run_check(self) -> bool: 525 | """Run the check command for the Airbyte connector.""" 526 | with TemporaryDirectory() as host_tmpdir: 527 | with open(f"{host_tmpdir}/config.json", "wb") as f: 528 | f.write(orjson.dumps(self.config.get("airbyte_config", {}))) 529 | runtime_conf_dir = host_tmpdir if self.is_native() else self.airbyte_mount_dir 530 | proc = subprocess.run( 531 | self.to_command( 532 | "check", 533 | "--config", 534 | f"{runtime_conf_dir}/config.json", 535 | docker_args=[ 536 | "--rm", 537 | "-i", 538 | "-v", 539 | f"{host_tmpdir}:{self.airbyte_mount_dir}", 540 | *self.docker_mounts, 541 | ], 542 | ), 543 | stdout=subprocess.PIPE, 544 | stderr=subprocess.PIPE, 545 | ) 546 | for line in proc.stdout.decode("utf-8").splitlines(): 547 | try: 548 | message = orjson.loads(line) 549 | except orjson.JSONDecodeError: 550 | if line: 551 | self.logger.warning("Could not parse message: %s", line) 552 | continue 553 | if message["type"] in (AirbyteMessage.LOG, AirbyteMessage.TRACE): 554 | self._process_log_message(message) 555 | elif message["type"] == AirbyteMessage.CONNECTION_STATUS: 556 | if message["connectionStatus"]["status"] == "SUCCEEDED": 557 | self.logger.info( 558 | "Configuration has been verified via the Airbyte check command." 559 | ) 560 | return True 561 | else: 562 | self.logger.error( 563 | "Connection check failed: %s", 564 | message["connectionStatus"]["message"], 565 | ) 566 | return False 567 | else: 568 | self.logger.warning("Unhandled message: %s", message) 569 | if proc.returncode != 0: 570 | raise AirbyteException( 571 | f"Connection check failed with return code {proc.returncode}:" 572 | f" {proc.stderr.decode()}" 573 | ) 574 | raise AirbyteException( 575 | "Could not verify connection, no connection status message received.\n" 576 | f"Stdout: {proc.stdout.decode('utf-8')}\n" 577 | f"Stderr: {proc.stderr.decode('utf-8')}" 578 | ) 579 | 580 | def run_connection_test(self) -> bool: 581 | """Run the connection test for the Airbyte connector.""" 582 | return self.run_check() 583 | 584 | @contextmanager 585 | def run_read(self) -> t.Iterator[subprocess.Popen]: 586 | """Run the read command for the Airbyte connector.""" 587 | with TemporaryDirectory() as host_tmpdir: 588 | with open(f"{host_tmpdir}/config.json", "wb") as config, open(f"{host_tmpdir}/catalog.json", 589 | "wb") as catalog: 590 | config.write(orjson.dumps(self.config.get("airbyte_config", {}))) 591 | catalog.write(orjson.dumps(self.configured_airbyte_catalog)) 592 | if self.airbyte_state: 593 | with open(f"{host_tmpdir}/state.json", "wb") as state: 594 | # Use the new airbyte state container if it exists. 595 | state_dict = self.airbyte_state 596 | if 'airbyte_state' in self.airbyte_state: 597 | # This is airbyte state V2 598 | state_dict = self.airbyte_state['airbyte_state'] 599 | 600 | self.logger.debug("Using state: %s", state_dict) 601 | state.write(orjson.dumps(state_dict)) 602 | 603 | runtime_conf_dir = host_tmpdir if self.is_native() else self.airbyte_mount_dir 604 | proc = subprocess.Popen( 605 | self.to_command( 606 | "read", 607 | "--config", 608 | f"{runtime_conf_dir}/config.json", 609 | "--catalog", 610 | f"{runtime_conf_dir}/catalog.json", 611 | *(["--state", f"{runtime_conf_dir}/state.json"] if self.airbyte_state else []), 612 | docker_args=[ 613 | "--rm", 614 | "-i", 615 | "-v", 616 | f"{host_tmpdir}:{self.airbyte_mount_dir}", 617 | *self.docker_mounts, 618 | ], 619 | ), 620 | stdout=subprocess.PIPE, 621 | stderr=subprocess.PIPE, 622 | ) 623 | try: 624 | # Context is held until EOF or exception 625 | yield proc 626 | finally: 627 | if not self.eof_received: 628 | proc.kill() 629 | self.logger.warning("Airbyte process terminated before EOF message received.") 630 | self.logger.debug("Waiting for Airbyte process to terminate.") 631 | returncode = proc.wait() 632 | if not self.eof_received and TapAirbyte.pipe_status is not PIPE_CLOSED: 633 | # If EOF was not received, the process was killed and we should raise an exception 634 | type_, value, _ = sys.exc_info() 635 | err = type_.__name__ if type_ else "UnknownError" 636 | raise AirbyteException(f"Airbyte process terminated early:\n{err}: {value}") 637 | if returncode != 0 and TapAirbyte.pipe_status is not PIPE_CLOSED: 638 | # If EOF was received, the process should have exited with return code 0 639 | raise AirbyteException( 640 | f"Airbyte process failed with return code {returncode}:" 641 | f" {proc.stderr.read() if proc.stderr else ''}" 642 | ) 643 | 644 | def _process_log_message(self, airbyte_message: t.Dict[str, t.Any]) -> None: 645 | """Process log messages from Airbyte.""" 646 | if airbyte_message["type"] == AirbyteMessage.LOG: 647 | self.logger.info(airbyte_message["log"]) 648 | elif airbyte_message["type"] == AirbyteMessage.TRACE: 649 | if airbyte_message["trace"].get("type") == "ERROR": 650 | exc = AirbyteException( 651 | airbyte_message["trace"]["error"].get("stack_trace", "Airbyte process failed.") 652 | ) 653 | self.logger.critical( 654 | airbyte_message["trace"]["error"]["message"], 655 | exc_info=exc, 656 | ) 657 | raise exc 658 | self.logger.debug(airbyte_message["trace"]) 659 | 660 | @property 661 | def image(self) -> str: 662 | """Get the Airbyte connector image.""" 663 | if not self._image: 664 | try: 665 | self._image: str = self.config["airbyte_spec"]["image"] 666 | except KeyError: 667 | raise AirbyteException( 668 | "Airbyte spec is missing required fields. Please ensure you are passing" 669 | " --config and that the passed config contains airbyte_spec." 670 | ) from KeyError 671 | return self._image 672 | 673 | @property 674 | def tag(self) -> str: 675 | """Get the Airbyte connector tag.""" 676 | if not self._tag: 677 | try: 678 | self._tag: str = t.cast(dict, self.config["airbyte_spec"]).get("tag", "latest") 679 | except KeyError: 680 | raise AirbyteException( 681 | "Airbyte spec is missing required fields. Please ensure you are passing" 682 | " --config and that the passed config contains airbyte_spec." 683 | ) from KeyError 684 | return self._tag 685 | 686 | @property 687 | def docker_mounts(self) -> t.List[str]: 688 | """Get the Docker mounts for the Airbyte connector.""" 689 | if not self._docker_mounts: 690 | configured_mounts = [] 691 | mounts = self.config.get("docker_mounts", []) 692 | mount: t.Dict[str, str] 693 | for mount in mounts: 694 | configured_mounts.extend( 695 | [ 696 | "--mount", 697 | ( 698 | f"source={mount['source']},target={mount['target']},type={mount.get('type', 'bind')}" 699 | ), 700 | ] 701 | ) 702 | self._docker_mounts: t.List[str] = configured_mounts 703 | return self._docker_mounts 704 | 705 | @property 706 | @lru_cache(maxsize=None) 707 | def airbyte_catalog(self) -> t.Dict[str, t.Any]: 708 | """Get the Airbyte catalog.""" 709 | with TemporaryDirectory() as host_tmpdir: 710 | with open(f"{host_tmpdir}/config.json", "wb") as f: 711 | f.write(orjson.dumps(self.config.get("airbyte_config", {}))) 712 | runtime_conf_dir = host_tmpdir if self.is_native() else self.airbyte_mount_dir 713 | proc = subprocess.run( 714 | self.to_command( 715 | "discover", 716 | "--config", 717 | f"{runtime_conf_dir}/config.json", 718 | docker_args=[ 719 | "--rm", 720 | "-i", 721 | "-v", 722 | f"{host_tmpdir}:{self.airbyte_mount_dir}", 723 | *self.docker_mounts, 724 | ], 725 | ), 726 | stdout=subprocess.PIPE, 727 | stderr=subprocess.PIPE, 728 | ) 729 | for line in proc.stdout.decode("utf-8").splitlines(): 730 | try: 731 | message = orjson.loads(line) 732 | except orjson.JSONDecodeError: 733 | continue 734 | if message["type"] in (AirbyteMessage.LOG, AirbyteMessage.TRACE): 735 | self._process_log_message(message) 736 | elif message["type"] == AirbyteMessage.CATALOG: 737 | return message["catalog"] 738 | if proc.returncode != 0: 739 | raise AirbyteException( 740 | f"Discover failed with return code {proc.returncode}: {proc.stderr.decode()}" 741 | ) 742 | raise AirbyteException( 743 | "Could not discover catalog, no catalog message received. \n" 744 | f"Stdout: {proc.stdout.decode('utf-8')}\n" 745 | f"Stderr: {proc.stderr.decode('utf-8')}" 746 | ) 747 | 748 | @property 749 | def configured_airbyte_catalog(self) -> t.Dict[str, t.Any]: 750 | """Get the Airbyte catalog with only selected streams.""" 751 | output = {"streams": []} 752 | for stream in self.airbyte_catalog["streams"]: 753 | entry = self.catalog.get_stream(stream["name"]) 754 | if entry is None: 755 | continue 756 | if entry.metadata.root.selected is False: 757 | continue 758 | try: 759 | sync_mode = REPLICATION_METHOD_MAP.get( 760 | (entry.replication_method or "FULL_REFRESH").upper(), 761 | stream["supported_sync_modes"][0], 762 | ) 763 | if sync_mode.lower() not in stream["supported_sync_modes"]: 764 | sync_mode = stream["supported_sync_modes"][0] 765 | except (IndexError, KeyError): 766 | sync_mode = "FULL_REFRESH" 767 | output["streams"].append( 768 | { 769 | "stream": stream, 770 | "sync_mode": sync_mode.lower(), 771 | "destination_sync_mode": NOOP_AIRBYTE_SYNC_MODE, 772 | } 773 | ) 774 | return output 775 | 776 | def load_state(self, state: t.Dict[str, t.Any]) -> None: 777 | """Load the state from the Airbyte source.""" 778 | super().load_state(state) 779 | self.airbyte_state = state 780 | 781 | def sync_all(self) -> None: 782 | """Sync all streams from the Airbyte source.""" 783 | stream: Stream 784 | self.eof_received = False 785 | for stream in self.streams.values(): 786 | if not stream.selected and not stream.has_selected_descendents: 787 | self.logger.info(f"Skipping deselected stream '{stream.name}'.") 788 | continue 789 | consumer = Thread(target=stream.sync, daemon=True) 790 | consumer.start() 791 | self.singer_consumers.append(consumer) 792 | t1 = time.perf_counter() 793 | with self.run_read() as airbyte_job: 794 | # Main processor loop 795 | if airbyte_job.stdout is None: 796 | raise AirbyteException("Could not start Airbyte process.") 797 | while TapAirbyte.pipe_status is not PIPE_CLOSED: 798 | message = airbyte_job.stdout.readline() 799 | if not message and airbyte_job.poll() is not None: 800 | self.eof_received = True 801 | break 802 | try: 803 | airbyte_message = orjson.loads(message) 804 | except orjson.JSONDecodeError: 805 | if message: 806 | self.logger.warning("Could not parse message: %s", message) 807 | continue 808 | if airbyte_message["type"] == AirbyteMessage.RECORD: 809 | stream_buffer: Queue = self.buffers.setdefault( 810 | airbyte_message["record"]["stream"], 811 | Queue(), 812 | ) 813 | stream_buffer.put_nowait(airbyte_message["record"]["data"]) 814 | elif airbyte_message["type"] in ( 815 | AirbyteMessage.LOG, 816 | AirbyteMessage.TRACE, 817 | ): 818 | self._process_log_message(airbyte_message) 819 | elif airbyte_message["type"] == AirbyteMessage.STATE: 820 | # See: https://docs.airbyte.com/understanding-airbyte/database-data-catalog 821 | # for how this state should be handled. 822 | state_message = deepcopy(airbyte_message["state"]) 823 | state_type = state_message["type"] 824 | 825 | if "airbyte_state" not in self.airbyte_state: 826 | self.airbyte_state["airbyte_state"] = [] 827 | 828 | # The airbyte_state_v2 here should adhere to the link above. 829 | existing_airbyte_state_v2: list[dict] = deepcopy(self.airbyte_state["airbyte_state"]) 830 | if state_type == "STREAM": 831 | stream_descriptor = state_message["stream"]["stream_descriptor"] 832 | stream_state = state_message["stream"]["stream_state"] 833 | 834 | # Update the state for this stream descriptor or add it to the list. 835 | found = False 836 | for existing_state in existing_airbyte_state_v2: 837 | if existing_state["type"] == "STREAM" and existing_state["stream"]["stream_descriptor"] == stream_descriptor: 838 | existing_state["stream"]["stream_state"] = stream_state 839 | found = True 840 | break 841 | if not found: 842 | existing_airbyte_state_v2.append({ 843 | "type": "STREAM", 844 | "stream": state_message["stream"] 845 | }) 846 | elif state_type == "GLOBAL": 847 | # Update the global state. 848 | found = False 849 | for existing_state in existing_airbyte_state_v2: 850 | if existing_state["type"] == "GLOBAL": 851 | existing_state["global"] = state_message["global"] 852 | found = True 853 | break 854 | if not found: 855 | existing_airbyte_state_v2.append({ 856 | "type": "GLOBAL", 857 | "global": state_message["global"] 858 | }) 859 | elif state_type == "LEGACY": 860 | # One record per connector. 861 | existing_airbyte_state_v2.clear() 862 | existing_airbyte_state_v2.append( 863 | { 864 | "type": "LEGACY", 865 | "legacy": state_message["legacy"] 866 | } 867 | ) 868 | 869 | if "data" in state_message: 870 | unpacked_state = state_message["data"] 871 | elif state_type == "STREAM": 872 | unpacked_state = state_message["stream"] 873 | elif state_type == "GLOBAL": 874 | unpacked_state = state_message["global"] 875 | elif state_type == "LEGACY": 876 | unpacked_state = state_message["legacy"] 877 | 878 | # Keep the legacy state behavior, but append the new state under a new key. 879 | # Deepcopy here since existing_airbyte_state_v2 can reference the same object. 880 | self.airbyte_state = deepcopy(unpacked_state) 881 | self.airbyte_state['airbyte_state'] = existing_airbyte_state_v2 882 | 883 | with STDOUT_LOCK: 884 | singer.write_message(singer.StateMessage(self.airbyte_state)) 885 | elif airbyte_message["type"] == AirbyteMessage.CONTROL: 886 | pass 887 | else: 888 | self.logger.warning("Unhandled message: %s", airbyte_message) 889 | # Daemon threads will be terminated when the main thread exits, 890 | # so we do not need to wait on them to join after SIGPIPE 891 | if TapAirbyte.pipe_status is not PIPE_CLOSED: 892 | self.logger.info("Waiting for sync threads to finish...") 893 | for sync in self.singer_consumers: 894 | sync.join() 895 | # Write final state if EOF was received from Airbyte 896 | if self.eof_received: 897 | with STDOUT_LOCK: 898 | singer.write_message(singer.StateMessage(self.airbyte_state)) 899 | t2 = time.perf_counter() 900 | for stream in self.streams.values(): 901 | stream.log_sync_costs() 902 | self.logger.info(f"Synced {len(self.streams)} streams in {t2 - t1:0.2f} seconds.") 903 | 904 | def discover_streams(self) -> t.List["AirbyteStream"]: 905 | """Discover streams from the Airbyte catalog.""" 906 | output_streams: t.List[AirbyteStream] = [] 907 | stream: t.Dict[str, t.Any] 908 | for stream in self.airbyte_catalog["streams"]: 909 | airbyte_stream = AirbyteStream( 910 | tap=self, 911 | name=stream["name"], 912 | schema=stream["json_schema"], 913 | ) 914 | try: 915 | # this is [str, ...?] in the Airbyte catalog 916 | if "cursor_field" in stream and isinstance(stream["cursor_field"][0], str): 917 | airbyte_stream.replication_key = stream["cursor_field"][0] 918 | elif ( 919 | "source_defined_cursor" in stream 920 | and isinstance(stream["source_defined_cursor"], bool) 921 | and stream["source_defined_cursor"] 922 | ): 923 | # The stream has a source defined cursor. Try using that 924 | if "default_cursor_field" in stream and isinstance( 925 | stream["default_cursor_field"][0], str 926 | ): 927 | airbyte_stream.replication_key = stream["default_cursor_field"][0] 928 | else: 929 | self.logger.warning( 930 | f"Stream {stream['name']} has a source defined cursor but no default_cursor_field." 931 | ) 932 | except IndexError: 933 | pass 934 | try: 935 | # this is [[str, ...]] in the Airbyte catalog 936 | if "primary_key" in stream and isinstance(stream["primary_key"][0], t.List): 937 | airbyte_stream.primary_keys = stream["primary_key"][0] 938 | elif "source_defined_primary_key" in stream and isinstance( 939 | stream["source_defined_primary_key"][0], t.List 940 | ): 941 | airbyte_stream.primary_keys = stream["source_defined_primary_key"][0] 942 | except IndexError: 943 | pass 944 | output_streams.append(airbyte_stream) 945 | return output_streams 946 | 947 | 948 | class AirbyteStream(Stream): 949 | """Stream class for Airbyte streams.""" 950 | 951 | def __init__(self, tap: TapAirbyte, schema: dict, name: str) -> None: 952 | super().__init__(tap, schema, name) 953 | self.parent = tap 954 | self._buffer: t.Optional[Queue] = None 955 | 956 | def _write_record_message(self, record: dict) -> None: 957 | for record_message in self._generate_record_messages(record): 958 | with STDOUT_LOCK: 959 | singer.write_message(record_message) 960 | 961 | def _write_state_message(self) -> None: 962 | pass 963 | 964 | def _increment_stream_state(self, *args, **kwargs) -> None: 965 | pass 966 | 967 | @property 968 | def buffer(self) -> Queue: 969 | """Get the buffer for the stream.""" 970 | if not self._buffer: 971 | while self.name not in self.parent.buffers: 972 | if self.parent.eof_received: 973 | # EOF received, no records for this stream 974 | self._buffer = Queue() 975 | break 976 | self.logger.debug(f"Waiting for records from Airbyte for stream {self.name}...") 977 | time.sleep(1) 978 | else: 979 | self._buffer = self.parent.buffers[self.name] 980 | return self._buffer 981 | 982 | def get_records(self, context: t.Optional[dict]) -> t.Iterable[dict]: 983 | """Get records from the stream.""" 984 | while ( 985 | self.parent.eof_received is False or not self.buffer.empty() 986 | ) and TapAirbyte.pipe_status is not PIPE_CLOSED: 987 | try: 988 | # The timeout permits the consumer to re-check the producer is alive 989 | yield self.buffer.get(timeout=1.0) 990 | except Empty: 991 | continue 992 | self.buffer.task_done() 993 | if self.name in self.parent.buffers: 994 | while not self.buffer.empty() and TapAirbyte.pipe_status is not PIPE_CLOSED: 995 | try: 996 | yield self.buffer.get(timeout=1.0) 997 | except Empty: 998 | break 999 | self.buffer.task_done() 1000 | 1001 | 1002 | if __name__ == "__main__": 1003 | TapAirbyte.cli() # type: ignore 1004 | -------------------------------------------------------------------------------- /tests/fixtures/SMEARGLE.singer: -------------------------------------------------------------------------------- 1 | {"type":"SCHEMA","stream":"pokemon","schema":{"properties":{"id":{"type":["null","integer"]},"name":{"type":["null","string"]},"base_experience":{"type":["null","integer"]},"height":{"type":["null","integer"]},"is_default ":{"type":["null","boolean"]},"order":{"type":["null","integer"]},"weight":{"type":["null","integer"]},"abilities":{"items":{"properties":{"is_hidden":{"type":["null","boolean"]},"slot":{"type":["null","integer"]},"ability":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]}},"type":["null","object"]},"type":["null","array"]},"forms":{"items":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]},"type":["null","array"]},"game_indices":{"items":{"properties":{"game_index":{"type":["null","integer"]},"version":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]}},"type":["null","object"]},"type":["null","array"]},"held_items":{"items":{"properties":{"item":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]},"version_details":{"items":{"properties":{"version":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]},"rarity":{"type":["null","integer"]}},"type":["null","object"]},"type":["null","array"]}},"type":["null","object"]},"type":["null","array"]},"location_area_encounters":{"type":["null","string"]},"moves":{"items":{"properties":{"move":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]},"version_group_details":{"items":{"properties":{"move_learn_method":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]},"version_group":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]},"level_learned_at":{"type":["null","integer"]}},"type":["null","object"]},"type":["null","array"]}},"type":["null","object"]},"type":["null","array"]},"sprites":{"properties":{"front_default":{"type":["null","string"]},"front_shiny":{"type":["null","string"]},"front_female":{"type":["null","string"]},"front_shiny_female":{"type":["null","string"]},"back_default":{"type":["null","string"]},"back_shiny":{"type":["null","string"]},"back_female":{"type":["null","string"]},"back_shiny_female":{"type":["null","string"]}},"type":["null","object"]},"species":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]},"stats":{"items":{"properties":{"stat":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]},"effort":{"type":["null","integer"]},"base_stat":{"type":["null","integer"]}},"type":["null","object"]},"type":["null","array"]},"types":{"items":{"properties":{"slot":{"type":["null","integer"]},"type":{"properties":{"name":{"type":["null","string"]},"url":{"type":["null","string"]}},"type":["null","object"]}},"type":["null","object"]},"type":["null","array"]}},"type":"object"},"key_properties":[]} 2 | {"type":"STATE","value":{"stream_descriptor":{"name":"pokemon","namespace":null},"stream_state":{}}} 3 | {"type":"RECORD","stream":"pokemon","record":{"abilities":[{"ability":{"name":"own-tempo","url":"https://pokeapi.co/api/v2/ability/20/"},"is_hidden":false,"slot":1},{"ability":{"name":"technician","url":"https://pokeapi.co/api/v2/ability/101/"},"is_hidden":false,"slot":2},{"ability":{"name":"moody","url":"https://pokeapi.co/api/v2/ability/141/"},"is_hidden":true,"slot":3}],"base_experience":88,"forms":[{"name":"smeargle","url":"https://pokeapi.co/api/v2/pokemon-form/235/"}],"game_indices":[{"game_index":235,"version":{"name":"gold","url":"https://pokeapi.co/api/v2/version/4/"}},{"game_index":235,"version":{"name":"silver","url":"https://pokeapi.co/api/v2/version/5/"}},{"game_index":235,"version":{"name":"crystal","url":"https://pokeapi.co/api/v2/version/6/"}},{"game_index":235,"version":{"name":"ruby","url":"https://pokeapi.co/api/v2/version/7/"}},{"game_index":235,"version":{"name":"sapphire","url":"https://pokeapi.co/api/v2/version/8/"}},{"game_index":235,"version":{"name":"emerald","url":"https://pokeapi.co/api/v2/version/9/"}},{"game_index":235,"version":{"name":"firered","url":"https://pokeapi.co/api/v2/version/10/"}},{"game_index":235,"version":{"name":"leafgreen","url":"https://pokeapi.co/api/v2/version/11/"}},{"game_index":235,"version":{"name":"diamond","url":"https://pokeapi.co/api/v2/version/12/"}},{"game_index":235,"version":{"name":"pearl","url":"https://pokeapi.co/api/v2/version/13/"}},{"game_index":235,"version":{"name":"platinum","url":"https://pokeapi.co/api/v2/version/14/"}},{"game_index":235,"version":{"name":"heartgold","url":"https://pokeapi.co/api/v2/version/15/"}},{"game_index":235,"version":{"name":"soulsilver","url":"https://pokeapi.co/api/v2/version/16/"}},{"game_index":235,"version":{"name":"black","url":"https://pokeapi.co/api/v2/version/17/"}},{"game_index":235,"version":{"name":"white","url":"https://pokeapi.co/api/v2/version/18/"}},{"game_index":235,"version":{"name":"black-2","url":"https://pokeapi.co/api/v2/version/21/"}},{"game_index":235,"version":{"name":"white-2","url":"https://pokeapi.co/api/v2/version/22/"}}],"height":12,"held_items":[],"id":235,"location_area_encounters":"https://pokeapi.co/api/v2/pokemon/235/encounters","moves":[{"move":{"name":"sketch","url":"https://pokeapi.co/api/v2/move/166/"},"version_group_details":[{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":31,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":51,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":61,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":71,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":81,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":91,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]}],"name":"smeargle","order":335,"species":{"name":"smeargle","url":"https://pokeapi.co/api/v2/pokemon-species/235/"},"sprites":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/235.png","back_female":null,"back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/shiny/235.png","back_shiny_female":null,"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/shiny/235.png","front_shiny_female":null,"other":{"dream_world":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/dream-world/235.svg","front_female":null},"home":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/shiny/235.png","front_shiny_female":null},"official-artwork":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/official-artwork/235.png"}},"versions":{"generation-i":{"red-blue":{"back_default":null,"back_gray":null,"back_transparent":null,"front_default":null,"front_gray":null,"front_transparent":null},"yellow":{"back_default":null,"back_gray":null,"back_transparent":null,"front_default":null,"front_gray":null,"front_transparent":null}},"generation-ii":{"crystal":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/235.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/shiny/235.png","back_shiny_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/shiny/235.png","back_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/235.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/235.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/shiny/235.png","front_shiny_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/shiny/235.png","front_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/235.png"},"gold":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/235.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/shiny/235.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/235.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/shiny/235.png","front_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/transparent/235.png"},"silver":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/235.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/shiny/235.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/235.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/shiny/235.png","front_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/transparent/235.png"}},"generation-iii":{"emerald":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/235.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/shiny/235.png"},"firered-leafgreen":{"back_default":null,"back_shiny":null,"front_default":null,"front_shiny":null},"ruby-sapphire":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/235.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/shiny/235.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/235.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/shiny/235.png"}},"generation-iv":{"diamond-pearl":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/235.png","back_female":null,"back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/shiny/235.png","back_shiny_female":null,"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/shiny/235.png","front_shiny_female":null},"heartgold-soulsilver":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/235.png","back_female":null,"back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/shiny/235.png","back_shiny_female":null,"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/shiny/235.png","front_shiny_female":null},"platinum":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/235.png","back_female":null,"back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/shiny/235.png","back_shiny_female":null,"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/shiny/235.png","front_shiny_female":null}},"generation-v":{"black-white":{"animated":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/235.gif","back_female":null,"back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/shiny/235.gif","back_shiny_female":null,"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/235.gif","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/shiny/235.gif","front_shiny_female":null},"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/235.png","back_female":null,"back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/shiny/235.png","back_shiny_female":null,"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/shiny/235.png","front_shiny_female":null}},"generation-vi":{"omegaruby-alphasapphire":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/shiny/235.png","front_shiny_female":null},"x-y":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/shiny/235.png","front_shiny_female":null}},"generation-vii":{"icons":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/icons/235.png","front_female":null},"ultra-sun-ultra-moon":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/235.png","front_female":null,"front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/shiny/235.png","front_shiny_female":null}},"generation-viii":{"icons":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-viii/icons/235.png","front_female":null}}}},"stats":[{"base_stat":55,"effort":0,"stat":{"name":"hp","url":"https://pokeapi.co/api/v2/stat/1/"}},{"base_stat":20,"effort":0,"stat":{"name":"attack","url":"https://pokeapi.co/api/v2/stat/2/"}},{"base_stat":35,"effort":0,"stat":{"name":"defense","url":"https://pokeapi.co/api/v2/stat/3/"}},{"base_stat":20,"effort":0,"stat":{"name":"special-attack","url":"https://pokeapi.co/api/v2/stat/4/"}},{"base_stat":45,"effort":0,"stat":{"name":"special-defense","url":"https://pokeapi.co/api/v2/stat/5/"}},{"base_stat":75,"effort":1,"stat":{"name":"speed","url":"https://pokeapi.co/api/v2/stat/6/"}}],"types":[{"slot":1,"type":{"name":"normal","url":"https://pokeapi.co/api/v2/type/1/"}}],"weight":580},"time_extracted":"2022-12-22T07:09:33.530897+00:00"} 4 | {"type":"STATE","value":{}} 5 | --------------------------------------------------------------------------------