├── .coveragerc ├── .flake8 ├── .github ├── dependabot.yml └── workflows │ └── main.yml ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── codecov.yml ├── graphqldb ├── __init__.py ├── adapter.py ├── db_engine_specs.py ├── dialect.py └── lib.py ├── mypy.ini ├── pip-compile-all.sh ├── pyproject.toml ├── requirements-dev.in ├── requirements-dev.txt ├── requirements.in ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── conftest.py ├── test_adapter.py ├── test_dialect.py └── test_lib.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | setup.py 4 | # We don't test this as we don't have Superset 5 | graphqldb/db_engine_specs.py 6 | 7 | [report] 8 | exclude_lines = 9 | # Have to re-enable the standard pragma 10 | pragma: no cover 11 | 12 | # Don't complain if tests don't hit defensive assertion code: 13 | raise NotImplementedError 14 | 15 | # These lines are not run by tests 16 | if typing.TYPE_CHECKING: 17 | if TYPE_CHECKING: 18 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203,R504 4 | exclude = 5 | # No need to traverse our git directory 6 | .git, 7 | # There's no value in checking cache directories 8 | __pycache__, 9 | per-file-ignores = 10 | tests/**:S101 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | open-pull-requests-limit: 10 8 | allow: 9 | - dependency-type: direct 10 | - dependency-type: indirect 11 | - package-ecosystem: "github-actions" 12 | directory: "/" 13 | schedule: 14 | interval: weekly 15 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | tags: 7 | - "*" 8 | pull_request: 9 | branches: ["main"] 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | steps: 15 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 16 | - uses: actions/checkout@v4 17 | 18 | - name: Set up Python 3.9 19 | uses: actions/setup-python@v5 20 | with: 21 | # Semantic version range syntax or exact version of a Python version 22 | python-version: "3.9" 23 | cache: "pip" 24 | cache-dependency-path: "requirements-dev.txt" 25 | 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip setuptools wheel 29 | pip install -r requirements-dev.txt 30 | 31 | - name: Test with black 32 | run: | 33 | black . --check 34 | 35 | - name: Test with flake8 36 | run: | 37 | flake8 . 38 | 39 | - name: Set up mypy cache 40 | uses: actions/cache@v4.0.2 41 | with: 42 | path: .mypy_cache 43 | key: mypy1-${{ hashFiles('./graphqldb/**/*.py') }}-${{ hashFiles('./tests/**/*.py') }} 44 | restore-keys: mypy1- 45 | 46 | - name: Test with mypy 47 | run: | 48 | mypy . 49 | 50 | - name: Install self 51 | run: | 52 | pip install --no-cache-dir --no-dependencies -e . 53 | 54 | - name: Test with pytest 55 | run: | 56 | pytest --cov=./ --cov-report=xml 57 | 58 | - name: Upload coverage to Codecov 59 | uses: codecov/codecov-action@v4 60 | with: 61 | file: ./coverage.xml 62 | flags: unittests 63 | env_vars: OS,PYTHON 64 | name: codecov-umbrella 65 | token: ${{ secrets.CODECOV_TOKEN }} 66 | fail_ci_if_error: true 67 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | multi_line_output=3 3 | include_trailing_comma=True 4 | force_grid_wrap=0 5 | use_parentheses=True 6 | ensure_newline_before_comments = True 7 | line_length=88 8 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/ambv/black 3 | rev: "23.3.0" 4 | hooks: 5 | - id: black 6 | language_version: python3 7 | - repo: https://github.com/PyCQA/isort 8 | rev: "5.12.0" 9 | hooks: 10 | - id: isort 11 | language_version: python3 12 | - repo: https://github.com/pycqa/flake8 13 | rev: "6.0.0" 14 | hooks: 15 | - id: flake8 16 | language_version: python3 17 | additional_dependencies: [ 18 | flake8-bandit==4.1.1, 19 | # No need for flake8-black 20 | flake8-bugbear==23.6.5, 21 | flake8-datetimez==20.10.0, 22 | flake8-debugger==4.1.2, 23 | # No need for flake8-isort 24 | flake8-print==5.0.0, 25 | flake8-return==1.1.3, 26 | ] 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Alex Rothberg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # graphql-db-api [![PyPI version](https://badge.fury.io/py/sqlalchemy-graphqlapi.svg)](https://badge.fury.io/py/sqlalchemy-graphqlapi) ![main workflow](https://github.com/cancan101/graphql-db-api/actions/workflows/main.yml/badge.svg) [![codecov](https://codecov.io/gh/cancan101/graphql-db-api/branch/main/graph/badge.svg?token=TOI17GOA2O)](https://codecov.io/gh/cancan101/graphql-db-api) 2 | 3 | A Python DB API 2.0 for GraphQL APIs 4 | 5 | This module allows you to query GraphQL APIs using SQL. 6 | 7 | ## SQLAlchemy support 8 | 9 | This module provides a SQLAlchemy dialect. 10 | 11 | ```python 12 | from sqlalchemy.engine import create_engine 13 | 14 | # Over HTTPS (default): 15 | engine_https = create_engine('graphql://host:port/path') 16 | 17 | # Over HTTP: 18 | engine_http = create_engine('graphql://host:port/path?is_https=0') 19 | 20 | # With a `Bearer` token in the `Authorization` header: 21 | engine_http = create_engine('graphql://:token@host:port/path') 22 | ``` 23 | 24 | ### Example Usage 25 | 26 | #### Querying Connections 27 | 28 | ```python 29 | from sqlalchemy import create_engine 30 | from sqlalchemy import text 31 | 32 | # We use GraphQL SWAPI (The Star Wars API) c/o Netlify: 33 | engine = create_engine('graphql://swapi-graphql.netlify.app/.netlify/functions/index') 34 | 35 | # Demonstration of requesting nested resource of homeworld 36 | # and then selecting fields from it 37 | query = "select name, homeworld__name from 'allPeople?include=homeworld'" 38 | 39 | with engine.connect() as connection: 40 | for row in connection.execute(text(query)): 41 | print(row) 42 | ``` 43 | 44 | #### Querying Lists 45 | 46 | We can mark a given GQL query as being a List when we query that "Table" using a query parameter: 47 | 48 | ```python 49 | from sqlalchemy import create_engine 50 | from sqlalchemy import text 51 | 52 | engine = create_engine('graphql://pet-library.moonhighway.com/') 53 | 54 | # The default assumes top level is a Connection. 55 | # For Lists, we must disable this: 56 | query = "select id, name from 'allPets?is_connection=0'" 57 | 58 | with engine.connect() as connection: 59 | for row in connection.execute(text(query)): 60 | print(row) 61 | ``` 62 | 63 | alternatively, we can set that at the `Engine` level: 64 | 65 | ```python 66 | from sqlalchemy import create_engine 67 | from sqlalchemy import text 68 | 69 | # We mark 'allPets' as being a List at the Engine level: 70 | engine = create_engine( 71 | 'graphql://pet-library.moonhighway.com/', 72 | list_queries=["allPets"], 73 | ) 74 | 75 | query = "select id, name from allPets" 76 | 77 | with engine.connect() as connection: 78 | for row in connection.execute(text(query)): 79 | print(row) 80 | ``` 81 | 82 | ## Superset support 83 | 84 | In order to use with Superset, install this package and then use the `graphql` protocol in the SQLAlchemy URI like: `graphql://swapi-graphql.netlify.app/.netlify/functions/index`. We install a [`db_engine_spec`](https://github.com/cancan101/graphql-db-api/blob/main/graphqldb/db_engine_specs.py) so Superset should recognize the driver. 85 | 86 | ## Roadmap 87 | 88 | - [x] Non-Connections top level 89 | - [x] Path traversal (basic) 90 | - [ ] Path traversal (basic + nested) 91 | - [ ] Path traversal (list / connection) 92 | - [x] Bearer Tokens in `Authorization` Header 93 | - [ ] Advanced Auth (e.g. with token refresh) 94 | - [ ] Passing Headers (e.g. Auth in other locations) 95 | - [ ] Filtering 96 | - [ ] Sorting 97 | - [x] Relay Pagination 98 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | -------------------------------------------------------------------------------- /graphqldb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cancan101/graphql-db-api/f2d10ae6c2be64080db788902411b93ff5c717aa/graphqldb/__init__.py -------------------------------------------------------------------------------- /graphqldb/adapter.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections import defaultdict 4 | from typing import ( 5 | Any, 6 | Collection, 7 | Dict, 8 | Iterator, 9 | List, 10 | Optional, 11 | Sequence, 12 | Tuple, 13 | TypedDict, 14 | Union, 15 | cast, 16 | ) 17 | from urllib.parse import parse_qs, urlparse 18 | 19 | from shillelagh.adapters.base import Adapter 20 | from shillelagh.fields import ( 21 | Boolean, 22 | Field, 23 | Filter, 24 | Float, 25 | Integer, 26 | ISODate, 27 | ISODateTime, 28 | ISOTime, 29 | String, 30 | ) 31 | from shillelagh.typing import RequestedOrder 32 | 33 | from .lib import get_last_query, run_query 34 | 35 | # ----------------------------------------------------------------------------- 36 | 37 | 38 | class MaybeNamed(TypedDict): 39 | name: Optional[str] 40 | 41 | 42 | class TypeInfo(MaybeNamed): 43 | ofType: Optional[Union[TypeInfo, MaybeNamed]] 44 | # technically an enum: 45 | kind: str 46 | 47 | 48 | class FieldInfo(TypedDict): 49 | name: str 50 | type: TypeInfo 51 | 52 | 53 | class TypeInfoWithFields(TypeInfo): 54 | fields: Optional[List[FieldInfo]] 55 | 56 | 57 | QueryArg = Union[str, int] 58 | 59 | # ----------------------------------------------------------------------------- 60 | 61 | 62 | def parse_gql_type(type_info: TypeInfo) -> Field: 63 | # TODO(cancan101): do we want to handle Nones here? 64 | name: Optional[str] = type_info["name"] 65 | if name == "String": 66 | return String() 67 | elif name == "ID": 68 | # TODO(cancan101): figure out if we want to map this to UUID, int, etc 69 | # This should probably be an API-level setting 70 | return String() 71 | elif name == "Int": 72 | return Integer() 73 | elif name == "Float": 74 | return Float() 75 | elif name == "Boolean": 76 | return Boolean() 77 | # These are extended scalars: 78 | elif name == "DateTime": 79 | # https://www.graphql-scalars.dev/docs/scalars/date-time 80 | return ISODateTime() 81 | elif name == "Date": 82 | # https://www.graphql-scalars.dev/docs/scalars/date 83 | return ISODate() 84 | elif name == "Time": 85 | # https://www.graphql-scalars.dev/docs/scalars/time 86 | return ISOTime() 87 | else: 88 | # TODO(cancan101): how do we want to handle other scalars? 89 | raise ValueError(f"Unknown type: {name}") 90 | 91 | 92 | def get_type_entries( 93 | field_obj: FieldInfo, 94 | *, 95 | data_types: Dict[str, TypeInfoWithFields], 96 | include: Collection[str], 97 | path: Optional[List[str]] = None, 98 | ) -> Dict[str, Field]: 99 | path = path or [] 100 | 101 | field_name = field_obj["name"] 102 | new_path = path + [field_name] 103 | 104 | field_obj_type = field_obj["type"] 105 | 106 | kind = field_obj_type["kind"] 107 | if kind == "SCALAR": 108 | field_field = parse_gql_type(field_obj_type) 109 | return {"__".join(new_path): field_field} 110 | elif kind == "NON_NULL": 111 | of_type = field_obj_type["ofType"] 112 | 113 | if of_type is None: 114 | raise ValueError("of_type is None") 115 | 116 | of_type_name = of_type["name"] 117 | if of_type_name is None: 118 | raise ValueError("of_type_name is None") 119 | 120 | return get_type_entries( 121 | FieldInfo( 122 | name=field_name, 123 | type=data_types[of_type_name], 124 | ), 125 | data_types=data_types, 126 | include=include, 127 | path=path, 128 | ) 129 | # TODO(cancan101): other types to handle: 130 | # LIST, ENUM, UNION, INTERFACE, OBJECT (implicitly handled) 131 | else: 132 | # Check to see if this is a requested include 133 | if field_name in include: 134 | ret = {} 135 | name = field_obj_type["name"] 136 | if name is None: 137 | raise ValueError(f"Unable to get type of: {field_name}") 138 | 139 | fields = data_types[name]["fields"] 140 | if fields is None: 141 | raise ValueError(f"Unable to get fields for: {name}") 142 | 143 | for field in fields: 144 | ret.update( 145 | get_type_entries( 146 | field, data_types=data_types, include=include, path=new_path 147 | ) 148 | ) 149 | return ret 150 | 151 | return {} 152 | 153 | 154 | # ----------------------------------------------------------------------------- 155 | 156 | 157 | # clean these up: 158 | def find_by_name(name: str, *, types: List[FieldInfo]) -> Optional[FieldInfo]: 159 | name_match = [x for x in types if x["name"] == name] 160 | if len(name_match) == 0: 161 | return None 162 | return name_match[0] 163 | 164 | 165 | def find_type_by_name(name: str, *, types: List[FieldInfo]) -> Optional[TypeInfo]: 166 | entry = find_by_name(name, types=types) 167 | if entry is None: 168 | return None 169 | return entry["type"] 170 | 171 | 172 | def get_edges_type_name(fields: List[FieldInfo]) -> Optional[str]: 173 | entry_type = find_type_by_name("edges", types=fields) 174 | if entry_type is None: 175 | return None 176 | edges_info = entry_type["ofType"] 177 | if edges_info is None: 178 | return None 179 | return edges_info["name"] 180 | 181 | 182 | def get_node_type_name(fields: List[FieldInfo]) -> Optional[str]: 183 | node_info = find_type_by_name("node", types=fields) 184 | if node_info is None: 185 | return None 186 | return node_info["name"] 187 | 188 | 189 | # ----------------------------------------------------------------------------- 190 | 191 | 192 | def extract_flattened_value(node: Dict[str, Any], field_name: str) -> Any: 193 | ret: Any = node 194 | for path in field_name.split("__"): 195 | if ret is None: 196 | return ret 197 | elif not isinstance(ret, dict): 198 | raise TypeError(f"{field_name} is not dict path") 199 | ret = ret.get(path) 200 | return ret 201 | 202 | 203 | def get_gql_fields(column_names: Sequence[str]) -> str: 204 | # TODO(cancan101): actually nest this 205 | def get_field_str(fields: List[str], root: Optional[str] = None) -> str: 206 | ret = " ".join(fields) 207 | if root is not None: 208 | ret = f"{root} {{{ret}}}" 209 | return ret 210 | 211 | mappings: Dict[Optional[str], List[str]] = defaultdict(list) 212 | for field in [x.split("__", 1) for x in column_names]: 213 | if len(field) == 1: 214 | mappings[None].append(field[-1]) 215 | else: 216 | mappings[field[0]].append(field[-1]) 217 | 218 | fields_str = " ".join( 219 | get_field_str(fields, root=root) for root, fields in mappings.items() 220 | ) 221 | return fields_str 222 | 223 | 224 | def _parse_query_arg(k: str, v: List[str]) -> Tuple[str, str]: 225 | if len(v) > 1: 226 | raise ValueError(f"{k} was specified {len(v)} times") 227 | 228 | return (k, v[0]) 229 | 230 | 231 | def _parse_query_args(query: Dict[str, List[str]]) -> Dict[str, QueryArg]: 232 | str_args = dict( 233 | _parse_query_arg(k[4:], v) for k, v in query.items() if k.startswith("arg_") 234 | ) 235 | int_args = dict( 236 | (k, int(v)) 237 | for k, v in ( 238 | _parse_query_arg(k[5:], v) 239 | for k, v in query.items() 240 | if k.startswith("iarg_") 241 | ) 242 | ) 243 | overlap = set(str_args.keys()) & set(int_args.keys()) 244 | if overlap: 245 | raise ValueError(f"{overlap} was specified in multiple arg sets") 246 | 247 | return dict(str_args, **int_args) 248 | 249 | 250 | def _format_arg(arg: QueryArg) -> str: 251 | if isinstance(arg, str): 252 | return f'"{arg}"' 253 | return str(arg) 254 | 255 | 256 | def _get_variable_argument_str(args: Dict[str, QueryArg]) -> str: 257 | return " ".join(f"{k}: {_format_arg(v)}" for k, v in args.items()) 258 | 259 | 260 | # ----------------------------------------------------------------------------- 261 | 262 | 263 | class GraphQLAdapter(Adapter): 264 | safe = True 265 | 266 | is_connection: bool 267 | 268 | def __init__( 269 | self, 270 | table: str, 271 | include: Collection[str], 272 | query_args: Dict[str, QueryArg], 273 | is_connection: Optional[bool], 274 | graphql_api: str, 275 | bearer_token: Optional[str] = None, 276 | pagination_relay: Optional[bool] = None, 277 | list_queries: Optional[List[str]] = None, 278 | ): 279 | super().__init__() 280 | 281 | # The query field name 282 | self.table = table 283 | 284 | self.include = set(include) 285 | self.query_args = query_args 286 | 287 | if is_connection is not None: 288 | self.is_connection = is_connection 289 | else: 290 | self.is_connection = list_queries is None or table not in list_queries 291 | 292 | self.graphql_api = graphql_api 293 | self.bearer_token = bearer_token 294 | 295 | if pagination_relay is True and self.is_connection is False: 296 | raise ValueError("pagination_relay True and is_connection False") 297 | # For now, default this to True. In the future, we can perhaps guess 298 | self.pagination_relay = True if pagination_relay is None else pagination_relay 299 | 300 | if self.is_connection: 301 | query_type_and_types_query = """{ 302 | __schema { 303 | queryType { 304 | fields { 305 | name 306 | type { 307 | name 308 | } 309 | } 310 | } 311 | types { 312 | name 313 | kind 314 | fields { 315 | name 316 | type { 317 | name 318 | kind 319 | ofType { 320 | name 321 | } 322 | } 323 | } 324 | } 325 | } 326 | }""" 327 | else: 328 | query_type_and_types_query = """{ 329 | __schema { 330 | queryType { 331 | fields { 332 | name 333 | type { 334 | name 335 | kind 336 | ofType { 337 | name 338 | kind 339 | ofType { 340 | kind 341 | name 342 | ofType { 343 | name 344 | } 345 | } 346 | } 347 | } 348 | } 349 | } 350 | types { 351 | name 352 | kind 353 | fields { 354 | name 355 | type { 356 | name 357 | kind 358 | ofType { 359 | name 360 | } 361 | } 362 | } 363 | } 364 | } 365 | }""" 366 | 367 | query_type_and_types = self.run_query(query=query_type_and_types_query) 368 | query_type_and_types_schema = query_type_and_types["__schema"] 369 | queries_return_fields: List[FieldInfo] = query_type_and_types_schema[ 370 | "queryType" 371 | ]["fields"] 372 | 373 | # find the matching query (a field on the query object) 374 | # TODO(cancan101): handle missing 375 | type_entry = find_type_by_name(self.table, types=queries_return_fields) 376 | if type_entry is None: 377 | raise ValueError(f"Unable to resolve type_entry for {self.table}") 378 | 379 | data_types_list: List[TypeInfoWithFields] = query_type_and_types_schema["types"] 380 | data_types: Dict[str, TypeInfoWithFields] = { 381 | t["name"]: t for t in data_types_list if t["name"] is not None 382 | } 383 | 384 | if self.is_connection: 385 | query_return_type_name = type_entry["name"] 386 | if query_return_type_name is None: 387 | raise ValueError( 388 | f"Unable to resolve query_return_type_name for {self.table}" 389 | ) 390 | 391 | query_return_fields = data_types[query_return_type_name]["fields"] 392 | if query_return_fields is None: 393 | raise ValueError("No fields found on query") 394 | 395 | # we are assuming a top level connection 396 | edges_type_name = get_edges_type_name(query_return_fields) 397 | if edges_type_name is None: 398 | raise ValueError("Unable to resolve edges_type_name") 399 | 400 | edges_fields = data_types[edges_type_name]["fields"] 401 | if edges_fields is None: 402 | raise ValueError("No fields found on edge") 403 | 404 | node_type_name = get_node_type_name(edges_fields) 405 | if node_type_name is None: 406 | raise ValueError("Unable to resolve node_type_name") 407 | 408 | else: 409 | # We are assuming it is NonNull of List of NonNull of item 410 | list_type = type_entry["ofType"] 411 | if list_type is None: 412 | raise ValueError("Unable to resolve list_type") 413 | 414 | # TODO(cancan101): put this info into type system 415 | list_type = cast(TypeInfo, list_type) 416 | 417 | item_container_type = list_type["ofType"] 418 | if item_container_type is None: 419 | raise ValueError("Unable to resolve item_container_type") 420 | 421 | # TODO(cancan101): put this info into type system 422 | item_container_type = cast(TypeInfo, item_container_type) 423 | 424 | node_type = item_container_type["ofType"] 425 | if node_type is None: 426 | raise ValueError("Unable to resolve node_type") 427 | 428 | node_type_name = node_type["name"] 429 | if node_type_name is None: 430 | raise ValueError("Unable to resolve node_type_name") 431 | 432 | node_fields = data_types[node_type_name]["fields"] 433 | if node_fields is None: 434 | raise ValueError("No fields found on node") 435 | 436 | self.columns: Dict[str, Field] = {} 437 | for node_field in node_fields: 438 | self.columns.update( 439 | get_type_entries( 440 | node_field, data_types=data_types, include=self.include 441 | ) 442 | ) 443 | 444 | @staticmethod 445 | def supports(uri: str, fast: bool = True, **kwargs: Any) -> Optional[bool]: 446 | # TODO the slow path here could connect to the GQL Server 447 | return True 448 | 449 | @staticmethod 450 | def parse_uri( 451 | table: str, 452 | ) -> Tuple[str, List[str], Dict[str, QueryArg], Optional[bool]]: 453 | """ 454 | This will pass in the first n args of __init__ for the Adapter 455 | """ 456 | parsed = urlparse(table) 457 | query_string = parse_qs(parsed.query) 458 | 459 | include_entry = query_string.get("include") 460 | is_connection_raw_qs = query_string.get("is_connection") 461 | if is_connection_raw_qs is None: 462 | is_connection = None 463 | else: 464 | is_connection = get_last_query(is_connection_raw_qs) != "0" 465 | 466 | include: List[str] = [] 467 | if include_entry: 468 | for i in include_entry: 469 | include.extend(i.split(",")) 470 | 471 | query_args = _parse_query_args(query_string) 472 | 473 | return (parsed.path, include, query_args, is_connection) 474 | 475 | def get_columns(self) -> Dict[str, Field]: 476 | return self.columns 477 | 478 | def run_query(self, query: str) -> Dict[str, Any]: 479 | return run_query(self.graphql_api, query=query, bearer_token=self.bearer_token) 480 | 481 | def get_data_connection( 482 | self, 483 | bounds: Dict[str, Filter], 484 | order: List[Tuple[str, RequestedOrder]], 485 | **kwargs: Any, 486 | ) -> Iterator[Dict[str, Any]]: 487 | fields_str = get_gql_fields(list(self.columns.keys())) 488 | query_args_user = dict(self.query_args) 489 | 490 | after = query_args_user.pop("after", None) 491 | 492 | # We loop for each page in the pagination 493 | while True: 494 | args = dict(query_args_user) 495 | if after is not None: 496 | args["after"] = after 497 | 498 | if args: 499 | variable_str = f"({_get_variable_argument_str(args)})" 500 | else: 501 | # Don't generate the () for empty list of args 502 | variable_str = "" 503 | 504 | if self.pagination_relay: 505 | page_info_str = "pageInfo {endCursor hasNextPage}" 506 | else: 507 | page_info_str = "" 508 | 509 | query = f"""query {{ 510 | {self.table}{variable_str}{{ 511 | edges{{ 512 | node{{ 513 | {fields_str} 514 | }} 515 | }} 516 | {page_info_str} 517 | }} 518 | }}""" 519 | query_data = self.run_query(query=query) 520 | query_data_connection = query_data[self.table] 521 | 522 | edges = query_data_connection["edges"] 523 | 524 | for edge in edges: 525 | node: Dict[str, Any] = edge["node"] 526 | 527 | yield {c: extract_flattened_value(node, c) for c in self.columns.keys()} 528 | 529 | if self.pagination_relay: 530 | page_info = query_data_connection["pageInfo"] 531 | if not page_info["hasNextPage"]: 532 | break 533 | after = page_info["endCursor"] 534 | else: 535 | # If there is no pagination being used, break immediately 536 | break 537 | 538 | def get_data_list( 539 | self, 540 | bounds: Dict[str, Filter], 541 | order: List[Tuple[str, RequestedOrder]], 542 | **kwargs: Any, 543 | ) -> Iterator[Dict[str, Any]]: 544 | fields_str = get_gql_fields(list(self.columns.keys())) 545 | 546 | if self.query_args: 547 | variable_str = f"({_get_variable_argument_str(self.query_args)})" 548 | else: 549 | # Don't generate the () for empty list of query_args 550 | variable_str = "" 551 | 552 | query = f"""query {{ 553 | {self.table}{variable_str}{{ 554 | {fields_str} 555 | }} 556 | }}""" 557 | query_data = self.run_query(query=query) 558 | nodes: List[Dict[str, Any]] = query_data[self.table] 559 | 560 | for node in nodes: 561 | yield {c: extract_flattened_value(node, c) for c in self.columns.keys()} 562 | 563 | def get_data( 564 | self, 565 | bounds: Dict[str, Filter], 566 | order: List[Tuple[str, RequestedOrder]], 567 | **kwargs: Any, 568 | ) -> Iterator[Dict[str, Any]]: 569 | if self.is_connection: 570 | return self.get_data_connection(bounds=bounds, order=order, **kwargs) 571 | else: 572 | return self.get_data_list(bounds=bounds, order=order, **kwargs) 573 | -------------------------------------------------------------------------------- /graphqldb/db_engine_specs.py: -------------------------------------------------------------------------------- 1 | # Taken from: https://github.com/apache/superset/blob/master/superset/db_engine_specs/gsheets.py # noqa: E501 2 | from superset.db_engine_specs.sqlite import SqliteEngineSpec 3 | 4 | 5 | class GraphQLEngineSpec(SqliteEngineSpec): 6 | """Engine for GraphQL API tables""" 7 | 8 | engine = "graphql" 9 | engine_name = "GraphQL" 10 | allows_joins = True 11 | allows_subqueries = True 12 | 13 | default_driver = "apsw" 14 | sqlalchemy_uri_placeholder = "graphql://" 15 | 16 | # TODO(cancan101): figure out what other spec items make sense here 17 | # See: https://preset.io/blog/building-database-connector/ 18 | -------------------------------------------------------------------------------- /graphqldb/dialect.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple 4 | 5 | from shillelagh.backends.apsw.dialects.base import APSWDialect 6 | 7 | if TYPE_CHECKING: 8 | from sqlalchemy.engine import Connection 9 | from sqlalchemy.engine.url import URL 10 | 11 | from .lib import extract_query, get_last_query, run_query 12 | 13 | # ----------------------------------------------------------------------------- 14 | 15 | ADAPTER_NAME = "graphql" 16 | 17 | # ----------------------------------------------------------------------------- 18 | 19 | 20 | class APSWGraphQLDialect(APSWDialect): 21 | supports_statement_cache = True 22 | 23 | def __init__( 24 | self, 25 | list_queries: Optional[List[str]] = None, 26 | **kwargs: Any, 27 | ): 28 | # We tell Shillelagh that this dialect supports just one adapter 29 | super().__init__(safe=True, adapters=[ADAPTER_NAME], **kwargs) 30 | 31 | self.list_queries = list_queries 32 | 33 | def get_table_names( 34 | self, 35 | connection: Connection, 36 | schema: Optional[str] = None, 37 | sqlite_include_internal: bool = False, 38 | **kwargs: Any, 39 | ) -> List[str]: 40 | url = connection.engine.url 41 | graphql_api = self.db_url_to_graphql_api(url) 42 | 43 | query = """{ 44 | __schema { 45 | queryType { 46 | fields { 47 | name 48 | } 49 | } 50 | } 51 | }""" 52 | bearer_token = self.db_url_to_graphql_bearer(url) 53 | data = run_query(graphql_api, query=query, bearer_token=bearer_token) 54 | 55 | # TODO(cancan101): filter out "non-Array" returns 56 | # This is tricky as Connections are non-Array 57 | return [field["name"] for field in data["__schema"]["queryType"]["fields"]] 58 | 59 | def db_url_to_graphql_api(self, url: URL) -> str: 60 | query = extract_query(url) 61 | is_https_param = query.get("is_https", "1") 62 | is_https = get_last_query(is_https_param) != "0" 63 | proto = "https" if is_https else "http" 64 | port_str = "" if url.port is None else f":{url.port}" 65 | return f"{proto}://{url.host}{port_str}/{url.database}" 66 | 67 | def db_url_to_graphql_bearer(self, url: URL) -> Optional[str]: 68 | return str(url.password) if url.password else None 69 | 70 | def create_connect_args( 71 | self, 72 | url: URL, 73 | ) -> Tuple[Tuple[()], Dict[str, Any]]: 74 | args, kwargs = super().create_connect_args(url) 75 | 76 | if "adapter_kwargs" in kwargs and kwargs["adapter_kwargs"] != {}: 77 | raise ValueError( 78 | f"Unexpected adapter_kwargs found: {kwargs['adapter_kwargs']}" 79 | ) 80 | 81 | graphql_api = self.db_url_to_graphql_api(url) 82 | bearer_token = self.db_url_to_graphql_bearer(url) 83 | 84 | query = extract_query(url) 85 | pagination_relay_param = query.get("is_relay") 86 | pagination_relay = ( 87 | get_last_query(pagination_relay_param) != "0" 88 | if pagination_relay_param is not None 89 | else None 90 | ) 91 | 92 | adapter_kwargs = { 93 | ADAPTER_NAME: { 94 | "graphql_api": graphql_api, 95 | "bearer_token": bearer_token, 96 | "pagination_relay": pagination_relay, 97 | "list_queries": self.list_queries, 98 | } 99 | } 100 | 101 | # this seems gross, esp the path override. unclear why memory has to be set here 102 | return args, {**kwargs, "path": ":memory:", "adapter_kwargs": adapter_kwargs} 103 | -------------------------------------------------------------------------------- /graphqldb/lib.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import urllib.parse 4 | from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Union 5 | 6 | import requests 7 | 8 | if TYPE_CHECKING: 9 | from sqlalchemy.engine.url import URL 10 | 11 | # ----------------------------------------------------------------------------- 12 | 13 | 14 | # Imported from: shillelagh.backends.apsw.dialects.gsheets 15 | def extract_query(url: URL) -> Dict[str, Union[str, Sequence[str]]]: 16 | """ 17 | Extract the query from the SQLAlchemy URL. 18 | """ 19 | if url.query: 20 | return dict(url.query) 21 | 22 | # there's a bug in how SQLAlchemy <1.4 handles URLs without hosts, 23 | # putting the query string as the host; handle that case here 24 | if url.host and url.host.startswith("?"): 25 | return dict(urllib.parse.parse_qsl(url.host[1:])) # pragma: no cover 26 | 27 | return {} 28 | 29 | 30 | def get_last_query(entry: Union[str, Sequence[str]]) -> str: 31 | if not isinstance(entry, str): 32 | entry = entry[-1] 33 | return entry 34 | 35 | 36 | # ----------------------------------------------------------------------------- 37 | 38 | 39 | def run_query( 40 | graphql_api: str, 41 | *, 42 | query: str, 43 | bearer_token: Optional[str] = None, 44 | ) -> Dict[str, Any]: 45 | headers: Dict[str, Any] = {} 46 | if bearer_token: 47 | headers["Authorization"] = f"Bearer {bearer_token}" 48 | 49 | # TODO(cancan101): figure out timeouts 50 | resp = requests.post( # noqa: S113 51 | graphql_api, json={"query": query}, headers=headers 52 | ) 53 | try: 54 | resp.raise_for_status() 55 | except requests.HTTPError as ex: 56 | # For now let's assume 400 will have errors 57 | # https://github.com/graphql/graphql-over-http/blob/main/spec/GraphQLOverHTTP.md#status-codes 58 | if ex.response.status_code != 400: 59 | raise 60 | 61 | resp_data = resp.json() 62 | 63 | if "errors" in resp_data: 64 | raise ValueError(resp_data["errors"]) 65 | 66 | return resp_data["data"] 67 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | exclude = (?x)( 3 | ^build/$ 4 | # Can't type check as we don't install Superset: 5 | | ^graphqldb/db_engine_specs\.py$ 6 | ) 7 | 8 | plugins = sqlalchemy.ext.mypy.plugin 9 | -------------------------------------------------------------------------------- /pip-compile-all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | 4 | pip-compile requirements.in --resolver=backtracking "$@" 5 | pip-compile requirements-dev.in --resolver=backtracking "$@" 6 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | target-version = ['py38'] 3 | exclude = ''' 4 | ( 5 | /( 6 | \.git # root of the project 7 | | \.mypy_cache 8 | )/ 9 | ) 10 | ''' 11 | -------------------------------------------------------------------------------- /requirements-dev.in: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | black 4 | flake8 5 | flake8-bandit 6 | # this is for local work; not used by pre-commit 7 | # We don't really need this at all as we can run black directly 8 | # flake8-black 9 | flake8-bugbear 10 | flake8-datetimez 11 | flake8-debugger 12 | flake8-isort 13 | flake8-print 14 | flake8-return 15 | isort 16 | mypy 17 | pip-tools 18 | pre-commit 19 | pytest 20 | pytest-cov 21 | responses 22 | sqlalchemy[mypy] 23 | types-requests 24 | types-setuptools 25 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.10 3 | # by the following command: 4 | # 5 | # pip-compile --resolver=backtracking requirements-dev.in 6 | # 7 | apsw==3.42.0.0 8 | # via 9 | # -r requirements.txt 10 | # shillelagh 11 | attrs==23.1.0 12 | # via 13 | # cattrs 14 | # flake8-bugbear 15 | # requests-cache 16 | bandit==1.7.8 17 | # via flake8-bandit 18 | black==23.3.0 19 | # via -r requirements-dev.in 20 | build==0.10.0 21 | # via pip-tools 22 | cattrs==23.2.3 23 | # via requests-cache 24 | certifi==2023.7.22 25 | # via 26 | # -r requirements.txt 27 | # requests 28 | cfgv==3.3.1 29 | # via pre-commit 30 | charset-normalizer==3.3.1 31 | # via 32 | # -r requirements.txt 33 | # requests 34 | click==8.1.7 35 | # via 36 | # black 37 | # pip-tools 38 | coverage[toml]==6.5.0 39 | # via pytest-cov 40 | distlib==0.3.7 41 | # via virtualenv 42 | exceptiongroup==1.2.0 43 | # via 44 | # cattrs 45 | # pytest 46 | filelock==3.12.2 47 | # via virtualenv 48 | flake8==6.0.0 49 | # via 50 | # -r requirements-dev.in 51 | # flake8-bandit 52 | # flake8-bugbear 53 | # flake8-datetimez 54 | # flake8-debugger 55 | # flake8-isort 56 | # flake8-print 57 | flake8-bandit==4.1.1 58 | # via -r requirements-dev.in 59 | flake8-bugbear==23.6.5 60 | # via -r requirements-dev.in 61 | flake8-datetimez==20.10.0 62 | # via -r requirements-dev.in 63 | flake8-debugger==4.1.2 64 | # via -r requirements-dev.in 65 | flake8-isort==6.0.0 66 | # via -r requirements-dev.in 67 | flake8-plugin-utils==1.3.3 68 | # via flake8-return 69 | flake8-print==5.0.0 70 | # via -r requirements-dev.in 71 | flake8-return==1.1.3 72 | # via -r requirements-dev.in 73 | greenlet==2.0.2 74 | # via 75 | # -r requirements.txt 76 | # shillelagh 77 | # sqlalchemy 78 | identify==2.5.30 79 | # via pre-commit 80 | idna==3.4 81 | # via 82 | # -r requirements.txt 83 | # requests 84 | iniconfig==2.0.0 85 | # via pytest 86 | isort==5.12.0 87 | # via 88 | # -r requirements-dev.in 89 | # flake8-isort 90 | markdown-it-py==3.0.0 91 | # via rich 92 | mccabe==0.7.0 93 | # via flake8 94 | mdurl==0.1.2 95 | # via markdown-it-py 96 | mypy==1.4.1 97 | # via 98 | # -r requirements-dev.in 99 | # sqlalchemy 100 | mypy-extensions==1.0.0 101 | # via 102 | # black 103 | # mypy 104 | nodeenv==1.8.0 105 | # via pre-commit 106 | packaging==23.1 107 | # via 108 | # -r requirements.txt 109 | # black 110 | # build 111 | # pytest 112 | # shillelagh 113 | pathspec==0.11.1 114 | # via black 115 | pbr==6.0.0 116 | # via stevedore 117 | pip-tools==7.1.0 118 | # via -r requirements-dev.in 119 | platformdirs==3.9.1 120 | # via 121 | # black 122 | # requests-cache 123 | # virtualenv 124 | pluggy==1.5.0 125 | # via pytest 126 | pre-commit==3.7.0 127 | # via -r requirements-dev.in 128 | pycodestyle==2.10.0 129 | # via 130 | # flake8 131 | # flake8-debugger 132 | # flake8-print 133 | pyflakes==3.0.1 134 | # via flake8 135 | pygments==2.15.1 136 | # via rich 137 | pyproject-hooks==1.0.0 138 | # via build 139 | pytest==7.4.0 140 | # via 141 | # -r requirements-dev.in 142 | # pytest-cov 143 | pytest-cov==4.1.0 144 | # via -r requirements-dev.in 145 | python-dateutil==2.8.2 146 | # via 147 | # -r requirements.txt 148 | # shillelagh 149 | pyyaml==6.0.1 150 | # via 151 | # bandit 152 | # pre-commit 153 | # responses 154 | requests==2.31.0 155 | # via 156 | # -r requirements.txt 157 | # requests-cache 158 | # responses 159 | # shillelagh 160 | requests-cache==1.2.0 161 | # via shillelagh 162 | responses==0.23.1 163 | # via -r requirements-dev.in 164 | rich==13.4.2 165 | # via bandit 166 | shillelagh==1.2.10 167 | # via -r requirements.txt 168 | six==1.16.0 169 | # via 170 | # -r requirements.txt 171 | # python-dateutil 172 | # url-normalize 173 | sqlalchemy[mypy]==2.0.19 174 | # via 175 | # -r requirements-dev.in 176 | # -r requirements.txt 177 | # shillelagh 178 | stevedore==5.2.0 179 | # via bandit 180 | tomli==2.0.1 181 | # via 182 | # black 183 | # build 184 | # coverage 185 | # mypy 186 | # pip-tools 187 | # pyproject-hooks 188 | # pytest 189 | types-pyyaml==6.0.12.20240311 190 | # via responses 191 | types-requests==2.31.0.1 192 | # via -r requirements-dev.in 193 | types-setuptools==68.0.0.0 194 | # via -r requirements-dev.in 195 | types-urllib3==1.26.25.14 196 | # via types-requests 197 | typing-extensions==4.7.1 198 | # via 199 | # -r requirements.txt 200 | # cattrs 201 | # mypy 202 | # shillelagh 203 | # sqlalchemy 204 | url-normalize==1.4.3 205 | # via requests-cache 206 | urllib3==2.0.7 207 | # via 208 | # -r requirements.txt 209 | # requests 210 | # requests-cache 211 | # responses 212 | virtualenv==20.24.5 213 | # via pre-commit 214 | wheel==0.40.0 215 | # via pip-tools 216 | 217 | # The following packages are considered to be unsafe in a requirements file: 218 | # pip 219 | # setuptools 220 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | requests 2 | shillelagh 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.10 3 | # by the following command: 4 | # 5 | # pip-compile --resolver=backtracking requirements.in 6 | # 7 | apsw==3.42.0.0 8 | # via shillelagh 9 | certifi==2023.7.22 10 | # via requests 11 | charset-normalizer==3.3.1 12 | # via requests 13 | greenlet==2.0.2 14 | # via 15 | # shillelagh 16 | # sqlalchemy 17 | idna==3.4 18 | # via requests 19 | packaging==23.1 20 | # via shillelagh 21 | python-dateutil==2.8.2 22 | # via shillelagh 23 | requests==2.31.0 24 | # via 25 | # -r requirements.in 26 | # shillelagh 27 | shillelagh==1.2.10 28 | # via -r requirements.in 29 | six==1.16.0 30 | # via python-dateutil 31 | sqlalchemy==2.0.19 32 | # via shillelagh 33 | typing-extensions==4.7.1 34 | # via 35 | # shillelagh 36 | # sqlalchemy 37 | urllib3==2.0.7 38 | # via requests 39 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from pathlib import Path 4 | from shutil import rmtree 5 | from typing import List, Tuple 6 | 7 | from setuptools import Command, find_packages, setup 8 | 9 | # ----------------------------------------------------------------------------- 10 | 11 | DESCRIPTION = "Python DB-API and SQLAlchemy interface for GraphQL APIs." 12 | VERSION = "0.0.1.dev5" 13 | 14 | # ----------------------------------------------------------------------------- 15 | 16 | # read the contents of your README file 17 | this_directory = Path(__file__).parent 18 | long_description = (this_directory / "README.md").read_text() 19 | long_description_content_type = "text/markdown; charset=UTF-8; variant=GFM" 20 | 21 | dist_directory = (this_directory / "dist").absolute() 22 | 23 | # ----------------------------------------------------------------------------- 24 | 25 | 26 | class BaseCommand(Command): 27 | user_options: List[Tuple[str, str, str]] = [] 28 | 29 | @staticmethod 30 | def status(s: str) -> None: 31 | """Prints things in bold.""" 32 | print("\033[1m{0}\033[0m".format(s)) # noqa: T201 33 | 34 | def system(self, command: str) -> None: 35 | os.system(command) # noqa: S605 36 | 37 | def initialize_options(self): 38 | pass 39 | 40 | def finalize_options(self): 41 | pass 42 | 43 | 44 | class BuildCommand(BaseCommand): 45 | """Support setup.py building.""" 46 | 47 | description = "Build the package." 48 | 49 | def run(self): 50 | try: 51 | self.status("Removing previous builds…") 52 | rmtree(dist_directory) 53 | except OSError: 54 | pass 55 | 56 | self.status("Building Source and Wheel (universal) distribution…") 57 | self.system("{0} -m build --sdist --wheel .".format(sys.executable)) 58 | 59 | self.status("Checking wheel contents…") 60 | self.system("check-wheel-contents dist/*.whl") 61 | 62 | self.status("Running twine check…") 63 | self.system("{0} -m twine check dist/*".format(sys.executable)) 64 | 65 | 66 | class UploadTestCommand(BaseCommand): 67 | """Support uploading to test PyPI.""" 68 | 69 | description = "Upload the package to the test PyPI." 70 | 71 | def run(self): 72 | self.status("Uploading the package to PyPi via Twine…") 73 | self.system( 74 | "twine upload --repository-url https://test.pypi.org/legacy/ dist/*" 75 | ) 76 | 77 | 78 | class UploadCommand(BaseCommand): 79 | """Support uploading to PyPI.""" 80 | 81 | description = "Upload the package to PyPI." 82 | 83 | def run(self): 84 | self.status("Uploading the package to PyPi via Twine…") 85 | self.system("twine upload dist/*") 86 | 87 | self.status("Pushing git tags…") 88 | self.system("git tag v{0}".format(VERSION)) 89 | self.system("git push --tags") 90 | 91 | 92 | # ----------------------------------------------------------------------------- 93 | 94 | setup( 95 | name="sqlalchemy-graphqlapi", 96 | version=VERSION, 97 | description=DESCRIPTION, 98 | long_description=long_description, 99 | long_description_content_type=long_description_content_type, 100 | author="Alex Rothberg", 101 | author_email="agrothberg@gmail.com", 102 | url="https://github.com/cancan101/graphql-db-api", 103 | packages=find_packages(exclude=("tests",)), 104 | entry_points={ 105 | "sqlalchemy.dialects": [ 106 | "graphql = graphqldb.dialect:APSWGraphQLDialect", 107 | ], 108 | "shillelagh.adapter": [ 109 | "graphql = graphqldb.adapter:GraphQLAdapter", 110 | ], 111 | }, 112 | install_requires=( 113 | "shillelagh >= 1.2.0", 114 | "requests >= 2.31.0", 115 | ), 116 | license="MIT", 117 | classifiers=[ 118 | # Trove classifiers 119 | # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers 120 | "Development Status :: 2 - Pre-Alpha", 121 | "License :: OSI Approved :: MIT License", 122 | "Programming Language :: Python", 123 | "Programming Language :: Python :: 3.8", 124 | "Programming Language :: Python :: 3.9", 125 | "Programming Language :: Python :: 3.10", 126 | "Programming Language :: Python :: 3.11", 127 | "Programming Language :: Python :: Implementation :: CPython", 128 | ], 129 | # $ setup.py publish support. 130 | cmdclass={ 131 | "buildit": BuildCommand, 132 | "uploadtest": UploadTestCommand, 133 | "upload": UploadCommand, 134 | }, 135 | ) 136 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cancan101/graphql-db-api/f2d10ae6c2be64080db788902411b93ff5c717aa/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from typing import Final, Generator 2 | 3 | import pytest 4 | import responses 5 | from sqlalchemy import create_engine 6 | from sqlalchemy.engine import Connection, Engine 7 | 8 | # ----------------------------------------------------------------------------- 9 | 10 | SWAPI_GRAPHQL_DB_URL: Final[ 11 | str 12 | ] = "graphql://swapi-graphql.netlify.app/.netlify/functions/index" 13 | PETSTORE_GRAPHQL_DB_URL: Final[str] = "graphql://pet-library.moonhighway.com/" 14 | 15 | # ----------------------------------------------------------------------------- 16 | 17 | 18 | @pytest.fixture 19 | def swapi_graphq_db_url() -> str: 20 | return SWAPI_GRAPHQL_DB_URL 21 | 22 | 23 | @pytest.fixture 24 | def swapi_engine(swapi_graphq_db_url: str) -> Engine: 25 | return create_engine(swapi_graphq_db_url) 26 | 27 | 28 | @pytest.fixture 29 | def swapi_connection(swapi_engine: Engine) -> Generator[Connection, None, None]: 30 | with swapi_engine.connect() as connection: 31 | yield connection 32 | 33 | 34 | @pytest.fixture 35 | def swapi_connection_no_relay( 36 | swapi_graphq_db_url: str, 37 | ) -> Generator[Connection, None, None]: 38 | swapi_engine_no_relay = create_engine(f"{swapi_graphq_db_url}?is_relay=0") 39 | with swapi_engine_no_relay.connect() as connection: 40 | yield connection 41 | 42 | 43 | @pytest.fixture 44 | def petstore_connection(swapi_engine: Engine) -> Generator[Connection, None, None]: 45 | petstore_engine = create_engine(PETSTORE_GRAPHQL_DB_URL) 46 | with petstore_engine.connect() as connection: 47 | yield connection 48 | 49 | 50 | @pytest.fixture 51 | def petstore_connection_on_engine( 52 | swapi_engine: Engine, 53 | ) -> Generator[Connection, None, None]: 54 | petstore_engine = create_engine(PETSTORE_GRAPHQL_DB_URL, list_queries=["allPets"]) 55 | with petstore_engine.connect() as connection: 56 | yield connection 57 | 58 | 59 | # ----------------------------------------------------------------------------- 60 | 61 | 62 | @pytest.fixture 63 | def mocked_responses() -> Generator[responses.RequestsMock, None, None]: 64 | with responses.RequestsMock() as rsps: 65 | yield rsps 66 | -------------------------------------------------------------------------------- /tests/test_adapter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from shillelagh.fields import ISODate, ISODateTime, String 3 | 4 | from graphqldb.adapter import ( 5 | TypeInfo, 6 | _get_variable_argument_str, 7 | _parse_query_args, 8 | extract_flattened_value, 9 | get_gql_fields, 10 | parse_gql_type, 11 | ) 12 | 13 | # ----------------------------------------------------------------------------- 14 | 15 | 16 | def test_get_gql_fields_single(): 17 | assert get_gql_fields(["foo"]) == "foo" 18 | 19 | 20 | def test_get_gql_fields_multiple(): 21 | assert get_gql_fields(["foo", "bar"]) == "foo bar" 22 | 23 | 24 | def test_get_gql_fields_nested(): 25 | assert get_gql_fields(["foo", "fooz__bar", "fooz__foo"]) == "foo fooz {bar foo}" 26 | 27 | 28 | def test_get_gql_fields_nested_grouping(): 29 | assert get_gql_fields(["fooz__bar", "foo", "fooz__foo"]) == "fooz {bar foo} foo" 30 | 31 | 32 | def test_get_gql_fields_nested_multiple(): 33 | assert ( 34 | get_gql_fields(["fooz__bar", "foo", "barzz__foo"]) 35 | == "fooz {bar} foo barzz {foo}" 36 | ) 37 | 38 | 39 | def test_extract_flattened_value(): 40 | data = {"foo": 1} 41 | assert extract_flattened_value(data, "foo") == 1 42 | assert extract_flattened_value(data, "bar") is None 43 | assert extract_flattened_value(data, "biz__bar") is None 44 | 45 | with pytest.raises(TypeError): 46 | assert extract_flattened_value(data, "foo__bar") is None 47 | 48 | 49 | def test_extract_flattened_value_nested(): 50 | data = {"foo": {"bar": 2}} 51 | assert extract_flattened_value(data, "foo__bar") == 2 52 | assert extract_flattened_value(data, "foo__bar2") is None 53 | 54 | 55 | def test_parse_gql_type(): 56 | assert ( 57 | type(parse_gql_type(TypeInfo(name="ID", ofType=None, kind="SCALAR"))) is String 58 | ) 59 | assert ( 60 | type(parse_gql_type(TypeInfo(name="DateTime", ofType=None, kind="SCALAR"))) 61 | is ISODateTime 62 | ) 63 | assert ( 64 | type(parse_gql_type(TypeInfo(name="Date", ofType=None, kind="SCALAR"))) 65 | is ISODate 66 | ) 67 | 68 | with pytest.raises(ValueError): 69 | parse_gql_type(TypeInfo(name="asdf", ofType=None, kind="SCALAR")) 70 | 71 | with pytest.raises(ValueError): 72 | parse_gql_type(TypeInfo(name=None, ofType=None, kind="SCALAR")) 73 | 74 | 75 | def test_get_variable_argument_str(): 76 | assert _get_variable_argument_str({"a": 1}) == "a: 1" 77 | assert _get_variable_argument_str({"a": 1, "b": "c"}) == 'a: 1 b: "c"' 78 | 79 | 80 | def test_parse_query_args(): 81 | assert _parse_query_args({"arg_foo": ["bar"]}) == {"foo": "bar"} 82 | assert _parse_query_args({"arg_foo": ["bar"], "iarg_baz": [33]}) == { 83 | "foo": "bar", 84 | "baz": 33, 85 | } 86 | 87 | with pytest.raises(ValueError): 88 | _parse_query_args({"arg_foo": ["bar", "baz"]}) 89 | 90 | # bad int 91 | with pytest.raises(ValueError): 92 | _parse_query_args({"iarg_foo": ["bar"]}) 93 | 94 | # dupe 95 | with pytest.raises(ValueError): 96 | _parse_query_args({"arg_foo": ["bar"], "iarg_foo": [3]}) 97 | -------------------------------------------------------------------------------- /tests/test_dialect.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import inspect, text 2 | from sqlalchemy.engine import Connection, Engine, make_url 3 | 4 | from graphqldb.dialect import APSWGraphQLDialect 5 | 6 | 7 | def test_create_engine(swapi_engine: Engine) -> None: 8 | pass 9 | 10 | 11 | def test_get_table_names_connections(swapi_connection: Connection) -> None: 12 | insp = inspect(swapi_connection) 13 | 14 | tables = insp.get_table_names() 15 | 16 | # TODO(cancan101): This should also test for tables shouldn't be here 17 | assert "allPlanets" in tables 18 | assert "allPeople" in tables 19 | assert "allSpecies" in tables 20 | 21 | 22 | def test_get_table_names_lists(petstore_connection: Connection) -> None: 23 | insp = inspect(petstore_connection) 24 | 25 | tables = insp.get_table_names() 26 | 27 | # TODO(cancan101): This should also test for tables shouldn't be here 28 | assert "allPets" in tables 29 | assert "allCustomers" in tables 30 | 31 | 32 | def test_query_connection(swapi_connection: Connection) -> None: 33 | result = swapi_connection.execute( 34 | text( 35 | """select 36 | name, 37 | height, 38 | mass, 39 | homeworld__name 40 | from 41 | 'allPeople?include=homeworld'""" 42 | ) 43 | ) 44 | assert len(list(result)) == 82 45 | 46 | 47 | def test_query_paginate(swapi_connection: Connection) -> None: 48 | result = swapi_connection.execute( 49 | text( 50 | """select 51 | id 52 | from 53 | 'allPeople?arg_after=YXJyYXljb25uZWN0aW9uOjA=&iarg_first=50'""" 54 | ) 55 | ) 56 | assert len(list(result)) == 81 57 | 58 | 59 | def test_query_no_paginate(swapi_connection_no_relay: Connection) -> None: 60 | """Test querying with no pagination enabled.""" 61 | result = swapi_connection_no_relay.execute( 62 | text( 63 | """select 64 | id 65 | from 66 | 'allPeople?iarg_first=3'""" 67 | ) 68 | ) 69 | assert len(list(result)) == 3 70 | 71 | 72 | def test_query_non_connection(petstore_connection: Connection) -> None: 73 | """Test querying against a non-connection (i.e. a List).""" 74 | result = petstore_connection.execute( 75 | text( 76 | """select 77 | id 78 | from 79 | 'allPets?is_connection=0'""" 80 | ) 81 | ) 82 | assert len(list(result)) == 25 83 | 84 | 85 | def test_query_non_connection_on_engine( 86 | petstore_connection_on_engine: Connection, 87 | ) -> None: 88 | """Test querying against a non-connection (i.e. a List).""" 89 | result = petstore_connection_on_engine.execute( 90 | text( 91 | """select 92 | id 93 | from 94 | allPets""" 95 | ) 96 | ) 97 | assert len(list(result)) == 25 98 | 99 | 100 | def test_db_url_to_graphql_api(): 101 | dialect = APSWGraphQLDialect() 102 | 103 | url_http = make_url("graphql://host:123/path?is_https=0") 104 | assert dialect.db_url_to_graphql_api(url_http) == "http://host:123/path" 105 | 106 | url_https = make_url("graphql://host:123/path?is_https=1") 107 | assert dialect.db_url_to_graphql_api(url_https) == "https://host:123/path" 108 | 109 | 110 | def test_create_connect_args(): 111 | dialect = APSWGraphQLDialect(list_queries=["abcd"]) 112 | 113 | url_http = make_url("graphql://:abcd@host:123/path?is_https=0&is_relay=1") 114 | 115 | _, kwargs = dialect.create_connect_args(url_http) 116 | adapter_kwargs = kwargs["adapter_kwargs"] 117 | kwargs_graphql = adapter_kwargs["graphql"] 118 | 119 | assert kwargs_graphql["graphql_api"] == "http://host:123/path" 120 | assert kwargs_graphql["bearer_token"] == "abcd" 121 | assert kwargs_graphql["pagination_relay"] is True 122 | assert kwargs_graphql["list_queries"] == ["abcd"] 123 | -------------------------------------------------------------------------------- /tests/test_lib.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import responses 3 | 4 | from graphqldb.lib import get_last_query, run_query 5 | 6 | # ----------------------------------------------------------------------------- 7 | 8 | SWAPI_API = "https://swapi-graphql.netlify.app/.netlify/functions/index" 9 | 10 | # ----------------------------------------------------------------------------- 11 | 12 | 13 | def test_run_query_error() -> None: 14 | with pytest.raises(ValueError): 15 | run_query( 16 | SWAPI_API, 17 | query="""{ 18 | allPeople { 19 | a 20 | } 21 | }""", 22 | ) 23 | 24 | 25 | def test_run_query_bearer_token(mocked_responses: responses.RequestsMock): 26 | mocked_responses.add(method=responses.POST, url=SWAPI_API, json={"data": {}}) 27 | run_query(SWAPI_API, query="{}", bearer_token="asdf") # noqa: S106 28 | assert mocked_responses.calls[0].request.headers["Authorization"] == "Bearer asdf" 29 | 30 | 31 | def test_get_last_query(): 32 | assert get_last_query("a") == "a" 33 | assert get_last_query(["a"]) == "a" 34 | assert get_last_query(["b", "d"]) == "d" 35 | --------------------------------------------------------------------------------