├── kaiba ├── __init__.py ├── py.typed ├── models │ ├── __init__.py │ ├── base.py │ ├── iterator.py │ ├── regex.py │ ├── slicing.py │ ├── data_fetcher.py │ ├── attribute.py │ ├── casting.py │ ├── kaiba_object.py │ ├── if_statement.py │ └── branching_object.py ├── casting │ ├── __init__.py │ ├── _cast_to_integer.py │ ├── _cast_to_decimal.py │ └── _cast_to_date.py ├── constants.py ├── process.py ├── iso.py ├── collection_handlers.py ├── handlers.py ├── mapper.py └── functions.py ├── tests ├── __init__.py ├── casting │ ├── __init__.py │ ├── test_cast_to_integer.py │ ├── test_cast_to_decimal.py │ └── test_cast_to_date.py ├── functions │ ├── __init__.py │ ├── test_apply_default.py │ ├── test_apply_regex.py │ ├── test_apply_casting.py │ ├── test_apply_seperator.py │ ├── test_apply_slicing.py │ └── test_apply_if_statements.py ├── handlers │ ├── __init__.py │ ├── test_handle_attribute.py │ └── test_handle_data_fetcher.py ├── collection_handlers │ ├── __init__.py │ ├── test_fetch_list_by_keys.py │ ├── test_set_value_in_dict.py │ └── test_fetch_data_by_keys.py ├── test_create_json_schema.py ├── validation │ ├── test_base.py │ ├── conftest.py │ ├── test_casting.py │ ├── test_branching_object.py │ ├── test_data_fetcher.py │ ├── test_attribute.py │ ├── test_slicing.py │ ├── test_regex.py │ ├── test_if_statement.py │ ├── test_iterator.py │ └── test_kaiba_object.py ├── iso │ ├── test_get_language_code.py │ ├── test_get_currency_code.py │ └── test_get_country_code.py ├── example.json ├── test_nested_iterables.py ├── json │ ├── valid.json │ ├── invalid.json │ ├── expected_regex.json │ ├── config_regex.json │ └── input_regex.json ├── test_process.py └── test_mapper.py ├── README.md ├── docs ├── changelog.md ├── contributing.md ├── usecases │ └── usecases.md ├── usage.md ├── index.md └── configuration.md ├── rtd_requirements.txt ├── .github ├── dependabot.yml └── workflows │ └── test.yml ├── .editorconfig ├── .pre-commit-config.yaml ├── mkdocs.yml ├── LICENSE ├── pyproject.toml ├── .gitignore ├── CONTRIBUTING.md ├── CHANGELOG.md └── setup.cfg /kaiba/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /kaiba/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | docs/index.md -------------------------------------------------------------------------------- /kaiba/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/casting/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/functions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/changelog.md: -------------------------------------------------------------------------------- 1 | ../CHANGELOG.md -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | ../CONTRIBUTING.md -------------------------------------------------------------------------------- /tests/collection_handlers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rtd_requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs-material 2 | -------------------------------------------------------------------------------- /tests/test_create_json_schema.py: -------------------------------------------------------------------------------- 1 | from kaiba.models.kaiba_object import KaibaObject 2 | 3 | 4 | def test_create_jsonschema_from_model(): 5 | """Test that we can create jsonschema.""" 6 | assert KaibaObject.schema_json(indent=2) 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 2 | 3 | version: 2 4 | updates: 5 | - package-ecosystem: "pip" 6 | directory: "/" 7 | schedule: 8 | interval: "weekly" 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Check http://editorconfig.org for more information 2 | # This is the main config file for this project: 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | end_of_line = lf 9 | indent_style = space 10 | insert_final_newline = true 11 | indent_size = 2 12 | 13 | [*.py] 14 | indent_size = 4 15 | 16 | [*.pyi] 17 | indent_size = 4 18 | -------------------------------------------------------------------------------- /kaiba/models/base.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | from typing import Union 3 | 4 | from pydantic import BaseModel, ConfigDict 5 | from pydantic.types import StrictBool, StrictInt, StrictStr 6 | 7 | AnyType = Union[StrictStr, StrictInt, StrictBool, Decimal, list, dict] 8 | StrInt = Union[StrictStr, StrictInt] 9 | 10 | 11 | class KaibaBaseModel(BaseModel): 12 | """Base model that forbids non defined attributes.""" 13 | 14 | model_config = ConfigDict(extra='forbid') 15 | -------------------------------------------------------------------------------- /tests/validation/test_base.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.casting import Casting 5 | 6 | 7 | def test_invalid(): 8 | """Test that extra attributes are not allowed.""" 9 | with pytest.raises(ValidationError) as ve: 10 | Casting(to='integer', bob='test') # type: ignore 11 | 12 | errors = ve.value.errors()[0] # noqa: WPS441 13 | assert errors['loc'] == ('bob',) 14 | assert errors['msg'] == 'Extra inputs are not permitted' 15 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v2.3.0 4 | hooks: 5 | - id: end-of-file-fixer 6 | - id: trailing-whitespace 7 | - repo: https://gitlab.com/pycqa/flake8 8 | rev: 3.7.8 9 | hooks: 10 | - id: flake8 11 | - repo: local 12 | hooks: 13 | - id: mypy 14 | name: Check mypy static types 15 | entry: poetry run mypy 16 | pass_filenames: true 17 | language: system 18 | types: [python] 19 | -------------------------------------------------------------------------------- /tests/validation/conftest.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture(scope='session') 7 | def valid() -> dict: 8 | """Load a valid example config.""" 9 | with open('tests/json/valid.json', 'r') as file_object: 10 | return json.loads(file_object.read()) 11 | 12 | 13 | @pytest.fixture(scope='session') 14 | def invalid() -> dict: 15 | """Load an invalid example config.""" 16 | with open('tests/json/invalid.json', 'r') as file_object: 17 | return json.loads(file_object.read()) 18 | -------------------------------------------------------------------------------- /kaiba/models/iterator.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pydantic import ConfigDict, Field 4 | 5 | from kaiba.models.base import KaibaBaseModel, StrInt 6 | 7 | 8 | class Iterator(KaibaBaseModel): 9 | """Allows for iterating lists at given path.""" 10 | 11 | alias: str 12 | path: List[StrInt] = Field(..., min_length=1) 13 | model_config = ConfigDict(json_schema_extra={ 14 | 'examples': [ 15 | { 16 | 'alias': 'an_item', 17 | 'path': ['path', 'to', 10, 'data'], 18 | }, 19 | ], 20 | }) 21 | -------------------------------------------------------------------------------- /kaiba/casting/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | from returns.result import safe 4 | 5 | from kaiba.casting._cast_to_date import cast_to_date 6 | from kaiba.casting._cast_to_decimal import cast_to_decimal 7 | from kaiba.casting._cast_to_integer import cast_to_integer 8 | from kaiba.models.casting import CastToOptions 9 | 10 | 11 | @safe 12 | def get_casting_function(cast_to: CastToOptions) -> Callable: 13 | """Return casting function depending on name.""" 14 | if cast_to == CastToOptions.INTEGER: 15 | return cast_to_integer 16 | 17 | elif cast_to == CastToOptions.DECIMAL: 18 | return cast_to_decimal 19 | 20 | return cast_to_date 21 | -------------------------------------------------------------------------------- /kaiba/constants.py: -------------------------------------------------------------------------------- 1 | from typing_extensions import Final 2 | 3 | # Casting 4 | INTEGER_CONTAINING_DECIMALS: Final[str] = 'integer_containing_decimals' 5 | YMD_DATE_FORMAT: Final = r'(^(yy|yyyy)[^\w]?mm[^\w]?dd$)' 6 | DMY_DATE_FORMAT: Final = r'(^dd[^\w]?mm[^\w]?(yy|yyyy)$)' 7 | MDY_DATE_FORMAT: Final = r'(^mm[^\w]?dd[^\w]?(yy|yyyy)$)' 8 | 9 | # Casting helpers 10 | COMMA: Final[str] = ',' 11 | PERIOD: Final[str] = '.' 12 | EMPTY: Final[str] = '' 13 | 14 | # ISO 15 | NAME: Final[str] = 'name' 16 | ALPHA_TWO: Final[str] = 'alpha_2' 17 | ALPHA_THREE: Final[str] = 'alpha_3' 18 | NUMERIC: Final[str] = 'numeric' 19 | OFFICIAL_NAME: Final[str] = 'official_name' 20 | INVALID: Final[str] = 'invalid' 21 | -------------------------------------------------------------------------------- /kaiba/models/regex.py: -------------------------------------------------------------------------------- 1 | from typing import List, Pattern, Union 2 | 3 | from pydantic import ConfigDict 4 | from pydantic.types import StrictInt 5 | 6 | from kaiba.models.base import KaibaBaseModel 7 | 8 | 9 | class Regex(KaibaBaseModel): 10 | """Use regular expression on data found by data_fetchers.""" 11 | 12 | expression: Pattern 13 | group: Union[StrictInt, List[StrictInt]] = 0 14 | model_config = ConfigDict(json_schema_extra={ 15 | 'examples': [ 16 | { 17 | 'expression': '[a-z]+', 18 | }, 19 | { 20 | 'expression': '([a-z])', 21 | 'group': [0, 3, 4], 22 | }, 23 | ], 24 | }) 25 | -------------------------------------------------------------------------------- /kaiba/models/slicing.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pydantic import ConfigDict, Field 4 | from pydantic.types import StrictInt 5 | 6 | from kaiba.models.base import KaibaBaseModel 7 | 8 | 9 | class Slicing(KaibaBaseModel): 10 | """Slice from inclusive to exclusive like python slice.""" 11 | 12 | slice_from: StrictInt = Field(alias='from') 13 | slice_to: Optional[StrictInt] = Field(None, alias='to') 14 | model_config = ConfigDict(json_schema_extra={ 15 | 'examples': [ 16 | { 17 | 'from': 3, 18 | }, 19 | { 20 | 'from': -5, 21 | }, 22 | { 23 | 'from': 3, 24 | 'to': 10, 25 | }, 26 | ], 27 | }) 28 | -------------------------------------------------------------------------------- /tests/validation/test_casting.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.casting import Casting, CastToOptions 5 | 6 | 7 | def test_validates(): # noqa: WPS218 8 | """Test that dict is marshalled to pydantic object.""" 9 | test = Casting( 10 | to='integer', 11 | ) 12 | assert test.to == CastToOptions.INTEGER 13 | assert test.original_format is None 14 | 15 | 16 | def test_invalid(): 17 | """Test that we get validation error with correct message.""" 18 | with pytest.raises(ValidationError) as ve: 19 | Casting(to='test') 20 | 21 | errors = ve.value.errors()[0] # noqa: WPS441 22 | assert errors['loc'] == ('to',) 23 | msg = errors['msg'] 24 | assert all(opt.value in msg for opt in CastToOptions) 25 | -------------------------------------------------------------------------------- /kaiba/casting/_cast_to_integer.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import decimal 4 | 5 | from returns.pipeline import flow 6 | from returns.pointfree import map_ 7 | from returns.result import ResultE 8 | 9 | from kaiba.casting._cast_to_decimal import cast_to_decimal # noqa: WPS436 10 | from kaiba.models.base import AnyType 11 | 12 | 13 | def cast_to_integer( 14 | value_to_cast: AnyType, 15 | original_format: str | None = None, 16 | ) -> ResultE[int]: 17 | """Cast input to integer.""" 18 | return flow( 19 | value_to_cast, 20 | cast_to_decimal, 21 | map_(_quantize_decimal), 22 | map_(int), 23 | ) 24 | 25 | 26 | def _quantize_decimal(number: decimal.Decimal) -> decimal.Decimal: 27 | """Quantize a decimal to whole number.""" 28 | return number.quantize(decimal.Decimal('1.')) 29 | -------------------------------------------------------------------------------- /tests/functions/test_apply_default.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | 3 | from kaiba.functions import apply_default 4 | 5 | 6 | def test_apply_default(): 7 | """Test if we get a default value.""" 8 | assert apply_default(None, 'default').unwrap() == 'default' 9 | 10 | 11 | def test_no_default_value(): 12 | """Test value returned when exists.""" 13 | assert apply_default('val', None).unwrap() == 'val' 14 | 15 | 16 | def test_no_values(): 17 | """Test returns Failure.""" 18 | test = apply_default(None, None) 19 | assert not is_successful(test) 20 | 21 | 22 | def test_bad_mapped_value(): 23 | """Test if we get a Failure when we give bad mapped value.""" 24 | test = apply_default(['array'], None) 25 | assert not is_successful(test) 26 | assert 'Unable to give default value' in str(test.failure()) 27 | -------------------------------------------------------------------------------- /tests/validation/test_branching_object.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.branching_object import BranchingObject 5 | 6 | 7 | def test_validates(): # noqa: WPS218 8 | """Test that dict is marshalled to pydantic object.""" 9 | test = BranchingObject( 10 | name='Name', 11 | ) 12 | assert test.name == 'Name' 13 | assert test.array is False 14 | assert isinstance(test.iterators, list) 15 | assert isinstance(test.branching_attributes, list) 16 | 17 | 18 | def test_invalid(): 19 | """Test that we get validation error with correct message.""" 20 | with pytest.raises(ValidationError) as ve: 21 | BranchingObject(array=False) # type: ignore 22 | 23 | errors = ve.value.errors()[0] # noqa: WPS441 24 | 25 | assert errors['loc'] == ('name',) 26 | assert errors['msg'] == 'Field required' 27 | -------------------------------------------------------------------------------- /tests/validation/test_data_fetcher.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.data_fetcher import DataFetcher 5 | 6 | 7 | def test_validates(): # noqa: WPS218 8 | """Test that dict is marshalled to pydantic object.""" 9 | test = DataFetcher( 10 | path=['test', 123], 11 | default='Default', 12 | ) 13 | assert test.path == ['test', 123] 14 | assert test.regex is None 15 | assert isinstance(test.if_statements, list) 16 | assert test.default == 'Default' 17 | 18 | 19 | def test_only_int_and_str_in_path(): 20 | """Test that giving an empty path is an error.""" 21 | with pytest.raises(ValidationError) as ve: 22 | DataFetcher(path=[12.2]) 23 | 24 | errors = ve.value.errors()[0] # noqa: WPS441 25 | 26 | assert errors['loc'] == ('path', 0, 'str') 27 | assert errors['msg'] == 'Input should be a valid string' 28 | -------------------------------------------------------------------------------- /tests/functions/test_apply_regex.py: -------------------------------------------------------------------------------- 1 | from kaiba.functions import apply_regex 2 | from kaiba.models.regex import Regex 3 | 4 | 5 | def test_regexp_get_empty_list_grup(): 6 | """Test regexp when group indeces are out of range.""" 7 | regexp = apply_regex( 8 | 'Hard work', 9 | Regex(**{'expression': 'r', 'group': []}), 10 | ) 11 | assert regexp.unwrap() == ['r', 'r'] 12 | 13 | 14 | def test_regexp_on_index_out_of_range(): 15 | """Test regexp when group indeces are out of range.""" 16 | regexp = apply_regex( 17 | 'Hard work', 18 | Regex(**{'expression': 'r', 'group': [1, 2, 3]}), 19 | ) 20 | assert isinstance(regexp.failure(), IndexError) is True 21 | assert regexp.failure().args == ('list index out of range',) 22 | 23 | 24 | def test_no_value_is_ok(): 25 | """When value is None we get a Success(None).""" 26 | assert apply_regex(None, Regex(**{'expression': 'a'})).unwrap() is None 27 | -------------------------------------------------------------------------------- /kaiba/process.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from pydantic import ValidationError 4 | from returns.functions import raise_exception 5 | from returns.result import Failure, ResultE 6 | 7 | from kaiba.mapper import map_data 8 | from kaiba.models.kaiba_object import KaibaObject 9 | 10 | 11 | def process( 12 | input_data: dict, 13 | configuration: dict, 14 | ) -> ResultE[Union[list, dict]]: 15 | """Validate configuration then process data.""" 16 | try: 17 | cfg = KaibaObject(**configuration) 18 | except ValidationError as ve: 19 | return Failure(ve) 20 | 21 | return map_data(input_data, cfg) 22 | 23 | 24 | def process_raise( 25 | input_data: dict, 26 | configuration: dict, 27 | ) -> Union[list, dict]: 28 | """Call Process and unwrap value if no error, otherwise raise.""" 29 | return process( 30 | input_data, 31 | configuration, 32 | ).alt( 33 | raise_exception, 34 | ).unwrap() 35 | -------------------------------------------------------------------------------- /tests/validation/test_attribute.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.attribute import Attribute 5 | 6 | 7 | def test_validates(): # noqa: WPS218 8 | """Test that dict is marshalled to pydantic object.""" 9 | test = Attribute( 10 | name='Name', 11 | default='Default', 12 | ) 13 | assert test.name == 'Name' 14 | assert test.default == 'Default' 15 | assert isinstance(test.data_fetchers, list) 16 | assert isinstance(test.if_statements, list) 17 | assert test.separator == '' 18 | assert test.casting is None 19 | 20 | 21 | def test_invalid(): 22 | """Test that we get validation error with correct message.""" 23 | with pytest.raises(ValidationError) as ve: 24 | Attribute(separator=' ') # type: ignore 25 | 26 | errors = ve.value.errors()[0] # noqa: WPS441 27 | 28 | assert errors['loc'] == ('name',) 29 | assert errors['msg'] == 'Field required' 30 | -------------------------------------------------------------------------------- /kaiba/models/data_fetcher.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List, Optional 2 | 3 | from pydantic import ConfigDict 4 | 5 | from kaiba.models.base import KaibaBaseModel, StrInt 6 | from kaiba.models.if_statement import IfStatement 7 | from kaiba.models.regex import Regex 8 | from kaiba.models.slicing import Slicing 9 | 10 | 11 | class DataFetcher(KaibaBaseModel): 12 | """Data fetcher lets you fetch data from the input.""" 13 | 14 | path: List[StrInt] = [] 15 | slicing: Optional[Slicing] = None 16 | regex: Optional[Regex] = None 17 | if_statements: List[IfStatement] = [] 18 | default: Optional[Any] = None 19 | model_config = ConfigDict(json_schema_extra={ 20 | 'examples': [ 21 | { 22 | 'path': ['path', 'to', 'data'], 23 | }, 24 | { 25 | 'path': ['path', 1, 2, 'my_val'], 26 | 'defualt': 'if no data was found this value is used', 27 | }, 28 | ], 29 | }) 30 | -------------------------------------------------------------------------------- /tests/collection_handlers/test_fetch_list_by_keys.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | 3 | from kaiba.collection_handlers import fetch_list_by_keys 4 | 5 | 6 | def test(): 7 | """Test that we can fetch key in dict.""" 8 | test: list = [{'key': ['val1']}, ['key']] 9 | assert fetch_list_by_keys(*test).unwrap() == ['val1'] 10 | 11 | 12 | def test_no_path_raises_value_error(): 13 | """Test that we get an error when we dont send a path.""" 14 | test: list = [{'key', 'val1'}, []] 15 | t_result = fetch_list_by_keys(*test) 16 | assert not is_successful(t_result) 17 | assert 'path list empty' in str(t_result.failure()) 18 | 19 | 20 | def test_that_found_value_must_be_list(): 21 | """Test that the value we find must be a list, expect error.""" 22 | test: list = [{'key': 'val1'}, ['key']] 23 | t_result = fetch_list_by_keys(*test) 24 | assert not is_successful(t_result) 25 | assert 'Non list data found' in str(t_result.failure()) 26 | -------------------------------------------------------------------------------- /kaiba/models/attribute.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from pydantic import ConfigDict 4 | 5 | from kaiba.models.base import AnyType, KaibaBaseModel 6 | from kaiba.models.casting import Casting 7 | from kaiba.models.data_fetcher import DataFetcher 8 | from kaiba.models.if_statement import IfStatement 9 | 10 | 11 | class Attribute(KaibaBaseModel): 12 | """Adds an attribute with the given name.""" 13 | 14 | name: str 15 | data_fetchers: List[DataFetcher] = [] 16 | separator: str = '' 17 | if_statements: List[IfStatement] = [] 18 | casting: Optional[Casting] = None 19 | default: Optional[AnyType] = None 20 | model_config = ConfigDict(json_schema_extra={ 21 | 'examples': [ 22 | { 23 | 'name': 'my_attribute', 24 | 'data_fetchers': [{ 25 | 'path': ['abc', 0], 26 | }], 27 | 'default': 'default_value', 28 | }, 29 | ], 30 | }) 31 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | strict: true 3 | site_name: kaiba 4 | site_description: Configurable json to json mapping 5 | theme: 6 | name: material 7 | palette: 8 | scheme: slate 9 | primary: green 10 | accent: deep-orange 11 | icon: 12 | logo: material/language-python 13 | language: en 14 | features: 15 | - navigation.tabs 16 | repo_name: kaiba-tech/kaiba 17 | repo_url: https://github.com/kaiba-tech/kaiba 18 | edit_uri: "" 19 | markdown_extensions: 20 | - admonition 21 | - codehilite: 22 | guess_lang: false 23 | - toc: 24 | permalink: true 25 | - pymdownx.magiclink 26 | - pymdownx.superfences 27 | - pymdownx.tabbed 28 | - pymdownx.tasklist: 29 | custom_checkbox: true 30 | nav: 31 | - Home: 32 | - Home: index.md 33 | - Usage: usage.md 34 | - Introduction: introduction.md 35 | - Configuration: configuration.md 36 | - Contributing: contributing.md 37 | - Changelog: changelog.md 38 | - Usecases: 39 | - Usecases: usecases/usecases.md 40 | -------------------------------------------------------------------------------- /kaiba/models/casting.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional 3 | 4 | from pydantic import ConfigDict 5 | 6 | from kaiba.models.base import KaibaBaseModel 7 | 8 | 9 | class CastToOptions(str, Enum): # noqa: WPS600 10 | """Types that we can cast a value to.""" 11 | 12 | STRING = 'string' # noqa: WPS115 13 | INTEGER = 'integer' # noqa: WPS115 14 | DECIMAL = 'decimal' # noqa: WPS115 15 | DATE = 'date' # noqa: WPS115 16 | 17 | 18 | class Casting(KaibaBaseModel): 19 | """Allows user to cast to type.""" 20 | 21 | to: CastToOptions 22 | original_format: Optional[str] = None 23 | model_config = ConfigDict(json_schema_extra={ 24 | 'examples': [ 25 | { 26 | 'to': 'integer', 27 | }, 28 | { 29 | 'to': 'date', 30 | 'original_format': 'ddmmyy', 31 | }, 32 | { 33 | 'to': 'date', 34 | 'original_format': 'yyyy.mm.dd', 35 | }, 36 | ], 37 | }) 38 | -------------------------------------------------------------------------------- /tests/functions/test_apply_casting.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | 3 | from kaiba.functions import apply_casting 4 | from kaiba.models.casting import Casting 5 | 6 | 7 | def test_value_but_cast_to_fails(): 8 | """No value should just fail with ValueError.""" 9 | test = apply_casting(None, Casting(**{'to': 'integer'})) 10 | assert not is_successful(test) 11 | assert isinstance(test.failure(), ValueError) 12 | assert 'value_to_cast is empty' in str(test.failure()) 13 | 14 | 15 | def test_string_fails_when_month_is_not_integer(): 16 | """Test threws ValueError when month out of range.""" 17 | test = apply_casting( 18 | '19.MM.12', 19 | Casting(**{ 20 | 'to': 'date', 21 | 'original_format': 'yy.mm.dd', 22 | }), 23 | ) 24 | expected = '{0}{1}'.format( 25 | 'Unable to cast (19.MM.12) to ISO date. ', 26 | "Exc(invalid literal for int() with base 10: '.M')", 27 | ) 28 | assert not is_successful(test) 29 | assert isinstance(test.failure(), ValueError) 30 | assert str(test.failure()) == expected 31 | -------------------------------------------------------------------------------- /kaiba/models/kaiba_object.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pydantic import ConfigDict 4 | from pydantic.types import StrictBool 5 | 6 | from kaiba.models.attribute import Attribute 7 | from kaiba.models.base import KaibaBaseModel 8 | from kaiba.models.branching_object import BranchingObject 9 | from kaiba.models.iterator import Iterator 10 | 11 | 12 | class KaibaObject(KaibaBaseModel): 13 | """Our main object.""" 14 | 15 | name: str 16 | array: StrictBool = False 17 | iterators: List[Iterator] = [] 18 | attributes: List[Attribute] = [] 19 | objects: List['KaibaObject'] = [] # noqa: WPS110 20 | branching_objects: List[BranchingObject] = [] 21 | model_config = ConfigDict(json_schema_extra={ 22 | 'examples': [ 23 | { 24 | 'name': 'object_name', 25 | 'attributes': [ 26 | { 27 | 'name': 'an_attribute', 28 | 'default': 'a value', 29 | }, 30 | ], 31 | }, 32 | ], 33 | }) 34 | 35 | 36 | KaibaObject.update_forward_refs() 37 | -------------------------------------------------------------------------------- /tests/validation/test_slicing.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.slicing import Slicing 5 | 6 | 7 | def test_validates(): # noqa: WPS218 8 | """Test that dict is marshalled to pydantic object.""" 9 | test = Slicing(**{'from': 0}) 10 | assert test.slice_from == 0 11 | assert test.slice_to is None 12 | 13 | 14 | def test_invalid(): 15 | """Test that we get validation error with correct message.""" 16 | with pytest.raises(ValidationError) as ve: 17 | # Ignore type on purpose since we want to check error 18 | Slicing(to=0) # type: ignore 19 | 20 | errors = ve.value.errors()[0] # noqa: WPS441 21 | assert errors['loc'] == ('from',) 22 | assert errors['msg'] == 'Field required' 23 | 24 | 25 | def test_invalid_type(): 26 | """Test that we get validation error with correct message.""" 27 | with pytest.raises(ValidationError) as ve: 28 | Slicing(**{'from': 'test'}) 29 | 30 | errors = ve.value.errors()[0] # noqa: WPS441 31 | assert errors['loc'] == ('from',) 32 | assert errors['msg'] == 'Input should be a valid integer' 33 | -------------------------------------------------------------------------------- /tests/validation/test_regex.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | import pytest 4 | from pydantic import ValidationError 5 | 6 | from kaiba.models.regex import Regex 7 | 8 | 9 | def test_validates(): # noqa: WPS218 10 | """Test that dict is marshalled to pydantic object.""" 11 | test = Regex(expression='[1-9]') 12 | assert test.expression == re.compile('[1-9]') 13 | assert test.group == 0 14 | 15 | 16 | def test_invalid_expression(): 17 | """Test that we get validation error with correct message.""" 18 | with pytest.raises(ValidationError) as ve: 19 | Regex(expression='abc[') 20 | 21 | errors = ve.value.errors()[0] # noqa: WPS441 22 | assert errors['loc'] == ('expression',) 23 | assert errors['msg'] == 'Input should be a valid regular expression' 24 | 25 | 26 | def test_invalid_group(): 27 | """Test that we get validation error with correct message.""" 28 | with pytest.raises(ValidationError) as ve: 29 | Regex(expression='[1-9]', group='test') 30 | 31 | errors = ve.value.errors()[0] # noqa: WPS441 32 | assert errors['loc'] == ('group', 'int') 33 | assert errors['msg'] == 'Input should be a valid integer' 34 | -------------------------------------------------------------------------------- /tests/iso/test_get_language_code.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from returns.primitives.exceptions import UnwrapFailedError 3 | 4 | from kaiba.iso import get_language_code 5 | 6 | 7 | def test_by_alpha_two(): 8 | """Test that we can fetch key in dict.""" 9 | assert get_language_code('NO').unwrap()['alpha_3'] == 'NOR' 10 | 11 | 12 | def test_by_alpha_three(): 13 | """Test that we can fetch key in dict.""" 14 | assert get_language_code('NOR').unwrap()['alpha_3'] == 'NOR' 15 | 16 | 17 | def test_by_name(): 18 | """Test getting by name.""" 19 | assert get_language_code('Norwegian').unwrap()['alpha_3'] == 'NOR' 20 | 21 | 22 | def test_bad_code(): 23 | """Assert raises with bad name.""" 24 | with pytest.raises(UnwrapFailedError): 25 | assert get_language_code('somelanguage').unwrap() 26 | 27 | 28 | def test_bad_alpha_two(): 29 | """Assert raises with bad alpha 2 value.""" 30 | with pytest.raises(UnwrapFailedError): 31 | assert get_language_code('XX').unwrap() 32 | 33 | 34 | def test_bad_alpha_three(): 35 | """Assert raises with bad alpha 3 value.""" 36 | with pytest.raises(UnwrapFailedError): 37 | assert get_language_code('XXX').unwrap() 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Kaiba Technologies AS 4 | 5 | Copyright (c) 2020 Greenbird Integration Technology AS 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | -------------------------------------------------------------------------------- /tests/validation/test_if_statement.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.if_statement import Conditions, IfStatement 5 | 6 | 7 | def test_validates(): # noqa: WPS218 8 | """Test that dict is marshalled to pydantic object.""" 9 | test = IfStatement( 10 | condition='is', 11 | target=None, 12 | then='then', 13 | ) 14 | assert test.condition == 'is' 15 | assert test.target is None 16 | assert test.then == 'then' 17 | assert test.otherwise is None 18 | 19 | 20 | def test_invalid_bad_condition_enum(): # noqa: WPS218 21 | """Test that we get validation error with correct message.""" 22 | with pytest.raises(ValidationError) as ve: 23 | IfStatement( # type: ignore 24 | condition='bad', 25 | otherwise='otherwise', 26 | ) 27 | 28 | condition = ve.value.errors()[0] # noqa: WPS441 29 | assert condition['loc'] == ('condition',) 30 | msg = condition['msg'] 31 | assert all(con.value in msg for con in Conditions) 32 | 33 | target = ve.value.errors()[1] # noqa: WPS441 34 | assert target['loc'] == ('target',) 35 | assert target['msg'] == 'Field required' 36 | 37 | then = ve.value.errors()[2] # noqa: WPS441 38 | assert then['loc'] == ('then',) 39 | assert then['msg'] == 'Field required' 40 | -------------------------------------------------------------------------------- /tests/collection_handlers/test_set_value_in_dict.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | 3 | from kaiba.collection_handlers import set_value_in_dict 4 | 5 | 6 | def test(): 7 | """Test that we can fetch key in dict.""" 8 | dictionary = {'key': 'val1'} 9 | test: list = ['val2', dictionary, ['key']] 10 | assert is_successful(set_value_in_dict(*test)) 11 | assert dictionary['key'] == 'val2' 12 | 13 | 14 | def test_multiple_path(): 15 | """Test that we can fetch key in dict.""" 16 | dictionary = {'key': {'key2': 'val1'}} 17 | test: list = ['val2', dictionary, ['key', 'key2']] 18 | assert is_successful(set_value_in_dict(*test)) 19 | assert dictionary['key']['key2'] == 'val2' 20 | 21 | 22 | def test_no_path_raises_value_error(): 23 | """Test that we get an error when we dont send a path.""" 24 | test: list = ['val2', {'key': ['val1']}, []] 25 | t_result = set_value_in_dict(*test) 26 | assert not is_successful(t_result) 27 | assert 'path list empty' in str(t_result.failure()) 28 | 29 | 30 | def test_path_to_missing_key_is_ok(): 31 | """Test that missing keys are okay.""" 32 | dictionary = {'key': ['val1']} 33 | test: list = ['val2', dictionary, ['bob']] 34 | t_result = set_value_in_dict(*test) 35 | assert is_successful(t_result) 36 | assert dictionary['bob'] == 'val2' # type: ignore 37 | -------------------------------------------------------------------------------- /docs/usecases/usecases.md: -------------------------------------------------------------------------------- 1 | In this section we try to explain some normal usecases for __Kaiba__. 2 | 3 | It is highly recommended that you go through the [introduction](../../introduction) before continuing 4 | 5 | ## Kaiba + CSV 6 | 7 | CSV is one of the most used filetypes when exchanging data by files. Here are some examples to look at when working with csv 8 | 9 | * [Transform CSV data to JSON]() 10 | * [Transform JSON to CSV]() 11 | * [Row Type CSV data]() 12 | 13 | 14 | ## Kaiba + XML 15 | 16 | XML is... ugh... but a lot of legacy systems expect XML as input and produces XML as output. We won't get rid of XML anytime soon, but atleast with Kaiba we can live with it. 17 | 18 | There are three things to look out for when working with XML. 19 | 20 | 1. __It's impossible to know if a child of an element is supposed to be an array or not.__ 21 | 2. __XML creates structure with Elements, but store data as either parameters or as text between element opening and closing tag.__ 22 | 23 | Because of this there are no 1 to 1 XML->JSON converter that will work for any XML. Conventions must be chosen but a good starting point is the Parker convention 24 | 25 | * [Restructure XML data with Kaiba]() 26 | * [From XML to CSV]() 27 | * [CSV to XML]() 28 | 29 | ## Other usecases 30 | 31 | Add an issue at our [issue tracker](https://github.com/kaiba-tech/kaiba/issues) for request for other usecases/examples 32 | -------------------------------------------------------------------------------- /tests/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": { 3 | "name": "schema", 4 | "array": true, 5 | "path_to_iterable": ["path_to", "some_list"], 6 | 7 | "attributes": [ 8 | { 9 | "name": "attr1", 10 | "mappings": [ 11 | { 12 | "path": ["path", "to", "value"], 13 | "default": "default value" 14 | } 15 | ], 16 | "separator": "", 17 | "if_statements": [ 18 | { 19 | "condition": "is", 20 | "target": "bob", 21 | "then": "arne" 22 | } 23 | ], 24 | "default": "another default" 25 | } 26 | ], 27 | "objects": [ 28 | { 29 | "name": "object1", 30 | "array": false, 31 | "attributes": [ 32 | { 33 | "name": "height", 34 | "default": "bob" 35 | } 36 | ] 37 | } 38 | ], 39 | "branching_objects": [ 40 | { 41 | "name": "object2", 42 | "array": true, 43 | "branching_attributes": [ 44 | [ 45 | { 46 | "name": "field_name", 47 | "default": "amount" 48 | }, 49 | { 50 | "name": "field_data", 51 | "mappings": [ 52 | { 53 | "path": ["path", "to", "amount"] 54 | } 55 | ] 56 | } 57 | ], 58 | [ 59 | { 60 | "name": "field_name", 61 | "default": "currency" 62 | }, 63 | { 64 | "name": "field_data", 65 | "default": "NOK" 66 | } 67 | ] 68 | ] 69 | } 70 | ] 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "kaiba" 3 | version = "3.0.1" 4 | description = "Configurable and documentable Json transformation and mapping" 5 | authors = ["Thomas Borgen "] 6 | 7 | license = "MIT" 8 | readme = "docs/index.md" 9 | repository = "https://github.com/kaiba-tech/kaiba" 10 | documentation = "https://kaiba-tech.github.io/kaiba/" 11 | 12 | keywords = [ 13 | "Json mapping", 14 | "data transformation", 15 | "json to json", 16 | "dict to dict", 17 | "configurable" 18 | ] 19 | 20 | classifiers = [ 21 | "Development Status :: 5 - Production/Stable", 22 | "Intended Audience :: Developers", 23 | "Operating System :: OS Independent", 24 | "Topic :: Software Development :: Libraries :: Python Modules", 25 | "Topic :: Utilities" 26 | ] 27 | 28 | [tool.poetry.dependencies] 29 | python = "^3.8.1" 30 | pycountry = ">=20.7.3,<23.0.0" 31 | returns = "^0.22.0" 32 | pydantic = "^2.4.2" 33 | 34 | [tool.poetry.group.dev.dependencies] 35 | mypy = "1.5" 36 | wemake-python-styleguide = "^0.18.0" 37 | pre-commit = "^2.16.0" 38 | simplejson = "^3.17.6" 39 | 40 | safety = "^2.3.5" 41 | 42 | pytest = "^7.4.2" 43 | pytest-cov = ">=3,<5" 44 | mkdocs-material = "^9.4.6" 45 | importlib-metadata = ">=4.8.2,<8.0.0" 46 | isort = "^5.10.1" 47 | 48 | [build-system] 49 | requires = ["poetry-core>=1.0.0"] 50 | build-backend = "poetry.core.masonry.api" 51 | 52 | [tool.isort] 53 | known_first_party = "kaiba" 54 | profile = "wemake" 55 | -------------------------------------------------------------------------------- /tests/functions/test_apply_seperator.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from returns.primitives.exceptions import UnwrapFailedError 3 | 4 | from kaiba.functions import apply_separator 5 | 6 | 7 | def test_separator(): 8 | """Test separator is applied between two values.""" 9 | test: list = [['val1', 'val2'], '-'] 10 | assert apply_separator(*test).unwrap() == 'val1-val2' 11 | 12 | 13 | def test_separator_one_value(): 14 | """Test when theres only one value, no separator should be applied.""" 15 | test: list = [['val1'], '-'] 16 | assert apply_separator(*test).unwrap() == 'val1' 17 | 18 | 19 | def test_one_integer_value_not_stringified(): 20 | """One value should allways return just the value uncasted.""" 21 | test: list = [[1], ''] 22 | assert apply_separator(*test).unwrap() == 1 23 | 24 | 25 | def test_one_integer_value_with_other_value(): 26 | """Two values no matter the type should be cast to string.""" 27 | test: list = [[1, 'val2'], '-'] 28 | assert apply_separator(*test).unwrap() == '1-val2' 29 | 30 | 31 | def test_no_value(): 32 | """When no value is given we should return Failure.""" 33 | test: list = [[], '-'] 34 | with pytest.raises(UnwrapFailedError): 35 | apply_separator(*test).unwrap() 36 | 37 | 38 | def test_no_separator(): 39 | """Test that no separator throws error.""" 40 | test: list = [['val1', 'val2'], None] 41 | with pytest.raises(UnwrapFailedError): 42 | apply_separator(*test).unwrap() 43 | -------------------------------------------------------------------------------- /tests/validation/test_iterator.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.iterator import Iterator 5 | 6 | 7 | def test_validates(): # noqa: WPS218 8 | """Test that dict is marshalled to pydantic object.""" 9 | test = Iterator( 10 | alias='test', 11 | path=['data'], 12 | ) 13 | assert test.alias == 'test' 14 | assert test.path == ['data'] 15 | 16 | 17 | def test_invalid(): 18 | """Test that we get validation error with correct message.""" 19 | with pytest.raises(ValidationError) as ve: 20 | Iterator(**{'alias': 'test'}) 21 | 22 | errors = ve.value.errors()[0] # noqa: WPS441 23 | 24 | assert errors['loc'] == ('path',) 25 | assert errors['msg'] == 'Field required' 26 | 27 | 28 | def test_empty_path_is_error(): 29 | """Test that giving an empty path is an error.""" 30 | with pytest.raises(ValidationError) as ve: 31 | Iterator(alias='test', path=[]) 32 | 33 | errors = ve.value.errors()[0] # noqa: WPS441 34 | 35 | assert errors['loc'] == ('path',) 36 | assert 'List should have at least 1 item' in errors['msg'] 37 | 38 | 39 | def test_only_int_and_str_in_path(): 40 | """Test that giving an empty path is an error.""" 41 | with pytest.raises(ValidationError) as ve: 42 | Iterator(alias='test', path=[12.2]) 43 | 44 | errors = ve.value.errors()[0] # noqa: WPS441 45 | 46 | assert errors['loc'] == ('path', 0, 'str') 47 | assert errors['msg'] == 'Input should be a valid string' 48 | -------------------------------------------------------------------------------- /tests/test_nested_iterables.py: -------------------------------------------------------------------------------- 1 | from kaiba.collection_handlers import iterable_data_handler 2 | from kaiba.models.iterator import Iterator 3 | 4 | 5 | def test_iterable_data_handler(): 6 | """Test that we can iterate multiple levels in one go.""" 7 | input_data = { 8 | 'data': [ 9 | {'nested': [ 10 | {'another': [ 11 | {'a': 'a'}, 12 | {'a': 'b'}, 13 | ]}, 14 | ]}, 15 | {'nested': [ 16 | {'another': [ 17 | {'a': 'c'}, 18 | {'a': 'd'}, 19 | ]}, 20 | ]}, 21 | ], 22 | } 23 | 24 | paths_to_iterables = [ 25 | Iterator(**{ 26 | 'alias': 'data', 27 | 'path': ['data'], 28 | }), 29 | Iterator(**{ 30 | 'alias': 'nested', 31 | 'path': ['data', 'nested'], 32 | }), 33 | Iterator(**{ 34 | 'alias': 'doesnotexist', 35 | 'path': ['does', 'not', 'exist'], 36 | }), 37 | Iterator(**{ 38 | 'alias': 'another', 39 | 'path': ['nested', 'another'], 40 | }), 41 | ] 42 | 43 | iterables = iterable_data_handler(input_data, paths_to_iterables).unwrap() 44 | assert len(iterables) == 4 45 | assert iterables[3]['another']['a'] == 'd' 46 | 47 | 48 | def test_iterable_no_paths_returns_failure(): 49 | """Test that when there are no paths we get a Failure.""" 50 | iterables = iterable_data_handler({}, []) 51 | assert 'No iterators' in str(iterables.failure()) 52 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | python-version: ["3.8", "3.9", "3.10", "3.11"] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v1 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | 25 | - name: Install and configure Poetry 26 | uses: snok/install-poetry@v1 27 | with: 28 | version: "1.3.1" 29 | virtualenvs-create: true 30 | virtualenvs-in-project: true 31 | - name: Set up cache 32 | uses: actions/cache@v1 33 | with: 34 | path: .venv 35 | key: venv-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} 36 | - name: Install dependencies 37 | run: | 38 | poetry install 39 | 40 | - name: Update pip to latest version 41 | run: | 42 | poetry run python -m pip install -U pip 43 | 44 | - name: Run code quality checks 45 | run: | 46 | poetry check 47 | poetry run pip check 48 | poetry run flake8 . 49 | poetry run mypy . 50 | poetry run safety check --full-report 51 | - name: Run pytest 52 | run: | 53 | poetry run pytest . 54 | - name: Upload coverage to Codecov 55 | if: matrix.python-version == 3.8 56 | uses: codecov/codecov-action@v1 57 | with: 58 | file: ./coverage.xml 59 | -------------------------------------------------------------------------------- /kaiba/casting/_cast_to_decimal.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import re 4 | from decimal import Decimal 5 | 6 | from returns.result import safe 7 | from typing_extensions import Final 8 | 9 | from kaiba.constants import COMMA, EMPTY, INTEGER_CONTAINING_DECIMALS, PERIOD 10 | from kaiba.models.base import AnyType 11 | 12 | _decimal_pattern: Final = re.compile(r'^([0-9]|-|\.|,)+$') 13 | _decimal_with_period_after_commas: Final = re.compile(r'^-?(\d+\,)*\d+\.\d+$') 14 | _decimal_with_comma_after_periods: Final = re.compile(r'^-?(\d+\.)*\d+\,\d+$') 15 | 16 | 17 | @safe 18 | def cast_to_decimal( 19 | value_to_cast: AnyType, 20 | original_format: str | None = None, 21 | ) -> Decimal: 22 | """Cast input to decimal.""" 23 | the_value = str(value_to_cast).replace(' ', EMPTY) 24 | 25 | if not _decimal_pattern.match(the_value): 26 | raise ValueError( 27 | "Illegal characters in value '{0}'".format(the_value), 28 | ) 29 | 30 | if original_format == INTEGER_CONTAINING_DECIMALS: 31 | return Decimal(the_value) / 100 32 | 33 | # ie 1234567,89 only comma as decimal separator 34 | if the_value.count(COMMA) == 1 and not the_value.count(PERIOD): 35 | return Decimal(the_value.replace(COMMA, PERIOD)) 36 | 37 | # ie 1,234,567.89 many commas followed by one period 38 | if _decimal_with_period_after_commas.match(the_value): 39 | return Decimal(the_value.replace(COMMA, EMPTY)) 40 | 41 | # ie 1.234.567,89 many periods followed by one comma 42 | if _decimal_with_comma_after_periods.match(the_value): 43 | return Decimal( 44 | the_value.replace(PERIOD, EMPTY).replace(COMMA, PERIOD), 45 | ) 46 | 47 | return Decimal(the_value) 48 | -------------------------------------------------------------------------------- /tests/collection_handlers/test_fetch_data_by_keys.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | from returns.result import Success 3 | 4 | from kaiba.collection_handlers import fetch_data_by_keys 5 | 6 | 7 | def test(): 8 | """Test that we can fetch key in dict.""" 9 | test: list = [{'key': 'val1'}, ['key']] 10 | assert fetch_data_by_keys(*test).unwrap() == 'val1' 11 | 12 | 13 | def test_two_keys(): 14 | """Test that we are able to map with multiple keys.""" 15 | test: list = [{'key1': {'key2': 'val1'}}, ['key1', 'key2']] 16 | assert fetch_data_by_keys(*test).unwrap() == 'val1' 17 | 18 | 19 | def test_find_dictionary(): 20 | """Test that giving path to dict is okay.""" 21 | test: list = [{'key': {'key1': 'val'}}, ['key']] 22 | assert fetch_data_by_keys(*test) == Success({'key1': 'val'}) 23 | 24 | 25 | def test_find_array(): 26 | """Test that giving path to array is okay.""" 27 | test: list = [{'key': ['val', 'val']}, ['key']] 28 | assert fetch_data_by_keys(*test) == Success(['val', 'val']) 29 | 30 | 31 | def test_no_such_key(): 32 | """Test Failure on missing key.""" 33 | test: list = [{'key': 'val1'}, ['missing']] 34 | t_result = fetch_data_by_keys(*test) 35 | assert not is_successful(t_result) 36 | assert 'missing' in str(t_result.failure()) 37 | 38 | 39 | def test_no_path(): 40 | """Test no path should return Failure.""" 41 | test: list = [{'key': 'val'}, []] 42 | t_result = fetch_data_by_keys(*test) 43 | assert not is_successful(t_result) 44 | assert 'path list empty' in str(t_result.failure()) 45 | 46 | 47 | def test_no_data(): 48 | """Test no data should return Failure.""" 49 | test: list = [{}, ['keys']] 50 | t_result = fetch_data_by_keys(*test) 51 | assert not is_successful(t_result) 52 | assert 'keys' in str(t_result.failure()) 53 | -------------------------------------------------------------------------------- /tests/iso/test_get_currency_code.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from returns.primitives.exceptions import UnwrapFailedError 3 | 4 | from kaiba.iso import get_currency_code 5 | 6 | 7 | def test_by_alpha_three_upper(): 8 | """Get currency by alpha 3 code.""" 9 | assert get_currency_code('NOK').unwrap()['alpha_3'] == 'NOK' 10 | 11 | 12 | def test_by_alpha_three_lower(): 13 | """Get currency by alpha 3 lower cased.""" 14 | assert get_currency_code('nok').unwrap()['alpha_3'] == 'NOK' 15 | 16 | 17 | def test_by_numeric(): 18 | """Test that we can fetch currency by numeric value.""" 19 | assert get_currency_code('578').unwrap()['alpha_3'] == 'NOK' 20 | 21 | 22 | def test_by_numeric_int(): 23 | """Test get by numeric with integer.""" 24 | assert get_currency_code(578).unwrap()['alpha_3'] == 'NOK' # noqa: Z432 25 | 26 | 27 | def test_by_numeric_below_hundred(): 28 | """Test get by numeric with number < 100.""" 29 | assert get_currency_code('8').unwrap()['alpha_3'] == 'ALL' 30 | 31 | 32 | def test_by_name(): 33 | """Test getting by name.""" 34 | assert get_currency_code('Norwegian Krone').unwrap()['alpha_3'] == 'NOK' 35 | 36 | 37 | def test_bad_code(): 38 | """Assert raises with bad name.""" 39 | with pytest.raises(UnwrapFailedError): 40 | assert get_currency_code('Noway').unwrap() 41 | 42 | 43 | def test_bad_numeric(): 44 | """Assert raises with bad numeric value.""" 45 | with pytest.raises(UnwrapFailedError): 46 | assert get_currency_code(123).unwrap() # noqa: Z432 47 | 48 | 49 | def test_bad_alpha_two(): 50 | """Assert raises with bad alpha 2 value.""" 51 | with pytest.raises(UnwrapFailedError): 52 | assert get_currency_code('XX').unwrap() 53 | 54 | 55 | def test_bad_alpha_three(): 56 | """Assert raises with bad alpha 3 value.""" 57 | with pytest.raises(UnwrapFailedError): 58 | assert get_currency_code('NAN').unwrap() 59 | -------------------------------------------------------------------------------- /tests/validation/test_kaiba_object.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from kaiba.models.kaiba_object import KaibaObject 5 | 6 | 7 | def test_validates_only_object(): # noqa: WPS218 8 | """Test that dict is marshalled to pydantic object.""" 9 | test = KaibaObject( 10 | name='Name', 11 | ) 12 | assert test.name == 'Name' 13 | assert test.array is False 14 | assert isinstance(test.iterators, list) 15 | assert isinstance(test.attributes, list) 16 | assert isinstance(test.branching_objects, list) 17 | assert isinstance(test.objects, list) 18 | 19 | 20 | def test_invalid_only_object(): 21 | """Test that we get validation error with correct message.""" 22 | with pytest.raises(ValidationError) as ve: 23 | KaibaObject(array=False) # type: ignore 24 | 25 | errors = ve.value.errors()[0] # noqa: WPS441 26 | 27 | assert errors['loc'] == ('name',) 28 | assert errors['msg'] == 'Field required' 29 | 30 | 31 | def test_validates(valid): 32 | """Test that we get dict back on valid validation.""" 33 | assert KaibaObject(**valid) 34 | 35 | 36 | def test_invalid(invalid): # noqa: WPS218 allow asserts 37 | """Test that we get a list of errors.""" 38 | with pytest.raises(ValidationError) as ve: 39 | KaibaObject(**invalid) 40 | 41 | errors = ve.value.errors() # noqa: WPS441 42 | 43 | assert errors[0]['loc'] == ( 44 | 'attributes', 0, 'if_statements', 0, 'condition', 45 | ) 46 | assert errors[0]['msg'] == 'Field required' 47 | 48 | assert errors[1]['loc'] == ( 49 | 'objects', 0, 'attributes', 0, 'deult', 50 | ) 51 | assert errors[1]['msg'] == 'Extra inputs are not permitted' 52 | 53 | assert errors[2]['loc'] == ( 54 | 'branching_objects', 0, 'branching_attributes', 0, 0, 'name', 55 | ) 56 | assert errors[2]['msg'] == 'Field required' 57 | 58 | # Should also complain about date original format not being correct 59 | -------------------------------------------------------------------------------- /kaiba/models/if_statement.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional 3 | 4 | from pydantic import ConfigDict, Field 5 | 6 | from kaiba.models.base import AnyType, KaibaBaseModel 7 | 8 | 9 | class Conditions(str, Enum): # noqa: WPS600 10 | """Conditions for if statements.""" 11 | 12 | IS = 'is' # noqa: WPS115 13 | NOT = 'not' # noqa: WPS115 14 | IN = 'in' # noqa: WPS115 15 | CONTAINS = 'contains' # noqa: WPS115 16 | 17 | 18 | class IfStatement(KaibaBaseModel): 19 | """If statements lets you conditionally change data.""" 20 | 21 | condition: Conditions 22 | target: Optional[AnyType] = Field(...) # ... = required but allow None 23 | then: Optional[AnyType] = Field(...) # ... = required but allow Nones 24 | otherwise: Optional[AnyType] = None # Should be any valid json value 25 | model_config = ConfigDict(json_schema_extra={ 26 | 'examples': [ 27 | { 28 | 'condition': 'is', 29 | 'target': 'target value', 30 | 'then': 'was target value', 31 | }, 32 | { 33 | 'condition': 'not', 34 | 'target': 'target_value', 35 | 'then': 'was not target value', 36 | 'otherwise': 'was target value', 37 | }, 38 | { 39 | 'condition': 'in', 40 | 'target': ['one', 'of', 'these'], 41 | 'then': 'was either one, of or these', 42 | 'otherwise': 'was none of those', 43 | }, 44 | { 45 | 'condition': 'in', 46 | 'target': 'a substring of this will be true', 47 | 'then': 'substrings also work', 48 | 'otherwise': 'was not a substring of target', 49 | }, 50 | { 51 | 'condition': 'contains', 52 | 'target': 'value', 53 | 'then': 'value was a substring of input', 54 | 'otherwise': 'value was not in the input', 55 | }, 56 | ], 57 | }) 58 | -------------------------------------------------------------------------------- /tests/json/valid.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "schema", 3 | "array": true, 4 | "iterators": [ 5 | {"alias": "iter_item", "path": [0, "test"]} 6 | ], 7 | "attributes": [ 8 | { 9 | "name": "attr1", 10 | "data_fetchers": [ 11 | { 12 | "path": ["path", "to", "value"], 13 | "default": "default value" 14 | } 15 | ], 16 | "separator": "", 17 | "if_statements": [ 18 | { 19 | "condition": "is", 20 | "target": "bob", 21 | "then": "arne" 22 | } 23 | ], 24 | "default": "another default" 25 | } 26 | ], 27 | "objects": [ 28 | { 29 | "name": "object1", 30 | "array": false, 31 | "attributes": [ 32 | { 33 | "name": "height", 34 | "default": "bob" 35 | } 36 | ] 37 | } 38 | ], 39 | "branching_objects": [ 40 | { 41 | "name": "object2", 42 | "array": true, 43 | "branching_attributes": [ 44 | [ 45 | { 46 | "name": "field_name", 47 | "default": "amount" 48 | }, 49 | { 50 | "name": "field_data", 51 | "data_fetchers": [ 52 | { 53 | "path": ["path", "to", "amount"] 54 | } 55 | ] 56 | } 57 | ], 58 | [ 59 | { 60 | "name": "field_name", 61 | "default": "currency" 62 | }, 63 | { 64 | "name": "field_data", 65 | "default": "NOK" 66 | } 67 | ] 68 | ] 69 | } 70 | ] 71 | } 72 | -------------------------------------------------------------------------------- /tests/iso/test_get_country_code.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from returns.primitives.exceptions import UnwrapFailedError 3 | 4 | from kaiba.iso import get_country_code 5 | 6 | 7 | def test(): 8 | """Test that we can fetch key in dict.""" 9 | assert get_country_code('NO').unwrap()['alpha_2'] == 'NO' 10 | 11 | 12 | def test_no_official_name(): 13 | """Some countries does not have official name. 14 | 15 | We should be able to get these and official_name should be `None` 16 | """ 17 | assert get_country_code('CA').unwrap()['official_name'] is None 18 | 19 | 20 | def test_by_numeric(): 21 | """Test that we can fetch currency by numeric value.""" 22 | assert get_country_code('578').unwrap()['numeric'] == '578' 23 | 24 | 25 | def test_by_numeric_int(): 26 | """Test get by numeric with integer.""" 27 | assert get_country_code(578).unwrap()['numeric'] == '578' # noqa: Z432 28 | 29 | 30 | def test_by_numeric_below_hundred(): 31 | """Test get by numeric with number < 100.""" 32 | assert get_country_code('8').unwrap()['numeric'] == '008' 33 | 34 | 35 | def test_by_name(): 36 | """Test getting by name.""" 37 | assert get_country_code('Norway').unwrap()['name'] == 'Norway' 38 | 39 | 40 | def test_by_official_name(): 41 | """Test getting by official name.""" 42 | test_value = get_country_code('Kingdom of Norway').unwrap()['official_name'] 43 | assert test_value == 'Kingdom of Norway' 44 | 45 | 46 | def test_bad_code(): 47 | """Assert raises with bad name.""" 48 | with pytest.raises(UnwrapFailedError): 49 | assert get_country_code('Noway').unwrap() 50 | 51 | 52 | def test_bad_numeric(): 53 | """Assert raises with bad numeric value.""" 54 | with pytest.raises(UnwrapFailedError): 55 | assert get_country_code(123).unwrap() # noqa: Z432 56 | 57 | 58 | def test_bad_alpha_two(): 59 | """Assert raises with bad alpha 2 value.""" 60 | with pytest.raises(UnwrapFailedError): 61 | assert get_country_code('XX').unwrap() 62 | 63 | 64 | def test_bad_alpha_three(): 65 | """Assert raises with bad alpha 3 value.""" 66 | with pytest.raises(UnwrapFailedError): 67 | assert get_country_code('XXX').unwrap() 68 | -------------------------------------------------------------------------------- /tests/casting/test_cast_to_integer.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | from typing_extensions import Final 3 | 4 | from kaiba.casting._cast_to_integer import cast_to_integer 5 | 6 | target: Final[int] = 123 7 | 8 | 9 | def test_cast_string(): 10 | """Cast a string with numbers to integer.""" 11 | assert cast_to_integer('123').unwrap() == target 12 | 13 | 14 | def test_cast_negative_string(): 15 | """Cast string with negative number to integer.""" 16 | assert cast_to_integer('-123').unwrap() == -target 17 | 18 | 19 | def test_cast_decimal_string(): 20 | """Cast a decimal string to integer.""" 21 | assert cast_to_integer( 22 | '123.0', 23 | 'decimal', 24 | ).unwrap() == target 25 | 26 | 27 | def test_cast_negative_decimal_string(): 28 | """Cast a negative decimal string to integer.""" 29 | assert cast_to_integer( 30 | '-123.0', 'decimal', 31 | ).unwrap() == -target 32 | 33 | 34 | def test_cast_decimal_string_rounds_up(): 35 | """Cast a decimal string >= .5 should round up.""" 36 | assert cast_to_integer( 37 | '122.5', 38 | 'decimal', 39 | ).unwrap() == target 40 | 41 | 42 | def test_once_more_cast_decimal_string_rounds_up(): 43 | """Cast a decimal string >= .5 should round up.""" 44 | assert cast_to_integer( 45 | '123.5', 46 | 'decimal', 47 | ).unwrap() == 124 48 | 49 | 50 | def test_cast_decimal_string_rounds_down(): 51 | """Cast a decimal string < .0 should round down.""" 52 | assert cast_to_integer( 53 | '123.49', 54 | 'decimal', 55 | ).unwrap() == target 56 | 57 | 58 | def test_abc_fails(): 59 | """Test that string with letters in fails.""" 60 | test = cast_to_integer('abc') 61 | assert not is_successful(test) 62 | assert isinstance(test.failure(), ValueError) 63 | assert 'Illegal characters in value' in str(test.failure()) 64 | 65 | 66 | def test_abc_with_decimal_argument_fails(): 67 | """Test that string with letters in fails when we supply 'decimal'.""" 68 | test = cast_to_integer( 69 | 'abc', 70 | 'decimal', 71 | ) 72 | assert not is_successful(test) 73 | assert isinstance(test.failure(), ValueError) 74 | assert 'Illegal characters in value' in str(test.failure()) 75 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # VSCode stuff: 58 | .DS_Store 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | db.sqlite3 64 | db.sqlite3-journal 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | # celery beat schedule file 97 | celerybeat-schedule 98 | 99 | # SageMath parsed files 100 | *.sage.py 101 | 102 | # Environments 103 | .env 104 | .venv 105 | env/ 106 | venv/ 107 | ENV/ 108 | env.bak/ 109 | venv.bak/ 110 | 111 | # Spyder project settings 112 | .spyderproject 113 | .spyproject 114 | 115 | # Rope project settings 116 | .ropeproject 117 | 118 | # mkdocs documentation 119 | /site 120 | 121 | # mypy 122 | .mypy_cache/ 123 | .dmypy.json 124 | dmypy.json 125 | 126 | # Pyre type checker 127 | .pyre/ 128 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## General Python info 4 | In this package we make extensive use of the [Returns library](https://github.com/dry-python/returns). Its a library that forces us to try and write None free code and also wraps exceptions. It changes return values to Result 'Monads' with Success and Failure return containers or Maybe and Nothing containers. This helps us to do railway-oriented-programming when working with mapping. 5 | 6 | 7 | ## New Environment Tools 8 | Lately we have gotten a few great python environment managers. The first being [PyEnv](https://github.com/pyenv/pyenv). Pyenv makes working with multiple versions of python easier. The second tool is [Poetry](https://poetry.eustace.io/). Poetry lets us create a lock file of all our dependencies, this means that both version of python and version of each dependency and its dependencies will be equal for everyone working on the project. It also uses the new pyproject.toml file which is the 'new' setup.py and requirements.txt in 1 file. Poetry also handles building and publishing. 9 | 10 | 11 | ## Setup the tools 12 | 13 | Get pyenv - pyenv lets you work with multiple versions of python. 14 | ```sh 15 | brew update 16 | brew install pyenv 17 | ``` 18 | 19 | If you are using bash, add the following to your `~/.bash_profile` to automatically load pyenv. If you are using another shell, run `pyenv init` and it will tell you how to set it up. 20 | ```sh 21 | eval "$(pyenv init -)" 22 | ``` 23 | 24 | Install a version of python 3.7+: This installs a clean python to pyenvs folders and lets us reference that as a 'base' in our virtualenvs. 25 | ```sh 26 | pyenv install 3.7.4 27 | ``` 28 | 29 | Get poetry - dependency management. 30 | ```sh 31 | curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python 32 | ``` 33 | 34 | Make poetry create virtualenvs inside of project folders. This makes it easier for IDE's to run in the correct virtualenv while debuging/running linters etc. 35 | ```sh 36 | poetry config virtualenvs.in-project true 37 | ``` 38 | 39 | ## Setup dev environment 40 | 41 | Activate pyenv for the current shell. 42 | ```sh 43 | pyenv shell 3.7.4 44 | ``` 45 | 46 | This creates a virtualenv and installs all dependencies including dev. 47 | ```sh 48 | poetry install 49 | ``` 50 | 51 | Now test that everything works. Poetry run runs a command in the virtualenv. 52 | ```sh 53 | poetry run pytest 54 | ``` 55 | 56 | Initialize pre-commit hooks for git. 57 | ```sh 58 | poetry run pre-commit install 59 | ``` 60 | -------------------------------------------------------------------------------- /kaiba/models/branching_object.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pydantic import ConfigDict 4 | from pydantic.types import StrictBool 5 | 6 | from kaiba.models.attribute import Attribute 7 | from kaiba.models.base import KaibaBaseModel 8 | from kaiba.models.iterator import Iterator 9 | 10 | 11 | class BranchingObject(KaibaBaseModel): 12 | """Lets you branch out attribute mappings. 13 | 14 | The branching object is a special kind of object where you can have 15 | different attributes per instance. 16 | 17 | ie: 18 | { 19 | "extra_fields": [ 20 | { 21 | "name": "field1", 22 | "value": "value at path x", 23 | }, 24 | { 25 | "name": "field2", 26 | "value": "value at path y" 27 | }, 28 | { 29 | "other_key": "whatever you want", 30 | } 31 | ] 32 | } 33 | 34 | Whereas normal objects you'd have to have both `name`, `value` and 35 | `other_key` in all instances. 36 | """ 37 | 38 | name: str 39 | array: StrictBool = False 40 | iterators: List[Iterator] = [] 41 | branching_attributes: List[List[Attribute]] = [] 42 | model_config = ConfigDict(json_schema_extra={ 43 | 'examples': [ 44 | { 45 | 'name': 'extra_fields', 46 | 'array': True, 47 | 'branching_attributes': [ 48 | [ 49 | { 50 | 'name': 'field_name', 51 | 'default': 'amount', 52 | }, 53 | { 54 | 'name': 'field_data', 55 | 'data_fetchers': [ 56 | { 57 | 'path': ['path', 'to', 'amount'], 58 | }, 59 | ], 60 | }, 61 | ], 62 | [ 63 | { 64 | 'name': 'field_name', 65 | 'default': 'currency', 66 | }, 67 | { 68 | 'name': 'field_data', 69 | 'data_fetchers': [ 70 | { 71 | 'path': ['path', 'to', 'currency'], 72 | }, 73 | ], 74 | 'default': 'NOK', 75 | }, 76 | ], 77 | ], 78 | }, 79 | ], 80 | }) 81 | -------------------------------------------------------------------------------- /kaiba/iso.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from pycountry import countries, currencies, languages 4 | from returns.result import safe 5 | 6 | from kaiba.constants import ALPHA_THREE, ALPHA_TWO, NAME, NUMERIC, OFFICIAL_NAME 7 | 8 | 9 | @safe 10 | def get_country_code(code: str | int) -> dict: 11 | """Find country by code or name.""" 12 | code = str(code).strip() 13 | country = countries.get(alpha_2=code.upper()) 14 | country = country or countries.get(alpha_3=code.upper()) 15 | country = country or countries.get(numeric=code.zfill(3)) 16 | country = country or countries.get(name=code) 17 | country = country or countries.get(official_name=code) 18 | 19 | if not country: 20 | raise ValueError( 21 | '{message}({country})'.format( 22 | message='Could not find country matching value', 23 | country=code, 24 | ), 25 | ) 26 | 27 | return { 28 | ALPHA_TWO: country.alpha_2.upper(), 29 | ALPHA_THREE: country.alpha_3.upper(), 30 | NAME: country.name, 31 | NUMERIC: country.numeric, 32 | OFFICIAL_NAME: _get_official_name(country).value_or(None), 33 | } 34 | 35 | 36 | @safe 37 | def _get_official_name(country) -> str: 38 | return country.official_name 39 | 40 | 41 | @safe 42 | def get_currency_code(code: str | int) -> dict: 43 | """Try to create a Currency object.""" 44 | code = str(code).strip() 45 | 46 | currency = currencies.get(alpha_3=code.upper()) 47 | currency = currency or currencies.get(numeric=code.zfill(3)) 48 | currency = currency or currencies.get(name=code) 49 | 50 | if not currency: 51 | raise ValueError( 52 | '{message}({code})'.format( 53 | message='Could not find currency matching code', 54 | code=code, 55 | ), 56 | ) 57 | 58 | return { 59 | ALPHA_THREE: currency.alpha_3.upper(), 60 | NUMERIC: currency.numeric, 61 | NAME: currency.name, 62 | } 63 | 64 | 65 | @safe 66 | def get_language_code(code: str) -> dict: 67 | """Try to create a Language object.""" 68 | code = str(code).strip() 69 | lan = languages.get(alpha_2=code.lower()) 70 | lan = lan or languages.get(alpha_3=code.lower()) 71 | lan = lan or languages.get(name=code) 72 | 73 | if not lan: 74 | raise ValueError( 75 | '{message}({code})'.format( 76 | message='Could not find language matching value', 77 | code=code, 78 | ), 79 | ) 80 | return { 81 | ALPHA_TWO: lan.alpha_2.upper(), 82 | ALPHA_THREE: lan.alpha_3.upper(), 83 | NAME: lan.name, 84 | 'scope': lan.scope, 85 | 'type': lan.type, 86 | } 87 | -------------------------------------------------------------------------------- /tests/casting/test_cast_to_decimal.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | 3 | from returns.pipeline import is_successful 4 | from typing_extensions import Final 5 | 6 | from kaiba.casting._cast_to_decimal import cast_to_decimal 7 | 8 | target: Final[Decimal] = Decimal('1234567.89') 9 | 10 | 11 | def test_string_with_one_period(): 12 | """Test normal decimal value with one period.""" 13 | assert cast_to_decimal( 14 | '1234567.89', 15 | ).unwrap() == target 16 | 17 | 18 | def test_string_with_one_comma(): 19 | """Test normal decimal value with one comma.""" 20 | assert cast_to_decimal( 21 | '1234567,89', 22 | ).unwrap() == target 23 | 24 | 25 | def test_string_with_period_and_space(): 26 | """Test space separated decimal number with 1 period.""" 27 | assert cast_to_decimal( 28 | '1 234 567.89', 29 | ).unwrap() == target 30 | 31 | 32 | def test_string_with_commas_and_period(): 33 | """Test comma as thousands separator with period as decimal.""" 34 | assert cast_to_decimal( 35 | '1,234,567.89', 36 | ).unwrap() == target 37 | 38 | 39 | def test_string_with_periods_and_comma(): 40 | """Test period as thousands separator with comma as decimal.""" 41 | assert cast_to_decimal( 42 | '1.234.567,89', 43 | ).unwrap() == target 44 | 45 | 46 | def test_string_with_no_period_nor_comma(): 47 | """Test an integer number will nicely become a decimal.""" 48 | test = cast_to_decimal('123456789') 49 | assert is_successful(test) 50 | assert test.unwrap() == Decimal('123456789') 51 | 52 | 53 | def test_with_integer_containing_decimals_format(): 54 | """Integer_containing_decimals format, should be divided by 100.""" 55 | test = cast_to_decimal( 56 | '123456789', 57 | 'integer_containing_decimals', 58 | ) 59 | assert test.unwrap() == target 60 | 61 | 62 | def test_abc_fails(): 63 | """Test that string with letters in fails.""" 64 | test = cast_to_decimal('abc') 65 | assert not is_successful(test) 66 | assert isinstance(test.failure(), ValueError) 67 | assert 'Illegal characters in value' in str(test.failure()) 68 | 69 | 70 | def test_abc_with_decimal_argument_fails(): 71 | """Test that string with letters in fails when we supply 'decimal'.""" 72 | test = cast_to_decimal( 73 | 'abc', 'integer_containing_decimals', 74 | ) 75 | assert not is_successful(test) 76 | assert isinstance(test.failure(), ValueError) 77 | assert 'Illegal characters in value' in str(test.failure()) 78 | 79 | 80 | def test_precision_is_maintained(): 81 | """Test high precision decimal value with one period.""" 82 | assert cast_to_decimal( 83 | '1234567.89123456789', 84 | ).unwrap() == Decimal('1234567.89123456789') 85 | -------------------------------------------------------------------------------- /tests/json/invalid.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "schema", 3 | "array": true, 4 | "iterators": [ 5 | {"alias": "iter_item", "path": [1, "some_list"]} 6 | ], 7 | "attributes": [ 8 | { 9 | "name": "attr1", 10 | "data_fetchers": [ 11 | { 12 | "path": ["path", "to", "value"], 13 | "default": "default value" 14 | } 15 | ], 16 | "separator": "", 17 | "if_statements": [ 18 | { 19 | "target": "bob", 20 | "then": "arne" 21 | } 22 | ], 23 | "default": "another default" 24 | } 25 | ], 26 | "objects": [ 27 | { 28 | "name": "object1", 29 | "array": false, 30 | "attributes": [ 31 | { 32 | "name": "height", 33 | "deult": "bob" 34 | }, 35 | { 36 | "name": "height", 37 | "data_fetchers": [ 38 | { 39 | "path": ["path"] 40 | } 41 | ], 42 | "casting": { 43 | "to": "date" 44 | } 45 | }, 46 | { 47 | "name": "test-bad-casting-format", 48 | "data_fetchers": [ 49 | { 50 | "path": ["path"] 51 | } 52 | ], 53 | "casting": { 54 | "to": "date", 55 | "original_format": "dd-mm-yyyyy" 56 | } 57 | } 58 | ] 59 | } 60 | ], 61 | "branching_objects": [ 62 | { 63 | "name": "object2", 64 | "array": true, 65 | "branching_attributes": [ 66 | [ 67 | { 68 | "default": "amount" 69 | }, 70 | { 71 | "name": "field_data", 72 | "data_fetchers": [ 73 | { 74 | "path": ["path", "to", "amount"] 75 | } 76 | ] 77 | } 78 | ], 79 | [ 80 | { 81 | "name": "field_name", 82 | "default": "currency" 83 | }, 84 | { 85 | "name": "field_data", 86 | "default": "NOK" 87 | } 88 | ] 89 | ] 90 | } 91 | ] 92 | } 93 | -------------------------------------------------------------------------------- /docs/usage.md: -------------------------------------------------------------------------------- 1 | The goal of this library is to make configurable data transformation(mapping) easy and flexible. 2 | 3 | We have decided to only support json to json mapping. This is because quite frankly its impossible to have configurable mapping that works on any format. We chose json because its quite easy to make anything into json and its quite easy to make json into anything. 4 | 5 | 6 | ## Running Kaiba 7 | 8 | There are multiple ways of running Kaiba with the most convenient for testing being with `kaiba-cli`. Then theres `kaiba-web` which is a webserver where you can post configuration and data, and it returns the mapped result. Last but not least, theres running `kaiba` as a package in your python code. This section will shed light on when to use which and add some quickstart examples. 9 | 10 | ### Kaiba CLI 11 | 12 | The Kaiba CLI is the easiest way to get started. All you need is python>=3.6 on your system. You also do not need to write any python code. 13 | 14 | usefull when: 15 | 16 | * Testing if Kaiba could be interesting for you 17 | * Need to trigger mapping with scheduling tools like Cron or Windows Service 18 | * You are using a different programming language, but can execute cmd scripts 19 | 20 | the [introduction](../introduction) uses kaiba-cli, head over there for a quick start 21 | 22 | ### Kaiba WEB 23 | 24 | Kaiba Web is a web api built with the [falcon framework](https://falconframework.org/). We have one-click deploy buttons for `GCP Run` and `Heroku`. It enables you to very easily set up a webserver where you can post configuration and raw data in the body and get mapped data in the body of the response. This also does not require to write any python code. 25 | 26 | Usefull when: 27 | 28 | * You can loadbalance it and deploy multiple instances 29 | * You are on a platform like GCP, AWS, Heroku 30 | * You are already in a microservice/webservice oriented environment 31 | 32 | Look at the [git repo](https://github.com/greenbird/kaiba-web) for deployment guide 33 | 34 | ### Kaiba python package 35 | 36 | If you already are using Python and just want to add Kaiba, you can easily import it into your program and run the code. This makes the most sense when you need to handle the data before or after the Kaiba transformation for example when you need to dump result to xml. 37 | 38 | Usefull when: 39 | 40 | * You need to do extensive handling of input data before mapping 41 | * You need to transform the output json into something else 42 | 43 | Caveats: 44 | Remember that the more you handle the data before or after Kaiba, the more you must document your changes to the data. 45 | 46 | To use Kaiba, import the process function and feed it your data and the configuration. 47 | 48 | ```python 49 | from kaiba.process import process 50 | 51 | your_data = [] 52 | your_config = {} 53 | 54 | result = process(your_data, your_config) 55 | ``` 56 | 57 | Notice that process expects `data: dict` and `configuration: dict` 58 | -------------------------------------------------------------------------------- /tests/functions/test_apply_slicing.py: -------------------------------------------------------------------------------- 1 | from kaiba.functions import apply_slicing 2 | from kaiba.models.slicing import Slicing 3 | 4 | 5 | def test_no_value_is_ok(): 6 | """When value is None we get a Success(None).""" 7 | assert apply_slicing(None, Slicing(**{'from': 0})) is None 8 | 9 | 10 | def test_middle_of_value(): 11 | """Test that we can get a value in middle of string.""" 12 | assert apply_slicing('test', Slicing(**{'from': 1, 'to': 3})) == 'es' 13 | 14 | 15 | def test_middle_to_end(): 16 | """Test that we can slice from middle to end of value.""" 17 | assert apply_slicing('test', Slicing(**{'from': 1})) == 'est' 18 | 19 | 20 | def test_start_to_middle(): 21 | """Test that we can slice from start to middle.""" 22 | assert apply_slicing('test', Slicing(**{'from': 0, 'to': 3})) == 'tes' 23 | 24 | 25 | def test_start_to_end(): 26 | """Test that we can slice from start to end.""" 27 | assert apply_slicing('test', Slicing(**{'from': 0, 'to': None})) == 'test' 28 | 29 | 30 | def test_negative_from(): 31 | """Test that a negative from value starts cutting at the end minus from.""" 32 | assert apply_slicing('012345', Slicing(**{'from': -2})) == '45' 33 | 34 | 35 | def test_negative_to(): 36 | """Test that a negative to value ends cut at end minus to.""" 37 | assert apply_slicing( 38 | '01234', 39 | Slicing(**{'from': 0, 'to': -2}), 40 | ) == '012' 41 | 42 | 43 | def test_int_is_stringified(): 44 | """Test that a non string value will be stringified before slice.""" 45 | assert apply_slicing(123, Slicing(**{'from': 2})) == '3' 46 | 47 | 48 | def test_float_is_stringified(): 49 | """Test that a float value is stringfied.""" 50 | assert apply_slicing(123.123, Slicing(**{'from': -3})) == '123' 51 | 52 | 53 | def test_boolean_is_stringified(): 54 | """Test that a boolean value is stringfied.""" 55 | assert apply_slicing( 56 | False, # noqa: WPS425 57 | Slicing(**{'from': 0, 'to': 1}), 58 | ) == 'F' 59 | 60 | 61 | def test_object_is_stringified(): 62 | """Test that an object is stringified.""" 63 | assert apply_slicing( 64 | {'test': 'bob'}, 65 | Slicing(**{'from': -5, 'to': -2}), 66 | ) == 'bob' 67 | 68 | 69 | def test_list(): 70 | """Test that we can slice a list and that its not cast to string.""" 71 | assert apply_slicing( 72 | [0, 1, 2], 73 | Slicing(**{'from': 1, 'to': None}), 74 | ) == [1, 2] 75 | 76 | 77 | def test_slice_range_longer_than_string(): 78 | """Test that slice range longer than the string length returns string.""" 79 | assert apply_slicing( 80 | '0123', 81 | Slicing(**{'from': 0, 'to': 50}), 82 | ) == '0123' 83 | 84 | 85 | def test_slice_range_on_range_out_of_string(): 86 | """Test that slice range out of the string.""" 87 | assert apply_slicing( 88 | '0123', 89 | Slicing(**{'from': 5, 'to': 10}), 90 | ) == '' 91 | -------------------------------------------------------------------------------- /tests/json/expected_regex.json: -------------------------------------------------------------------------------- 1 | { 2 | "game": { 3 | "event": "Live Chess", 4 | "site": "Chess.com", 5 | "result": "1/2-1/2", 6 | "eco": "A00", 7 | "moves": "1. c3 {[%clk 0:03:00]} 1... e6 {[%clk 0:03:00]} 2. Qb3 {[%clk 0:02:57.1]} 2... Ne7 {[%clk 0:02:58.4]} 3. Nf3 {[%clk 0:02:47.7]} 3... d5 {[%clk 0:02:57.9]} 4. e3 {[%clk 0:02:45.6]} 4... Nd7 {[%clk 0:02:57.3]} 5. Be2 {[%clk 0:02:42.3]} 5... a6 {[%clk 0:02:56.9]} 6. O-O {[%clk 0:02:36.6]} 6... b5 {[%clk 0:02:56.3]} 7. d4 {[%clk 0:02:35.1]} 7... Bb7 {[%clk 0:02:55.7]} 8. Qc2 {[%clk 0:02:28.4]} 8... g6 {[%clk 0:02:55.3]} 9. Nbd2 {[%clk 0:02:25.4]} 9... Bg7 {[%clk 0:02:54.6]} 10. Nb3 {[%clk 0:02:23.6]} 10... c6 {[%clk 0:02:54.2]} 11. a3 {[%clk 0:02:21.3]} 11... a5 {[%clk 0:02:52]} 12. a4 {[%clk 0:02:17.9]} 12... bxa4 {[%clk 0:02:49.7]} 13. Rxa4 {[%clk 0:02:16.2]} 13... Nb6 {[%clk 0:02:45.5]} 14. Rxa5 {[%clk 0:02:12.4]} 14... Rxa5 {[%clk 0:02:43.9]} 15. Nxa5 {[%clk 0:02:11.4]} 15... Nc4 {[%clk 0:02:40.4]} 16. Nxb7 {[%clk 0:02:07.8]} 16... Qb6 {[%clk 0:02:36.9]} 17. Bxc4 {[%clk 0:01:59.4]} 17... dxc4 {[%clk 0:02:34.1]} 18. Nd6+ {[%clk 0:01:58.1]} 18... Kd7 {[%clk 0:02:30.1]} 19. Nxc4 {[%clk 0:01:55.2]} 19... Qa6 {[%clk 0:02:28.6]} 20. b4 {[%clk 0:01:49.9]} 20... Ra8 {[%clk 0:02:24.4]} 21. Qd3 {[%clk 0:01:45]} 21... Nd5 {[%clk 0:02:14.7]} 22. Nce5+ {[%clk 0:01:42.2]} 22... Bxe5 {[%clk 0:02:12.2]} 23. Nxe5+ {[%clk 0:01:37.6]} 23... Ke7 {[%clk 0:02:11.1]} 24. Qxa6 {[%clk 0:01:34.8]} 24... Rxa6 {[%clk 0:02:08.5]} 25. c4 {[%clk 0:01:32.9]} 25... Nxb4 {[%clk 0:02:05]} 26. Bd2 {[%clk 0:01:32]} 26... Nc2 {[%clk 0:01:59.5]} 27. h3 {[%clk 0:01:30.3]} 27... Kf6 {[%clk 0:01:55.7]} 28. Rb1 {[%clk 0:01:28.2]} 28... c5 {[%clk 0:01:43.9]} 29. dxc5 {[%clk 0:01:26.3]} 29... Kxe5 {[%clk 0:01:41.1]} 30. Bc3+ {[%clk 0:01:22.3]} 30... Ke4 {[%clk 0:01:39.6]} 31. Rb7 {[%clk 0:01:16.4]} 31... Kd3 {[%clk 0:01:37.9]} 32. Be5 {[%clk 0:01:11.9]} 32... Kxc4 {[%clk 0:01:35.7]} 33. Rxf7 {[%clk 0:01:07.8]} 33... Kxc5 {[%clk 0:01:34.2]} 34. Rxh7 {[%clk 0:01:06.7]} 34... Kd5 {[%clk 0:01:32.4]} 35. Bg3 {[%clk 0:01:03.4]} 35... Ra1+ {[%clk 0:01:28.5]} 36. Kh2 {[%clk 0:00:59.9]} 36... e5 {[%clk 0:01:24.4]} 37. Rh6 {[%clk 0:00:55.1]} 37... g5 {[%clk 0:01:21.8]} 38. Rh5 {[%clk 0:00:52.4]} 38... Ke4 {[%clk 0:01:17.7]} 39. Rxg5 {[%clk 0:00:51.4]} 39... Nb4 {[%clk 0:01:11.7]} 40. Rxe5+ {[%clk 0:00:48.3]} 40... Kd3 {[%clk 0:01:09.9]} 41. Rb5 {[%clk 0:00:44.8]} 41... Kc4 {[%clk 0:01:06.8]} 42. Rb8 {[%clk 0:00:39.9]} 42... Nd3 {[%clk 0:01:02.3]} 43. e4 {[%clk 0:00:35.4]} 43... Kd4 {[%clk 0:00:58.8]} 44. e5 {[%clk 0:00:33.6]} 44... Nxf2 {[%clk 0:00:53]} 45. Bxf2+ {[%clk 0:00:31]} 45... Kxe5 {[%clk 0:00:52.1]} 46. Rd8 {[%clk 0:00:23.4]} 46... Kf5 {[%clk 0:00:51.2]} 47. g4+ {[%clk 0:00:22.2]} 47... Kf4 {[%clk 0:00:49.1]} 48. h4 {[%clk 0:00:20.5]} 48... Kxg4 {[%clk 0:00:47.5]} 49. Rd4+ {[%clk 0:00:19.4]} 49... Kf3 {[%clk 0:00:44.7]} 50. h5 {[%clk 0:00:17.5]} 50... Ra6 {[%clk 0:00:41.5]} 51. h6 {[%clk 0:00:15.4]} 51... Rxh6+ {[%clk 0:00:37.6]} 52. Rh4 {[%clk 0:00:13.9]} 52... Rxh4+ {[%clk 0:00:34.9]} 53. Bxh4 {[%clk 0:00:12.8]} 1/2-1/2" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /tests/json/config_regex.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "root", 3 | "array": false, 4 | "objects": [ 5 | { 6 | "name": "game", 7 | "array": false, 8 | "attributes": [ 9 | { 10 | "name": "event", 11 | "data_fetchers": [ 12 | { 13 | "path": ["pgn"], 14 | "regex": { 15 | "expression": "Event \\\"[\\w\\d ]+\\\"" 16 | }, 17 | "slicing": { 18 | "from": 7, 19 | "to": -1 20 | } 21 | } 22 | ] 23 | }, 24 | { 25 | "name": "site", 26 | "data_fetchers": [ 27 | { 28 | "path": ["pgn"], 29 | "regex": { 30 | "expression": "Site \\\"[\\w\\d. ]+\\\"" 31 | }, 32 | "slicing": { 33 | "from": 6, 34 | "to": -1 35 | } 36 | } 37 | ] 38 | }, 39 | { 40 | "name": "result", 41 | "data_fetchers": [ 42 | { 43 | "path": ["pgn"], 44 | "regex": { 45 | "expression": "Result \\\"[\\w\\d\/ -]+\\\"" 46 | }, 47 | "slicing": { 48 | "from": 8, 49 | "to": -1 50 | } 51 | } 52 | ] 53 | }, 54 | { 55 | "name": "eco", 56 | "data_fetchers": [ 57 | { 58 | "path": ["pgn"], 59 | "regex": { 60 | "expression": "ECO \\\"[\\w\\d ]+\\\"" 61 | }, 62 | "slicing": { 63 | "from": 5, 64 | "to": -1 65 | } 66 | } 67 | ] 68 | }, 69 | { 70 | "name": "moves", 71 | "data_fetchers": [ 72 | { 73 | "path": ["pgn"], 74 | "regex": { 75 | "expression": "\\s1\\..*" 76 | }, 77 | "slicing": { 78 | "from": 1 79 | } 80 | } 81 | ] 82 | } 83 | ] 84 | } 85 | ] 86 | } 87 | -------------------------------------------------------------------------------- /kaiba/collection_handlers.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Union 2 | 3 | from returns.result import Failure, ResultE, Success, safe 4 | 5 | from kaiba.models.base import AnyType 6 | from kaiba.models.iterator import Iterator 7 | 8 | 9 | @safe 10 | def set_value_in_dict( 11 | new_value: AnyType, 12 | collection: Dict[str, Any], 13 | path: List[str], 14 | ) -> None: 15 | """Set value in a dict(pass by ref) by path.""" 16 | if not path: 17 | raise ValueError('path list empty') 18 | 19 | for key in path[:-1]: 20 | # this will return a Failure[KeyError] if not found 21 | collection = collection[key] 22 | 23 | collection[path[-1]] = new_value 24 | 25 | 26 | @safe 27 | def fetch_data_by_keys( 28 | collection: Union[Dict[str, AnyType], List[AnyType]], 29 | path: List[Union[str, int]], 30 | ) -> AnyType: 31 | """Find data in collection by following a list of path.""" 32 | if not path: 33 | raise ValueError('path list empty') 34 | 35 | for key in path: 36 | # this will return a Failure[KeyError] if not found 37 | collection = collection[key] # type: ignore 38 | 39 | return collection 40 | 41 | 42 | @safe 43 | def fetch_list_by_keys( 44 | collection: Dict[str, AnyType], 45 | path: List[Union[str, int]], 46 | ) -> list: 47 | """Find data that *must* be a list else it fails. 48 | 49 | Example 50 | >>> fetch_list_by_keys( 51 | ... {'object': {'some_list': ['1']}}, ['object', 'some_list'], 52 | ... ).unwrap() 53 | ['1'] 54 | """ 55 | if not path: 56 | raise ValueError('path list empty') 57 | 58 | for key in path: 59 | # this will return a Failure[KeyError] if not found 60 | collection = collection[key] # type: ignore 61 | 62 | if isinstance(collection, list): # type: ignore 63 | return collection # type: ignore 64 | 65 | raise ValueError('Non list data found: ', str(collection)) 66 | 67 | 68 | def iterable_data_handler( 69 | raw_data: dict, 70 | iterators: List[Iterator], 71 | ) -> ResultE[list]: 72 | """Iterate and create all combinations from list of iterators.""" 73 | if not iterators: 74 | return Failure(ValueError('No iterators')) 75 | 76 | iterable, rest = iterators[0], iterators[1:] 77 | 78 | if not rest: 79 | return create_iterable(raw_data, iterable) 80 | 81 | my_list: list = [] 82 | 83 | for iterable_list in create_iterable(raw_data, iterable).unwrap(): 84 | 85 | iterable_data_handler(iterable_list, rest).map( 86 | my_list.extend, 87 | ) 88 | 89 | return Success(my_list) 90 | 91 | 92 | def create_iterable( 93 | input_data: dict, 94 | iterable: Iterator, 95 | ) -> ResultE[list]: 96 | """Return set of set of data per entry in list at iterable[path].""" 97 | return fetch_list_by_keys( 98 | input_data, 99 | iterable.path, 100 | ).map( 101 | lambda collections: [ 102 | { 103 | **input_data, 104 | **{iterable.alias: collection}, 105 | } 106 | for collection in collections 107 | ], 108 | ).lash( 109 | lambda _: Success([input_data]), 110 | ) 111 | -------------------------------------------------------------------------------- /kaiba/handlers.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Union 2 | 3 | from returns.curry import partial 4 | from returns.pipeline import flow, is_successful 5 | from returns.pointfree import bind, lash, map_ 6 | from returns.result import ResultE, Success, safe 7 | 8 | from kaiba.collection_handlers import fetch_data_by_keys 9 | from kaiba.functions import ( 10 | apply_casting, 11 | apply_default, 12 | apply_if_statements, 13 | apply_regex, 14 | apply_separator, 15 | apply_slicing, 16 | ) 17 | from kaiba.models.attribute import Attribute 18 | from kaiba.models.base import AnyType 19 | from kaiba.models.data_fetcher import DataFetcher 20 | 21 | 22 | def handle_data_fetcher( 23 | collection: Union[Dict[str, Any], List[Any]], 24 | cfg: DataFetcher, 25 | ) -> ResultE[AnyType]: 26 | """Find a data at path or produce a value. 27 | 28 | return value can be: 29 | - value found at path 30 | - value found but sliced 31 | - value found applied to regular expression 32 | - conditional value depending on if statements 33 | - default value if all the above still produces `None` 34 | 35 | Flow description: 36 | 37 | find data from path or None -> 38 | apply regular expression -> 39 | apply slicing -> 40 | apply if statements -> 41 | return default value if Failure else mapped value 42 | """ 43 | return flow( 44 | collection, 45 | partial(fetch_data_by_keys, path=cfg.path), 46 | lash(lambda _: Success(None)), # type: ignore 47 | bind( 48 | partial( 49 | apply_regex, regex=cfg.regex, 50 | ), 51 | ), 52 | lash(lambda _: Success(None)), # type: ignore 53 | map_(partial( 54 | apply_slicing, slicing=cfg.slicing, 55 | )), 56 | bind(partial( 57 | apply_if_statements, statements=cfg.if_statements, 58 | )), 59 | lash( # type: ignore 60 | lambda _: apply_default(cfg.default), 61 | ), 62 | ) 63 | 64 | 65 | def handle_attribute( 66 | collection: Union[Dict[str, Any], List[Any]], 67 | cfg: Attribute, 68 | ) -> ResultE[AnyType]: 69 | """Handle one attribute with data fetchers, ifs, casting and default value. 70 | 71 | flow description: 72 | 73 | Fetch once for all data in Attribute.data_fetchers -> 74 | Apply separator to values if there are more than 1 75 | Failure -> fix to Success(None) 76 | Apply if statements 77 | Success -> Cast Value 78 | Failure -> apply default value 79 | 80 | Return Result 81 | """ 82 | fetched_values = [ 83 | fetched.unwrap() 84 | for fetched in # noqa: WPS361 85 | [ 86 | handle_data_fetcher(collection, data_fetcher) 87 | for data_fetcher in cfg.data_fetchers 88 | ] 89 | if is_successful(fetched) 90 | ] 91 | 92 | # partially declare if statement and casting functions 93 | ifs = partial(apply_if_statements, statements=cfg.if_statements) 94 | 95 | cast = safe(lambda the_value: the_value) 96 | if cfg.casting: 97 | cast = partial(apply_casting, casting=cfg.casting) # type: ignore 98 | 99 | return flow( 100 | apply_separator(fetched_values, separator=cfg.separator), 101 | lash(lambda _: Success(None)), # type: ignore 102 | bind(ifs), 103 | bind(cast), 104 | lash( # type: ignore 105 | lambda _: apply_default(default=cfg.default), 106 | ), 107 | ) 108 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Version history 2 | 3 | | Change | Bumps | 4 | | - | - | 5 | | Breaking | major | 6 | | New Feature | minor | 7 | | otherwise | patch | 8 | 9 | 10 | ## Latest Changes 11 | 12 | ## Version 3.0.1 downstream mypy type support. 13 | 14 | ### Upgrades 15 | 16 | * Adds a py.typed marker so that when using this package, mypy recognizes it as a typed package. 17 | 18 | ## Version 3.0.0 @dataclass from attr bug and no more callable objects. 19 | 20 | We were using `@dataclass` decorator from attr, which caused kaiba not to run when used in an environment that did not have attr installed. Thanks to @ChameleonTartu for finding and reporting the bug. 21 | 22 | Non-private code has been changed which is why this is a major version. In the future we will make sure to have clearer line between what is private internal kaiba code and what is the interface that users will use. 23 | 24 | ### Internal 25 | 26 | * We are now callable object free. From now on, we will only use functions. 27 | 28 | 29 | ## Version 2.0.0 Upgrade dependencies, support python 3.10, 3.11 30 | 31 | This major version bump was needed because we upgrade `pydantic` to version 2 from version 1. This changed a lot of error messages in schema validation. If anyone was depending on those, it would break their code. In the same update we are updating a lot of other dependencies too. Most importantly going from `returns` 0.14 to 0.22. This might mean we can make the functional code look better in the future. We are also now support python 3.10 and 3.11 and test them like we do 3.8 and 3.9. 32 | 33 | ### Breaking 34 | 35 | * Upgrades to `pydantic` version 2. This is a breaking change because it changes the output of validation errors. 36 | 37 | ### Upgrades 38 | 39 | * Bumps `returns` to `0.22` 40 | 41 | ### Internal 42 | 43 | * Update `mypy` version 44 | * CI - Github workflow tests now targets `main` branch instead of `master` 45 | 46 | 47 | ## Version 1.0.0 Release: Kaiba 48 | 49 | Kaiba is a data transformation tool written in Python that uses a DTL(Data Transformation Language) expressed in normal JSON to govern output structure, data fetching and data transformation. 50 | 51 | ### Features 52 | 53 | * Mapping by configuration File. 54 | * Looping/Iterating data from multiple places to create 1 or many objects 55 | * Combine multiple values to one. 56 | * Default values 57 | * If statements 58 | * conditions: is, not, in, contains 59 | * can match any valid json value including objects and lists 60 | * Casting 61 | * integer, decimal, iso date 62 | * Regular Expressions 63 | * get whole regex result 64 | * choose matching groups 65 | * Slicing 66 | * Slice/Substring strings or arrays 67 | 68 | ### Changelog 69 | 70 | * Restructures pydantic models 71 | * Rename Mapping->DataFetcher 72 | * Rename Attribute.mappings->Attribute.data_fetchers 73 | * Rename Regexp->Regex 74 | * Rename Regex.search to Regex.expression 75 | * Rename Iterable->Iterator 76 | * Rename iterables->iterators 77 | * Simplify typing 78 | 79 | 80 | ## Version 0.3.0 - Migrate to pydantic 81 | 82 | This version changes how we validate our json configuration and also how we parse it. Using pydantic it is much easier to handle to handle typing and the code in general. 83 | 84 | * Removes json schema adds pydantic for config validation 85 | 86 | ## Version 0.2.1 - Schema troubles 87 | 88 | Kaiba is forked from the greenbird/piri @ version 2.2.0 89 | 90 | Fixes problems with Schema validation 91 | 92 | * In attribute make sure either name + mappings or name + default is required 93 | * In mappings make sure that length of path is above 1 if default is not provided. 94 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | format = wemake 3 | show-source = True 4 | doctests = True 5 | enable-extensions = G 6 | statistics = False 7 | 8 | # Plugins: 9 | accept-encodings = utf-8 10 | max-complexity = 6 11 | max-line-length = 80 12 | radon-max-cc = 10 13 | radon-no-assert = True 14 | radon-show-closures = True 15 | 16 | # wemake-python-styleguide 17 | max-methods = 8 18 | ignore = 19 | # Skip docstrings for now. 20 | RST 21 | # Skip docstrings for now. 22 | DAR 23 | # empty __init__ files ok. 24 | D104 25 | # Don't use docstrings in modules. 26 | D100 27 | # WrongLoopIterTypeViolation, fix this later 28 | WPS335 29 | 30 | exclude = 31 | # Trash and cache: 32 | .git 33 | __pycache__ 34 | .venv 35 | .eggs 36 | *.egg 37 | *.md 38 | docs/*.py 39 | 40 | per-file-ignores = 41 | # in Tests: 42 | # S101: allow asserts 43 | # WPS202: Allow more module members 44 | # WPS204: OverusedExpressionViolation 45 | # WPS226: OverusedStringViolation 46 | # WPS432: magic numbers are okay in tests 47 | # WPS114: Underscore name pattern is okay in tests 48 | # WPS442: Allow name overshaddowing in conftest fixtures 49 | # WPS517: Allow unpacking **{} for pydantic models in tests 50 | tests/*.py: S101, WPS202, WPS204, WPS226, WPS432, WPS114, WPS442, WPS517, WPS436 51 | 52 | # in casting tests: 53 | # WPS436: Allow protected module import 54 | # tests/casting/*.py: 55 | 56 | # In casting module: 57 | # WPS412: Allow `__init__.py` module with logic 58 | # WPS436: Allow protected module imports 59 | kaiba/casting/__init__.py: WPS412, WPS436 60 | 61 | # In pydantic schema file until we split it up 62 | # WPS202: Allow too many module memebers 63 | # WPS402: Allow too many noqa's 64 | kaiba/pydantic_schema.py: WPS202 WPS402 65 | 66 | # WPS431: Allow nested class in pydantic schema 67 | # WPS306: Allow not inheriting class 68 | # WPS226: Allow string constant over-use 69 | kaiba/models/*.py: WPS431, WPS306, WPS226 70 | 71 | # In functions.py ignore too many imports 72 | # WPS201: Found module with too many imports 73 | kaiba/functions.py: WPS201 74 | 75 | [isort] 76 | # See https://github.com/timothycrosley/isort#multi-line-output-modes 77 | multi_line_output = 3 78 | include_trailing_comma = true 79 | default_section = FIRSTPARTY 80 | # Should be: 80 - 1 81 | line_length = 79 82 | 83 | [tool:pytest] 84 | # py.test options: 85 | norecursedirs = *.egg .eggs dist build docs .tox .git __pycache__ 86 | 87 | addopts = 88 | --doctest-modules 89 | --cov=kaiba 90 | --cov-report=term 91 | --cov-report=html 92 | --cov-report=xml 93 | --cov-branch 94 | --cov-fail-under=100 95 | # --mypy-ini-file=setup.cfg 96 | 97 | 98 | [mypy] 99 | # mypy configurations: http://bit.ly/2zEl9WI 100 | 101 | # Plugins, includes custom:, add this plugin some time 102 | # plugins = 103 | # returns.contrib.mypy.decorator_plugin 104 | plugins = 105 | returns.contrib.mypy.returns_plugin, pydantic.mypy 106 | 107 | # We have disabled this checks due to some problems with `mypy` type 108 | # system, it does not look like it will be fixed soon. 109 | # disallow_any_explicit = True 110 | # disallow_any_generics = True 111 | 112 | allow_redefinition = false 113 | check_untyped_defs = true 114 | ignore_errors = false 115 | ignore_missing_imports = true 116 | implicit_reexport = false 117 | local_partial_types = true 118 | no_implicit_optional = true 119 | strict_equality = true 120 | strict_optional = true 121 | warn_no_return = true 122 | warn_redundant_casts = true 123 | warn_unreachable = true 124 | warn_unused_configs = true 125 | warn_unused_ignores = true 126 | -------------------------------------------------------------------------------- /tests/handlers/test_handle_attribute.py: -------------------------------------------------------------------------------- 1 | import decimal 2 | 3 | from kaiba.handlers import handle_attribute 4 | from kaiba.models.attribute import Attribute 5 | 6 | 7 | def test_get_key_in_dict(): 8 | """Test that we can fetch key in dict.""" 9 | input_data = {'key': 'val1'} 10 | config = Attribute(**{ 11 | 'name': 'attrib', 12 | 'data_fetchers': [ 13 | {'path': ['key']}, 14 | ], 15 | }) 16 | 17 | assert handle_attribute( 18 | input_data, 19 | config, 20 | ).unwrap() == 'val1' 21 | 22 | 23 | def test_casting_to_decimal(): 24 | """Test that we can cast a string value to decimal.""" 25 | input_data = {'key': '1,123,123.12'} 26 | config = Attribute(**{ 27 | 'name': 'attrib', 28 | 'data_fetchers': [ 29 | {'path': ['key']}, 30 | ], 31 | 'casting': {'to': 'decimal'}, 32 | }) 33 | 34 | assert handle_attribute( 35 | input_data, 36 | config, 37 | ).unwrap() == decimal.Decimal('1123123.12') 38 | 39 | 40 | def test_regex_is_applied_to_attribute(): 41 | """Test that we can expression by pattern.""" 42 | input_data: dict = {'game': '1. e4 e5 ... 14. Rxe8+ Rxe8'} 43 | config = Attribute(**{ 44 | 'name': 'moves', 45 | 'data_fetchers': [ 46 | { 47 | 'path': ['game'], 48 | 'regex': { 49 | 'expression': '(Rxe8)', 50 | 'group': 1, 51 | }, 52 | }, 53 | ], 54 | }) 55 | assert handle_attribute( 56 | input_data, 57 | config, 58 | ).unwrap() == 'Rxe8' 59 | 60 | 61 | def test_regex_is_not_applied_to_attribute(): 62 | """Test that we don't lose data when expression by pattern fails.""" 63 | input_data: dict = {'game': '1. d4 d5'} 64 | config = Attribute(**{ 65 | 'name': 'moves', 66 | 'data_fetchers': [ 67 | { 68 | 'path': ['game'], 69 | 'regex': { 70 | 'expression': '(d6)', 71 | 'group': 0, 72 | }, 73 | }, 74 | ], 75 | }) 76 | regex = handle_attribute(input_data, config) 77 | assert isinstance(regex.failure(), ValueError) is True 78 | assert regex.failure().args == ('Default value should not be `None`',) 79 | 80 | 81 | def test_all(): 82 | """Test a full attribute schema.""" 83 | input_data = {'key': 'val1', 'key2': 'val2'} 84 | config = Attribute(**{ 85 | 'name': 'attrib', 86 | 'data_fetchers': [ 87 | { 88 | 'path': ['key'], 89 | 'if_statements': [ 90 | { 91 | 'condition': 'is', 92 | 'target': 'val1', 93 | 'then': None, 94 | }, 95 | ], 96 | 'default': 'default', 97 | }, 98 | { 99 | 'path': ['key2'], 100 | 'if_statements': [ 101 | { 102 | 'condition': 'is', 103 | 'target': 'val2', 104 | 'then': 'if', 105 | }, 106 | ], 107 | }, 108 | ], 109 | 'separator': '-', 110 | 'if_statements': [ 111 | { 112 | 'condition': 'is', 113 | 'target': 'default-if', 114 | 'then': None, 115 | }, 116 | ], 117 | 'default': 'default2', 118 | }) 119 | 120 | assert handle_attribute( 121 | input_data, 122 | config, 123 | ).unwrap() == 'default2' 124 | -------------------------------------------------------------------------------- /tests/handlers/test_handle_data_fetcher.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | from returns.result import Success 3 | 4 | from kaiba.handlers import handle_data_fetcher 5 | from kaiba.models.data_fetcher import DataFetcher 6 | 7 | 8 | def test_get_string_value_from_key(): 9 | """Test that we can find value.""" 10 | input_data = {'key': 'val1'} 11 | config = DataFetcher(**{'path': ['key']}) 12 | 13 | assert handle_data_fetcher( 14 | input_data, 15 | config, 16 | ) == Success('val1') 17 | 18 | 19 | def test_get_array_value_from_key(): 20 | """Test that we can find an array.""" 21 | input_data = {'key': ['array']} 22 | config = DataFetcher(**{'path': ['key']}) 23 | 24 | assert handle_data_fetcher( 25 | input_data, 26 | config, 27 | ) == Success(['array']) 28 | 29 | 30 | def test_get_object_value_from_key(): 31 | """Test that we can find an object.""" 32 | input_data = {'key': {'obj': 'val1'}} 33 | config = DataFetcher(**{'path': ['key']}) 34 | 35 | assert handle_data_fetcher( 36 | input_data, 37 | config, 38 | ) == Success({'obj': 'val1'}) 39 | 40 | 41 | def test_default_value_is_used(): 42 | """Test that we get a default value when no path and no ifs.""" 43 | input_data = {'key': 'val'} 44 | config = DataFetcher(**{'default': 'default'}) 45 | 46 | assert handle_data_fetcher( 47 | input_data, config, 48 | ).unwrap() == 'default' 49 | 50 | 51 | def test_regex_is_applied(): 52 | """Test that we can search by pattern.""" 53 | input_data: dict = {'game': '8. d4 Re8 ... 14. Rxe8+ Rxe8 15. h3'} # noqa: E501 54 | config = DataFetcher(**{ 55 | 'path': ['game'], 56 | 'regex': { 57 | 'expression': '(Rxe8.*)', 58 | }, 59 | }) 60 | assert handle_data_fetcher( 61 | input_data, 62 | config, 63 | ).unwrap() == 'Rxe8+ Rxe8 15. h3' 64 | 65 | 66 | def test_regex_is_applied_on_group_as_list(): 67 | """Test that we can search by pattern when it is a list.""" 68 | input_data: dict = {'game': '1. e4 e5 6. Qe2+ Qe7 7. Qxe7+ Kxe7 8. d4 Re8'} 69 | config = DataFetcher(**{ 70 | 'path': ['game'], 71 | 'regex': { 72 | 'expression': r'(e\d)+', 73 | 'group': [0, 1, 6], 74 | }, 75 | }) 76 | assert handle_data_fetcher( 77 | input_data, 78 | config, 79 | ).unwrap() == ['e4', 'e5', 'e8'] 80 | 81 | 82 | def test_slicing_is_applied(): 83 | """Test that applying slicing works.""" 84 | input_data = {'key': 'value'} 85 | config = DataFetcher(**{ 86 | 'path': ['key'], 87 | 'slicing': { 88 | 'from': 2, 89 | 'to': 3, 90 | }, 91 | }) 92 | assert handle_data_fetcher( 93 | input_data, config, 94 | ).unwrap() == 'l' 95 | 96 | 97 | def test_if_statements_are_applied(): 98 | """Test that applying if statements works.""" 99 | input_data = {'key': 'val'} 100 | config = DataFetcher(**{ 101 | 'if_statements': [{ 102 | 'condition': 'is', 103 | 'target': None, 104 | 'then': 'otherval', 105 | }], 106 | 'default': 'bob', 107 | }) 108 | assert handle_data_fetcher( 109 | input_data, config, 110 | ).unwrap() == 'otherval' 111 | 112 | 113 | def test_default_value_not_none(): 114 | """Test that providing bad data returns Failure instance.""" 115 | failure = handle_data_fetcher( 116 | {'fail': 'failure'}, 117 | DataFetcher(**{'path': [], 'default': None}), 118 | ) 119 | assert not is_successful(failure) 120 | assert 'Default value should not be `None`' in str(failure.failure()) 121 | -------------------------------------------------------------------------------- /tests/json/input_regex.json: -------------------------------------------------------------------------------- 1 | { 2 | "black": { 3 | "@id": "https://api.chess.com/pub/player/chameleoniasa", 4 | "rating": 1273, 5 | "result": "insufficient", 6 | "username": "ChameleonIASA" 7 | }, 8 | "end_time": 1347534562, 9 | "fen": "8/8/8/8/7B/5k2/7K/8 b - -", 10 | "pgn": "[Event \"Live Chess\"]\n[Site \"Chess.com\"]\n[Date \"2012.09.13\"]\n[Round \"-\"]\n[White \"GAURAV480\"]\n[Black \"ChameleonIASA\"]\n[Result \"1/2-1/2\"]\n[ECO \"A00\"]\n[ECOUrl \"https://www.chess.com/openings/Saragossa-Opening\"]\n[CurrentPosition \"8/8/8/8/7B/5k2/7K/8 b - -\"]\n[Timezone \"UTC\"]\n[UTCDate \"2012.09.13\"]\n[UTCTime \"11:03:43\"]\n[WhiteElo \"1283\"]\n[BlackElo \"1273\"]\n[TimeControl \"180\"]\n[Termination \"Game drawn by insufficient material\"]\n[StartTime \"11:03:43\"]\n[EndDate \"2012.09.13\"]\n[EndTime \"11:09:22\"]\n[Link \"https://www.chess.com/live/game/361066365\"]\n\n1. c3 {[%clk 0:03:00]} 1... e6 {[%clk 0:03:00]} 2. Qb3 {[%clk 0:02:57.1]} 2... Ne7 {[%clk 0:02:58.4]} 3. Nf3 {[%clk 0:02:47.7]} 3... d5 {[%clk 0:02:57.9]} 4. e3 {[%clk 0:02:45.6]} 4... Nd7 {[%clk 0:02:57.3]} 5. Be2 {[%clk 0:02:42.3]} 5... a6 {[%clk 0:02:56.9]} 6. O-O {[%clk 0:02:36.6]} 6... b5 {[%clk 0:02:56.3]} 7. d4 {[%clk 0:02:35.1]} 7... Bb7 {[%clk 0:02:55.7]} 8. Qc2 {[%clk 0:02:28.4]} 8... g6 {[%clk 0:02:55.3]} 9. Nbd2 {[%clk 0:02:25.4]} 9... Bg7 {[%clk 0:02:54.6]} 10. Nb3 {[%clk 0:02:23.6]} 10... c6 {[%clk 0:02:54.2]} 11. a3 {[%clk 0:02:21.3]} 11... a5 {[%clk 0:02:52]} 12. a4 {[%clk 0:02:17.9]} 12... bxa4 {[%clk 0:02:49.7]} 13. Rxa4 {[%clk 0:02:16.2]} 13... Nb6 {[%clk 0:02:45.5]} 14. Rxa5 {[%clk 0:02:12.4]} 14... Rxa5 {[%clk 0:02:43.9]} 15. Nxa5 {[%clk 0:02:11.4]} 15... Nc4 {[%clk 0:02:40.4]} 16. Nxb7 {[%clk 0:02:07.8]} 16... Qb6 {[%clk 0:02:36.9]} 17. Bxc4 {[%clk 0:01:59.4]} 17... dxc4 {[%clk 0:02:34.1]} 18. Nd6+ {[%clk 0:01:58.1]} 18... Kd7 {[%clk 0:02:30.1]} 19. Nxc4 {[%clk 0:01:55.2]} 19... Qa6 {[%clk 0:02:28.6]} 20. b4 {[%clk 0:01:49.9]} 20... Ra8 {[%clk 0:02:24.4]} 21. Qd3 {[%clk 0:01:45]} 21... Nd5 {[%clk 0:02:14.7]} 22. Nce5+ {[%clk 0:01:42.2]} 22... Bxe5 {[%clk 0:02:12.2]} 23. Nxe5+ {[%clk 0:01:37.6]} 23... Ke7 {[%clk 0:02:11.1]} 24. Qxa6 {[%clk 0:01:34.8]} 24... Rxa6 {[%clk 0:02:08.5]} 25. c4 {[%clk 0:01:32.9]} 25... Nxb4 {[%clk 0:02:05]} 26. Bd2 {[%clk 0:01:32]} 26... Nc2 {[%clk 0:01:59.5]} 27. h3 {[%clk 0:01:30.3]} 27... Kf6 {[%clk 0:01:55.7]} 28. Rb1 {[%clk 0:01:28.2]} 28... c5 {[%clk 0:01:43.9]} 29. dxc5 {[%clk 0:01:26.3]} 29... Kxe5 {[%clk 0:01:41.1]} 30. Bc3+ {[%clk 0:01:22.3]} 30... Ke4 {[%clk 0:01:39.6]} 31. Rb7 {[%clk 0:01:16.4]} 31... Kd3 {[%clk 0:01:37.9]} 32. Be5 {[%clk 0:01:11.9]} 32... Kxc4 {[%clk 0:01:35.7]} 33. Rxf7 {[%clk 0:01:07.8]} 33... Kxc5 {[%clk 0:01:34.2]} 34. Rxh7 {[%clk 0:01:06.7]} 34... Kd5 {[%clk 0:01:32.4]} 35. Bg3 {[%clk 0:01:03.4]} 35... Ra1+ {[%clk 0:01:28.5]} 36. Kh2 {[%clk 0:00:59.9]} 36... e5 {[%clk 0:01:24.4]} 37. Rh6 {[%clk 0:00:55.1]} 37... g5 {[%clk 0:01:21.8]} 38. Rh5 {[%clk 0:00:52.4]} 38... Ke4 {[%clk 0:01:17.7]} 39. Rxg5 {[%clk 0:00:51.4]} 39... Nb4 {[%clk 0:01:11.7]} 40. Rxe5+ {[%clk 0:00:48.3]} 40... Kd3 {[%clk 0:01:09.9]} 41. Rb5 {[%clk 0:00:44.8]} 41... Kc4 {[%clk 0:01:06.8]} 42. Rb8 {[%clk 0:00:39.9]} 42... Nd3 {[%clk 0:01:02.3]} 43. e4 {[%clk 0:00:35.4]} 43... Kd4 {[%clk 0:00:58.8]} 44. e5 {[%clk 0:00:33.6]} 44... Nxf2 {[%clk 0:00:53]} 45. Bxf2+ {[%clk 0:00:31]} 45... Kxe5 {[%clk 0:00:52.1]} 46. Rd8 {[%clk 0:00:23.4]} 46... Kf5 {[%clk 0:00:51.2]} 47. g4+ {[%clk 0:00:22.2]} 47... Kf4 {[%clk 0:00:49.1]} 48. h4 {[%clk 0:00:20.5]} 48... Kxg4 {[%clk 0:00:47.5]} 49. Rd4+ {[%clk 0:00:19.4]} 49... Kf3 {[%clk 0:00:44.7]} 50. h5 {[%clk 0:00:17.5]} 50... Ra6 {[%clk 0:00:41.5]} 51. h6 {[%clk 0:00:15.4]} 51... Rxh6+ {[%clk 0:00:37.6]} 52. Rh4 {[%clk 0:00:13.9]} 52... Rxh4+ {[%clk 0:00:34.9]} 53. Bxh4 {[%clk 0:00:12.8]} 1/2-1/2", 11 | "rated": true, 12 | "rules": "chess", 13 | "time_class": "blitz", 14 | "time_control": "180", 15 | "url": "https://www.chess.com/live/game/361066365", 16 | "white": { 17 | "@id": "https://api.chess.com/pub/player/gaurav480", 18 | "rating": 1283, 19 | "result": "insufficient", 20 | "username": "GAURAV480" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /kaiba/mapper.py: -------------------------------------------------------------------------------- 1 | import decimal 2 | from typing import List, Optional, Union 3 | 4 | from returns.curry import partial 5 | from returns.maybe import maybe 6 | from returns.pipeline import is_successful 7 | from returns.result import safe 8 | 9 | from kaiba.collection_handlers import iterable_data_handler 10 | from kaiba.handlers import handle_attribute 11 | from kaiba.models.attribute import Attribute 12 | from kaiba.models.branching_object import BranchingObject 13 | from kaiba.models.kaiba_object import KaibaObject 14 | 15 | decimal.getcontext().rounding = decimal.ROUND_HALF_UP 16 | 17 | 18 | @safe 19 | def map_data( 20 | input_data: dict, 21 | configuration: KaibaObject, 22 | ) -> Union[list, dict]: 23 | """Map entrypoint. 24 | 25 | Try to get iterable data 26 | if that fails then just run map_object normally, but make it into an array 27 | if array is true. 28 | 29 | If we had iterable data, iterate that data and run map_object with the 30 | current iteration data added to the root of the input_data dictionary 31 | """ 32 | iterate_data = iterable_data_handler( 33 | input_data, configuration.iterators, 34 | ) 35 | 36 | if not is_successful(iterate_data): 37 | 38 | return map_object( 39 | input_data, 40 | configuration, 41 | ).map( 42 | partial(set_array, array=configuration.array), 43 | ).unwrap() 44 | 45 | mapped_objects: List[dict] = [] 46 | 47 | # find returns function to work with iterators 48 | for iteration in iterate_data.unwrap(): 49 | map_object( 50 | iteration, 51 | configuration, 52 | ).map( 53 | mapped_objects.append, 54 | ) 55 | 56 | return mapped_objects 57 | 58 | 59 | def set_array( 60 | input_data: dict, 61 | array: bool, 62 | ) -> Union[List[dict], dict]: 63 | """Return data wrapped in array if if array=True.""" 64 | if array: 65 | return [input_data] 66 | return input_data 67 | 68 | 69 | @maybe 70 | def map_object( 71 | input_data: dict, 72 | configuration: KaibaObject, 73 | ) -> Optional[dict]: 74 | """Map one object. 75 | 76 | One object has a collections of: 77 | Attribute mappings, 78 | Nested object mappings, 79 | Branching object mappings, 80 | 81 | All functions we call return a dictionary with the mapped values 82 | so all we have to do is to call update on a shared object_data dict. 83 | 84 | return example: 85 | return { 86 | 'attrib': 'val', 87 | 'object1': {'attrib1': 'val'} 88 | 'branching_object1: [{'attrib1': 'val'}] 89 | } 90 | """ 91 | object_data: dict = {} 92 | 93 | map_attributes( 94 | input_data, configuration.attributes, 95 | ).map(object_data.update) 96 | 97 | map_objects( 98 | input_data, configuration.objects, 99 | ).map(object_data.update) 100 | 101 | map_branching_objects( 102 | input_data, configuration.branching_objects, 103 | ).map(object_data.update) 104 | 105 | # need this as long as empty dict is not seen as None by returns.maybe 106 | return object_data or None 107 | 108 | 109 | @maybe 110 | def map_attributes( 111 | input_data: dict, 112 | configuration: List[Attribute], 113 | ) -> Optional[dict]: 114 | """For all attributes map attribute. 115 | 116 | name of attribute should be set 117 | { 118 | 'attribute1': 'value', 119 | 'attribute2': 'value2', 120 | } 121 | """ 122 | attributes: dict = {} 123 | 124 | for attribute_cfg in configuration: 125 | attribute_value = handle_attribute(input_data, attribute_cfg) 126 | 127 | if is_successful(attribute_value): 128 | attributes[attribute_cfg.name] = attribute_value.unwrap() 129 | 130 | return attributes or None 131 | 132 | 133 | @maybe 134 | def map_objects( 135 | input_data: dict, 136 | configuration: List[KaibaObject], 137 | ) -> Optional[dict]: 138 | """For all objects map object. 139 | 140 | name of object should be set. 141 | { 142 | 'name1': object, 143 | 'name2': object2, 144 | } 145 | """ 146 | mapped_objects: dict = {} 147 | 148 | for object_cfg in configuration: 149 | object_value = map_data(input_data, object_cfg) 150 | 151 | if is_successful(object_value): 152 | mapped_objects[object_cfg.name] = object_value.unwrap() 153 | 154 | return mapped_objects or None 155 | 156 | 157 | @maybe 158 | def map_branching_attributes( 159 | input_data: dict, 160 | b_attributes: List[List[Attribute]], 161 | ) -> Optional[List[dict]]: 162 | """Map branching attributes. 163 | 164 | Branching attributes are a list of attribute mappings that will be 165 | mapped to the same name in branching object. 166 | """ 167 | mapped_attributes: List[dict] = [] 168 | 169 | for sub_cfg in b_attributes: 170 | map_attributes( 171 | input_data, sub_cfg, 172 | ).map( 173 | mapped_attributes.append, 174 | ) 175 | 176 | # need this as long as empty dict is not seen as None by returns.maybe 177 | return mapped_attributes or None 178 | 179 | 180 | @maybe 181 | def map_branching_objects( 182 | input_data: dict, 183 | configuration: List[BranchingObject], 184 | ) -> Optional[dict]: 185 | """Map branching object. 186 | 187 | Branching object is a case where we want to create the same object multiple 188 | times, however we want to find the data in different places. 189 | """ 190 | mapped_objects: dict = {} 191 | 192 | for b_object in configuration: 193 | mapped = map_branching_attributes( 194 | input_data, b_object.branching_attributes, 195 | ) 196 | 197 | if is_successful(mapped): 198 | mapped_objects[b_object.name] = mapped.unwrap() 199 | 200 | # need this as long as empty dict is not seen as None by returns.maybe 201 | return mapped_objects or None 202 | -------------------------------------------------------------------------------- /kaiba/casting/_cast_to_date.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import re 3 | 4 | from returns.pipeline import flow 5 | from returns.pointfree import alt, bind, lash 6 | from returns.result import Failure, ResultE, safe 7 | from typing_extensions import Final 8 | 9 | from kaiba.models.base import AnyType 10 | 11 | _yymmdd_pattern: Final = re.compile(r'^yy[^\w]?mm[^\w]?dd$') 12 | _ddmmyy_pattern: Final = re.compile(r'^dd[^\w]?mm[^\w]?yy$') 13 | _mmddyy_pattern: Final = re.compile(r'^mm[^\w]?dd[^\w]?yy$') 14 | 15 | _mmddyyyy_pattern: Final = re.compile(r'^mm[^\w]?dd[^\w]?yyyy$') 16 | _ddmmyyyy_pattern: Final = re.compile(r'^dd[^\w]?mm[^\w]?yyyy$') 17 | _yyyymmdd_pattern: Final = re.compile(r'^yyyy[^\w]?mm[^\w]?dd$') 18 | _iso_pattern: Final = re.compile(r'^\d{4}-\d{2}-\d{2}') 19 | 20 | _error_message: Final = 'Unable to cast ({value}) to ISO date. Exc({failure})' 21 | 22 | 23 | def cast_to_date( 24 | value_to_cast: AnyType, 25 | original_format: str, 26 | ) -> ResultE[str]: 27 | r""" 28 | purpose: Convert string date into ISO format date. 29 | 30 | :input value_to_cast: the string to be converted to a date 31 | :input original_format: a string describing the format the data string 32 | is before convertion. 33 | :raises ValueError: when not able to convert or value_to_cast 34 | does not match original_format. 35 | regex_tips ^ = start of string 36 | $ = end of string 37 | [^[0-9]] = matches anything not [0-9] 38 | [^\d] = matches anything not digits 39 | \d = matches digits 40 | {n} = matches previous match n amount of times 41 | () = grouper, groups up stuff for use in replace. 42 | """ 43 | date_value = str(value_to_cast) 44 | return flow( 45 | date_value, 46 | _value_is_iso, 47 | lash( # type: ignore 48 | lambda _: _cast_with_millennia( 49 | date_value, 50 | original_format=original_format, 51 | ), 52 | ), 53 | lash( # type: ignore 54 | lambda _: _cast_with_no_millennia( 55 | date_value, 56 | original_format=original_format, 57 | ), 58 | ), 59 | bind(_validate_date), 60 | alt( # type: ignore 61 | lambda failure: ValueError( 62 | _error_message.format( 63 | value=date_value, 64 | failure=failure, 65 | ), 66 | ), 67 | ), 68 | ) 69 | 70 | 71 | @safe 72 | def _value_is_iso(value_to_cast: str) -> str: 73 | if _iso_pattern.match(value_to_cast): 74 | return value_to_cast 75 | raise ValueError('Unable to cast to ISO date') 76 | 77 | 78 | def _cast_with_millennia( 79 | value_to_cast: str, 80 | original_format: str, 81 | ) -> ResultE[str]: 82 | # mm[.]dd[.]yyyy any separator 83 | if _mmddyyyy_pattern.match(original_format): 84 | return _apply_regex_sub( 85 | r'(\d{2})[^\w]?(\d{2})[^\w]?(\d{4})', 86 | r'\3-\1-\2', 87 | value_to_cast, 88 | ) 89 | 90 | # dd[.]mm[.]yyyy any separator 91 | if _ddmmyyyy_pattern.match(original_format): 92 | return _apply_regex_sub( 93 | r'(\d{2})[^\w]?(\d{2})[^\w]?(\d{4})', 94 | r'\3-\2-\1', 95 | value_to_cast, 96 | ) 97 | 98 | # yyyy[.]mm[.]dd any separator 99 | if _yyyymmdd_pattern.match(original_format): 100 | return _apply_regex_sub( 101 | r'(\d{4})[^\w]?(\d{2})[^\w]?(\d{2})', 102 | r'\1-\2-\3', 103 | value_to_cast, 104 | ) 105 | 106 | return Failure( 107 | ValueError( 108 | 'Unable to case to milennia format: {value}'.format( 109 | value=value_to_cast, 110 | ), 111 | ), 112 | ) 113 | 114 | 115 | def _cast_with_no_millennia( 116 | value_to_cast: str, 117 | original_format: str, 118 | ) -> ResultE[str]: 119 | no_millenia_patterns = { 120 | _yymmdd_pattern: ( 121 | r'(\d{2})[^\w]?(\d{2})[^\w]?(\d{2})', 122 | r'\3\2\1', 123 | ), 124 | _ddmmyy_pattern: ( 125 | r'(\d{2})[^\w]?(\d{2})[^\w]?(\d{2})', 126 | r'\1\2\3', 127 | ), 128 | _mmddyy_pattern: ( 129 | r'(\d{2})[^\w]?(\d{2})[^\w]?(\d{2})', 130 | r'\2\1\3', 131 | ), 132 | } 133 | for no_millenia_pattern in no_millenia_patterns: # noqa: WPS528 134 | if no_millenia_pattern.match(original_format): 135 | pattern, arrangement = no_millenia_patterns[no_millenia_pattern] 136 | return _apply_regex_sub( 137 | pattern, arrangement, value_to_cast, 138 | ).bind( 139 | _convert_ddmmyy_to_iso_date, 140 | ) 141 | 142 | return Failure( 143 | ValueError( 144 | 'Unable to cast to no millennia format: {value}'.format( 145 | value=value_to_cast, 146 | ), 147 | ), 148 | ) 149 | 150 | 151 | @safe 152 | def _validate_date(date_string: str) -> str: 153 | return datetime.date( 154 | *map( 155 | int, 156 | date_string.split('-'), 157 | ), 158 | ).isoformat() 159 | 160 | 161 | @safe 162 | def _apply_regex_sub( 163 | pattern: str, 164 | arrangement: str, 165 | input_string: str, 166 | ) -> str: 167 | return re.sub(pattern, arrangement, input_string) 168 | 169 | 170 | @safe # noqa: WPS210, Hard to reduce variables 171 | def _convert_ddmmyy_to_iso_date( # noqa: WPS210 172 | date_string: str, 173 | ) -> str: 174 | 175 | in_day = int(date_string[:2]) 176 | in_month = int(date_string[2:4]) 177 | in_year = int(date_string[4:6]) 178 | 179 | today = datetime.date.today() 180 | # think of this 'century' variable as century - 1. as in: 181 | # 2018 = 2000, 1990 = 1900 182 | century = (today.year // 100) * 100 183 | # yy = last two digits of year. 2018 = 18, 1990 = 90 184 | yy = today.year % 100 185 | if in_year > yy: 186 | century -= 100 187 | year = century + in_year 188 | date_obj = datetime.date(year, in_month, in_day) 189 | 190 | return date_obj.isoformat() 191 | -------------------------------------------------------------------------------- /tests/casting/test_cast_to_date.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | from typing_extensions import Final 3 | 4 | from kaiba.casting._cast_to_date import cast_to_date 5 | 6 | target_after_2000: Final['str'] = '2019-09-07' 7 | target_before_2000: Final['str'] = '1994-06-08' 8 | 9 | 10 | def test_string_yyyymmdd_no_delimiter(): 11 | """Test that yyyymmdd pattern is accepted.""" 12 | assert cast_to_date( 13 | '20190907', 14 | 'yyyymmdd', 15 | ).unwrap() == target_after_2000 16 | 17 | 18 | def test_string_ddmmyyyy_no_delimiter(): 19 | """Test that ddmmyyyy pattern is accepted.""" 20 | assert cast_to_date( 21 | '07092019', 22 | 'ddmmyyyy', 23 | ).unwrap() == target_after_2000 24 | 25 | 26 | def test_string_mmddyyyy_no_delimiter(): 27 | """Test that mmddyyyy pattern is accepted.""" 28 | assert cast_to_date( 29 | '09072019', 30 | 'mmddyyyy', 31 | ).unwrap() == target_after_2000 32 | 33 | 34 | def test_string_yyyymmdd_with_hyphen(): 35 | """Test that yyyy-mm-dd pattern is accepted.""" 36 | assert cast_to_date( 37 | '2019-09-07', 38 | 'yyyy-mm-dd', 39 | ).unwrap() == target_after_2000 40 | 41 | 42 | def test_string_yyyymmdd_with_back_slash(): 43 | """Test that yyyy/mm/dd pattern is accepted.""" 44 | assert cast_to_date( 45 | '2019/09/07', 46 | 'yyyy/mm/dd', 47 | ).unwrap() == target_after_2000 48 | 49 | 50 | def test_string_yyyymmdd_with_dots(): 51 | """Test that yyyy.mm.dd pattern is accepted.""" 52 | assert cast_to_date( 53 | '2019.09.07', 54 | 'yyyy.mm.dd', 55 | ).unwrap() == target_after_2000 56 | 57 | 58 | def test_string_ddmmyyyy_with_hyphen(): 59 | """Test that dd-mm-yyyy pattern is accepted.""" 60 | assert cast_to_date( 61 | '07-09-2019', 62 | 'dd-mm-yyyy', 63 | ).unwrap() == target_after_2000 64 | 65 | 66 | def test_string_ddmmmyyyy_with_back_slash(): 67 | """Test that dd/mm/yyyy pattern is accepted.""" 68 | assert cast_to_date( 69 | '07/09/2019', 70 | 'dd/mm/yyyy', 71 | ).unwrap() == target_after_2000 72 | 73 | 74 | def test_string_ddmmmyyyy_with_dots(): 75 | """Test that dd.mm.yyyy pattern is accepted.""" 76 | assert cast_to_date( 77 | '07.09.2019', 78 | 'dd.mm.yyyy', 79 | ).unwrap() == target_after_2000 80 | 81 | 82 | def test_string_mmddyyyy_hyphen(): 83 | """Test that mm-dd-yyyy pattern is accepted.""" 84 | assert cast_to_date( 85 | '09-07-2019', 86 | 'mm-dd-yyyy', 87 | ).unwrap() == target_after_2000 88 | 89 | 90 | def test_string_mmddyyyy_back_slash(): 91 | """Test that mm/dd/yyyy pattern is accepted.""" 92 | assert cast_to_date( 93 | '09/07/2019', 94 | 'mm/dd/yyyy', 95 | ).unwrap() == target_after_2000 96 | 97 | 98 | def test_string_mmddyyyy_dots(): 99 | """Test that mm.dd.yyyy pattern is accepted.""" 100 | assert cast_to_date( 101 | '09.07.2019', 102 | 'mm.dd.yyyy', 103 | ).unwrap() == target_after_2000 104 | 105 | 106 | def test_string_mmddyy_no_delimiter_after_2000(): 107 | """Test that mmddyy pattern is accepted.""" 108 | assert cast_to_date( 109 | '090719', 110 | 'mmddyy', 111 | ).unwrap() == target_after_2000 112 | 113 | 114 | def test_string_mmddyy_no_delimiter_before_2000(): 115 | """Test that mm.dd.yy pattern is accepted.""" 116 | assert cast_to_date( 117 | '060894', 118 | 'mmddyy', 119 | ).unwrap() == target_before_2000 120 | 121 | 122 | def test_string_yymmdd_no_delimiter_after_2000(): 123 | """Test that yymmdd pattern is accepted.""" 124 | assert cast_to_date( 125 | '190907', 126 | 'yymmdd', 127 | ).unwrap() == target_after_2000 128 | 129 | 130 | def test_string_yymmdd_no_delimiter_before_2000(): 131 | """Test that yymmdd pattern is accepted.""" 132 | assert cast_to_date( 133 | '940608', 134 | 'yymmdd', 135 | ).unwrap() == target_before_2000 136 | 137 | 138 | def test_string_ddmmyy_no_delimiter_after_2000(): 139 | """Test that ddmmyy pattern is accepted.""" 140 | assert cast_to_date( 141 | '070919', 142 | 'ddmmyy', 143 | ).unwrap() == target_after_2000 144 | 145 | 146 | def test_string_ddmmyy_no_delimiter_before_2000(): 147 | """Test that ddmmyy pattern is accepted.""" 148 | assert cast_to_date( 149 | '080694', 150 | 'ddmmyy', 151 | ).unwrap() == target_before_2000 152 | 153 | 154 | def test_string_mmddyy_with_dots_after_2000(): 155 | """Test that mm.dd.yy pattern is accepted.""" 156 | assert cast_to_date( 157 | '09.07.19', 158 | 'mm.dd.yy', 159 | ).unwrap() == target_after_2000 160 | 161 | 162 | def test_string_mmddyy_with_dots_before_2000(): 163 | """Test that mm.dd.yy pattern is accepted.""" 164 | assert cast_to_date( 165 | '06.08.94', 166 | 'mm.dd.yy', 167 | ).unwrap() == target_before_2000 168 | 169 | 170 | def test_string_ddmmyy_with_dots_after_2000(): 171 | """Test that dd.mm.yy pattern is accepted.""" 172 | assert cast_to_date( 173 | '07.09.19', 174 | 'dd.mm.yy', 175 | ).unwrap() == target_after_2000 176 | 177 | 178 | def test_string_ddmmyy_with_dots_before_2000(): 179 | """Test that dd.mm.yy pattern is accepted.""" 180 | assert cast_to_date( 181 | '08.06.94', 182 | 'dd.mm.yy', 183 | ).unwrap() == target_before_2000 184 | 185 | 186 | def test_string_yymmdd_with_dots_after_2000(): 187 | """Test that yy.mm.dd pattern is accepted.""" 188 | assert cast_to_date( 189 | '19.09.07', 190 | 'yy.mm.dd', 191 | ).unwrap() == target_after_2000 192 | 193 | 194 | def test_string_yymmdd_with_dots_before_2000(): 195 | """Test that yy.mm.dd pattern is accepted.""" 196 | assert cast_to_date( 197 | '94.06.08', 198 | 'yy.mm.dd', 199 | ).unwrap() == target_before_2000 200 | 201 | 202 | def test_string_fails_as_invalid_date(): 203 | """Test threws ValueError when invalid date passed.""" 204 | test = cast_to_date( 205 | '994.06.08', 206 | 'yyy.mm.dd', 207 | ) 208 | assert not is_successful(test) 209 | assert isinstance(test.failure(), ValueError) 210 | assert 'Unable to cast (994.06.08) to ISO date. Exc(Unable to cast to no millennia format: 994.06.08)' in str( # noqa: E501 211 | test.failure(), 212 | ) 213 | 214 | 215 | def test_string_fails_when_month_out_of_range(): 216 | """Test threws ValueError when month out of range.""" 217 | test = cast_to_date( 218 | '19.14.12', 219 | 'yy.mm.dd', 220 | ) 221 | assert not is_successful(test) 222 | assert isinstance(test.failure(), ValueError) 223 | assert 'Unable to cast (19.14.12) to ISO date. Exc(month must be in 1..12)' in str( # noqa: E501 224 | test.failure(), 225 | ) 226 | 227 | 228 | def test_string_fails_when_month_is_not_integer(): 229 | """Test threws ValueError when month out of range.""" 230 | test = cast_to_date( 231 | '19.MM.12', 232 | 'yy.mm.dd', 233 | ) 234 | expected = '{0}{1}'.format( 235 | 'Unable to cast (19.MM.12) to ISO date. ', 236 | "Exc(invalid literal for int() with base 10: '.M')", 237 | ) 238 | assert not is_successful(test) 239 | assert isinstance(test.failure(), ValueError) 240 | assert str(test.failure()) == expected 241 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Kaiba 2 | 3 | Kaiba is a data transformation tool written in Python that uses a DTL(Data Transformation Language) expressed in normal JSON to govern output structure, data fetching and data transformation. 4 | ___ 5 | ![test](https://github.com/kaiba-tech/kaiba/workflows/test/badge.svg) 6 | [![codecov](https://codecov.io/gh/kaiba-tech/kaiba/branch/master/graph/badge.svg)](https://codecov.io/gh/kaiba-tech/kaiba) 7 | [![Python Version](https://img.shields.io/pypi/pyversions/kaiba.svg)](https://pypi.org/project/kaiba/) 8 | [![wemake-python-styleguide](https://img.shields.io/badge/style-wemake-000000.svg)](https://github.com/wemake-services/wemake-python-styleguide) 9 | ___ 10 | 11 | **Documentation 12 | ([Stable](https://kaiba.readthedocs.io/) | 13 | [Latest](https://kaiba.readthedocs.io/en/latest/)) | 14 | [Source Code](https://github.com/kaiba-tech/kaiba) | 15 | [Task Tracker](https://github.com/kaiba-tech/kaiba/issues)** 16 | 17 | ## What is Kaiba 18 | 19 | Kaiba is a JSON to JSON mapper. That means that we read input JSON and create output JSON. How the output is created is based on instructions from a configuration file. The configuration file governs the the output structure and tells Kaiba where in the input to find data and where to place it in the output. In addition to this Kaiba supports data transformation with `data casting`, `regular expressions`, `if conditions`, `combination of data from multiple places` and of course setting `default` values. 20 | 21 | __This enables you to change any input into the output you desire.__ 22 | 23 | ## The Kaiba App 24 | 25 | The kaiba App is currently in development 26 | 27 | [app.kaiba.tech](https://app.kaiba.tech) 28 | 29 | The app provides a user interface for creating Kaiba configurations. With the app you can map in real time easily and create the kaiba config. 30 | 31 | ## The Kaiba API 32 | 33 | The kaiba api is open for anyone to try, you send your data and the configuration and get mapped data response. 34 | 35 | [api.kaiba.tech/docs](https://api.kaiba.tech/docs) 36 | 37 | ## Typical usecases 38 | 39 | * You `GET` data from api, but need to transform it for your backend system 40 | * `POST`ing data to an api that needs data on a different format than what your system produces 41 | * All your backends speak different language? pipe it through __Kaiba__ 42 | * Customer delivers weirdly formatted data? Use __Kaiba__ to make it sexy 43 | * Have CSV but need nicely structured JSON? make CSV into a JSON list and transform it with __Kaiba__ 44 | * Have XML but need to change it? make it into JSON, transform it with __Kaiba__ and then dump it to XML again. 45 | * Customers legacy system needs CSV. Use __Kaiba__ to transform your nicely structured JSON data into a JSON List that can be easily dumped to CSV 46 | 47 | ## Official Open kaiba Solutions 48 | 49 | [kaiba-cli](https://github.com/kaiba-tech/kaiba-cli), commandline interface for file to file mapping. 50 | 51 | [kaiba-api](https://github.com/kaiba-tech/kaiba-api), FastAPI driven rest server that maps data with kaiba 52 | 53 | ## Enterprise solutions 54 | 55 | Coming... 56 | 57 | ## Goal 58 | 59 | The goal of this library is to make JSON to JSON transformation/mapping easy, configurable and documentable. We achieve this by using a simple but feature-rich JSON configuration which then also acts as documentation and as a contract between parties. 60 | 61 | ## Why 62 | 63 | Kaiba was born because we really dislike mapping. Documenting whatever decisions made in your code so that some product owner understands it is also _no me gusto_. Transforming data from one format to another is something software engineers do allmost daily... It should be easy! And documenting it shouldn't be something you have to worry about. 64 | 65 | After the Worst POC in History we never wanted to do mapping by scripts and code again. This lead to the idea that it should be possible to create a file which governs how the structure should look and how the data should be transformed. This would then be the `single source of truth` and with Kaiba we have achieved this. 66 | 67 | We believe that this will make collaboration between teams faster and easier. Use Kaiba to agree with data formats between Front-end and Back-end. Between the 3rd party system and your back-end. You can even use Kaiba for testing existing integrations ;-) 68 | 69 | ## Features 70 | 71 | * Mapping with configuration File. 72 | * [JSON Schema](https://json-schema.org/) validation of the config file. 73 | * Structurally Transform JSON 74 | * Combine multiple values to one. 75 | * Default values 76 | * If statements 77 | * is, contains, in, not 78 | * Casting 79 | * integer, decimal, iso date 80 | * Regular Expressions 81 | 82 | ## Contributing 83 | Please see [contribute](https://kaiba.readthedocs.io/en/stable/contributing) 84 | 85 | ## Installation 86 | 87 | Package is on pypi. Use pip or poetry to install 88 | 89 | ```sh 90 | pip install kaiba 91 | ``` 92 | ```sh 93 | poetry add kaiba 94 | ``` 95 | 96 | ## Introduction 97 | 98 | Have a look at our introduction course [here](https://kaiba.readthedocs.io/en/stable/introduction) 99 | 100 | ## Quickstart 101 | ```python 102 | import simplejson 103 | 104 | from kaiba.process import process 105 | 106 | my_config = { 107 | 'name': 'schema', 108 | 'array': False, 109 | 'objects': [ 110 | { 111 | 'name': 'invoices', 112 | 'array': True, 113 | 'iterators': [ 114 | { 115 | 'alias': 'invoice', 116 | 'path': ['root', 'invoices'], 117 | }, 118 | ], 119 | 'attributes': [ 120 | { 121 | 'name': 'amount', 122 | 'data_fetchers': [ 123 | { 124 | 'path': ['invoice', 'amount'], 125 | }, 126 | ], 127 | 'casting': { 128 | 'to': 'decimal', 129 | 'original_format': 'integer_containing_decimals', 130 | }, 131 | 'default': 0, 132 | }, 133 | { 134 | 'name': 'debtor', 135 | 'data_fetchers': [ 136 | { 137 | 'path': ['root', 'customer', 'first_name'], 138 | }, 139 | { 140 | 'path': ['root', 'customer', 'last_name'], 141 | }, 142 | ], 143 | 'separator': ' ', 144 | }, 145 | ], 146 | 'objects': [], 147 | }, 148 | ], 149 | } 150 | 151 | example_data = { 152 | 'root': { 153 | 'customer': { 154 | 'first_name': 'John', 155 | 'last_name': 'Smith', 156 | }, 157 | 'invoices': [ 158 | { 159 | 'amount': 10050, 160 | }, 161 | { 162 | 'amount': 20050, 163 | }, 164 | { 165 | 'amount': -15005, 166 | }, 167 | ], 168 | }, 169 | } 170 | 171 | mapped_data = process(example_data, my_config) 172 | 173 | with open('resultfile.json', 'w') as output_file: 174 | output_file.write(simplejson.dumps(mapped_data)) 175 | 176 | ``` 177 | 178 | contents of resultfile.json 179 | ```json 180 | { 181 | "invoices": [ 182 | { 183 | "amount": 100.5, 184 | "debtor": "John Smith" 185 | }, 186 | { 187 | "amount": 200.5, 188 | "debtor": "John Smith" 189 | }, 190 | { 191 | "amount": -150.05, 192 | "debtor": "John Smith" 193 | } 194 | ] 195 | } 196 | ``` 197 | -------------------------------------------------------------------------------- /kaiba/functions.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import re 4 | from decimal import Decimal 5 | from typing import Any, List, Optional, Union 6 | 7 | from returns.pipeline import flow 8 | from returns.pointfree import bind 9 | from returns.result import Failure, ResultE, safe 10 | 11 | from kaiba.casting import get_casting_function 12 | from kaiba.models.base import AnyType 13 | from kaiba.models.casting import Casting 14 | from kaiba.models.if_statement import Conditions, IfStatement 15 | from kaiba.models.regex import Regex 16 | from kaiba.models.slicing import Slicing 17 | 18 | ValueTypes = (str, int, float, bool, Decimal) 19 | 20 | 21 | @safe 22 | def apply_if_statements( 23 | if_value: Optional[AnyType], 24 | statements: List[IfStatement], 25 | ) -> Optional[AnyType]: 26 | """Apply if statements to a value. 27 | 28 | :param if_value: The value to use when evaluating if statements 29 | :type if_value: AnyType 30 | 31 | :param statements: :term:`statements` collection of if operations 32 | :type statements: List[Dict[str, Any]] 33 | 34 | :return: Success/Failure containers 35 | :rtype: AnyType 36 | 37 | one if object looks like this 38 | 39 | .. code-block:: json 40 | 41 | { 42 | "condition": "is", 43 | "target": "foo", 44 | "then": "bar", 45 | "otherwise": "no foo" - optional 46 | } 47 | 48 | The if statements are chained so that the next works on the output of the 49 | previous. If no "otherwise" is provided then the original value or value 50 | from the previous operation will be returned. 51 | 52 | Example 53 | >>> apply_if_statements( 54 | ... '1', [ 55 | ... IfStatement( 56 | ... **{'condition': 'is', 'target': '1', 'then': '2'} 57 | ... ) 58 | ... ], 59 | ... ).unwrap() == '2' 60 | True 61 | >>> apply_if_statements( 62 | ... 'a', 63 | ... [IfStatement(**{ 64 | ... 'condition': 'is', 65 | ... 'target': '1', 66 | ... 'then': '2', 67 | ... 'otherwise': '3' 68 | ... })], 69 | ... ).unwrap() == '3' 70 | True 71 | 72 | """ 73 | for statement in statements: 74 | 75 | if_value = _apply_statement( 76 | if_value, statement, 77 | ) 78 | 79 | if if_value is None: 80 | raise ValueError('If statement failed or produced `None`') 81 | 82 | return if_value 83 | 84 | 85 | def _apply_statement( 86 | if_value: Optional[AnyType], 87 | statement: IfStatement, 88 | ) -> Optional[AnyType]: 89 | evaluation: bool = False 90 | 91 | condition = statement.condition 92 | target = statement.target 93 | 94 | if condition == Conditions.IS: 95 | evaluation = if_value == target 96 | 97 | if condition == Conditions.NOT: 98 | evaluation = if_value != target 99 | 100 | if condition == Conditions.IN: 101 | evaluation = if_value in target # type: ignore 102 | 103 | if condition == Conditions.CONTAINS: 104 | list_or_dict = isinstance(if_value, (dict, list)) 105 | evaluation = list_or_dict and target in if_value # type: ignore 106 | evaluation = evaluation or not list_or_dict and str(target) in str(if_value) # noqa: E501 E262 107 | if evaluation: 108 | return statement.then 109 | 110 | return statement.otherwise or if_value 111 | 112 | 113 | @safe 114 | def apply_separator( 115 | mapped_values: List[AnyType], 116 | separator: str, 117 | ) -> AnyType: 118 | """Apply separator between the values of a List[Any]. 119 | 120 | :param mapped_values: The list of values to join with the separator 121 | :type mapped_values: List[AnyType] 122 | 123 | :param separator: :term:`separator` value to join mapped_values list with 124 | :type separator: str 125 | 126 | :return: Success/Failure containers 127 | :rtype: AnyType 128 | 129 | Example 130 | >>> from returns.pipeline import is_successful 131 | >>> apply_separator(['a', 'b', 'c'], ' ').unwrap() 132 | 'a b c' 133 | >>> apply_separator([1, 'b', True], ' ').unwrap() 134 | '1 b True' 135 | >>> is_successful(apply_separator([], ' ')) 136 | False 137 | 138 | """ 139 | if not mapped_values: 140 | raise ValueError('mapped_values is empty') 141 | 142 | if len(mapped_values) == 1: 143 | return mapped_values[0] 144 | 145 | return separator.join([str(mapped) for mapped in mapped_values]) 146 | 147 | 148 | def apply_slicing( 149 | value_to_slice: Optional[Any], 150 | slicing: Slicing | None, 151 | ) -> Optional[AnyType]: 152 | """Slice value from index to index. 153 | 154 | :param value_to_slice: The value to slice 155 | :type value_to_slice: Any 156 | 157 | :param slicing: :term:`slicing` object 158 | :type slicing: dict 159 | 160 | 161 | :return: Success/Failure containers 162 | :rtype: Any 163 | 164 | Example 165 | >>> apply_slicing('123', Slicing(**{'from': 1})) 166 | '23' 167 | >>> apply_slicing('test', Slicing(**{'from': 1, 'to': 3})) 168 | 'es' 169 | """ 170 | if value_to_slice is None: 171 | return value_to_slice 172 | 173 | if not slicing: 174 | return value_to_slice 175 | 176 | if not isinstance(value_to_slice, list): 177 | value_to_slice = str(value_to_slice) 178 | 179 | return value_to_slice[slicing.slice_from:slicing.slice_to] 180 | 181 | 182 | @safe 183 | def apply_regex( # noqa: WPS212, WPS234 184 | value_to_match: Optional[AnyType], 185 | regex: Regex | None, 186 | ) -> Union[List[AnyType], AnyType, None]: 187 | r"""Match value by a certain regex pattern. 188 | 189 | :param value_to_match: The value to match 190 | :type value_to_match: AnyType 191 | 192 | :param regex: :term: `matching` object which has parameters for 193 | regex match 194 | :type regex: Regex 195 | 196 | :return: Success/Failure container 197 | :rtype: AnyType 198 | 199 | Example 200 | >>> apply_regex( 201 | ... 'abcdef', 202 | ... Regex(**{'expression': '(?<=abc)def'}) 203 | ... ).unwrap() 204 | 'def' 205 | >>> apply_regex( 206 | ... 'Isaac Newton, physicist', 207 | ... Regex(**{'expression': r'(\w+)', 'group': 1}), 208 | ... ).unwrap() 209 | 'Newton' 210 | >>> apply_regex( 211 | ... 'Isaac Newton, physicist', 212 | ... Regex(**{'expression': r'(\w+)', 'group': [1, 2]}), 213 | ... ).unwrap() 214 | ['Newton', 'physicist'] 215 | >>> apply_regex(None, Regex(**{'expression': 'a+'})).unwrap() 216 | >>> apply_regex('Open-source matters', None).unwrap() 217 | 'Open-source matters' 218 | """ 219 | if value_to_match is None: 220 | return value_to_match 221 | 222 | if not regex or not regex.expression: 223 | return value_to_match 224 | 225 | pattern = regex.expression 226 | groups = re.finditer( 227 | pattern, 228 | value_to_match, # type: ignore 229 | ) 230 | matches: list = [gr.group(0) for gr in groups] 231 | num_group: Union[int, List[int]] = regex.group 232 | if isinstance(num_group, list): 233 | if not num_group: 234 | return matches 235 | return [matches[ind] for ind in num_group] # typing: ignore 236 | return matches[num_group] 237 | 238 | 239 | def apply_casting( 240 | value_to_cast: Optional[AnyType], 241 | casting: Casting, 242 | ) -> ResultE[AnyType]: 243 | """Casting one type of code to another. 244 | 245 | :param casting: :term:`casting` object 246 | :type casting: dict 247 | 248 | :param value_to_cast: The value to cast to casting['to'] 249 | :type value_to_cast: AnyType 250 | 251 | :return: Success/Failure containers 252 | :rtype: AnyType 253 | 254 | Example 255 | >>> apply_casting('123', Casting(**{'to': 'integer'})).unwrap() 256 | 123 257 | >>> apply_casting('123.12', Casting(**{'to': 'decimal'})).unwrap() 258 | Decimal('123.12') 259 | """ 260 | if value_to_cast is None: 261 | return Failure(ValueError('value_to_cast is empty')) 262 | 263 | return flow( 264 | casting.to, 265 | get_casting_function, 266 | bind( # type: ignore 267 | lambda function: function( # type: ignore 268 | value_to_cast, casting.original_format, 269 | ), 270 | ), 271 | ) 272 | 273 | 274 | @safe 275 | def apply_default( 276 | mapped_value: Optional[AnyType] = None, 277 | default: Optional[AnyType] = None, 278 | ) -> AnyType: 279 | """Apply default value if exists and if mapped value is None. 280 | 281 | :param mapped_value: If this value is *None* the default value is returned 282 | :type mapped_value: Optional[AnyType] 283 | 284 | :param default: :term:`default` value to return if mapped value is None 285 | :type default: Optional[AnyType] 286 | 287 | :return: Success/Failure containers 288 | :rtype: AnyType 289 | 290 | If default *is not* none and mapped_value *is* None then return default 291 | else if mapped_value is not in accepted ValueTypes then throw an error 292 | else return mapped value 293 | 294 | Example 295 | >>> apply_default('test', None).unwrap() == 'test' 296 | True 297 | >>> apply_default('nope', 'test').unwrap() == 'nope' 298 | True 299 | """ 300 | if default is not None: 301 | if mapped_value is None: 302 | return default 303 | 304 | if mapped_value is None: 305 | raise ValueError('Default value should not be `None`') 306 | 307 | if not isinstance(mapped_value, ValueTypes): 308 | raise ValueError('Unable to give default value') 309 | 310 | return mapped_value 311 | -------------------------------------------------------------------------------- /tests/functions/test_apply_if_statements.py: -------------------------------------------------------------------------------- 1 | from returns.result import Success 2 | 3 | from kaiba.functions import apply_if_statements 4 | from kaiba.models.if_statement import IfStatement 5 | 6 | 7 | def test_if_is(): 8 | """Test that 1 if (is) statement works.""" 9 | test: list = [ 10 | 'target_value', 11 | [ 12 | IfStatement(**{ 13 | 'condition': 'is', 14 | 'target': 'target_value', 15 | 'then': 'value2', 16 | }), 17 | ], 18 | ] 19 | assert apply_if_statements(*test) == Success('value2') 20 | 21 | 22 | def test_if_is_condition_false(): 23 | """Test if condition False.""" 24 | test: list = [ 25 | 'not_target_value', 26 | [ 27 | IfStatement(**{ 28 | 'condition': 'is', 29 | 'target': 'target_value', 30 | 'then': 'value2', 31 | }), 32 | ], 33 | ] 34 | assert apply_if_statements(*test) == Success('not_target_value') 35 | 36 | 37 | def test_if_is_condition_array_value(): 38 | """Test that we can do if is statement on arrays.""" 39 | test: list = [ 40 | ['target_value'], 41 | [ 42 | IfStatement(**{ 43 | 'condition': 'is', 44 | 'target': ['target_value'], 45 | 'then': 'value2', 46 | }), 47 | ], 48 | ] 49 | assert apply_if_statements(*test) == Success('value2') 50 | 51 | 52 | def test_if_is_condition_objects_value(): 53 | """Test that we can do if is statement on objectss.""" 54 | test: list = [ 55 | {'val': 'target'}, 56 | [ 57 | IfStatement(**{ 58 | 'condition': 'is', 59 | 'target': {'val': 'target'}, 60 | 'then': 'value2', 61 | }), 62 | ], 63 | ] 64 | assert apply_if_statements(*test) == Success('value2') 65 | 66 | 67 | def test_if_in(): 68 | """Test that 1 if (in) statement works.""" 69 | test: list = [ 70 | 'target_value', 71 | [ 72 | IfStatement(**{ 73 | 'condition': 'in', 74 | 'target': ['target_value'], 75 | 'then': 'value2', 76 | }), 77 | ], 78 | ] 79 | assert apply_if_statements(*test) == Success('value2') 80 | 81 | 82 | def test_if_in_condition_false(): 83 | """Test if in condition False.""" 84 | test: list = [ 85 | 'not_target_value', 86 | [ 87 | IfStatement(**{ 88 | 'condition': 'in', 89 | 'target': 'target_value', 90 | 'then': 'value2', 91 | }), 92 | ], 93 | ] 94 | assert apply_if_statements(*test) == Success('not_target_value') 95 | 96 | 97 | def test_if_in_condition_array_value(): 98 | """Test that we can do if in statement on arrays.""" 99 | test: list = [ 100 | ['target_value'], 101 | [ 102 | IfStatement(**{ 103 | 'condition': 'in', 104 | 'target': [['target_value']], 105 | 'then': 'value2', 106 | }), 107 | ], 108 | ] 109 | assert apply_if_statements(*test) == Success('value2') 110 | 111 | 112 | def test_if_in_condition_objects_value(): 113 | """Test that we can do if is statement on objectss.""" 114 | test: list = [ 115 | {'val': 'target'}, 116 | [ 117 | IfStatement(**{ 118 | 'condition': 'in', 119 | 'target': [{'val': 'target'}], 120 | 'then': 'value2', 121 | }), 122 | ], 123 | ] 124 | assert apply_if_statements(*test) == Success('value2') 125 | 126 | 127 | def test_if_not(): 128 | """Test that 1 if (not) statement works.""" 129 | test: list = [ 130 | 'target_value', 131 | [ 132 | IfStatement(**{ 133 | 'condition': 'not', 134 | 'target': 'not_target', 135 | 'then': 'value2', 136 | }), 137 | ], 138 | ] 139 | assert apply_if_statements(*test) == Success('value2') 140 | 141 | 142 | def test_if_not_condition_false(): 143 | """Test if not condition False.""" 144 | test: list = [ 145 | 'target_value', 146 | [ 147 | IfStatement(**{ 148 | 'condition': 'not', 149 | 'target': 'target_value', 150 | 'then': 'value2', 151 | }), 152 | ], 153 | ] 154 | assert apply_if_statements(*test) == Success('target_value') 155 | 156 | 157 | def test_if_not_condition_array_value(): 158 | """Test that we can do if not statement on arrays.""" 159 | test: list = [ 160 | ['target_value'], 161 | [ 162 | IfStatement(**{ 163 | 'condition': 'not', 164 | 'target': ['not_target'], 165 | 'then': 'value2', 166 | }), 167 | ], 168 | ] 169 | assert apply_if_statements(*test) == Success('value2') 170 | 171 | 172 | def test_if_not_condition_array_value_with_int(): 173 | """Test that we can do if not statement on arrays.""" 174 | test: list = [ 175 | [123], 176 | [ 177 | IfStatement(**{ 178 | 'condition': 'is', 179 | 'target': [123], 180 | 'then': 'value2', 181 | }), 182 | ], 183 | ] 184 | assert apply_if_statements(*test) == Success('value2') 185 | 186 | 187 | def test_if_not_condition_objects_value(): 188 | """Test that we can do if not statement on objectss.""" 189 | test: list = [ 190 | {'val': 'target'}, 191 | [ 192 | IfStatement(**{ 193 | 'condition': 'not', 194 | 'target': {'val': 'tarnot'}, 195 | 'then': 'value2', 196 | }), 197 | ], 198 | ] 199 | assert apply_if_statements(*test) == Success('value2') 200 | 201 | 202 | def test_if_contains(): 203 | """Test that 1 if (contains) statement works.""" 204 | test: list = [ 205 | 'target_value', 206 | [ 207 | IfStatement(**{ 208 | 'condition': 'contains', 209 | 'target': '_value', 210 | 'then': 'value2', 211 | }), 212 | ], 213 | ] 214 | assert apply_if_statements(*test) == Success('value2') 215 | 216 | 217 | def test_if_contains_condition_false(): 218 | """Test if contains condition False.""" 219 | test: list = [ 220 | 'not_target_value', 221 | [ 222 | IfStatement(**{ 223 | 'condition': 'contains', 224 | 'target': 'does_not_contain', 225 | 'then': 'value2', 226 | }), 227 | ], 228 | ] 229 | assert apply_if_statements(*test) == Success('not_target_value') 230 | 231 | 232 | def test_if_contains_condition_array_value(): 233 | """Test that we can do if contains statement on arrays.""" 234 | test: list = [ 235 | ['value', 'target'], 236 | [ 237 | IfStatement(**{ 238 | 'condition': 'contains', 239 | 'target': 'target', 240 | 'then': 'value2', 241 | }), 242 | ], 243 | ] 244 | assert apply_if_statements(*test) == Success('value2') 245 | 246 | 247 | def test_if_contains_condition_objects_value(): 248 | """Test that we can do if contains statement on objectss.""" 249 | test: list = [ 250 | {'val': 'target'}, 251 | [ 252 | IfStatement(**{ 253 | 'condition': 'contains', 254 | 'target': 'val', 255 | 'then': 'value2', 256 | }), 257 | ], 258 | ] 259 | assert apply_if_statements(*test) == Success('value2') 260 | 261 | 262 | def test_if_contains_objects_in_array_value(): 263 | """Test that we can do if contains statement on objectss.""" 264 | test: list = [ 265 | [{'val': 'target'}], 266 | [ 267 | IfStatement(**{ 268 | 'condition': 'contains', 269 | 'target': {'val': 'target'}, 270 | 'then': 'value2', 271 | }), 272 | ], 273 | ] 274 | assert apply_if_statements(*test) == Success('value2') 275 | 276 | 277 | def test_if_contains_array_does_not_stringify(): 278 | """Test that we can do if contains statement on array[objects]. 279 | 280 | However its very important that list and object tests does not do 281 | string casting for check since it would give the check two possible 282 | ways to do the check and there should only be one. 283 | for objects the 'in' checks if the key exist 284 | for arrays the 'in' checks if the element exist inside the aray 285 | for everything else we will stringify the test value. 286 | """ 287 | test: list = [ 288 | [{'val': 'target'}], 289 | [ 290 | IfStatement(**{ 291 | 'condition': 'contains', 292 | 'target': 'target', 293 | 'then': 'value2', 294 | }), 295 | ], 296 | ] 297 | assert apply_if_statements(*test) == Success([{'val': 'target'}]) 298 | 299 | 300 | def test_if_contains_works_with_non_strings(): 301 | """Test that we can do if contains statement on objectss.""" 302 | test: list = [ 303 | 123, 304 | [ 305 | IfStatement(**{ 306 | 'condition': 'contains', 307 | 'target': 123, 308 | 'then': 'value2', 309 | }), 310 | ], 311 | ] 312 | assert apply_if_statements(*test) == Success('value2') 313 | 314 | 315 | def test_if_chained(): 316 | """Test that two if (is) statement works.""" 317 | test: list = [ 318 | 'target_value', 319 | [ 320 | IfStatement(**{ 321 | 'condition': 'is', 322 | 'target': 'target_value', 323 | 'then': 'value2', 324 | }), 325 | IfStatement(**{ 326 | 'condition': 'is', 327 | 'target': 'value2', 328 | 'then': 'value3', 329 | }), 330 | ], 331 | ] 332 | assert apply_if_statements(*test) == Success('value3') 333 | 334 | 335 | def test_if_failed_condition_goes_to_otherwise(): 336 | """Test that we get the then value when condition fails.""" 337 | test: list = [ 338 | 'not_target_value', 339 | [ 340 | IfStatement(**{ 341 | 'condition': 'is', 342 | 'target': 'target_value', 343 | 'then': 'no', 344 | 'otherwise': 'yes', 345 | }), 346 | ], 347 | ] 348 | assert apply_if_statements(*test) == Success('yes') 349 | -------------------------------------------------------------------------------- /docs/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration Json data 2 | The configuration governs not only where to find data, but also the structure of the output which will mirror the structure in the configuration json. 3 | 4 | The two main components of the configuration json is the object and attributes. An object can contain nested objects and/or attributes. In the attribute part of the file is where you actually tell the mapper where to find data. In the object you are deciding the structure and also telling the mapper if there are iterable data anywhere that needs to be iterated to create multiple instances. 5 | 6 | 7 | !!! warning 8 | This document is a bit outdated since its not updated after switch to pydantic models. 9 | 10 | ## Object 11 | 12 | An object has a name, it can have attributes, nested objects or a special type of objects called [branching objects](#branching-object). It will also know if itself is an array and the path to where the input data can be iterated to create multiple objects. 13 | 14 | | name | type | description | comment | 15 | | --- | --- | --- | --- | 16 | | __name__ | str | name of the key it will get in parent object | the root will not get a name | 17 | | __array__ | bool | tells the mapper if this should be an array or not | | 18 | | iterables | array[[iterable](#iterable)] | Lets you iterate over lists in input data and apply configuration to every iteration of the lists | | 19 | | _attributes_ | array[[attribute](#attribute)] | An array of this objects attribute mappings | | 20 | | _objects_ | array[[object](#object)] | Here you can nest more objects. | | 21 | | _branching_objects_ | array[[branching object](#branching-object)] | Array of a special kind of object | rarely used | 22 | 23 | 24 | ```json 25 | { 26 | "name": "object_name", 27 | "array": true, 28 | "iterables": [], 29 | "objects": [], 30 | "branching_objects": [], 31 | "attributes": [] 32 | } 33 | ``` 34 | 35 | ## Iterable 36 | 37 | Iterables are your bread and butter for looping lists in the input data. For each an every one of the elements in the list we apply the current object and its childrens mapping configuration. 38 | 39 | | name | type | description | 40 | | --- | --- | --- | 41 | | __alias__ | str | The aliased name the current iteration will get in the data which mapping is applied to | 42 | | __path__ | array[str\|int] | path to the iterable list/array in the input data | 43 | 44 | ```json 45 | { 46 | "alias": "keyname", 47 | "path": ["path", "to", "list"] 48 | } 49 | ``` 50 | [Explanation of path](#explanation-of-path) 51 | 52 | [Explanation of Iterables coming](#) 53 | 54 | ## Attribute 55 | 56 | The attributes are like 'color' of a car or 'amount' in an invoice. Attributes are have a name ('amount'), a number of mappings, separator, if statements, casting and a default value if all else fails. 57 | 58 | | name | type | description | default | 59 | | --- | --- | --- | --- | 60 | | __name__ | str | The name it will get in the parent object | | 61 | | _mappings_ | array[[mapping](#mapping)] | list of mapping objects which is where to find data | `[]` | 62 | | separator | str | string to separate each value in case multiple are found in mapping step | `''` | 63 | | if_statements | array[[if statement](#if-statement)] | If statements that can change data based on conditions | `[]` | 64 | | casting | [casting](#casting) | Lets you cast data to a spesific type [int, decimal, date] | `{}` | 65 | | _default_ | Any | If after all mapping, if statements and casting the result is None this value is used | `None` | 66 | 67 | ```json 68 | { 69 | "name": "attribute_name", 70 | "mappings": [], 71 | "separator": "", 72 | "if_statements": [], 73 | "casting": {}, 74 | "default": "default value" 75 | } 76 | ``` 77 | 78 | ## Mapping 79 | 80 | This is the only place where actual interaction with the input data is done. 81 | 82 | | name | type | description | default | 83 | | --- | --- | --- | --- | 84 | | _path_ | array[str\|int] | path to data you want to retrieve. | `[]` | 85 | | if_statements | array[[if statement](#if-statement)] | If statements that can change data based on conditions | `[]` | 86 | | _default_ | Any | If no value is found or value is None after if_statements then this value is used | `None` | 87 | 88 | !!! note 89 | either `path` or `default` must contain a something 90 | 91 | ### Explanation of path 92 | 93 | You add a list of `strings` or `integers` that will get you to your data. so for example if you needed to get to the second element in the list called `my_list` in the following json then your `path` will be `["my_list", 1]` and you will get the value `index1` 94 | 95 | ```json 96 | { 97 | "my_list": ["index0", "index1"] 98 | } 99 | ``` 100 | 101 | * if_statements: list of [if statements](#if-statement) that can change the data depending on conditions 102 | * default: a default value if none is found or value found is ```None``` 103 | 104 | ```json 105 | { 106 | "path": ["path", "to", "data"], 107 | "if_statements": [], 108 | "default": "default" 109 | } 110 | ``` 111 | >input({'path': { 'to': { 'data': 'value'}}}) -> 'value' 112 | 113 | >input({'path': { 'does_not_exist'}}) -> 'default' 114 | 115 | >input() -> 'default' 116 | 117 | ## Regexp 118 | 119 | Let's you match values by certain patterns in `search` and specify what matches and in what order to return in `group`. 120 | 121 | !!! Note 122 | The default `group` value is `0`. It will return the first match. To return all values `group` should be equal to `[]`. You can also specify `group` as `[1, 3, 2]`, which will return 2nd, 4th and 3rd matched elements in exact order. To return 5th element from the end, you would need to get all matches `[]` and slice over array. Slicing over array's [issue](https://github.com/greenbird/piri/issues/121) 123 | 124 | | name | type | description | default | 125 | | --- | --- | --- | --- | 126 | | __search__ | string | Pattern for string matching | | 127 | | group | integer\|array | Index/-ces of matching sinstrings to return | 0 | 128 | 129 | ```json 130 | { 131 | "search": "(i\w)", 132 | "group": [0, 2, 1] 133 | } 134 | ``` 135 | 136 | > input('Vladimir Kramnik') -> ['im', 'ik', 'ir'] 137 | 138 | !!! Note 139 | Values are returned as a string when `group` is integer or as an array of strings if `group` is an array. 140 | 141 | ## Slicing 142 | 143 | Lets you slice a value from index `from` to index `to`. Slicing is implemented exactly like pythons string[x:x] slicing. This means that when `from` is negative you count back from the end, and if `to` is `null` or left out then we consume the rest of the string. 144 | 145 | 146 | | name | type | description | default | 147 | | --- | --- | --- | --- | 148 | | __from__ | int | Where to cut value from counted from `0` | | 149 | | to | int\|null | To what index we cut to, leave key/value out or set value=`null` to go to end of string | | 150 | 151 | ```json 152 | { 153 | "from": 1, 154 | "to": 3, 155 | } 156 | ``` 157 | > input('hello') -> 'el' 158 | 159 | !!! Note 160 | All values are turned into `string` before slicing is applied. This lets you also slice any values independant on their original type in the input data. If the config Slicing object is empty, this str casting is also skipped. Any result after slicing is also a string. So if you need a different format use [casting to change it](#casting) 161 | 162 | ## If Statement 163 | 164 | This is where you can change found(or not found) data to something else based on a condition. They are chained in the sense that what the first one produces will be the input to the next one. Thus if you want the original value if the first one fails, then leave out ```otherwise``` 165 | 166 | 167 | | name | type | description | default | 168 | | --- | --- | --- | --- | 169 | | __condition__ | one of ["is", "not", "in", "contains"] | What condition to use when checking `value` against `target` | | 170 | | __target__ | str\|number\|bool\|array\|object | Target what we do our condition against ie: `value == target` when condition is `is` | | 171 | | __then__ | str\|number\|bool\|array\|object | value that we will return if the condition is true | | 172 | | otherwise | str\|number\|bool\|array\|object | Optional value that we can return if the condition is false | `None` | 173 | 174 | ```json 175 | { 176 | "condition": "is", 177 | "target": "1", 178 | "then": "first_type", 179 | "otherwise": "default_type" 180 | } 181 | ``` 182 | > input('2') -> 'default_type' 183 | 184 | > input('1') -> 'first_type' 185 | 186 | If statements work a bit different depending on what value you expect to get. This table shows the exact operators used for all valuetypes 187 | 188 | | condition | valuetype | code | comment | 189 | | --- | --- | --- | --- | 190 | | is | all | `value == target` | | 191 | | not | all | `value != target` | | 192 | | in | array\|object | `value in target` | target must be an array or a string. where value must then equal one of the array `items` or a substring when target is a string. This means that the array values can be arrays or objects for when checking against array and object values | 193 | | contains | object | `target in value` | For objects this means that the `target is a key in object` | 194 | | contains | array | `target in value` | For arrays this means that `target equals one or more item in the array` | 195 | | contains | rest | `target in str(value)` | For the rest we string cast the value so that we can do this check for the rest of the types | 196 | 197 | 198 | ## Casting 199 | The casting object lets you cast whatever value is found to some new value. Currently integer, decimal and date are supported and original format is optional helper data that we need for some special cases where the format of the input value cannot be asserted automatically. 200 | 201 | | name | type | description | default | 202 | | --- | --- | --- | --- | 203 | | __to__ | one of ["integer", "decimal", "date"] | What type to cast the value to | | 204 | | original_format | "integer_containing_decimals" or spesific date format(see below)" | For some values we need to specify extra information in order to correctly cast it.| `None` | 205 | 206 | __about original format__ 207 | 208 | !!! note 209 | When `to` is `date` then original_format is `required`. 210 | 211 | | when to is | original format | description | 212 | | --- | --- | --- | 213 | | decimal | integer_containing_decimals | is used when some integer value should be casted to decimal and we need to divide it by 100 | 214 | | date | `yyyy.mm.dd` `yy.mm.dd` `yymmdd` `dd.mm.yyyy` `dd.mm.yy` `ddmmyy` | The format of the input date. `.` means any delimiter. Output is always iso-date yyyy-mm-dd | 215 | 216 | 217 | __Examples__ 218 | ```json 219 | { 220 | "to": "decimal", 221 | "original_format": "integer_containing_decimals" 222 | } 223 | ``` 224 | `"10050" -> Decimal(100.50)` 225 | 226 | ```json 227 | { 228 | "to": "date", 229 | "original_format": "ddmmyyyy" 230 | } 231 | ``` 232 | `"01012001" -> "2010-01-01"` 233 | 234 | 235 | ## Branching Object 236 | The branching object is a special object that does not have attributes or object childs but has a special branching_attributes child. The point of this object is to make sure that we can map data from different sources into the same element. for example, we have an object called "extradata" with the attributes 'name' and 'data'. This is kind of a field that can _be_ many things. like 'name' = 'extra_address_line1', and another one with 'extra_address_line2'. This must then get its data from different places, and thats what these branching objects are for. 237 | 238 | 239 | | name | type | description | 240 | | --- | --- | --- | 241 | | __name__ | str | Name of the object | 242 | | __array__ | bool | if it should be an array or not | 243 | | iterables | array[[iterable](#iterable)] | Lets you iterate over lists in input data and apply configuration to every iteration of the lists | 244 | | __branching_attributes__ | array[array[[attribute](#attribute)]] | list of list of attributes where each list of attributes will create a branching object. | 245 | 246 | 247 | __Example__ 248 | 249 | ```json 250 | { 251 | "name": "extradata", 252 | "array": true, 253 | "branching_attributes": [ 254 | [ 255 | { 256 | "name": "name", 257 | "default": "extra_address_line1" 258 | }, 259 | { 260 | "name": "data", 261 | "mappings": [{"path": ["list", "to", "line1", "value"]}] 262 | } 263 | ], 264 | [ 265 | { 266 | "name": "name", 267 | "default": "extra_address_line2" 268 | }, 269 | { 270 | "name": "data", 271 | "mappings": [{"path": ["list", "to", "line2", "value"]}] 272 | } 273 | ] 274 | ] 275 | } 276 | ``` 277 | 278 | This will produce: 279 | 280 | ```json 281 | { 282 | "extradata": [ 283 | { 284 | "name": "extra_address_line1", 285 | "data": "address value 1" 286 | }, 287 | { 288 | "name": "extra_address_line2", 289 | "data": "address value 2" 290 | } 291 | ] 292 | } 293 | ``` 294 | -------------------------------------------------------------------------------- /tests/test_process.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | from pydantic import ValidationError 5 | 6 | from kaiba.process import process_raise 7 | 8 | 9 | def test_creating_key_to_name(): 10 | """Test that we can fetch key in dict.""" 11 | input_data = {'key': 'test name'} 12 | config = { 13 | 'name': 'root', 14 | 'array': False, 15 | 'attributes': [ 16 | { 17 | 'name': 'name', 18 | 'data_fetchers': [ 19 | { 20 | 'path': ['key'], 21 | }, 22 | ], 23 | }, 24 | ], 25 | } 26 | 27 | assert process_raise( 28 | input_data, 29 | config, 30 | ) == {'name': 'test name'} 31 | 32 | 33 | def test_bad_config_gives_failure(): 34 | """Test that we can fetch key in dict.""" 35 | input_data = {'key': 'test name'} 36 | config = { 37 | 'namme': 'root', 38 | 'attributesadfs': [ 39 | { 40 | 'name': 'name', 41 | 'data_fetchers': [ 42 | { 43 | 'path': ['key'], 44 | }, 45 | ], 46 | }, 47 | ], 48 | } 49 | with pytest.raises(ValidationError) as ve: 50 | process_raise(input_data, config) 51 | 52 | assert ve.match('name') # noqa: WPS441 53 | assert ve.match('Field required') # noqa: WPS441 54 | 55 | 56 | def test_array_true_but_no_loop_gives_array(): 57 | """Test that we get an array if we set array = true in object.""" 58 | input_data = {'key': 'test name'} 59 | config = { 60 | 'name': 'root', 61 | 'array': True, 62 | 'attributes': [ 63 | { 64 | 'name': 'name', 65 | 'data_fetchers': [ 66 | { 67 | 'path': ['key'], 68 | }, 69 | ], 70 | }, 71 | ], 72 | } 73 | 74 | assert process_raise( 75 | input_data, 76 | config, 77 | ) == [{'name': 'test name'}] 78 | 79 | 80 | def test_double_repeatable(): 81 | """Test that we can map nested repeatable objects.""" 82 | config = { 83 | 'name': 'root', 84 | 'array': True, 85 | 'iterators': [ 86 | { 87 | 'alias': 'journals', 88 | 'path': ['journals'], 89 | }, 90 | ], 91 | 'attributes': [ 92 | { 93 | 'name': 'journal_id', 94 | 'data_fetchers': [ 95 | { 96 | 'path': ['journals', 'journal', 'id'], 97 | }, 98 | ], 99 | }, 100 | ], 101 | 'objects': [ 102 | { 103 | 'name': 'invoices', 104 | 'array': True, 105 | 'iterators': [ 106 | { 107 | 'alias': 'invoices', 108 | 'path': ['journals', 'journal', 'invoices'], 109 | }, 110 | ], 111 | 'attributes': [ 112 | { 113 | 'name': 'amount', 114 | 'data_fetchers': [ 115 | { 116 | 'path': ['invoices', 'amount'], 117 | }, 118 | ], 119 | }, 120 | ], 121 | }, 122 | ], 123 | } 124 | input_data = { 125 | 'journals': [ 126 | { 127 | 'journal': { 128 | 'id': 1, 129 | 'invoices': [{'amount': 1.1}, {'amount': 1.2}], 130 | }, 131 | }, 132 | { 133 | 'journal': { 134 | 'id': 2, 135 | 'invoices': [{'amount': 1.3}, {'amount': 1.4}], 136 | }, 137 | }, 138 | ], 139 | } 140 | expected_result = [ 141 | { 142 | 'journal_id': 1, 143 | 'invoices': [ 144 | {'amount': 1.1}, 145 | {'amount': 1.2}, 146 | ], 147 | }, 148 | { 149 | 'journal_id': 2, 150 | 'invoices': [ 151 | {'amount': 1.3}, 152 | {'amount': 1.4}, 153 | ], 154 | }, 155 | ] 156 | 157 | assert process_raise( 158 | input_data, 159 | config, 160 | ) == expected_result 161 | 162 | 163 | def test_mapping_where_data_is_not_found(): 164 | """Test that when we map and don't find data its okay.""" 165 | config = { 166 | 'name': 'root', 167 | 'array': True, 168 | 'iterators': [ 169 | { 170 | 'alias': 'journals', 171 | 'path': ['journals'], 172 | }, 173 | ], 174 | 'attributes': [ 175 | { 176 | 'name': 'journal_id', 177 | 'data_fetchers': [ 178 | { 179 | 'path': ['journals', 'journal', 'id'], 180 | }, 181 | ], 182 | }, 183 | ], 184 | 'objects': [ 185 | { 186 | 'name': 'invoices', 187 | 'array': True, 188 | 'iterators': [ 189 | { 190 | 'alias': 'invoices', 191 | 'path': ['journals', 'journal', 'invoices'], 192 | }, 193 | ], 194 | 'attributes': [ 195 | { 196 | 'name': 'amount', 197 | 'data_fetchers': [ 198 | { 199 | 'path': ['invoices', 'amount'], 200 | }, 201 | ], 202 | }, 203 | ], 204 | }, 205 | ], 206 | 'branching_objects': [ 207 | { 208 | 'name': 'extrafield', 209 | 'array': True, 210 | 'branching_attributes': [ 211 | [ 212 | { 213 | 'name': 'datavalue', 214 | 'data_fetchers': [ 215 | { 216 | 'path': ['extra', 'extra1'], 217 | }, 218 | ], 219 | }, 220 | ], 221 | ], 222 | }, 223 | ], 224 | } 225 | input_data = { 226 | 'journals': [ 227 | { 228 | 'journal': { 229 | 'id': 1, 230 | 'invoices': [{}, {'amount': 1.2}], 231 | }, 232 | }, 233 | { 234 | 'journal': { 235 | 'id': 2, 236 | }, 237 | }, 238 | ], 239 | } 240 | expected_result = [ 241 | { 242 | 'journal_id': 1, 243 | 'invoices': [ 244 | {'amount': 1.2}, 245 | ], 246 | }, 247 | { 248 | 'journal_id': 2, 249 | 'invoices': [], 250 | }, 251 | ] 252 | 253 | assert process_raise( 254 | input_data, 255 | config, 256 | ) == expected_result 257 | 258 | 259 | def test_most_features(): 260 | """Test that we can fetch key in dict.""" 261 | config = { 262 | 'name': 'schema', 263 | 'array': False, 264 | 'attributes': [ 265 | { 266 | 'name': 'name', 267 | 'data_fetchers': [ 268 | { 269 | 'path': ['key'], 270 | 'if_statements': [ 271 | { 272 | 'condition': 'is', 273 | 'target': 'val1', 274 | 'then': None, 275 | }, 276 | ], 277 | 'default': 'default', 278 | }, 279 | { 280 | 'path': ['key2'], 281 | 'if_statements': [ 282 | { 283 | 'condition': 'is', 284 | 'target': 'val2', 285 | 'then': 'if', 286 | }, 287 | ], 288 | }, 289 | ], 290 | 'separator': '-', 291 | 'if_statements': [ 292 | { 293 | 'condition': 'is', 294 | 'target': 'default-if', 295 | 'then': None, 296 | }, 297 | ], 298 | 'default': 'default2', 299 | }, 300 | ], 301 | 'objects': [ 302 | { 303 | 'name': 'address', 304 | 'array': False, 305 | 'attributes': [ 306 | { 307 | 'name': 'address1', 308 | 'data_fetchers': [ 309 | { 310 | 'path': ['a1'], 311 | }, 312 | ], 313 | }, 314 | { 315 | 'name': 'address2', 316 | 'data_fetchers': [ 317 | { 318 | 'path': ['a2'], 319 | }, 320 | ], 321 | }, 322 | ], 323 | }, 324 | { 325 | 'name': 'people', 326 | 'array': True, 327 | 'iterators': [ 328 | { 329 | 'alias': 'persons', 330 | 'path': ['persons'], 331 | }, 332 | ], 333 | 'attributes': [ 334 | { 335 | 'name': 'firstname', 336 | 'data_fetchers': [ 337 | { 338 | 'path': ['persons', 'name'], 339 | }, 340 | ], 341 | }, 342 | ], 343 | }, 344 | ], 345 | 'branching_objects': [ 346 | { 347 | 'name': 'extrafield', 348 | 'array': True, 349 | 'branching_attributes': [ 350 | [ 351 | { 352 | 'name': 'dataname', 353 | 'default': 'one', 354 | }, 355 | { 356 | 'name': 'datavalue', 357 | 'data_fetchers': [ 358 | { 359 | 'path': ['extra', 'extra1'], 360 | }, 361 | ], 362 | }, 363 | ], 364 | [ 365 | { 366 | 'name': 'dataname', 367 | 'default': 'two', 368 | }, 369 | { 370 | 'name': 'datavalue', 371 | 'data_fetchers': [ 372 | { 373 | 'path': ['extra', 'extra2'], 374 | }, 375 | ], 376 | }, 377 | ], 378 | ], 379 | }, 380 | ], 381 | } 382 | input_data = { 383 | 'key': 'val1', 384 | 'key2': 'val2', 385 | 'a1': 'a1', 386 | 'a2': 'a2', 387 | 'persons': [{'name': 'john'}, {'name': 'bob'}], 388 | 'extra': { 389 | 'extra1': 'extra1val', 390 | 'extra2': 'extra2val', 391 | }, 392 | } 393 | expected_result = { 394 | 'name': 'default2', 395 | 'address': { 396 | 'address1': 'a1', 397 | 'address2': 'a2', 398 | }, 399 | 'people': [ 400 | {'firstname': 'john'}, 401 | {'firstname': 'bob'}, 402 | ], 403 | 'extrafield': [ 404 | {'dataname': 'one', 'datavalue': 'extra1val'}, 405 | {'dataname': 'two', 'datavalue': 'extra2val'}, 406 | ], 407 | } 408 | 409 | assert process_raise( 410 | input_data, 411 | config, 412 | ) == expected_result 413 | 414 | 415 | def test_regex_feature(): # noqa: WPS210 416 | """Test Regexp on the example from the docs.""" 417 | with open('tests/json/config_regex.json', 'r') as config_file: 418 | config = json.load(config_file) 419 | 420 | with open('tests/json/input_regex.json', 'r') as input_file: 421 | input_data = json.load(input_file) 422 | 423 | with open('tests/json/expected_regex.json', 'r') as expected_file: 424 | expected_result = json.load(expected_file) 425 | 426 | assert process_raise( 427 | input_data, 428 | config, 429 | ) == expected_result 430 | -------------------------------------------------------------------------------- /tests/test_mapper.py: -------------------------------------------------------------------------------- 1 | from returns.pipeline import is_successful 2 | 3 | from kaiba.mapper import map_data 4 | from kaiba.models.kaiba_object import KaibaObject 5 | 6 | 7 | def test_creating_key_to_name(): 8 | """Test that we can fetch key in dict.""" 9 | input_data = {'key': 'test name'} 10 | config = KaibaObject( 11 | name='root', 12 | array=False, 13 | attributes=[ 14 | { 15 | 'name': 'name', 16 | 'data_fetchers': [ 17 | { 18 | 'path': ['key'], 19 | }, 20 | ], 21 | }, 22 | ], 23 | ) 24 | 25 | assert map_data( 26 | input_data, 27 | config, 28 | ).unwrap() == {'name': 'test name'} 29 | 30 | 31 | def test_pydantic_makes_float_into_decimal(): 32 | """Test that a json float is turned into Decimal.""" 33 | config = KaibaObject(**{ 34 | 'name': 'root', 35 | 'attributes': [ 36 | { 37 | 'name': 'name', 38 | 'default': 123.123, 39 | }, 40 | ], 41 | }) 42 | 43 | # Decimal has is_finite 44 | assert config.attributes[0].default.is_finite() # type: ignore 45 | 46 | 47 | def test_array_true_but_no_loop_gives_array(): 48 | """Test that we get an array if we set array = true in object.""" 49 | input_data = {'key': 'test name'} 50 | config = KaibaObject(**{ 51 | 'name': 'root', 52 | 'array': True, 53 | 'attributes': [ 54 | { 55 | 'name': 'name', 56 | 'data_fetchers': [ 57 | { 58 | 'path': ['key'], 59 | }, 60 | ], 61 | }, 62 | ], 63 | }) 64 | 65 | assert map_data( 66 | input_data, 67 | config, 68 | ).unwrap() == [{'name': 'test name'}] 69 | 70 | 71 | def test_missing_data_gives_nothing(): 72 | """Test that we get an array if we set array = true in object.""" 73 | input_data = {'key': 'test name'} 74 | config = KaibaObject(**{ 75 | 'name': 'root', 76 | 'array': True, 77 | 'attributes': [ 78 | { 79 | 'name': 'name', 80 | 'data_fetchers': [ 81 | { 82 | 'path': ['missing'], 83 | }, 84 | ], 85 | }, 86 | ], 87 | }) 88 | 89 | assert not is_successful(map_data( 90 | input_data, 91 | config, 92 | )) 93 | 94 | 95 | def test_missing_data_creates_no_object(): 96 | """Test that if an object mapping result is empty we create now 'key'.""" 97 | input_data = {'key': 'test name'} 98 | config = KaibaObject(**{ 99 | 'name': 'root', 100 | 'array': True, 101 | 'attributes': [ 102 | { 103 | 'name': 'an_attribute', 104 | 'default': 'val', 105 | }, 106 | ], 107 | 'objects': [ 108 | { 109 | 'name': 'test', 110 | 'array': False, 111 | 'attributes': [ 112 | { 113 | 'name': 'name', 114 | 'data_fetchers': [ 115 | { 116 | 'path': ['missing'], 117 | }, 118 | ], 119 | }, 120 | ], 121 | }, 122 | ], 123 | }) 124 | 125 | expected_result = [{ 126 | 'an_attribute': 'val', 127 | }] 128 | 129 | assert map_data( 130 | input_data, 131 | config, 132 | ).unwrap() == expected_result 133 | 134 | 135 | def test_double_repeatable(): 136 | """Test that we can map nested repeatable objects.""" 137 | config = KaibaObject(**{ 138 | 'name': 'root', 139 | 'array': True, 140 | 'iterators': [ 141 | { 142 | 'alias': 'journals', 143 | 'path': ['journals'], 144 | }, 145 | ], 146 | 'attributes': [ 147 | { 148 | 'name': 'journal_id', 149 | 'data_fetchers': [ 150 | { 151 | 'path': ['journals', 'journal', 'id'], 152 | }, 153 | ], 154 | }, 155 | ], 156 | 'objects': [ 157 | { 158 | 'name': 'invoices', 159 | 'array': True, 160 | 'iterators': [ 161 | { 162 | 'alias': 'invoices', 163 | 'path': ['journals', 'journal', 'invoices'], 164 | }, 165 | ], 166 | 'attributes': [ 167 | { 168 | 'name': 'amount', 169 | 'data_fetchers': [ 170 | { 171 | 'path': ['invoices', 'amount'], 172 | }, 173 | ], 174 | }, 175 | ], 176 | }, 177 | ], 178 | }) 179 | input_data = { 180 | 'journals': [ 181 | { 182 | 'journal': { 183 | 'id': 1, 184 | 'invoices': [{'amount': 1.1}, {'amount': 1.2}], 185 | }, 186 | }, 187 | { 188 | 'journal': { 189 | 'id': 2, 190 | 'invoices': [{'amount': 1.3}, {'amount': 1.4}], 191 | }, 192 | }, 193 | ], 194 | } 195 | expected_result = [ 196 | { 197 | 'journal_id': 1, 198 | 'invoices': [ 199 | {'amount': 1.1}, 200 | {'amount': 1.2}, 201 | ], 202 | }, 203 | { 204 | 'journal_id': 2, 205 | 'invoices': [ 206 | {'amount': 1.3}, 207 | {'amount': 1.4}, 208 | ], 209 | }, 210 | ] 211 | 212 | assert map_data( 213 | input_data, 214 | config, 215 | ).unwrap() == expected_result 216 | 217 | 218 | def test_mapping_where_data_is_not_found(): 219 | """Test that when we map and don't find data its okay.""" 220 | config = KaibaObject(**{ 221 | 'name': 'root', 222 | 'array': True, 223 | 'iterators': [ 224 | { 225 | 'alias': 'journals', 226 | 'path': ['journals'], 227 | }, 228 | ], 229 | 'attributes': [ 230 | { 231 | 'name': 'journal_id', 232 | 'data_fetchers': [ 233 | { 234 | 'path': ['journals', 'journal', 'id'], 235 | }, 236 | ], 237 | }, 238 | ], 239 | 'objects': [ 240 | { 241 | 'name': 'invoices', 242 | 'array': True, 243 | 'iterators': [ 244 | { 245 | 'alias': 'invoices', 246 | 'path': ['journals', 'journal', 'invoices'], 247 | }, 248 | ], 249 | 'attributes': [ 250 | { 251 | 'name': 'amount', 252 | 'data_fetchers': [ 253 | { 254 | 'path': ['invoices', 'amount'], 255 | }, 256 | ], 257 | }, 258 | ], 259 | }, 260 | ], 261 | 'branching_objects': [ 262 | { 263 | 'name': 'extrafield', 264 | 'array': True, 265 | 'branching_attributes': [ 266 | [ 267 | { 268 | 'name': 'datavalue', 269 | 'data_fetchers': [ 270 | { 271 | 'path': ['extra', 'extra1'], 272 | }, 273 | ], 274 | }, 275 | ], 276 | ], 277 | }, 278 | ], 279 | }) 280 | input_data = { 281 | 'journals': [ 282 | { 283 | 'journal': { 284 | 'id': 1, 285 | 'invoices': [{}, {'amount': 1.2}], 286 | }, 287 | }, 288 | { 289 | 'journal': { 290 | 'id': 2, 291 | }, 292 | }, 293 | ], 294 | } 295 | expected_result = [ 296 | { 297 | 'journal_id': 1, 298 | 'invoices': [ 299 | {'amount': 1.2}, 300 | ], 301 | }, 302 | { 303 | 'journal_id': 2, 304 | 'invoices': [], 305 | }, 306 | ] 307 | 308 | assert map_data( 309 | input_data, 310 | config, 311 | ).unwrap() == expected_result 312 | 313 | 314 | def test_most_features(): 315 | """Test that we can fetch key in dict.""" 316 | config = KaibaObject(**{ 317 | 'name': 'schema', 318 | 'array': False, 319 | 'attributes': [ 320 | { 321 | 'name': 'name', 322 | 'data_fetchers': [ 323 | { 324 | 'path': ['key'], 325 | 'if_statements': [ 326 | { 327 | 'condition': 'is', 328 | 'target': 'val1', 329 | 'then': None, 330 | }, 331 | ], 332 | 'default': 'default', 333 | }, 334 | { 335 | 'path': ['key2'], 336 | 'if_statements': [ 337 | { 338 | 'condition': 'is', 339 | 'target': 'val2', 340 | 'then': 'if', 341 | }, 342 | ], 343 | }, 344 | ], 345 | 'separator': '-', 346 | 'if_statements': [ 347 | { 348 | 'condition': 'is', 349 | 'target': 'default-if', 350 | 'then': None, 351 | }, 352 | ], 353 | 'default': 'default2', 354 | }, 355 | ], 356 | 'objects': [ 357 | { 358 | 'name': 'address', 359 | 'array': False, 360 | 'attributes': [ 361 | { 362 | 'name': 'address1', 363 | 'data_fetchers': [ 364 | { 365 | 'path': ['a1'], 366 | }, 367 | ], 368 | }, 369 | { 370 | 'name': 'address2', 371 | 'data_fetchers': [ 372 | { 373 | 'path': ['a2'], 374 | }, 375 | ], 376 | }, 377 | ], 378 | }, 379 | { 380 | 'name': 'people', 381 | 'array': True, 382 | 'iterators': [ 383 | { 384 | 'alias': 'persons', 385 | 'path': ['persons'], 386 | }, 387 | ], 388 | 'attributes': [ 389 | { 390 | 'name': 'firstname', 391 | 'data_fetchers': [ 392 | { 393 | 'path': ['persons', 'name'], 394 | }, 395 | ], 396 | }, 397 | ], 398 | }, 399 | ], 400 | 'branching_objects': [ 401 | { 402 | 'name': 'extrafield', 403 | 'array': True, 404 | 'branching_attributes': [ 405 | [ 406 | { 407 | 'name': 'dataname', 408 | 'default': 'one', 409 | }, 410 | { 411 | 'name': 'datavalue', 412 | 'data_fetchers': [ 413 | { 414 | 'path': ['extra', 'extra1'], 415 | }, 416 | ], 417 | }, 418 | ], 419 | [ 420 | { 421 | 'name': 'dataname', 422 | 'default': 'two', 423 | }, 424 | { 425 | 'name': 'datavalue', 426 | 'data_fetchers': [ 427 | { 428 | 'path': ['extra', 'extra2'], 429 | }, 430 | ], 431 | }, 432 | ], 433 | ], 434 | }, 435 | ], 436 | }) 437 | input_data = { 438 | 'key': 'val1', 439 | 'key2': 'val2', 440 | 'a1': 'a1', 441 | 'a2': 'a2', 442 | 'persons': [{'name': 'john'}, {'name': 'bob'}], 443 | 'extra': { 444 | 'extra1': 'extra1val', 445 | 'extra2': 'extra2val', 446 | }, 447 | } 448 | expected_result = { 449 | 'name': 'default2', 450 | 'address': { 451 | 'address1': 'a1', 452 | 'address2': 'a2', 453 | }, 454 | 'people': [ 455 | {'firstname': 'john'}, 456 | {'firstname': 'bob'}, 457 | ], 458 | 'extrafield': [ 459 | {'dataname': 'one', 'datavalue': 'extra1val'}, 460 | {'dataname': 'two', 'datavalue': 'extra2val'}, 461 | ], 462 | } 463 | 464 | assert map_data( 465 | input_data, 466 | config, 467 | ).unwrap() == expected_result 468 | --------------------------------------------------------------------------------