├── .coveragerc ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── CONTRIBUTING.md ├── CONTRIBUTORS ├── LICENSE ├── MANIFEST.in ├── README.md ├── data_schema ├── __init__.py ├── apps.py ├── convert_value.py ├── docs │ └── release_notes.rst ├── exceptions.py ├── field_schema_type.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_auto_20150705_1714.py │ ├── 0003_auto_20151013_1556.py │ ├── 0004_fieldschema_case.py │ ├── 0005_auto_20180302_2356.py │ ├── 0006_auto_20220818_1425.py │ ├── 0007_auto_20230418_2042.py │ └── __init__.py ├── models.py ├── tests │ ├── __init__.py │ ├── convert_value_tests.py │ ├── model_tests.py │ └── tests.py └── version.py ├── manage.py ├── publish.py ├── requirements ├── requirements-testing.txt └── requirements.txt ├── run_tests.py ├── settings.py ├── setup.cfg └── setup.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = 4 | */migrations/* 5 | data_schema/version.py 6 | source = data_schema 7 | [report] 8 | exclude_lines = 9 | # Have to re-enable the standard pragma 10 | pragma: no cover 11 | 12 | # Don't complain if tests don't hit defensive assertion code: 13 | raise NotImplementedError 14 | fail_under = 100 15 | show_missing = 1 16 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # copied from django-cte 2 | name: data_schema tests 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master,develop] 8 | 9 | jobs: 10 | tests: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python: ['3.7', '3.8', '3.9'] 16 | # Time to switch to pytest or nose2?? 17 | # nosetests is broken on 3.10 18 | # AttributeError: module 'collections' has no attribute 'Callable' 19 | # https://github.com/nose-devs/nose/issues/1099 20 | django: 21 | - 'Django~=3.2.0' 22 | - 'Django~=4.0.0' 23 | - 'Django~=4.1.0' 24 | - 'Django~=4.2.0' 25 | experimental: [false] 26 | # include: 27 | # - python: '3.9' 28 | # django: 'https://github.com/django/django/archive/refs/heads/main.zip#egg=Django' 29 | # experimental: true 30 | # # NOTE this job will appear to pass even when it fails because of 31 | # # `continue-on-error: true`. Github Actions apparently does not 32 | # # have this feature, similar to Travis' allow-failure, yet. 33 | # # https://github.com/actions/toolkit/issues/399 34 | exclude: 35 | - python: '3.7' 36 | django: 'Django~=4.0.0' 37 | - python: '3.7' 38 | django: 'Django~=4.1.0' 39 | - python: '3.7' 40 | django: 'Django~=4.2.0' 41 | services: 42 | postgres: 43 | image: postgres:latest 44 | env: 45 | POSTGRES_DB: postgres 46 | POSTGRES_PASSWORD: postgres 47 | POSTGRES_USER: postgres 48 | ports: 49 | - 5432:5432 50 | options: >- 51 | --health-cmd pg_isready 52 | --health-interval 10s 53 | --health-timeout 5s 54 | --health-retries 5 55 | steps: 56 | - uses: actions/checkout@v2 57 | - uses: actions/setup-python@v2 58 | with: 59 | python-version: ${{ matrix.python }} 60 | - name: Setup 61 | run: | 62 | python --version 63 | pip install --upgrade pip wheel 64 | pip install -r requirements/requirements.txt 65 | pip install -r requirements/requirements-testing.txt 66 | pip install "${{ matrix.django }}" 67 | pip freeze 68 | - name: Run tests 69 | env: 70 | DB_SETTINGS: >- 71 | { 72 | "ENGINE":"django.db.backends.postgresql_psycopg2", 73 | "NAME":"data_schema", 74 | "USER":"postgres", 75 | "PASSWORD":"postgres", 76 | "HOST":"localhost", 77 | "PORT":"5432" 78 | } 79 | run: | 80 | coverage run manage.py test data_schema 81 | coverage report --fail-under=99 82 | continue-on-error: ${{ matrix.experimental }} 83 | - name: Check style 84 | run: flake8 data_schema 85 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python files 2 | *.pyc 3 | 4 | # Vim files 5 | *.swp 6 | *.swo 7 | 8 | # Coverage files 9 | .coverage 10 | htmlcov/ 11 | 12 | # Setuptools distribution folder. 13 | /dist/ 14 | 15 | # Python egg metadata, regenerated from source files by setuptools. 16 | /*.egg-info 17 | /*.egg 18 | .eggs/ 19 | 20 | # Virtualenv 21 | env/ 22 | venv/ 23 | 24 | # OSX 25 | .DS_Store 26 | 27 | # pycharm 28 | .idea 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | Contributions and issues are most welcome! All issues and pull requests are 3 | handled through GitHub on the 4 | [ambitioninc repository](https://github.com/ambitioninc/django-data-schema/issues). 5 | Also, please check for any existing issues before filing a new one. If you have 6 | a great idea but it involves big changes, please file a ticket before making a 7 | pull request! We want to make sure you don't spend your time coding something 8 | that might not fit the scope of the project. 9 | 10 | ## Running the tests 11 | 12 | To get the source source code and run the unit tests, run: 13 | ```bash 14 | git clone git://github.com/ambitioninc/django-data-schema.git 15 | cd django-data-schema 16 | virtualenv env 17 | . env/bin/activate 18 | python setup.py install 19 | coverage run setup.py test 20 | coverage report --fail-under=100 21 | ``` 22 | 23 | While 100% code coverage does not make a library bug-free, it significantly 24 | reduces the number of easily caught bugs! Please make sure coverage is at 100% 25 | before submitting a pull request! 26 | 27 | ## Code Quality 28 | 29 | For code quality, please run flake8: 30 | ```bash 31 | pip install flake8 32 | flake8 . 33 | ``` 34 | 35 | ## Code Styling 36 | Please arrange imports with the following style 37 | 38 | ```python 39 | # Standard library imports 40 | import os 41 | 42 | # Third party package imports 43 | from unittest.mock import patch 44 | from django.conf import settings 45 | 46 | # Local package imports 47 | from data_schema.version import __version__ 48 | ``` 49 | 50 | Please follow 51 | [Google's python style](http://google-styleguide.googlecode.com/svn/trunk/pyguide.html) 52 | guide wherever possible. 53 | 54 | 55 | 56 | ## Release Checklist 57 | 58 | Before a new release, please go through the following checklist: 59 | 60 | * Bump version in data_schema/version.py 61 | * Git tag the version 62 | * Upload to pypi: 63 | ```bash 64 | pip install wheel 65 | python setup.py sdist bdist_wheel upload 66 | ``` 67 | 68 | ## Vulnerability Reporting 69 | 70 | For any security issues, please do NOT file an issue or pull request on GitHub! 71 | Please contact [security@ambition.com](mailto:security@ambition.com) with the 72 | GPG key provided on [Ambition's website](http://ambition.com/security/). 73 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | Wes Kendall @wesleykendall wes.kendall@ambition.com (primary) 2 | Wes Okes @wesokes wes.okes@gmail.com 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Ambition 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | recursive-include requirements * 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/ambitioninc/django-data-schema.png)](https://travis-ci.org/ambitioninc/django-data-schema) 2 | 3 | # Django Data Schema 4 | Django data schema is a lightweight Django app for defining the schema for a model, dictionary, or list. 5 | By describing a schema on a piece of data, this allows other applications to easily reference 6 | fields of models or fields in dictionaries (or their related json fields). 7 | 8 | Django data schema also takes care of all conversions under the hood, such as parsing datetime strings, converting strings to numeric values, using default values when values don't exist, and so on. 9 | 10 | 1. [Installation](#installation) 11 | 2. [Model Overview](#model-overview) 12 | 3. [Examples](#examples) 13 | 14 | ## Installation 15 | 16 | ```python 17 | pip install django-data-schema 18 | ``` 19 | 20 | ## Model Overview 21 | Django data schema defines three models for building schemas on data. These models are ``DataSchema``, 22 | ``FieldSchema``, and ``FieldOptional``. 23 | 24 | The ``DataSchema`` model provides a ``model_content_type`` field that points to a Django ``ContentType`` model. 25 | This field represents which object this schema is modeling. If the field is None, it is assumed that 26 | this schema models an object such as a dictionary or list. 27 | 28 | After the enclosing ``DataSchema`` has been defined, various ``FieldSchema`` models can reference the main 29 | data schema. ``FieldSchema`` models provide the following attributes: 30 | 31 | - ``field_key``: The name of the field. Used to identify a field in a dictionary or model. 32 | - ``field_position``: The position of the field. Used to identify a field in a list. 33 | - ``uniqueness_order``: The order of this field in the uniqueness constraint of the schema. Defaults to None. 34 | - ``field_type``: The type of field. More on the field types below. 35 | - ``field_format``: An optional formatting string for the field. Used differently depending on the field type and documented more below. 36 | - ``default_value``: If the field returns None, this default value will be returned instead. 37 | 38 | A ``FieldSchema`` object must specify its data type. While data of a given type can be stored in different formats, 39 | django-data-schema normalizes the data when accessing it through ``get_value``, described below. The available 40 | types are listed in the ``FieldSchemaType`` class. These types are listed here, with the type they normalize to: 41 | 42 | - ``FieldSchemaType.DATE``: A python ``date`` object from the ``datetime`` module. Currently returned as a ``datetime`` object. 43 | - ``FieldSchemaType.DATETIME``: A python ``datetime`` object from the ``datetime`` module. 44 | - ``FieldSchemaType.INT``: A python ``int``. 45 | - ``FieldSchemaType.FLOAT``: A python ``float``. 46 | - ``FieldSchemaType.STRING``: A python ``str``. 47 | - ``FieldSchemaType.BOOLEAN``: A python ``bool``. 48 | 49 | These fields provide the necessary conversion mechanisms when accessing data via ``FieldSchema.get_value``. Differences in how the ``get_value`` function operates are detailed below. 50 | 51 | ### Using get_value on DATE or DATETIME fields 52 | The ``get_value`` function has the following behavior on DATE and DATETIME fields: 53 | 54 | - If called on a Python ``int`` or ``float`` value, the numeric value will be passed to the ``datetime.utcfromtimestamp`` function. 55 | - If called on a ``string`` or ``unicode`` value, the string will be stripped of all trailing and leading whitespace. If the string is empty, the default value (or None) will be used. If the string is not empty, it will be passed to dateutil's ``parse`` function. If the ``field_format`` field is specified on the ``FieldSchema`` object, it will be passed to the ``strptime`` function instead. 56 | - If called on an aware datetime object (or a string with a timezone), it will be converted to naive UTC time. 57 | - If called on None, the default value (or None) is returned. 58 | 59 | ### Using get_value on INT or FLOAT fields 60 | The ``get_value`` function has the following behavior on INT and FLOAT fields: 61 | 62 | - If called on a ``string`` or ``unicode`` value, the string will be stripped of all non-numeric numbers except for periods. If the string is blank, the default value (or None) will be returned. If not, the string will then be passed to ``int()`` or ``float()``. 63 | - If called on an ``int`` or ``float``, the value will be passed to the ``int()`` or ``float()`` function. 64 | - No other values can be converted. The ``field_format`` parameter is ignored. 65 | - If called on None, the default value (or None) is returned. 66 | 67 | ### Using get_value on a STRING field 68 | The ``get_value`` function has the following behavior on a STRING field: 69 | 70 | - If called on a ``string`` or ``unicode`` value, the string will be stripped of all trailing and leading whitespace. If a ``field_format`` is specified, the string is then be matched to the regex. If it passes, the string is returned. If not, None is returned and the default value is used (or None). 71 | - All other types are passed to the ``str()`` function. 72 | - If called on None, the default value (or None) is returned. 73 | 74 | ### Using get_value on a BOOLEAN field 75 | The ``get_value`` function has the following behavior on a BOOLEAN field: 76 | 77 | - Bool data types will return True or False 78 | - Truthy looking string values return True ('t', 'T', 'true', 'True', 'TRUE', 1, '1') 79 | - Falsy looking string values return False ('f', 'F', 'false', 'False', 'FALSE', 0, '0') 80 | - If called on None, the default value (or None) is returned. 81 | 82 | ## Examples 83 | 84 | A data schema can be created like the following: 85 | 86 | ```python 87 | from data_schema import DataSchema, FieldSchema, FieldSchemaType 88 | 89 | user_login_schema = DataSchema.objects.create() 90 | user_id_field = FieldSchema.objects.create( 91 | data_schema=user_login_schema, field_key='user_id', uniqueness_order=1, field_type=FieldSchemaType.STRING) 92 | login_time_field = FieldSchema.objects.create( 93 | data_schema=user_login_schema, field_key='login_time', field_type=FieldSchemaType.DATETIME) 94 | ``` 95 | 96 | The above example represents the schema of a user login. In this schema, the user id field provides the uniqueness 97 | constraint of the data. The uniqueness constraint can then easily be accessed by simply doing the following. 98 | 99 | ```python 100 | unique_fields = user_login_schema.get_unique_fields() 101 | ``` 102 | 103 | The above function returns the unique fields in the order in which they were specified, allowing the user to 104 | generate a unique ID for the data. 105 | 106 | To obtain values of data using the schema, one can use the ``get_value`` function as follows: 107 | 108 | ```python 109 | data = { 110 | 'user_id': 'my_user_id', 111 | 'login_time': 1396396800, 112 | } 113 | 114 | print login_time_field.get_value(data) 115 | 2014-04-02 00:00:00 116 | ``` 117 | 118 | Note that the ``get_value`` function looks at the type of data object and uses the proper access method. If the 119 | data object is a ``dict``, it accesses it using ``data[field_key]``. If it is an object, it accesses it with 120 | ``getattr(data, field_key)``. An array is accessed as ``data[field_position]``. 121 | 122 | Here's another example of parsing datetime objects in an array with a format string. 123 | 124 | ```python 125 | string_time_field_schema = FieldSchema.objects.create( 126 | data_schema=data_schema, field_key='time', field_position=1, field_type=FieldSchemaType.DATETIME, field_format='%Y-%m-%d %H:%M:%S') 127 | 128 | print string_time_field_schema.get_value(['value', '2013-04-12 12:12:12']) 129 | 2013-04-12 12:12:12 130 | ``` 131 | 132 | Note that if you are parsing numerical fields, Django data schema will strip out any non-numerical values, allowing the user to get values of currency-based numbers and other formats. 133 | 134 | ```python 135 | revenue_field_schema = FieldSchema.objects.create( 136 | data_schema=data_schema, field_key='revenue', field_type=FieldSchemaType.FLOAT) 137 | 138 | print revenue_field_schema.get_value({'revenue': '$15,000,456.23'}) 139 | 15000456.23 140 | ``` 141 | 142 | Note that ``FieldSchema`` objects have an analogous ``set_value`` method for setting the value of a field. 143 | The ``set_value`` method does not do any data conversions, so when calling this method, be sure to use a value 144 | that is in the correct format. 145 | -------------------------------------------------------------------------------- /data_schema/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .version import __version__ 3 | 4 | -------------------------------------------------------------------------------- /data_schema/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class DataSchemaConfig(AppConfig): 5 | name = 'data_schema' 6 | verbose_name = "Django Data Schema" 7 | -------------------------------------------------------------------------------- /data_schema/convert_value.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functions for handling conversions of values from one type to another. 3 | """ 4 | from datetime import datetime 5 | import re 6 | 7 | from dateutil.parser import parse 8 | import fleming 9 | import pytz 10 | 11 | from data_schema.field_schema_type import FieldSchemaType, FieldSchemaCase 12 | from data_schema.exceptions import InvalidDateFormatException 13 | 14 | 15 | class ValueConverter(object): 16 | """ 17 | A generic value converter. 18 | """ 19 | NUMBER_TYPES = (float, complex, int) 20 | 21 | def __init__(self, field_schema_type, python_type): 22 | # Set the FieldSchemaType value 23 | self._field_schema_type = field_schema_type 24 | 25 | # Set the python type to convert the value to 26 | self._python_type = python_type 27 | 28 | def is_string(self, value): 29 | """ 30 | Returns True if the value is a string. 31 | """ 32 | return isinstance(value, str) 33 | 34 | def is_numeric(self, value): 35 | """ 36 | Returns True if the value is numeric. 37 | """ 38 | return isinstance(value, self.NUMBER_TYPES) 39 | 40 | def _preprocess_value(self, value, format_str, transform_case=None): 41 | """ 42 | Preprocesses the value depending on the field schema type. 43 | """ 44 | # Strip all strings by default 45 | if self.is_string(value): 46 | value = value.strip() 47 | 48 | # Convert blank strings to None if the schema type isn't a string 49 | if self._field_schema_type != FieldSchemaType.STRING and not value: 50 | value = None 51 | 52 | return value 53 | 54 | def _convert_value(self, value, format_str): 55 | """ 56 | Converts the value into the format of the field schema type. 57 | """ 58 | return self._python_type(value) 59 | 60 | def __call__(self, value, format_str, default_value, transform_case=None): 61 | """ 62 | Converts a provided value to the configured python type. 63 | """ 64 | value = self._preprocess_value(value, format_str, transform_case=transform_case) 65 | 66 | # Set the value to the default if it is None and there is a default 67 | value = default_value if value is None and default_value is not None else value 68 | 69 | try: 70 | # Convert the value if it isn't None 71 | return self._convert_value(value, format_str) if value is not None else None 72 | except Exception as e: 73 | # Attach additional information to the exception to make higher level error handling easier 74 | e.bad_value = value 75 | e.expected_type = self._field_schema_type 76 | raise e 77 | 78 | 79 | class BooleanConverter(ValueConverter): 80 | """ 81 | Converts strings to a boolean value or None 82 | """ 83 | TRUE_VALUES = frozenset(('t', 'T', 'true', 'True', 'TRUE', True, 1, '1',)) 84 | FALSE_VALUES = frozenset(('f', 'F', 'false', 'False', 'FALSE', False, 0, '0',)) 85 | 86 | def _convert_value(self, value, format_str): 87 | if value in self.TRUE_VALUES: 88 | return True 89 | elif value in self.FALSE_VALUES: 90 | return False 91 | return None 92 | 93 | 94 | class NumericConverter(ValueConverter): 95 | """ 96 | Converts numeric values (floats and ints). 97 | """ 98 | # A compiled regex for extracting non-numeric characters 99 | NON_NUMERIC_REGEX = re.compile(r'[^\d\.\-eE]+') 100 | 101 | def _preprocess_value(self, value, format_str, transform_case=None): 102 | """ 103 | Strips out any non-numeric characters for numeric values if they are a string. 104 | """ 105 | value = super(NumericConverter, self)._preprocess_value(value, format_str, transform_case=transform_case) 106 | if self.is_string(value): 107 | value = self.NON_NUMERIC_REGEX.sub('', value) or None 108 | 109 | return value 110 | 111 | def _convert_value(self, value, format_str): 112 | converted_value = super()._convert_value(value, format_str) 113 | 114 | if converted_value in [float('inf'), float('-inf')]: 115 | raise ValueError('inf not a valid value for numeric data') 116 | 117 | return converted_value 118 | 119 | 120 | class DurationConverter(ValueConverter): 121 | """ 122 | Converts durations from [hh]:mm:ss format to integer number of seconds 123 | """ 124 | TIME_FORMAT_DURATION_REGEXP = re.compile(r'^\d{1,2}:\d{1,2}(:\d{1,2})?$') 125 | 126 | def __call__(self, value, format_str, default_value, transform_case=None): 127 | if self.is_string(value) and self.TIME_FORMAT_DURATION_REGEXP.match(value) is not None: 128 | return super(DurationConverter, self).__call__(value, format_str, default_value, transform_case) 129 | return NumericConverter(FieldSchemaType.INT, int)(value, format_str, default_value, transform_case) 130 | 131 | def _convert_value(self, value, format_str): 132 | duration_constituents = value.split(':') 133 | value = int(duration_constituents[-2]) * 60 + int(duration_constituents[-1]) 134 | if len(duration_constituents) == 3: 135 | value += int(duration_constituents[0]) * 3600 136 | return value 137 | 138 | 139 | class DatetimeConverter(ValueConverter): 140 | """ 141 | Converts datetime values (date and datetime). 142 | """ 143 | def _convert_value(self, value, format_str): 144 | """ 145 | Formats datetimes based on the input type. If the input is a string, uses strptime and the 146 | format string specified or dateutil.parse if no format is specified. If the input is numeric, 147 | it assumes it is a unix timestamp. This function also takes care of converting any 148 | aware datetimes to naive UTC. 149 | """ 150 | try: 151 | value = datetime.utcfromtimestamp(float(value)) 152 | except Exception: 153 | pass 154 | if self.is_string(value): 155 | value = datetime.strptime(value, format_str) if format_str else parse(value) 156 | 157 | # It is assumed that value is a datetime here. If it isn't a datetime, then it is a bad value like 158 | # a number that is too large to be parsed as an integer 159 | if type(value) != datetime: 160 | raise InvalidDateFormatException(f'Invalid date format: {value}') 161 | 162 | # Convert any aware datetime objects to naive utc 163 | return value if value.tzinfo is None else fleming.convert_to_tz(value, pytz.utc, return_naive=True) 164 | 165 | 166 | class DateFlooredConverter(DatetimeConverter): 167 | """ 168 | Floors datetime values (date and datetime) to date 169 | """ 170 | def _convert_value(self, value, format_str): 171 | value = super(DateFlooredConverter, self)._convert_value(value, format_str) 172 | return fleming.floor(value, day=1) 173 | 174 | 175 | class StringConverter(ValueConverter): 176 | """ 177 | Converts string values. 178 | """ 179 | def _preprocess_value(self, value, format_str, transform_case=None): 180 | """ 181 | Performs additional regex matching for any provided format string. If the value 182 | does not match the format string, None is returned. 183 | """ 184 | value = super(StringConverter, self)._preprocess_value(value, format_str, transform_case=transform_case) 185 | if self.is_string(value) and format_str: 186 | value = value if re.match(format_str, value) else None 187 | 188 | if value and transform_case: 189 | if transform_case == FieldSchemaCase.LOWER: 190 | value = value.lower() 191 | else: 192 | value = value.upper() 193 | 194 | return value 195 | 196 | 197 | # Create a mapping of the field schema types to their associated converters 198 | FIELD_SCHEMA_CONVERTERS = { 199 | FieldSchemaType.DATE: DatetimeConverter(FieldSchemaType.DATE, datetime), 200 | FieldSchemaType.DATETIME: DatetimeConverter(FieldSchemaType.DATETIME, datetime), 201 | FieldSchemaType.DATE_FLOORED: DateFlooredConverter(FieldSchemaType.DATE_FLOORED, datetime), 202 | FieldSchemaType.INT: NumericConverter(FieldSchemaType.INT, int), 203 | FieldSchemaType.FLOAT: NumericConverter(FieldSchemaType.FLOAT, float), 204 | FieldSchemaType.STRING: StringConverter(FieldSchemaType.STRING, str), 205 | FieldSchemaType.BOOLEAN: BooleanConverter(FieldSchemaType.BOOLEAN, bool), 206 | FieldSchemaType.DURATION: DurationConverter(FieldSchemaType.DURATION, int), 207 | } 208 | 209 | 210 | def convert_value(field_schema_type, value, format_str=None, default_value=None, transform_case=None): 211 | """ 212 | Converts a value to a type with an optional format string. 213 | """ 214 | return FIELD_SCHEMA_CONVERTERS[field_schema_type](value, format_str, default_value, transform_case) 215 | -------------------------------------------------------------------------------- /data_schema/docs/release_notes.rst: -------------------------------------------------------------------------------- 1 | Release Notes 2 | 3 | v2.1.0 4 | ------ 5 | * Drop django 2 6 | * Add django 4.2 7 | 8 | v2.0.1 9 | ------ 10 | * Add blank=True to null model fields 11 | 12 | v2.0.0 13 | ------ 14 | * drop python 3.6 15 | * python 3.8, 3.9 16 | * django 3.0, 3.1, 3.2, 4.0, 4.1 17 | 18 | v1.5.0 19 | ------ 20 | * Update display_name field on FieldSchema to no longer have a char limit. 21 | 22 | v1.4.0 23 | ------ 24 | * Raise InvalidDateFormatException when trying to parse a timestamp that is too large. 25 | * remove django 2.0 26 | * remove django 2.1 27 | 28 | v1.3.0 29 | ------ 30 | * INF, -INF no longer allowed as numeric field values 31 | 32 | v1.2.0 33 | ------ 34 | * Python 3.7 35 | * Django 2.1 36 | * Django 2.2 37 | 38 | v1.1.0 39 | ------ 40 | * Add tox to support more versions 41 | 42 | v1.0.1 43 | ------ 44 | * Lengthened size of field key 45 | 46 | v1.0.0 47 | ------ 48 | * Remove python 2.7 support 49 | * Remove python 3.4 support 50 | * Remove Django 1.9 support 51 | * Remove Django 1.10 support 52 | * Add Django 2.0 support 53 | 54 | v0.15.0 55 | ------ 56 | * Add python 3.6 support 57 | * Drop Django 1.8 support 58 | * Add Django 1.10 support 59 | * Add Django 1.11 support 60 | -------------------------------------------------------------------------------- /data_schema/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class InvalidDateFormatException(Exception): 4 | pass 5 | -------------------------------------------------------------------------------- /data_schema/field_schema_type.py: -------------------------------------------------------------------------------- 1 | class FieldSchemaType(object): 2 | """ 3 | Specifies all of the field schema types supported. 4 | """ 5 | DATE = 'DATE' 6 | DATETIME = 'DATETIME' 7 | DATE_FLOORED = 'DATE_FLOORED' 8 | INT = 'INT' 9 | FLOAT = 'FLOAT' 10 | STRING = 'STRING' 11 | BOOLEAN = 'BOOLEAN' 12 | DURATION = 'DURATION' 13 | 14 | @classmethod 15 | def choices(cls): 16 | """ 17 | An alphabetical list of the types. This must be alphabetical for the 18 | database default on the Field Schema model's field_type field 19 | """ 20 | def is_internal(x): 21 | return x.startswith('__') and x.endswith('__') 22 | 23 | types = [(val, val) for val in cls.__dict__ if not is_internal(val) and val != 'choices'] 24 | 25 | types.sort() 26 | return types 27 | 28 | 29 | class FieldSchemaCase(object): 30 | LOWER = 'LOWER' 31 | UPPER = 'UPPER' 32 | -------------------------------------------------------------------------------- /data_schema/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django.db import models, migrations 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('contenttypes', '0001_initial'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='DataSchema', 16 | fields=[ 17 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 18 | ('model_content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, default=None, to='contenttypes.ContentType', null=True)), 19 | ], 20 | options={ 21 | }, 22 | bases=(models.Model,), 23 | ), 24 | migrations.CreateModel( 25 | name='FieldOption', 26 | fields=[ 27 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 28 | ('value', models.CharField(max_length=128)), 29 | ], 30 | options={ 31 | }, 32 | bases=(models.Model,), 33 | ), 34 | migrations.CreateModel( 35 | name='FieldSchema', 36 | fields=[ 37 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 38 | ('field_key', models.CharField(max_length=64)), 39 | ('display_name', models.CharField(default=None, max_length=64, null=True)), 40 | ('uniqueness_order', models.IntegerField(null=True)), 41 | ('field_position', models.IntegerField(null=True)), 42 | ('field_format', models.CharField(default=None, max_length=64, null=True, blank=True)), 43 | ('default_value', models.CharField(default=None, max_length=128, null=True, blank=True)), 44 | ('field_type', models.CharField(choices=[('BOOLEAN', 'BOOLEAN'), ('DATE', 'DATE'), ('DATETIME', 'DATETIME'), ('FLOAT', 'FLOAT'), ('INT', 'INT'), ('STRING', 'STRING')], max_length=32)), 45 | ('has_options', models.BooleanField(default=False)), 46 | ('data_schema', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data_schema.DataSchema')), 47 | ], 48 | options={ 49 | }, 50 | bases=(models.Model,), 51 | ), 52 | migrations.AlterUniqueTogether( 53 | name='fieldschema', 54 | unique_together=set([('data_schema', 'field_key')]), 55 | ), 56 | migrations.AddField( 57 | model_name='fieldoption', 58 | name='field_schema', 59 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data_schema.FieldSchema'), 60 | preserve_default=True, 61 | ), 62 | ] 63 | -------------------------------------------------------------------------------- /data_schema/migrations/0002_auto_20150705_1714.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django.db import models, migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('data_schema', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='fieldschema', 15 | name='display_name', 16 | field=models.CharField(default=b'', max_length=64, blank=True), 17 | ), 18 | migrations.AlterUniqueTogether( 19 | name='fieldoption', 20 | unique_together=set([('field_schema', 'value')]), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /data_schema/migrations/0003_auto_20151013_1556.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django.db import models, migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('data_schema', '0002_auto_20150705_1714'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='fieldoption', 15 | name='value', 16 | field=models.CharField(max_length=1024), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /data_schema/migrations/0004_fieldschema_case.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.2 on 2016-07-13 20:42 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('data_schema', '0003_auto_20151013_1556'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='fieldschema', 16 | name='transform_case', 17 | field=models.CharField(blank=True, default=None, max_length=5, null=True), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /data_schema/migrations/0005_auto_20180302_2356.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.10 on 2018-03-02 23:56 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('data_schema', '0004_fieldschema_case'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='fieldschema', 16 | name='display_name', 17 | field=models.CharField(blank=True, default='', max_length=64), 18 | ), 19 | migrations.AlterField( 20 | model_name='fieldschema', 21 | name='field_key', 22 | field=models.CharField(max_length=255), 23 | ), 24 | migrations.AlterField( 25 | model_name='fieldschema', 26 | name='field_type', 27 | field=models.CharField(choices=[('BOOLEAN', 'BOOLEAN'), ('DATE', 'DATE'), ('DATETIME', 'DATETIME'), ('DATE_FLOORED', 'DATE_FLOORED'), ('DURATION', 'DURATION'), ('FLOAT', 'FLOAT'), ('INT', 'INT'), ('STRING', 'STRING')], max_length=32), 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /data_schema/migrations/0006_auto_20220818_1425.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.28 on 2022-08-18 14:25 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('data_schema', '0005_auto_20180302_2356'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='fieldschema', 15 | name='display_name', 16 | field=models.TextField(blank=True, default=''), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /data_schema/migrations/0007_auto_20230418_2042.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.18 on 2023-04-18 20:42 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('contenttypes', '0002_remove_content_type_name'), 11 | ('data_schema', '0006_auto_20220818_1425'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='dataschema', 17 | name='model_content_type', 18 | field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype'), 19 | ), 20 | migrations.AlterField( 21 | model_name='fieldschema', 22 | name='field_position', 23 | field=models.IntegerField(blank=True, null=True), 24 | ), 25 | migrations.AlterField( 26 | model_name='fieldschema', 27 | name='uniqueness_order', 28 | field=models.IntegerField(blank=True, null=True), 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /data_schema/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ambitioninc/django-data-schema/9afe963828cbf2e45d42d1e7c258e5c93a6ffc69/data_schema/migrations/__init__.py -------------------------------------------------------------------------------- /data_schema/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.contenttypes.models import ContentType 2 | from django.db import models, transaction 3 | from manager_utils import ManagerUtilsManager, sync 4 | 5 | from data_schema.convert_value import convert_value 6 | from data_schema.field_schema_type import FieldSchemaType 7 | 8 | 9 | class DataSchemaManager(ManagerUtilsManager): 10 | """ 11 | A model manager for data schemas. Caches related attributes of data schemas. 12 | """ 13 | def get_queryset(self): 14 | return super(DataSchemaManager, self).get_queryset().select_related( 15 | 'model_content_type').prefetch_related('fieldschema_set') 16 | 17 | 18 | class DataSchema(models.Model): 19 | """Define a schema information about a unit of data, such as a 20 | dictionary, list or model. 21 | 22 | ``FieldSchema`` objects are created referencing a ``DataSchema`` 23 | to describe individual fields within the unit of data. 24 | """ 25 | # The content type of the django model for which this schema is related. If None, this schema is 26 | # for a dictionary of data. 27 | model_content_type = models.ForeignKey(ContentType, null=True, blank=True, default=None, on_delete=models.CASCADE) 28 | 29 | # A custom model manager that caches objects 30 | objects = DataSchemaManager() 31 | 32 | @transaction.atomic 33 | def update(self, **updates): 34 | """ 35 | Takes a template of updates and updates the data schema. The following are possible keyword 36 | arguments. 37 | 38 | 'model_content_type': The string path of a model that this schema represents (or None), 39 | 'fieldschema_set': [{ 40 | 'field_key': The field key of the field, 41 | 'display_name': The display name for the field (or the 'field_key' by default), 42 | 'field_type': The FieldSchemaType type of the field, 43 | 'uniqueness_order': The order this field is in the uniquessness constraint (or None by default), 44 | 'field_position': The position of this field if it can be parsed by an array (or None by default), 45 | 'field_format': The format of this field (or None by default), 46 | 'default_value': The default value of this field (or None by default), 47 | 'fieldoption_set': The set of options for the field schema (optional), 48 | }, { 49 | Additional field schemas... 50 | }] 51 | """ 52 | if 'model_content_type' in updates: 53 | self.model_content_type = updates['model_content_type'] 54 | 55 | self.save() 56 | 57 | if 'fieldschema_set' in updates: 58 | # Sync the field schema models 59 | field_schemas = [ 60 | FieldSchema( 61 | data_schema=self, 62 | field_key=fs_values['field_key'], 63 | display_name=fs_values.get('display_name', ''), 64 | field_type=fs_values['field_type'], 65 | uniqueness_order=fs_values.get('uniqueness_order', None), 66 | field_position=fs_values.get('field_position', None), 67 | field_format=fs_values.get('field_format', None), 68 | default_value=fs_values.get('default_value', None), 69 | has_options=bool(('fieldoption_set' in fs_values) and fs_values['fieldoption_set']), 70 | transform_case=fs_values.get('transform_case', None), 71 | ) 72 | for fs_values in updates['fieldschema_set'] 73 | ] 74 | 75 | sync( 76 | self.fieldschema_set.all(), 77 | field_schemas, 78 | ['field_key'], 79 | [ 80 | 'display_name', 'field_key', 'field_type', 'uniqueness_order', 'field_position', 81 | 'field_format', 'default_value', 'has_options', 'transform_case' 82 | ] 83 | ) 84 | 85 | # Sync the options of the field schema models if they are present 86 | for fs_values in updates['fieldschema_set']: 87 | if 'fieldoption_set' in fs_values: 88 | fs = self.fieldschema_set.get(field_key=fs_values['field_key']) 89 | sync(fs.fieldoption_set.all(), [ 90 | FieldOption( 91 | field_schema=fs, 92 | value=f_option, 93 | ) 94 | for f_option in fs_values['fieldoption_set'] 95 | ], ['value'], ['value']) 96 | 97 | def get_unique_fields(self): 98 | """ 99 | Gets all of the fields that create the uniqueness constraint for a metric record. 100 | """ 101 | if not hasattr(self, '_unique_fields'): 102 | # Instead of querying the reverse relationship directly, assume that it has been cached 103 | # with prefetch_related and go through all fields. 104 | setattr(self, '_unique_fields', [ 105 | field for field in self.fieldschema_set.all() if field.uniqueness_order is not None 106 | ]) 107 | self._unique_fields.sort(key=lambda k: k.uniqueness_order or 0) 108 | return self._unique_fields 109 | 110 | def get_fields(self): 111 | """ 112 | Gets all fields in the schema. Note - dont use django's order_by since we are caching the fieldschema_set 113 | beforehand. 114 | """ 115 | return sorted(self.fieldschema_set.all(), key=lambda k: k.field_position or 0) 116 | 117 | def _get_field_map(self): 118 | """ 119 | Returns a cached mapping of field keys to their field schemas. 120 | """ 121 | if not hasattr(self, '_field_map'): 122 | self._field_map = { 123 | field.field_key: field for field in self.get_fields() 124 | } 125 | return self._field_map 126 | 127 | def get_value(self, obj, field_key): 128 | """ 129 | Given an object and a field key, return the value of the field in the object. 130 | """ 131 | try: 132 | return self._get_field_map()[field_key].get_value(obj) 133 | except Exception as e: 134 | # Attach additional information to the exception to make higher level error handling easier 135 | e.field_key = field_key 136 | raise e 137 | 138 | def set_value(self, obj, field_key, value): 139 | """ 140 | Given an object and a field key, set the value of the field in the object. 141 | """ 142 | return self._get_field_map()[field_key].set_value(obj, value) 143 | 144 | 145 | class FieldSchema(models.Model): 146 | """ 147 | Specifies the schema for a field in a piece of data. 148 | """ 149 | class Meta: 150 | unique_together = ('data_schema', 'field_key') 151 | 152 | def __str__(self): 153 | return u'{0} - {1} - {2}'.format(self.id, self.field_key, self.display_name) 154 | 155 | # The data schema to which this field belongs 156 | data_schema = models.ForeignKey(DataSchema, on_delete=models.CASCADE) 157 | 158 | # The key for the field in the data 159 | field_key = models.CharField(max_length=255) 160 | 161 | # Optional way to display the field. defaults to the field_key 162 | display_name = models.TextField(blank=True, default='') 163 | 164 | # The order in which this field appears in the UID for the record. It is null if it does 165 | # not appear in the uniqueness constraint 166 | uniqueness_order = models.IntegerField(null=True, blank=True) 167 | 168 | # The position of the field. This ordering is relevant when parsing a list of fields into 169 | # a dictionary with the field names as keys 170 | field_position = models.IntegerField(null=True, blank=True) 171 | 172 | # The type of field. The available choices are present in the FieldSchemaType class 173 | field_type = models.CharField(max_length=32, choices=FieldSchemaType.choices()) 174 | 175 | # If the field is a string and needs to be converted to another type, this string specifies 176 | # the format for a field 177 | field_format = models.CharField(null=True, blank=True, default=None, max_length=64) 178 | 179 | # This field provides a default value to be used for the field in the case that it is None. 180 | default_value = models.CharField(null=True, blank=True, default=None, max_length=128) 181 | 182 | # Flag to indicate if setting values should be validated against an option list 183 | has_options = models.BooleanField(default=False) 184 | 185 | # Only applies to string data types. Valid options are upper and lower 186 | transform_case = models.CharField(null=True, default=None, blank=True, max_length=5) 187 | 188 | # Use django manager utils to manage FieldSchema objects 189 | objects = ManagerUtilsManager() 190 | 191 | def set_value(self, obj, value): 192 | """ 193 | Given an object, set the value of the field in that object. 194 | """ 195 | if self.has_options: 196 | # Build a set of possible values by calling self.get_value on the stored value options. This will 197 | # make sure they are the right data type because they are stored as strings 198 | values = set( 199 | self.get_value({ 200 | self.field_key: field_option.value 201 | }) 202 | for field_option in self.fieldoption_set.all() 203 | ) 204 | if value not in values: 205 | raise Exception('Invalid option for {0}'.format(self.field_key)) 206 | 207 | if isinstance(obj, list): 208 | obj[self.field_position] = value 209 | elif isinstance(obj, dict): 210 | obj[self.field_key] = value 211 | else: 212 | setattr(obj, self.field_key, value) 213 | 214 | def get_value(self, obj): 215 | """ 216 | Given an object, return the value of the field in that object. 217 | """ 218 | if isinstance(obj, list): 219 | value = obj[self.field_position] if 0 <= self.field_position < len(obj) else None 220 | elif isinstance(obj, dict): 221 | value = obj[self.field_key] if self.field_key in obj else None 222 | else: 223 | value = getattr(obj, self.field_key) if hasattr(obj, self.field_key) else None 224 | 225 | return convert_value(self.field_type, value, self.field_format, self.default_value, self.transform_case) 226 | 227 | def save(self, *args, **kwargs): 228 | if not self.display_name: 229 | self.display_name = self.field_key 230 | super(FieldSchema, self).save(*args, **kwargs) 231 | 232 | 233 | class FieldOption(models.Model): 234 | """ 235 | Specifies a set of possible values that a field schema can have 236 | """ 237 | field_schema = models.ForeignKey(FieldSchema, on_delete=models.CASCADE) 238 | value = models.CharField(max_length=1024) 239 | 240 | class Meta: 241 | unique_together = ('field_schema', 'value') 242 | -------------------------------------------------------------------------------- /data_schema/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ambitioninc/django-data-schema/9afe963828cbf2e45d42d1e7c258e5c93a6ffc69/data_schema/tests/__init__.py -------------------------------------------------------------------------------- /data_schema/tests/convert_value_tests.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from django.test import SimpleTestCase 4 | 5 | from data_schema.models import FieldSchemaType 6 | from data_schema.convert_value import convert_value 7 | from data_schema.exceptions import InvalidDateFormatException 8 | 9 | 10 | class ConvertValueExceptionTest(SimpleTestCase): 11 | def test_get_value_exception(self): 12 | """ 13 | Tests that when we fail to parse a value, we get a ValueError with additional information attached. 14 | """ 15 | bad_value = '-' 16 | 17 | with self.assertRaises(ValueError) as ctx: 18 | convert_value(FieldSchemaType.INT, bad_value) 19 | 20 | self.assertEquals(bad_value, ctx.exception.bad_value) 21 | self.assertEquals(FieldSchemaType.INT, ctx.exception.expected_type) 22 | 23 | 24 | class BooleanConverterTest(SimpleTestCase): 25 | 26 | def test_convert_value_true(self): 27 | """ 28 | Verifies true string values are True 29 | """ 30 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, 't')) 31 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, 'T')) 32 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, 'true')) 33 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, 'True')) 34 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, 'TRUE')) 35 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, True)) 36 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, 1)) 37 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, '1')) 38 | 39 | def test_convert_value_false(self): 40 | """ 41 | Verifies false string values are False 42 | """ 43 | self.assertFalse(convert_value(FieldSchemaType.BOOLEAN, 'f')) 44 | self.assertFalse(convert_value(FieldSchemaType.BOOLEAN, 'F')) 45 | self.assertFalse(convert_value(FieldSchemaType.BOOLEAN, 'false')) 46 | self.assertFalse(convert_value(FieldSchemaType.BOOLEAN, 'False')) 47 | self.assertFalse(convert_value(FieldSchemaType.BOOLEAN, 'FALSE')) 48 | self.assertFalse(convert_value(FieldSchemaType.BOOLEAN, False)) 49 | self.assertFalse(convert_value(FieldSchemaType.BOOLEAN, 0)) 50 | self.assertFalse(convert_value(FieldSchemaType.BOOLEAN, '0')) 51 | 52 | def test_convert_value_empty(self): 53 | """ 54 | Verifies that any other value returns None 55 | """ 56 | self.assertIsNone(convert_value(FieldSchemaType.BOOLEAN, None)) 57 | self.assertIsNone(convert_value(FieldSchemaType.BOOLEAN, '')) 58 | self.assertIsNone(convert_value(FieldSchemaType.BOOLEAN, 'string')) 59 | self.assertIsNone(convert_value(FieldSchemaType.BOOLEAN, 5)) 60 | 61 | def test_convert_datetime(self): 62 | """ 63 | Verifies that datetime field type attempts to coerce to timestamp before 64 | attempting to parse the string as a date string 65 | """ 66 | # still 67 | self.assertIsInstance(convert_value(FieldSchemaType.DATETIME, 1447251508), datetime) 68 | self.assertIsInstance(convert_value(FieldSchemaType.DATETIME, 1447251508.1234), datetime) 69 | self.assertIsInstance(convert_value(FieldSchemaType.DATETIME, 1.447251508e9), datetime) 70 | self.assertIsInstance(convert_value(FieldSchemaType.DATETIME, '1447251508'), datetime) 71 | self.assertIsInstance(convert_value(FieldSchemaType.DATETIME, '1447251508.1234'), datetime) 72 | self.assertIsInstance(convert_value(FieldSchemaType.DATETIME, '1.447251508e9'), datetime) 73 | # parses date strings 74 | self.assertIsInstance(convert_value(FieldSchemaType.DATETIME, '2015-11-09 15:30:00'), datetime) 75 | 76 | def test_convert_value_default(self): 77 | """ 78 | Verifies that the default value will be used if the passed value is null 79 | """ 80 | self.assertTrue(convert_value(FieldSchemaType.BOOLEAN, None, default_value=True)) 81 | self.assertIsNone(convert_value(FieldSchemaType.BOOLEAN, 'invalid', default_value=True)) 82 | 83 | 84 | class DurationConverterTest(SimpleTestCase): 85 | 86 | def test_convert_valid_simple_number(self): 87 | """ 88 | Verifies that a simple number conversion will be used 89 | """ 90 | self.assertEqual(70, convert_value(FieldSchemaType.DURATION, '70')) 91 | 92 | def test_convert_minutes_seconds(self): 93 | """ 94 | Verifies that a string of the format mm:ss will be converted correctly 95 | """ 96 | self.assertEqual(65, convert_value(FieldSchemaType.DURATION, '01:05')) 97 | self.assertEqual(65, convert_value(FieldSchemaType.DURATION, '1:5')) 98 | self.assertEqual(65, convert_value(FieldSchemaType.DURATION, '01:5')) 99 | 100 | def test_convert_hours_minutes_seconds(self): 101 | """ 102 | Verifies that a string of the format hh:mm:ss will be converted correctly 103 | """ 104 | self.assertEqual(7265, convert_value(FieldSchemaType.DURATION, '02:01:05')) 105 | self.assertEqual(7265, convert_value(FieldSchemaType.DURATION, '02:01:5')) 106 | self.assertEqual(7265, convert_value(FieldSchemaType.DURATION, '02:1:5')) 107 | self.assertEqual(7265, convert_value(FieldSchemaType.DURATION, '2:1:5')) 108 | 109 | def test_convert_invalid(self): 110 | """ 111 | Verifies that an altogether invalid string results in a value of None 112 | """ 113 | self.assertIsNone(convert_value(FieldSchemaType.DURATION, 'sup')) 114 | self.assertIsNone(convert_value(FieldSchemaType.DURATION, ':::')) 115 | 116 | 117 | class DateFlooredConverterTest(SimpleTestCase): 118 | 119 | def test_convert(self): 120 | """ 121 | Verify that the value of a datetime field is floored to the date 122 | """ 123 | self.assertEqual(datetime(2017, 3, 1, 0), convert_value(FieldSchemaType.DATE_FLOORED, '2017-03-01T10:30.000Z')) 124 | self.assertEqual(datetime(2017, 3, 1, 0), convert_value(FieldSchemaType.DATE_FLOORED, '2017-03-01 10:30.00')) 125 | self.assertEqual(datetime(2017, 3, 1, 0), convert_value(FieldSchemaType.DATE_FLOORED, '2017-03-01')) 126 | 127 | def test_convert_too_large_integer(self): 128 | """ 129 | Verifies that InvalidDateFormatException is raised instead of a generic attribute error 130 | """ 131 | with self.assertRaises(InvalidDateFormatException) as context: 132 | convert_value(FieldSchemaType.DATE_FLOORED, 3333333333333333333333333) 133 | 134 | self.assertEqual(str(context.exception), 'Invalid date format: 3333333333333333333333333') 135 | -------------------------------------------------------------------------------- /data_schema/tests/model_tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django_dynamic_fixture import G 3 | 4 | from data_schema.models import FieldSchema, DataSchema 5 | 6 | 7 | class FieldSchemaTest(TestCase): 8 | 9 | def test_unicode(self): 10 | field = FieldSchema(field_key='key', display_name='Field', id=10) 11 | 12 | self.assertEqual(str(field), u'10 - key - Field') 13 | 14 | def test_display_name_no_character_limit(self): 15 | """ 16 | Test to ensure that we are no longer running into the 64 character limit for display_name on the FieldSchema 17 | """ 18 | data_schema = G(DataSchema) 19 | field_schema = FieldSchema( 20 | data_schema=data_schema, 21 | field_key='testing_id', 22 | display_name='Testing this is longer than 64 characters and we are no longer failing') 23 | 24 | field_schema.save() 25 | 26 | self.assertEqual( 27 | field_schema.display_name, 28 | 'Testing this is longer than 64 characters and we are no longer failing' 29 | ) 30 | -------------------------------------------------------------------------------- /data_schema/tests/tests.py: -------------------------------------------------------------------------------- 1 | from copy import copy 2 | from datetime import datetime 3 | from functools import partial 4 | 5 | from django.contrib.contenttypes.models import ContentType 6 | from django.test import TestCase 7 | from django_dynamic_fixture import G 8 | from unittest.mock import patch 9 | import pytz 10 | 11 | from data_schema.models import DataSchema, FieldSchema, FieldOption 12 | from data_schema.field_schema_type import FieldSchemaCase, FieldSchemaType 13 | from data_schema.convert_value import ValueConverter 14 | 15 | 16 | class ValueConverterTest(TestCase): 17 | def test_is_numeric(self): 18 | converter = ValueConverter(FieldSchemaType.FLOAT, float) 19 | self.assertTrue(converter.is_numeric(0)) 20 | self.assertTrue(converter.is_numeric(100)) 21 | self.assertTrue(converter.is_numeric(1.34)) 22 | self.assertTrue(converter.is_numeric(1.34e2)) 23 | self.assertFalse(converter.is_numeric('foo')) 24 | self.assertFalse(converter.is_numeric({'foo': 'bar'})) 25 | 26 | 27 | class FieldSchemaTypeTest(TestCase): 28 | def test_choices(self): 29 | expected_choices = set([ 30 | ('DATE', 'DATE'), 31 | ('DATETIME', 'DATETIME'), 32 | ('DATE_FLOORED', 'DATE_FLOORED'), 33 | ('INT', 'INT'), 34 | ('FLOAT', 'FLOAT'), 35 | ('STRING', 'STRING'), 36 | ('BOOLEAN', 'BOOLEAN'), 37 | ('DURATION', 'DURATION'), 38 | ]) 39 | self.assertEquals(expected_choices, set(FieldSchemaType.choices())) 40 | 41 | def test_alphabetical(self): 42 | choices = FieldSchemaType.choices() 43 | 44 | sorted_choices = copy(choices) 45 | sorted_choices.sort() 46 | 47 | self.assertListEqual(choices, sorted_choices) 48 | 49 | 50 | class DataSchemaUpdateTest(TestCase): 51 | """ 52 | Tests the update method in the DataSchema model. 53 | """ 54 | def test_update_no_values(self): 55 | ds = DataSchema() 56 | ds.update() 57 | self.assertIsNotNone(ds.id) 58 | 59 | def test_update_with_model_ctype_none(self): 60 | ds = DataSchema() 61 | ds.update(model_content_type=None) 62 | self.assertIsNone(ds.model_content_type) 63 | 64 | def test_update_with_model_ctype_not_none(self): 65 | ds = DataSchema() 66 | ds.update(model_content_type=ContentType.objects.get_for_model(ds)) 67 | self.assertEquals(ds.model_content_type, ContentType.objects.get_for_model(ds)) 68 | 69 | def test_empty_field_schema_set(self): 70 | ds = DataSchema() 71 | ds.update(fieldschema_set=[]) 72 | self.assertEquals(FieldSchema.objects.count(), 0) 73 | 74 | def test_empty_field_schema_set_with_preexisting(self): 75 | ds = G(DataSchema) 76 | G(FieldSchema, data_schema=ds) 77 | ds.update(fieldschema_set=[]) 78 | self.assertEquals(FieldSchema.objects.count(), 0) 79 | 80 | def test_field_schema_set_creation_with_basic_values(self): 81 | ds = DataSchema() 82 | ds.update(fieldschema_set=[{ 83 | 'field_key': 'email', 84 | 'field_type': 'STRING' 85 | }]) 86 | fs = ds.fieldschema_set.get() 87 | self.assertEquals(fs.field_key, 'email') 88 | self.assertEquals(fs.field_type, 'STRING') 89 | 90 | def test_field_schema_set_preexisting_values(self): 91 | ds = G(DataSchema) 92 | G(FieldSchema, field_key='email', display_name='Email!', data_schema=ds) 93 | G(FieldSchema, field_key='hi', data_schema=ds) 94 | 95 | ds.update(fieldschema_set=[{ 96 | 'field_key': 'email', 97 | 'field_type': 'STRING', 98 | 'display_name': 'Email', 99 | 'uniqueness_order': 1, 100 | 'field_position': 1, 101 | 'field_format': 'format', 102 | 'default_value': '', 103 | 'transform_case': FieldSchemaCase.LOWER, 104 | }, { 105 | 'field_key': 'date', 106 | 'field_type': 'DATETIME', 107 | 'display_name': 'Date', 108 | 'uniqueness_order': 2, 109 | 'field_position': 2, 110 | 'field_format': 'format2', 111 | 'default_value': 'default', 112 | }]) 113 | 114 | self.assertEquals(FieldSchema.objects.count(), 2) 115 | fs = ds.fieldschema_set.all().order_by('field_key')[0] 116 | self.assertEquals(fs.field_key, 'date') 117 | self.assertEquals(fs.field_type, 'DATETIME') 118 | self.assertEquals(fs.display_name, 'Date') 119 | self.assertEquals(fs.uniqueness_order, 2) 120 | self.assertEquals(fs.field_position, 2) 121 | self.assertEquals(fs.field_format, 'format2') 122 | self.assertEquals(fs.default_value, 'default') 123 | self.assertFalse(fs.has_options) 124 | self.assertIsNone(fs.transform_case) 125 | 126 | fs = ds.fieldschema_set.all().order_by('field_key')[1] 127 | self.assertEquals(fs.field_key, 'email') 128 | self.assertEquals(fs.field_type, 'STRING') 129 | self.assertEquals(fs.display_name, 'Email') 130 | self.assertEquals(fs.uniqueness_order, 1) 131 | self.assertEquals(fs.field_position, 1) 132 | self.assertEquals(fs.field_format, 'format') 133 | self.assertEquals(fs.default_value, '') 134 | self.assertFalse(fs.has_options) 135 | self.assertEquals(fs.transform_case, FieldSchemaCase.LOWER) 136 | 137 | def test_field_schema_set_preexisting_values_w_options(self): 138 | ds = G(DataSchema) 139 | G(FieldSchema, field_key='email', display_name='Email!', data_schema=ds) 140 | G(FieldSchema, field_key='hi', data_schema=ds) 141 | 142 | ds.update(fieldschema_set=[{ 143 | 'field_key': 'email', 144 | 'field_type': 'STRING', 145 | 'display_name': 'Email', 146 | 'uniqueness_order': 1, 147 | 'field_position': 1, 148 | 'field_format': 'format', 149 | 'default_value': '', 150 | 'fieldoption_set': ['option1', 'option2'], 151 | }, { 152 | 'field_key': 'date', 153 | 'field_type': 'DATETIME', 154 | 'display_name': 'Date', 155 | 'uniqueness_order': 2, 156 | 'field_position': 2, 157 | 'field_format': 'format2', 158 | 'default_value': 'default', 159 | 'fieldoption_set': ['option3', 'option4'], 160 | }]) 161 | 162 | self.assertEquals(FieldSchema.objects.count(), 2) 163 | fs = ds.fieldschema_set.all().order_by('field_key')[0] 164 | self.assertEquals(fs.field_key, 'date') 165 | self.assertEquals(fs.field_type, 'DATETIME') 166 | self.assertEquals(fs.display_name, 'Date') 167 | self.assertEquals(fs.uniqueness_order, 2) 168 | self.assertEquals(fs.field_position, 2) 169 | self.assertEquals(fs.field_format, 'format2') 170 | self.assertEquals(fs.default_value, 'default') 171 | self.assertTrue(fs.has_options) 172 | self.assertEquals(set(['option3', 'option4']), set(fs.fieldoption_set.values_list('value', flat=True))) 173 | 174 | fs = ds.fieldschema_set.all().order_by('field_key')[1] 175 | self.assertEquals(fs.field_key, 'email') 176 | self.assertEquals(fs.field_type, 'STRING') 177 | self.assertEquals(fs.display_name, 'Email') 178 | self.assertEquals(fs.uniqueness_order, 1) 179 | self.assertEquals(fs.field_position, 1) 180 | self.assertEquals(fs.field_format, 'format') 181 | self.assertEquals(fs.default_value, '') 182 | self.assertTrue(fs.has_options) 183 | self.assertEquals(set(['option1', 'option2']), set(fs.fieldoption_set.values_list('value', flat=True))) 184 | 185 | 186 | class DataSchemaTest(TestCase): 187 | """ 188 | Tests the DataSchema model. 189 | """ 190 | def test_get_value_exception(self): 191 | """ 192 | Tests that when we fail to parse a value, we get a ValueError with additional information attached. 193 | """ 194 | bad_value = '-' 195 | field_key = 'number' 196 | data_schema = G(DataSchema) 197 | G( 198 | FieldSchema, field_key='number', field_position=0, field_type=FieldSchemaType.INT, 199 | data_schema=data_schema) 200 | 201 | with self.assertRaises(ValueError) as ctx: 202 | data_schema.get_value({field_key: bad_value}, field_key) 203 | 204 | self.assertEquals(field_key, ctx.exception.field_key) 205 | self.assertEquals(bad_value, ctx.exception.bad_value) 206 | self.assertEquals(FieldSchemaType.INT, ctx.exception.expected_type) 207 | 208 | def test_get_unique_fields_no_fields(self): 209 | """ 210 | Tests the get_unique_fields function when there are no fields defined. 211 | """ 212 | data_schema = G(DataSchema) 213 | self.assertEquals(data_schema.get_unique_fields(), []) 214 | 215 | def test_get_unique_fields_no_unique_fields(self): 216 | """ 217 | Tests the get_unique_fields function when there are fields defined, but 218 | none of them have a unique constraint. 219 | """ 220 | data_schema = G(DataSchema) 221 | G(FieldSchema, data_schema=data_schema) 222 | G(FieldSchema, data_schema=data_schema) 223 | 224 | self.assertEquals(data_schema.get_unique_fields(), []) 225 | 226 | def test_get_unique_fields_one(self): 227 | """ 228 | Tests retrieving one unique field. 229 | """ 230 | data_schema = G(DataSchema) 231 | field = G(FieldSchema, data_schema=data_schema, uniqueness_order=1) 232 | G(FieldSchema, data_schema=data_schema) 233 | 234 | self.assertEquals(data_schema.get_unique_fields(), [field]) 235 | 236 | def test_get_unique_fields_three(self): 237 | """ 238 | Tests retrieving three unique fields. 239 | """ 240 | data_schema = G(DataSchema) 241 | field1 = G(FieldSchema, data_schema=data_schema, uniqueness_order=1) 242 | field2 = G(FieldSchema, data_schema=data_schema, uniqueness_order=3) 243 | field3 = G(FieldSchema, data_schema=data_schema, uniqueness_order=2) 244 | G(FieldSchema, data_schema=data_schema) 245 | 246 | self.assertEquals(data_schema.get_unique_fields(), [field1, field3, field2]) 247 | 248 | def test_optimal_queries_get_unique_fields(self): 249 | """ 250 | Tests that get_unique_fields incurs no additional queries when caching the 251 | schema with the model manager. 252 | """ 253 | data_schema = G(DataSchema) 254 | field1 = G(FieldSchema, data_schema=data_schema, uniqueness_order=1) 255 | field2 = G(FieldSchema, data_schema=data_schema, uniqueness_order=3) 256 | field3 = G(FieldSchema, data_schema=data_schema, uniqueness_order=2) 257 | G(FieldSchema, data_schema=data_schema) 258 | 259 | data_schema = DataSchema.objects.get(id=data_schema.id) 260 | 261 | with self.assertNumQueries(0): 262 | self.assertEquals(data_schema.get_unique_fields(), [field1, field3, field2]) 263 | 264 | def test_cached_unique_fields(self): 265 | """ 266 | Tests that get_unique_fields function caches the unique fields. 267 | """ 268 | data_schema = G(DataSchema) 269 | field1 = G(FieldSchema, data_schema=data_schema, uniqueness_order=1) 270 | field2 = G(FieldSchema, data_schema=data_schema, uniqueness_order=3) 271 | field3 = G(FieldSchema, data_schema=data_schema, uniqueness_order=2) 272 | G(FieldSchema, data_schema=data_schema) 273 | 274 | data_schema = DataSchema.objects.get(id=data_schema.id) 275 | 276 | self.assertFalse(hasattr(data_schema, '_unique_fields')) 277 | self.assertEquals(data_schema.get_unique_fields(), [field1, field3, field2]) 278 | self.assertTrue(hasattr(data_schema, '_unique_fields')) 279 | self.assertEquals(data_schema.get_unique_fields(), [field1, field3, field2]) 280 | 281 | def test_get_fields_no_fields(self): 282 | """ 283 | Tests the get_fields function when there are no fields defined. 284 | """ 285 | data_schema = G(DataSchema) 286 | self.assertEquals(data_schema.get_fields(), []) 287 | 288 | def test_get_fields_one(self): 289 | """ 290 | Tests retrieving one field. 291 | """ 292 | data_schema = G(DataSchema) 293 | field = G(FieldSchema, data_schema=data_schema) 294 | G(FieldSchema) 295 | 296 | self.assertEquals(data_schema.get_fields(), [field]) 297 | 298 | def test_get_fields_three(self): 299 | """ 300 | Tests retrieving three fields. 301 | """ 302 | data_schema = G(DataSchema) 303 | field1 = G(FieldSchema, data_schema=data_schema) 304 | field2 = G(FieldSchema, data_schema=data_schema) 305 | field3 = G(FieldSchema, data_schema=data_schema, uniqueness_order=1) 306 | G(FieldSchema) 307 | 308 | self.assertEquals(set(data_schema.get_fields()), set([field1, field2, field3])) 309 | 310 | def test_get_fields_with_field_ordering(self): 311 | """ 312 | Tests that obtaining fields with a field position returns them in the proper 313 | order. 314 | """ 315 | data_schema = G(DataSchema) 316 | field1 = G(FieldSchema, data_schema=data_schema, field_position=2) 317 | field2 = G(FieldSchema, data_schema=data_schema, field_position=3) 318 | field3 = G(FieldSchema, data_schema=data_schema, field_position=1) 319 | G(FieldSchema) 320 | 321 | self.assertEquals(data_schema.get_fields(), [field3, field1, field2]) 322 | 323 | def test_optimal_queries_get_fields(self): 324 | """ 325 | Tests that get_fields incurs no additional queries when caching the 326 | schema with the model manager. 327 | """ 328 | data_schema = G(DataSchema) 329 | field1 = G(FieldSchema, data_schema=data_schema, uniqueness_order=1) 330 | field2 = G(FieldSchema, data_schema=data_schema, uniqueness_order=3) 331 | field3 = G(FieldSchema, data_schema=data_schema, uniqueness_order=2) 332 | G(FieldSchema) 333 | 334 | data_schema = DataSchema.objects.get(id=data_schema.id) 335 | 336 | with self.assertNumQueries(0): 337 | self.assertEquals(set(data_schema.get_fields()), set([field1, field3, field2])) 338 | 339 | def test_set_value_list(self): 340 | """ 341 | Tests setting the value of a list. 342 | """ 343 | data_schema = G(DataSchema) 344 | G(FieldSchema, data_schema=data_schema, field_key='field_key', field_position=1) 345 | val = ['hello', 'worlds'] 346 | data_schema.set_value(val, 'field_key', 'world') 347 | self.assertEquals(val, ['hello', 'world']) 348 | 349 | def test_set_value_obj(self): 350 | """ 351 | Tests setting the value of an object. 352 | """ 353 | class Input: 354 | field_key = 'none' 355 | data_schema = G(DataSchema) 356 | G(FieldSchema, data_schema=data_schema, field_key='field_key') 357 | obj = Input() 358 | data_schema.set_value(obj, 'field_key', 'value') 359 | self.assertEquals(obj.field_key, 'value') 360 | 361 | def test_set_value_dict(self): 362 | """ 363 | Tests setting the value of a dict. 364 | """ 365 | data_schema = G(DataSchema) 366 | G(FieldSchema, data_schema=data_schema, field_key='field_key') 367 | val = {'field_key': 'value1'} 368 | data_schema.set_value(val, 'field_key', 'value') 369 | self.assertEquals(val['field_key'], 'value') 370 | 371 | @patch('data_schema.models.convert_value', spec_set=True) 372 | def test_get_value_dict(self, convert_value_mock): 373 | """ 374 | Tests getting the value of a field when the object is a dictionary. 375 | """ 376 | data_schema = G(DataSchema) 377 | G(FieldSchema, field_key='field_key', field_type=FieldSchemaType.STRING, data_schema=data_schema) 378 | obj = { 379 | 'field_key': 'value', 380 | } 381 | data_schema.get_value(obj, 'field_key') 382 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, 'value', None, None, None) 383 | 384 | def test_get_value_dict_cached(self): 385 | """ 386 | Tests getting the value of a field twice (i.e. the cache gets used) 387 | """ 388 | data_schema = G(DataSchema) 389 | G(FieldSchema, field_key='field_key', data_schema=data_schema, field_type=FieldSchemaType.STRING) 390 | obj = { 391 | 'field_key': 'none', 392 | } 393 | value = data_schema.get_value(obj, 'field_key') 394 | self.assertEquals(value, 'none') 395 | value = data_schema.get_value(obj, 'field_key') 396 | self.assertEquals(value, 'none') 397 | 398 | @patch('data_schema.models.convert_value', spec_set=True) 399 | def test_get_value_obj(self, convert_value_mock): 400 | """ 401 | Tests the get_value function with an object as input. 402 | """ 403 | class Input: 404 | field_key = 'value' 405 | 406 | data_schema = G(DataSchema) 407 | G( 408 | FieldSchema, field_key='field_key', field_type=FieldSchemaType.STRING, field_format='format', 409 | data_schema=data_schema) 410 | data_schema.get_value(Input(), 'field_key') 411 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, 'value', 'format', None, None) 412 | 413 | @patch('data_schema.models.convert_value', spec_set=True) 414 | def test_get_value_list(self, convert_value_mock): 415 | """ 416 | Tests the get_value function with a list as input. 417 | """ 418 | data_schema = G(DataSchema) 419 | G( 420 | FieldSchema, field_key='field_key', field_position=1, field_type=FieldSchemaType.STRING, 421 | data_schema=data_schema) 422 | data_schema.get_value(['hello', 'world'], 'field_key') 423 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, 'world', None, None, None) 424 | 425 | 426 | class FieldSchemaTest(TestCase): 427 | """ 428 | Tests functionality in the FieldSchema model. 429 | """ 430 | def test_set_value_list(self): 431 | """ 432 | Tests setting the value of a list. 433 | """ 434 | field_schema = G(FieldSchema, field_key='field_key', field_position=1) 435 | val = ['hello', 'worlds'] 436 | field_schema.set_value(val, 'world') 437 | self.assertEquals(val, ['hello', 'world']) 438 | 439 | def test_set_value_obj(self): 440 | """ 441 | Tests setting the value of an object. 442 | """ 443 | class Input: 444 | field_key = 'none' 445 | field_schema = G(FieldSchema, field_key='field_key') 446 | obj = Input() 447 | field_schema.set_value(obj, 'value') 448 | self.assertEquals(obj.field_key, 'value') 449 | 450 | def test_set_value_dict(self): 451 | """ 452 | Tests setting the value of a dict. 453 | """ 454 | field_schema = G(FieldSchema, field_key='field_key') 455 | val = {'field_key': 'value1'} 456 | field_schema.set_value(val, 'value') 457 | self.assertEquals(val['field_key'], 'value') 458 | 459 | @patch('data_schema.models.convert_value', spec_set=True) 460 | def test_get_value_dict(self, convert_value_mock): 461 | """ 462 | Tests getting the value of a field when the object is a dictionary. 463 | """ 464 | field_schema = G(FieldSchema, field_key='field_key', field_type=FieldSchemaType.STRING) 465 | field_schema.get_value({'field_key': 'hello'}) 466 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, 'hello', None, None, None) 467 | 468 | @patch('data_schema.models.convert_value', spec_set=True) 469 | def test_get_value_obj(self, convert_value_mock): 470 | """ 471 | Tests the get_value function with an object as input. 472 | """ 473 | class Input: 474 | field_key = 'value' 475 | 476 | field_schema = G(FieldSchema, field_key='field_key', field_type=FieldSchemaType.STRING, field_format='format') 477 | field_schema.get_value(Input()) 478 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, 'value', 'format', None, None) 479 | 480 | @patch('data_schema.models.convert_value', spec_set=True) 481 | def test_get_value_list(self, convert_value_mock): 482 | """ 483 | Tests the get_value function with a list as input. 484 | """ 485 | field_schema = G(FieldSchema, field_key='field_key', field_position=1, field_type=FieldSchemaType.STRING) 486 | field_schema.get_value(['hello', 'world']) 487 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, 'world', None, None, None) 488 | 489 | @patch('data_schema.models.convert_value', spec_set=True) 490 | def test_get_value_dict_non_extant(self, convert_value_mock): 491 | """ 492 | Tests getting the value of a field when the object is a dictionary and it doesn't exist in the dict. 493 | """ 494 | field_schema = G(FieldSchema, field_key='field_key_bad', field_type=FieldSchemaType.STRING) 495 | field_schema.get_value({'field_key': 'hello'}) 496 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, None, None, None, None) 497 | 498 | @patch('data_schema.models.convert_value', spec_set=True) 499 | def test_get_value_obj_non_extant(self, convert_value_mock): 500 | """ 501 | Tests the get_value function with an object as input and the field key is not in the object. 502 | """ 503 | class Input: 504 | field_key = 'value' 505 | 506 | field_schema = G( 507 | FieldSchema, field_key='field_key_bad', field_type=FieldSchemaType.STRING, field_format='format') 508 | field_schema.get_value(Input()) 509 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, None, 'format', None, None) 510 | 511 | @patch('data_schema.models.convert_value', spec_set=True) 512 | def test_get_value_list_non_extant_negative(self, convert_value_mock): 513 | """ 514 | Tests the get_value function with a list as input and the input position is negative. 515 | """ 516 | field_schema = G(FieldSchema, field_key='field_key', field_position=-1, field_type=FieldSchemaType.STRING) 517 | field_schema.get_value(['hello', 'world']) 518 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, None, None, None, None) 519 | 520 | @patch('data_schema.models.convert_value', spec_set=True) 521 | def test_get_value_list_non_extant_out_of_range(self, convert_value_mock): 522 | """ 523 | Tests the get_value function with a list as input and the input position is greater than the 524 | length of the list. 525 | """ 526 | field_schema = G(FieldSchema, field_key='field_key', field_position=2, field_type=FieldSchemaType.STRING) 527 | field_schema.get_value(['hello', 'world']) 528 | convert_value_mock.assert_called_once_with(FieldSchemaType.STRING, None, None, None, None) 529 | 530 | def test_set_display_name(self): 531 | """ 532 | Tests that a display name is left alone if different than the field_key 533 | """ 534 | field_schema = G(FieldSchema, field_key='test', display_name='display') 535 | self.assertEqual('test', field_schema.field_key) 536 | self.assertEqual('display', field_schema.display_name) 537 | 538 | def test_set_display_name_empty(self): 539 | """ 540 | Tests that the field_key is copied to the display name if there is no display name set when saving 541 | """ 542 | field_schema = G(FieldSchema, field_key='test') 543 | self.assertEqual('test', field_schema.field_key) 544 | self.assertEqual('test', field_schema.display_name) 545 | 546 | 547 | class DateFieldSchemaTest(TestCase): 548 | """ 549 | Tests the DATE type for field schemas. 550 | """ 551 | def test_no_format_string(self): 552 | """ 553 | Tests when there is a string input with no format string. 554 | """ 555 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATE) 556 | val = field_schema.get_value({'time': '2013/04/02 9:25 PM'}) 557 | self.assertEquals(val, datetime(2013, 4, 2, 21, 25)) 558 | 559 | def test_none(self): 560 | """ 561 | Tests getting a value of None. 562 | """ 563 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATE, field_format='%Y-%m-%d') 564 | val = field_schema.get_value({'time': None}) 565 | self.assertEquals(val, None) 566 | 567 | def test_blank(self): 568 | """ 569 | Tests blank strings of input. 570 | """ 571 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATE, field_format='%Y-%m-%d') 572 | val = field_schema.get_value({'time': ' '}) 573 | self.assertEquals(val, None) 574 | 575 | def test_padded_date_with_format(self): 576 | """ 577 | Tests a date that is padded and has a format string. 578 | """ 579 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATE, field_format='%Y-%m-%d') 580 | val = field_schema.get_value({'time': ' 2013-04-05 '}) 581 | self.assertEquals(val, datetime(2013, 4, 5)) 582 | 583 | def test_get_value_date(self): 584 | """ 585 | Tests getting the value when the input is already a date object. 586 | """ 587 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATE) 588 | val = field_schema.get_value({'time': datetime(2013, 4, 4)}) 589 | self.assertEquals(val, datetime(2013, 4, 4)) 590 | 591 | def test_get_value_int(self): 592 | """ 593 | Tests getting the date value of an int. Assumed to be a utc timestamp. 594 | """ 595 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATE) 596 | val = field_schema.get_value({'time': 1399486805}) 597 | self.assertEquals(val, datetime(2014, 5, 7, 18, 20, 5)) 598 | 599 | def test_get_value_float(self): 600 | """ 601 | Tests getting the date value of an float. Assumed to be a utc timestamp. 602 | """ 603 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATE) 604 | val = field_schema.get_value({'time': 1399486805.0}) 605 | self.assertEquals(val, datetime(2014, 5, 7, 18, 20, 5)) 606 | 607 | def test_get_value_formatted(self): 608 | """ 609 | Tests getting a formatted date value. 610 | """ 611 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATE, field_format='%Y-%m-%d') 612 | val = field_schema.get_value({'time': '2013-04-05'}) 613 | self.assertEquals(val, datetime(2013, 4, 5)) 614 | 615 | 616 | class DatetimeFieldSchemaTest(TestCase): 617 | """ 618 | Tests the DATETIME type for field schemas. 619 | """ 620 | def test_default_value_blank(self): 621 | """ 622 | Tests when a default value is used and there is a blank string. 623 | """ 624 | field_schema = G( 625 | FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME, default_value='2013/04/02 9:25 PM') 626 | val = field_schema.get_value({'time': ' '}) 627 | self.assertEquals(val, datetime(2013, 4, 2, 21, 25)) 628 | 629 | def test_default_value_null(self): 630 | """ 631 | Tests when a default value is used and there is a null object. 632 | """ 633 | field_schema = G( 634 | FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME, default_value='2013/04/02 9:25 PM') 635 | val = field_schema.get_value({'time': None}) 636 | self.assertEquals(val, datetime(2013, 4, 2, 21, 25)) 637 | 638 | def test_no_format_string(self): 639 | """ 640 | Tests when there is a string input with no format string. 641 | """ 642 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME) 643 | val = field_schema.get_value({'time': '2013/04/02 9:25 PM'}) 644 | self.assertEquals(val, datetime(2013, 4, 2, 21, 25)) 645 | 646 | def test_datetime_with_tz_dateutil(self): 647 | """ 648 | Tests that a datetime with a tz is converted back to naive UTC after using dateutil for parsing. 649 | """ 650 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME) 651 | val = field_schema.get_value({'time': '2013/04/02 09:25:00+0400'}) 652 | self.assertEquals(val, datetime(2013, 4, 2, 5, 25)) 653 | 654 | def test_datetime_with_tz(self): 655 | """ 656 | Tests that a datetime with a tz is converted back to naive UTC. 657 | """ 658 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME) 659 | val = field_schema.get_value({'time': datetime(2013, 4, 2, 9, 25, tzinfo=pytz.utc)}) 660 | self.assertEquals(val, datetime(2013, 4, 2, 9, 25)) 661 | 662 | def test_none(self): 663 | """ 664 | Tests getting a value of None. 665 | """ 666 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME, field_format='%Y-%m-%d') 667 | val = field_schema.get_value({'time': None}) 668 | self.assertEquals(val, None) 669 | 670 | def test_blank(self): 671 | """ 672 | Tests blank strings of input. 673 | """ 674 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME, field_format='%Y-%m-%d') 675 | val = field_schema.get_value({'time': ' '}) 676 | self.assertEquals(val, None) 677 | 678 | def test_get_value_date(self): 679 | """ 680 | Tests getting the value when the input is already a date object. 681 | """ 682 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME) 683 | val = field_schema.get_value({'time': datetime(2013, 4, 4)}) 684 | self.assertEquals(val, datetime(2013, 4, 4)) 685 | 686 | def test_get_value_int(self): 687 | """ 688 | Tests getting the date value of an int. Assumed to be a utc timestamp. 689 | """ 690 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME) 691 | val = field_schema.get_value({'time': 1399486805}) 692 | self.assertEquals(val, datetime(2014, 5, 7, 18, 20, 5)) 693 | 694 | def test_get_value_float(self): 695 | """ 696 | Tests getting the date value of an float. Assumed to be a utc timestamp. 697 | """ 698 | field_schema = G(FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME) 699 | val = field_schema.get_value({'time': 1399486805.0}) 700 | self.assertEquals(val, datetime(2014, 5, 7, 18, 20, 5)) 701 | 702 | def test_get_value_formatted(self): 703 | """ 704 | Tests getting a formatted date value. 705 | """ 706 | field_schema = G( 707 | FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME, field_format='%Y-%m-%d %H:%M:%S') 708 | val = field_schema.get_value({'time': '2013-04-05 12:12:12'}) 709 | self.assertEquals(val, datetime(2013, 4, 5, 12, 12, 12)) 710 | 711 | def test_get_value_formatted_unicode(self): 712 | """ 713 | Tests getting a formatted date in unicode. 714 | """ 715 | field_schema = G( 716 | FieldSchema, field_key='time', field_type=FieldSchemaType.DATETIME, field_format='%Y-%m-%d %H:%M:%S') 717 | val = field_schema.get_value({'time': u'2013-04-05 12:12:12'}) 718 | self.assertEquals(val, datetime(2013, 4, 5, 12, 12, 12)) 719 | 720 | 721 | class IntFieldSchemaTest(TestCase): 722 | """ 723 | Tests the INT type for field schemas. 724 | """ 725 | def test_negative_string(self): 726 | """ 727 | Tests parsing a negative string number. 728 | """ 729 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.INT) 730 | val = field_schema.get_value({'val': '-1'}) 731 | self.assertEquals(val, -1) 732 | 733 | def test_none(self): 734 | """ 735 | Tests getting a value of None. 736 | """ 737 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.INT) 738 | val = field_schema.get_value({'val': None}) 739 | self.assertEquals(val, None) 740 | 741 | def test_blank(self): 742 | """ 743 | Tests blank strings of input. 744 | """ 745 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.INT) 746 | val = field_schema.get_value({'val': ' '}) 747 | self.assertEquals(val, None) 748 | 749 | def test_get_value_non_numeric_str(self): 750 | """ 751 | Tests getting the value of a string that has currency information. 752 | """ 753 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.INT) 754 | val = field_schema.get_value({'val': ' $15,000,456 Dollars '}) 755 | self.assertAlmostEquals(val, 15000456) 756 | 757 | def test_get_value_str(self): 758 | """ 759 | Tests getting the value when the input is a string. 760 | """ 761 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.INT) 762 | val = field_schema.get_value({'val': '1'}) 763 | self.assertEquals(val, 1) 764 | 765 | def test_get_value_int(self): 766 | """ 767 | Tests getting the value when it is an int. 768 | """ 769 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.INT) 770 | val = field_schema.get_value({'val': 5}) 771 | self.assertEquals(val, 5) 772 | 773 | def test_get_value_float(self): 774 | """ 775 | Tests getting the date value of a float. 776 | """ 777 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.INT) 778 | val = field_schema.get_value({'val': 5.2}) 779 | self.assertEquals(val, 5) 780 | 781 | 782 | class StringFieldSchemaTest(TestCase): 783 | """ 784 | Tests the STRING type for field schemas. 785 | """ 786 | def test_bad_unicode_input(self): 787 | """ 788 | Unicode special chars should be handled properly. 789 | """ 790 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING) 791 | val = field_schema.get_value({'val': u'\u2019'}) 792 | self.assertEquals(val, u'\u2019') 793 | 794 | def test_unicode_input(self): 795 | """ 796 | Unicode should be handled properly. 797 | """ 798 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING) 799 | val = field_schema.get_value({'val': u' '}) 800 | self.assertEquals(val, '') 801 | 802 | def test_matching_format(self): 803 | """ 804 | Tests returning a string that matches a format. 805 | """ 806 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING, field_format=r'^[\d\.]+$') 807 | val = field_schema.get_value({'val': '23.45'}) 808 | self.assertEquals(val, '23.45') 809 | 810 | def test_non_matching_format(self): 811 | """ 812 | Tests returning a string that matches a format. 813 | """ 814 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING, field_format=r'^[\d\.]+$') 815 | val = field_schema.get_value({'val': '23,45'}) 816 | self.assertEquals(val, None) 817 | 818 | def test_matching_format_limit_length(self): 819 | """ 820 | Tests returning a string that matches a format of a limited length number. 821 | """ 822 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING, field_format=r'^[\d]{1,5}$') 823 | val = field_schema.get_value({'val': '2345'}) 824 | self.assertEquals(val, '2345') 825 | val = field_schema.get_value({'val': '23456'}) 826 | self.assertEquals(val, '23456') 827 | 828 | def test_non_matching_format_limit_length(self): 829 | """ 830 | Tests returning a string that matches a format of a limited length number. 831 | """ 832 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING, field_format=r'^[\d]{1,5}$') 833 | val = field_schema.get_value({'val': '234567'}) 834 | self.assertEquals(val, None) 835 | 836 | def test_none(self): 837 | """ 838 | Tests getting a value of None. 839 | """ 840 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING) 841 | val = field_schema.get_value({'val': None}) 842 | self.assertEquals(val, None) 843 | 844 | def test_blank(self): 845 | """ 846 | Tests blank strings of input. Contrary to other formats, the string field schema 847 | returns a blank string instead of None (since blank strings are valid strings). 848 | """ 849 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING) 850 | val = field_schema.get_value({'val': ' '}) 851 | self.assertEquals(val, '') 852 | 853 | def test_strip_whitespaces(self): 854 | """ 855 | Tests that getting a string results in its leading and trailing whitespace being 856 | stripped. 857 | """ 858 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING) 859 | val = field_schema.get_value({'val': ' 1 2 3 '}) 860 | self.assertEquals(val, '1 2 3') 861 | 862 | def test_get_value_str(self): 863 | """ 864 | Tests getting the value when the input is a string. 865 | """ 866 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING) 867 | val = field_schema.get_value({'val': '1'}) 868 | self.assertEquals(val, '1') 869 | 870 | def test_get_value_int(self): 871 | """ 872 | Tests getting the value when it is an int. 873 | """ 874 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING) 875 | val = field_schema.get_value({'val': 5}) 876 | self.assertEquals(val, '5') 877 | 878 | def test_get_value_float(self): 879 | """ 880 | Tests getting the date value of a float. 881 | """ 882 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.STRING) 883 | val = field_schema.get_value({'val': 5.2}) 884 | self.assertEquals(val, '5.2') 885 | 886 | def test_lowercase(self): 887 | """ 888 | Tests that the string is converted to lowercase 889 | """ 890 | field_schema = G( 891 | FieldSchema, field_key='val', field_type=FieldSchemaType.STRING, transform_case=FieldSchemaCase.LOWER 892 | ) 893 | val = field_schema.get_value({'val': 'Value'}) 894 | self.assertEquals(val, 'value') 895 | 896 | def test_uppercase(self): 897 | """ 898 | Tests that the string is converted to uppercase 899 | """ 900 | field_schema = G( 901 | FieldSchema, field_key='val', field_type=FieldSchemaType.STRING, transform_case=FieldSchemaCase.UPPER 902 | ) 903 | val = field_schema.get_value({'val': 'Value'}) 904 | self.assertEquals(val, 'VALUE') 905 | 906 | 907 | class FloatFieldSchemaTest(TestCase): 908 | """ 909 | Tests the FLOAT type for field schemas. 910 | """ 911 | def test_positive_scientific_notation(self): 912 | """ 913 | Tests that positive scientific notation strings are parsed. 914 | """ 915 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 916 | val = field_schema.get_value({'val': '1.1E2'}) 917 | self.assertEquals(val, 110) 918 | 919 | def test_positive_scientific_notation_small_e(self): 920 | """ 921 | Tests that positive scientific notation strings are parsed with a lowercase e. 922 | """ 923 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 924 | val = field_schema.get_value({'val': '1.1e2'}) 925 | self.assertEquals(val, 110) 926 | 927 | def test_inf_invalid(self): 928 | """ 929 | Verifies that an inf or -inf value will raise a value error 930 | """ 931 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 932 | self.assertRaises(ValueError, partial(field_schema.get_value, {'val': '5e9099'})) # results in inf 933 | self.assertRaises(ValueError, partial(field_schema.get_value, {'val': '-5e9099'})) # results in -inf 934 | 935 | def test_negative_scientific_notation(self): 936 | """ 937 | Tests that negative scientific notation strings are parsed. 938 | """ 939 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 940 | val = field_schema.get_value({'val': '-1.1E-2'}) 941 | self.assertEquals(val, -0.011) 942 | 943 | def test_negative_string(self): 944 | """ 945 | Tests parsing a negative string number. 946 | """ 947 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 948 | val = field_schema.get_value({'val': '-1.1'}) 949 | self.assertEquals(val, -1.1) 950 | 951 | def test_none(self): 952 | """ 953 | Tests getting a value of None. 954 | """ 955 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 956 | val = field_schema.get_value({'val': None}) 957 | self.assertEquals(val, None) 958 | 959 | def test_blank(self): 960 | """ 961 | Tests blank strings of input. 962 | """ 963 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 964 | val = field_schema.get_value({'val': ' '}) 965 | self.assertEquals(val, None) 966 | 967 | def test_get_value_non_numeric_str(self): 968 | """ 969 | Tests getting the value of a string that has currency information. 970 | """ 971 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 972 | val = field_schema.get_value({'val': ' $15,000,456.34 Dollars '}) 973 | self.assertAlmostEquals(val, 15000456.34) 974 | 975 | def test_get_value_non_numeric_unicode(self): 976 | """ 977 | Tests getting the value of a unicode object that has currency information. 978 | """ 979 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 980 | val = field_schema.get_value({'val': u' $15,000,456.34 Dollars '}) 981 | self.assertAlmostEquals(val, 15000456.34) 982 | 983 | def test_get_value_str(self): 984 | """ 985 | Tests getting the value when the input is a string. 986 | """ 987 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 988 | val = field_schema.get_value({'val': '1'}) 989 | self.assertAlmostEquals(val, 1.0) 990 | 991 | def test_get_value_int(self): 992 | """ 993 | Tests getting the value when it is an int. 994 | """ 995 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 996 | val = field_schema.get_value({'val': 5}) 997 | self.assertAlmostEquals(val, 5.0) 998 | 999 | def test_get_value_float(self): 1000 | """ 1001 | Tests getting the date value of a float. 1002 | """ 1003 | field_schema = G(FieldSchema, field_key='val', field_type=FieldSchemaType.FLOAT) 1004 | val = field_schema.get_value({'val': 5.2}) 1005 | self.assertAlmostEquals(val, 5.2) 1006 | 1007 | 1008 | class FieldOptionTest(TestCase): 1009 | 1010 | def test_set_valid_value(self): 1011 | """ 1012 | The field schema should have defined options and a valid option should be set 1013 | """ 1014 | field_schema = G(FieldSchema, field_type=FieldSchemaType.STRING, field_key='my_key', has_options=True) 1015 | G(FieldOption, field_schema=field_schema, value='one') 1016 | G(FieldOption, field_schema=field_schema, value='two') 1017 | item = { 1018 | 'my_key': None, 1019 | } 1020 | field_schema.set_value(item, 'one') 1021 | self.assertEqual('one', item['my_key']) 1022 | 1023 | def test_set_invalid_value(self): 1024 | """ 1025 | The field schema should have defined options and an invalid option should be set 1026 | """ 1027 | field_schema = G(FieldSchema, field_type=FieldSchemaType.STRING, field_key='my_key', has_options=True) 1028 | G(FieldOption, field_schema=field_schema, value='one') 1029 | G(FieldOption, field_schema=field_schema, value='two') 1030 | item = { 1031 | 'my_key': None, 1032 | } 1033 | with self.assertRaises(Exception): 1034 | field_schema.set_value(item, 'three') 1035 | self.assertIsNone(item['my_key']) 1036 | 1037 | def test_set_value_different_type(self): 1038 | """ 1039 | The field schema should be a different type and it should validate correctly 1040 | """ 1041 | field_schema = G(FieldSchema, field_type=FieldSchemaType.INT, field_key='my_key', has_options=True) 1042 | G(FieldOption, field_schema=field_schema, value='1') 1043 | G(FieldOption, field_schema=field_schema, value='2') 1044 | item = { 1045 | 'my_key': None, 1046 | } 1047 | field_schema.set_value(item, 1) 1048 | self.assertEqual(1, item['my_key']) 1049 | -------------------------------------------------------------------------------- /data_schema/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '2.1.0' 2 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | from settings import configure_settings 5 | 6 | 7 | if __name__ == '__main__': 8 | configure_settings() 9 | 10 | from django.core.management import execute_from_command_line 11 | 12 | execute_from_command_line(sys.argv) 13 | -------------------------------------------------------------------------------- /publish.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | subprocess.call(['rm', '-r', 'dist/']) 4 | subprocess.call(['python', '-m', 'pip', 'install', 'build', 'twine']) 5 | subprocess.call(['python', '-m', 'build']) 6 | subprocess.call(['twine', 'check', 'dist/*']) 7 | subprocess.call(['twine', 'upload', 'dist/*']) 8 | subprocess.call(['rm', '-r', 'dist/']) 9 | -------------------------------------------------------------------------------- /requirements/requirements-testing.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | #coveralls 3 | django-dynamic-fixture 4 | django-nose 5 | psycopg2 6 | flake8 7 | -------------------------------------------------------------------------------- /requirements/requirements.txt: -------------------------------------------------------------------------------- 1 | Django>=3.2 2 | django-manager-utils>=3.1.0 3 | fleming>=0.7.0 4 | python-dateutil>=2.2 5 | -------------------------------------------------------------------------------- /run_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides the ability to run test on a standalone Django app. 3 | """ 4 | import sys 5 | from optparse import OptionParser 6 | from settings import configure_settings 7 | 8 | 9 | # Configure the default settings 10 | configure_settings() 11 | 12 | # Django nose must be imported here since it depends on the settings being configured 13 | from django_nose import NoseTestSuiteRunner 14 | 15 | 16 | def run_tests(*test_args, **kwargs): 17 | if not test_args: 18 | test_args = ['data_schema'] 19 | 20 | kwargs.setdefault('interactive', False) 21 | 22 | test_runner = NoseTestSuiteRunner(**kwargs) 23 | 24 | failures = test_runner.run_tests(test_args) 25 | sys.exit(failures) 26 | 27 | 28 | if __name__ == '__main__': 29 | parser = OptionParser() 30 | parser.add_option('--verbosity', dest='verbosity', action='store', default=1, type=int) 31 | (options, args) = parser.parse_args() 32 | 33 | run_tests(*args, **options.__dict__) 34 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | 4 | from django.conf import settings 5 | 6 | 7 | def configure_settings(): 8 | """ 9 | Configures settings for manage.py and for run_tests.py. 10 | """ 11 | if not settings.configured: 12 | # Determine the database settings depending on if a test_db var is set in CI mode or not 13 | test_db = os.environ.get('DB', None) 14 | if test_db is None: 15 | db_config = { 16 | 'ENGINE': 'django.db.backends.postgresql', 17 | 'NAME': 'data_schema', 18 | 'USER': 'postgres', 19 | 'PASSWORD': '', 20 | 'HOST': 'db', 21 | } 22 | elif test_db == 'postgres': 23 | db_config = { 24 | 'ENGINE': 'django.db.backends.postgresql', 25 | 'NAME': 'data_schema', 26 | 'USER': 'postgres', 27 | 'PASSWORD': '', 28 | 'HOST': 'db', 29 | } 30 | else: 31 | raise RuntimeError('Unsupported test DB {0}'.format(test_db)) 32 | 33 | # Check env for db override (used for github actions) 34 | if os.environ.get('DB_SETTINGS'): 35 | db_config = json.loads(os.environ.get('DB_SETTINGS')) 36 | 37 | settings.configure( 38 | TEST_RUNNER='django_nose.NoseTestSuiteRunner', 39 | NOSE_ARGS=['--nocapture', '--nologcapture', '--verbosity=1'], 40 | DATABASES={ 41 | 'default': db_config, 42 | }, 43 | MIDDLEWARE_CLASSES={}, 44 | INSTALLED_APPS=( 45 | 'django.contrib.auth', 46 | 'django.contrib.contenttypes', 47 | 'django.contrib.sessions', 48 | 'data_schema', 49 | 'data_schema.tests', 50 | ), 51 | DEBUG=False, 52 | DDF_FILL_NULLABLE_FIELDS=False, 53 | SECRET_KEY='foo', 54 | ) 55 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | exclude = build,docs,venv,env,*.egg,migrations,south_migrations 4 | max-complexity = 10 5 | ignore = E402 6 | 7 | [bdist_wheel] 8 | universal = 1 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # import multiprocessing to avoid this bug (http://bugs.python.org/issue15881#msg170215) 2 | import multiprocessing 3 | assert multiprocessing 4 | import re 5 | from setuptools import setup, find_packages 6 | 7 | 8 | def get_version(): 9 | """ 10 | Extracts the version number from the version.py file. 11 | """ 12 | VERSION_FILE = 'data_schema/version.py' 13 | mo = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', open(VERSION_FILE, 'rt').read(), re.M) 14 | if mo: 15 | return mo.group(1) 16 | else: 17 | raise RuntimeError('Unable to find version string in {0}.'.format(VERSION_FILE)) 18 | 19 | 20 | def get_lines(file_path): 21 | return open(file_path, 'r').read().split('\n') 22 | 23 | 24 | install_requires = get_lines('requirements/requirements.txt') 25 | tests_require = get_lines('requirements/requirements-testing.txt') 26 | 27 | 28 | setup( 29 | name='django-data-schema', 30 | version=get_version(), 31 | description='Schemas over dictionaries and models in Django', 32 | long_description=open('README.md').read(), 33 | long_description_content_type='text/markdown', 34 | url='https://github.com/ambitioninc/django-data-schema', 35 | author='Wes Kendall', 36 | author_email='opensource@ambition.com', 37 | keywords='Django Data Schema', 38 | packages=find_packages(), 39 | classifiers=[ 40 | 'Programming Language :: Python', 41 | 'Programming Language :: Python :: 3.7', 42 | 'Programming Language :: Python :: 3.8', 43 | 'Programming Language :: Python :: 3.9', 44 | 'Intended Audience :: Developers', 45 | 'License :: OSI Approved :: BSD License', 46 | 'Operating System :: OS Independent', 47 | 'Framework :: Django', 48 | 'Framework :: Django :: 3.2', 49 | 'Framework :: Django :: 4.0', 50 | 'Framework :: Django :: 4.1', 51 | 'Framework :: Django :: 4.2', 52 | ], 53 | license='MIT', 54 | install_requires=install_requires, 55 | tests_require=tests_require, 56 | test_suite='run_tests.run_tests', 57 | include_package_data=True, 58 | ) 59 | --------------------------------------------------------------------------------