├── ckanext ├── fluent │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ └── test_helpers.py │ ├── templates │ │ └── scheming │ │ │ ├── display_snippets │ │ │ ├── fluent_text.html │ │ │ ├── fluent_markdown.html │ │ │ ├── fluent_link.html │ │ │ └── fluent_tags.html │ │ │ ├── form_snippets │ │ │ ├── fluent_help_text.html │ │ │ ├── fluent_markdown.html │ │ │ ├── fluent_tags.html │ │ │ └── fluent_text.html │ │ │ └── error_snippets │ │ │ └── fluent_text.html │ ├── plugins.py │ ├── presets.json │ ├── fluent_scheming.yaml │ ├── helpers.py │ └── validators.py └── __init__.py ├── .gitignore ├── dev-requirements.txt ├── MANIFEST.in ├── requirements.txt ├── docs ├── multilingual-form.png └── multilingual-tags.png ├── CHANGELOG.md ├── setup.py ├── test.ini ├── COPYING ├── COPYING.fr ├── .github └── workflows │ └── test.yml └── README.md /ckanext/fluent/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ckanext/fluent/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | mock 2 | pytest-ckan 3 | pytest-cov 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include ckanext *.html *.json *.yaml 2 | include README.md CHANGELOG.md 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming 2 | six 3 | -------------------------------------------------------------------------------- /docs/multilingual-form.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ckan/ckanext-fluent/HEAD/docs/multilingual-form.png -------------------------------------------------------------------------------- /docs/multilingual-tags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ckan/ckanext-fluent/HEAD/docs/multilingual-tags.png -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/display_snippets/fluent_text.html: -------------------------------------------------------------------------------- 1 | {{ h.scheming_language_text(data[field.field_name]) }} 2 | -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/display_snippets/fluent_markdown.html: -------------------------------------------------------------------------------- 1 | {{ h.render_markdown( 2 | h.scheming_language_text(data[field.field_name])) }} 3 | -------------------------------------------------------------------------------- /ckanext/__init__.py: -------------------------------------------------------------------------------- 1 | # this is a namespace package 2 | try: 3 | import pkg_resources 4 | pkg_resources.declare_namespace(__name__) 5 | except ImportError: 6 | import pkgutil 7 | __path__ = pkgutil.extend_path(__path__, __name__) 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.0.1 2 | 3 | 2014-09-03 4 | 5 | * initial release (forked from multilingual fields branch of ckanext-scheming) 6 | 7 | ## 1.0.0 8 | 9 | 2015-10-14 10 | 11 | * first stable release 12 | * scheming presets: fluent_text, fluent_tags, fluent_markdown, fluent_link 13 | -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/display_snippets/fluent_link.html: -------------------------------------------------------------------------------- 1 | {{ h.link_to(h.scheming_language_text(data[field.field_name]), 2 | url=h.scheming_language_text(data[field.field_name]), 3 | rel=field.display_property, 4 | target='_blank', 5 | **field.get('display_attributes', {}) )}} 6 | -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/form_snippets/fluent_help_text.html: -------------------------------------------------------------------------------- 1 | {% import 'macros/form.html' as form %} 2 | 3 | {%- if field.fluent_help_text and lang in field.fluent_help_text -%} 4 | {{- form.info( 5 | text=h.scheming_language_text(field.fluent_help_text[lang]), 6 | inline=field.get('help_inline', false) 7 | ) -}} 8 | {%- elif field.help_text -%} 9 | {{- form.info( 10 | text=h.scheming_language_text(field.help_text), 11 | inline=field.get('help_inline', false) 12 | ) -}} 13 | {%- endif -%} 14 | -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/display_snippets/fluent_tags.html: -------------------------------------------------------------------------------- 1 | {# slight abuse of scheming_language_text for selecting the desired 2 | language version #} 3 | {%- set value = h.scheming_language_text(data[field.field_name]) -%} 4 |
5 | {% block tag_list %} 6 | 15 | {% endblock %} 16 |
17 | -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/form_snippets/fluent_markdown.html: -------------------------------------------------------------------------------- 1 | {% import 'macros/form.html' as form %} 2 | 3 | {%- for lang in h.fluent_form_languages(field, entity_type, object_type, schema) -%} 4 | {% call form.markdown( 5 | field.field_name + '-' + lang, 6 | id='field-' + field.field_name + '-' + lang, 7 | label=h.fluent_form_label(field, lang), 8 | placeholder=h.scheming_language_text(field.form_placeholder, lang), 9 | value=data[field.field_name + '-' + lang] 10 | or data.get(field.field_name, {})[lang], 11 | error=errors[field.field_name + '-' + lang], 12 | attrs=field.form_attrs or {}, 13 | is_required=h.scheming_field_required(field) 14 | ) %} 15 | {%- snippet 'scheming/form_snippets/fluent_help_text.html', 16 | field=field, 17 | lang=lang -%} 18 | {% endcall %} 19 | {%- endfor -%} 20 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import sys, os 3 | 4 | version = '1.0.0' 5 | 6 | setup( 7 | name='ckanext-fluent', 8 | version=version, 9 | description="Multilingual fields for CKAN", 10 | long_description=""" 11 | """, 12 | classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers 13 | keywords='', 14 | author='Government of Canada', 15 | author_email='ian@excess.org', 16 | url='', 17 | license='MIT', 18 | packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 19 | namespace_packages=['ckanext'], 20 | include_package_data=True, 21 | zip_safe=False, 22 | install_requires=[ 23 | # -*- Extra requirements: -*- 24 | ], 25 | entry_points=\ 26 | """ 27 | [ckan.plugins] 28 | fluent=ckanext.fluent.plugins:FluentPlugin 29 | """, 30 | ) 31 | -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/form_snippets/fluent_tags.html: -------------------------------------------------------------------------------- 1 | {% import 'macros/form.html' as form %} 2 | 3 | {%- for lang in h.fluent_form_languages(field, entity_type, object_type, schema) -%} 4 | {% call form.input( 5 | field.field_name + '-' + lang, 6 | id='field-' + field.field_name + '-' + lang, 7 | label=h.fluent_form_label(field, lang), 8 | placeholder=h.scheming_language_text(field.form_placeholder), 9 | value=data[field.field_name + '-' + lang] 10 | or ','.join(data.get(field.field_name, {}).get(lang, [])), 11 | error=errors[field.field_name + '-' + lang], 12 | classes=['control-medium'], 13 | attrs=field.form_attrs if 'form_attrs' in field else {}, 14 | is_required=h.scheming_field_required(field) 15 | ) %} 16 | {%- snippet 'scheming/form_snippets/fluent_help_text.html', 17 | field=field, 18 | lang=lang -%} 19 | {% endcall %} 20 | {%- endfor -%} 21 | -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/form_snippets/fluent_text.html: -------------------------------------------------------------------------------- 1 | {% import 'macros/form.html' as form %} 2 | 3 | {%- for lang in h.fluent_form_languages(field, entity_type, object_type, schema) -%} 4 | {% call form.input( 5 | field.field_name + '-' + lang, 6 | id='field-' + field.field_name + '-' + lang, 7 | label=h.fluent_form_label(field, lang), 8 | placeholder=h.scheming_language_text(field.form_placeholder, lang), 9 | value=data[field.field_name + '-' + lang] 10 | or data.get(field.field_name, {})[lang], 11 | error=errors[field.field_name + '-' + lang], 12 | classes=['control-medium'], 13 | attrs=dict({"class": "form-control"}, **(field.get('form_attrs', {}))), 14 | is_required=h.scheming_field_required(field) 15 | ) %} 16 | {%- snippet 'scheming/form_snippets/fluent_help_text.html', 17 | field=field, 18 | lang=lang -%} 19 | {% endcall %} 20 | {%- endfor -%} 21 | -------------------------------------------------------------------------------- /ckanext/fluent/templates/scheming/error_snippets/fluent_text.html: -------------------------------------------------------------------------------- 1 | {%- set my_errors = [] -%} 2 | {%- for key, errors in unprocessed.items() -%} 3 | {%- if key.startswith(field.field_name + '-') -%} 4 | {%- do my_errors.append(key) -%} 5 | {%- endif -%} 6 | {%- endfor -%} 7 | 8 | {# List errors in the language order from the schema #} 9 | {%- for lang in h.fluent_form_languages(field, entity_type, object_type, schema) -%} 10 | {%- set key = field.field_name + '-' + lang -%} 11 | {%- if key in my_errors -%} 12 | {%- set errors = unprocessed.pop(key) -%} 13 |
  • {{ 14 | h.fluent_form_label(field, lang)}}: {{ errors[0] }} 15 | {%- do my_errors.remove(key) -%} 16 | {%- endif -%} 17 | {%- endfor -%} 18 | 19 | {%- for key in my_errors | sort -%} 20 | {%- set errors = unprocessed.pop(key) -%} 21 | {%- set lang = key[(field.field_name + '-') | length:] -%} 22 |
  • {{ 23 | h.fluent_form_label(field, lang)}}: {{ errors[0] }} 24 | {%- endfor -%} 25 | -------------------------------------------------------------------------------- /test.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | debug = false 3 | smtp_server = localhost 4 | error_email_from = paste@localhost 5 | 6 | [server:main] 7 | use = egg:Paste#http 8 | host = 0.0.0.0 9 | port = 5000 10 | 11 | [app:celery] 12 | CELERY_ALWAYS_EAGER = True 13 | 14 | [app:main] 15 | use = config:../ckan/test-core.ini 16 | solr_url = http://127.0.0.1:8983/solr 17 | 18 | ckan.plugins = scheming_datasets scheming_groups scheming_organizations fluent 19 | scheming.presets = ckanext.scheming:presets.json 20 | ckanext.fluent:presets.json 21 | 22 | # Logging configuration 23 | [loggers] 24 | keys = root, ckan, sqlalchemy 25 | 26 | [handlers] 27 | keys = console 28 | 29 | [formatters] 30 | keys = generic 31 | 32 | [logger_root] 33 | level = WARN 34 | handlers = console 35 | 36 | [logger_ckan] 37 | qualname = ckan 38 | handlers = 39 | level = INFO 40 | propagate = 0 41 | 42 | [logger_sqlalchemy] 43 | handlers = 44 | qualname = sqlalchemy.engine 45 | level = WARN 46 | 47 | [logger_harvest] 48 | level = WARNING 49 | handlers = console 50 | qualname = ckanext.harvest 51 | propagate = 0 52 | 53 | [handler_console] 54 | class = StreamHandler 55 | args = (sys.stdout,) 56 | level = NOTSET 57 | formatter = generic 58 | 59 | [formatter_generic] 60 | format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s 61 | -------------------------------------------------------------------------------- /ckanext/fluent/plugins.py: -------------------------------------------------------------------------------- 1 | import ckan.plugins as p 2 | from ckan.plugins.toolkit import add_template_directory, h 3 | 4 | from ckanext.fluent import validators, helpers 5 | 6 | 7 | 8 | class FluentPlugin(p.SingletonPlugin): 9 | p.implements(p.IValidators) 10 | p.implements(p.IConfigurer) 11 | p.implements(p.ITemplateHelpers) 12 | 13 | def update_config(self, config): 14 | """ 15 | We have some form snippets that support ckanext-scheming 16 | """ 17 | add_template_directory(config, 'templates') 18 | 19 | def get_helpers(self): 20 | template_helpers = { 21 | 'fluent_form_languages': helpers.fluent_form_languages, 22 | 'fluent_form_label': helpers.fluent_form_label, 23 | 'scheming_missing_required_fields': 24 | helpers.scheming_missing_required_fields, 25 | } 26 | return template_helpers 27 | 28 | def get_validators(self): 29 | return { 30 | 'fluent_text': validators.fluent_text, 31 | 'fluent_text_output': 32 | validators.fluent_text_output, 33 | 'fluent_tags': validators.fluent_tags, 34 | 'fluent_tags_output': 35 | validators.fluent_tags_output, 36 | 'fluent_core_translated_output': 37 | validators.fluent_core_translated_output, 38 | } 39 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | ckanext-fluent - Terms and Conditions of Use 2 | 3 | Unless otherwise noted, computer program source code of ckanext-fluent is 4 | covered under Crown Copyright, Government of Canada, and is distributed under the MIT License. 5 | 6 | 7 | MIT License 8 | 9 | Copyright (c) Her Majesty the Queen in Right of Canada, represented by the President of the Treasury 10 | Board, 2013-2018 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and 13 | associated documentation files (the "Software"), to deal in the Software without restriction, 14 | including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 15 | and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, 16 | subject to the following conditions: 17 | 18 | The above copyright notice and this permission notice shall be included in all copies or substantial 19 | portions of the Software. 20 | 21 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT 22 | NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 23 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 24 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 25 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /COPYING.fr: -------------------------------------------------------------------------------- 1 | ckanext-fluent - Conditions régissant l'utilisation 2 | 3 | Sauf indication contraire, le code source de la ckanext-fluent 4 | est protégé par le droit d'auteur de la Couronne du gouvernement du Canada et distribué 5 | sous la licence MIT. 6 | 7 | 8 | Licence MIT 9 | 10 | (c) Droit d'auteur – Sa Majesté la Reine du chef du Canada, représentée par le président du Conseil 11 | du Trésor, 2013-2018 12 | 13 | La présente autorise toute personne d'obtenir gratuitement une copie du présent logiciel et des 14 | documents connexes (le « logiciel »), de traiter le logiciel sans restriction, y compris, mais sans 15 | s'y limiter, les droits d'utiliser, de copier, de modifier, de fusionner, de publier, de distribuer, 16 | d'accorder une sous licence et de vendre des copies dudit logiciel, et de permettre aux personnes 17 | auxquelles le logiciel est fourni de le faire, selon les conditions suivantes : 18 | 19 | L'avis de droit d'auteur ci dessus et le présent avis de permission seront inclus dans toutes les copies 20 | et les sections importantes du logiciel. 21 | 22 | LE LOGICIEL EST FOURNI « TEL QUEL », SANS AUCUNE GARANTIE, EXPRESSE OU IMPLICITE, Y COMPRIS, MAIS SANS 23 | S'Y LIMITER, LA GARANTIE DE QUALITÉ MARCHANDE, L'ADAPTATION À UN USAGE PARTICULIER ET L'ABSENCE DE 24 | CONTREFAÇON. EN AUCUN CAS LES AUTEURS OU LES DÉTENTEURS DU DROIT D'AUTEUR NE SERONT TENUS RESPONSABLES 25 | DE TOUTE DEMANDE, DOMMAGE OU BRIS DE CONTRAT, DÉLIT CIVIL OU TOUT AUTRE MANQUEMENT LIÉ AU LOGICIEL, 26 | À SON UTILISATION OU À D'AUTRES ÉCHANGES LIÉS AU LOGICIEL. 27 | -------------------------------------------------------------------------------- /ckanext/fluent/tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | from ckanext.fluent.helpers import ( 2 | fluent_form_languages, 3 | fluent_alternate_languages, 4 | fluent_form_label 5 | ) 6 | 7 | 8 | class TestFluentHelpers(object): 9 | 10 | def test_fluent_form_languages_field(self): 11 | field = { 12 | 'form_languages': ['en', 'es'] 13 | } 14 | res = fluent_form_languages(field=field) 15 | assert res == ['en', 'es'] 16 | 17 | def test_fluent_form_languages_schema(self): 18 | schema = { 19 | 'form_languages': ['en', 'fr'] 20 | } 21 | res = fluent_form_languages(schema=schema) 22 | assert res == ['en', 'fr'] 23 | 24 | def test_fluent_alternate_languages_field(self): 25 | field = { 26 | 'alternate_languages': {'en': ['en-GB']} 27 | } 28 | res = fluent_alternate_languages(field=field) 29 | assert res == {'en': ['en-GB']} 30 | 31 | def test_fluent_alternate_languages_schema(self): 32 | schema = { 33 | 'alternate_languages': {'en': ['en-GB']} 34 | } 35 | res = fluent_alternate_languages(schema=schema) 36 | assert res == {'en': ['en-GB']} 37 | 38 | def test_fluent_form_label_exists(self): 39 | field = { 40 | 'field_name': 'fname', 41 | 'fluent_form_label': { 42 | 'en': 'English label', 43 | 'fr': 'French label' 44 | } 45 | } 46 | lang = 'en' 47 | res = fluent_form_label(field, lang) 48 | assert res == 'English label' 49 | 50 | def test_fluent_form_label_not_exists(self): 51 | field = { 52 | 'field_name': 'fname', 53 | 'fluent_form_label': { 54 | 'en': 'English label', 55 | 'fr': 'French label' 56 | }, 57 | 'label': 'Standard label' 58 | } 59 | lang = 'es' 60 | res = fluent_form_label(field, lang) 61 | assert res == 'ES Standard label' 62 | -------------------------------------------------------------------------------- /ckanext/fluent/presets.json: -------------------------------------------------------------------------------- 1 | { 2 | "scheming_presets_version": 1, 3 | "about": "scheming presets for fluent fields", 4 | "about_url": "http://github.com/open-data/ckanext-fluent", 5 | "presets": [ 6 | { 7 | "preset_name": "fluent_core_translated", 8 | "values": { 9 | "form_snippet": "fluent_text.html", 10 | "display_snippet": "fluent_text.html", 11 | "error_snippet": "fluent_text.html", 12 | "validators": "fluent_text", 13 | "output_validators": "fluent_core_translated_output" 14 | } 15 | }, 16 | { 17 | "preset_name": "fluent_text", 18 | "values": { 19 | "form_snippet": "fluent_text.html", 20 | "display_snippet": "fluent_text.html", 21 | "error_snippet": "fluent_text.html", 22 | "validators": "fluent_text", 23 | "output_validators": "fluent_text_output" 24 | } 25 | }, 26 | { 27 | "preset_name": "fluent_tags", 28 | "values": { 29 | "form_snippet": "fluent_tags.html", 30 | "display_snippet": "fluent_tags.html", 31 | "error_snippet": "fluent_text.html", 32 | "validators": "fluent_tags", 33 | "output_validators": "fluent_tags_output", 34 | "form_attrs": { 35 | "style": "width: 100%", 36 | "class": "form-control" 37 | } 38 | } 39 | }, 40 | { 41 | "preset_name": "fluent_markdown", 42 | "values": { 43 | "form_snippet": "fluent_markdown.html", 44 | "display_snippet": "fluent_markdown.html", 45 | "error_snippet": "fluent_text.html", 46 | "validators": "fluent_text", 47 | "output_validators": "fluent_text_output" 48 | } 49 | }, 50 | { 51 | "preset_name": "fluent_link", 52 | "values": { 53 | "form_snippet": "fluent_text.html", 54 | "display_snippet": "fluent_link.html", 55 | "error_snippet": "fluent_text.html", 56 | "validators": "fluent_text", 57 | "output_validators": "fluent_text_output" 58 | } 59 | } 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [push, pull_request] 3 | jobs: 4 | lint: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - uses: actions/setup-python@v5 9 | with: 10 | python-version: '3.9' 11 | - name: Install requirements 12 | run: pip install flake8 pycodestyle 13 | - name: Check syntax 14 | run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan 15 | 16 | test: 17 | needs: lint 18 | strategy: 19 | matrix: 20 | ckan-version: ["2.11", "2.10"] 21 | fail-fast: false 22 | 23 | name: CKAN ${{ matrix.ckan-version }} 24 | runs-on: ubuntu-latest 25 | container: 26 | image: ckan/ckan-dev:${{ matrix.ckan-version }} 27 | services: 28 | solr: 29 | image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 30 | postgres: 31 | image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} 32 | env: 33 | POSTGRES_USER: postgres 34 | POSTGRES_PASSWORD: postgres 35 | POSTGRES_DB: postgres 36 | options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 37 | redis: 38 | image: redis:3 39 | env: 40 | CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test 41 | CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test 42 | CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test 43 | CKAN_SOLR_URL: http://solr:8983/solr/ckan 44 | CKAN_REDIS_URL: redis://redis:6379/1 45 | 46 | steps: 47 | - uses: actions/checkout@v3 48 | - name: Install requirements 49 | run: | 50 | pip install -r requirements.txt 51 | pip install -r dev-requirements.txt 52 | pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming 53 | pip install -e . 54 | # Replace default path to CKAN core config file with the one on the container 55 | sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini 56 | - name: Setup extension (CKAN >= 2.9) 57 | if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} 58 | run: | 59 | ckan -c test.ini db init 60 | - name: Setup extension (CKAN < 2.9) 61 | if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} 62 | run: | 63 | paster --plugin=ckan db init -c test.ini 64 | - name: Run tests 65 | run: pytest --ckan-ini=test.ini --cov=ckanext.fluent --disable-warnings ckanext/fluent/tests 66 | -------------------------------------------------------------------------------- /ckanext/fluent/fluent_scheming.yaml: -------------------------------------------------------------------------------- 1 | scheming_version: 1 2 | dataset_type: fluent-test 3 | about_url: http://github.com/open-data/ckanext-fluent 4 | form_languages: [en, fr] 5 | dataset_fields: 6 | - field_name: title_translated 7 | label: 8 | en: Title 9 | fr: Titre 10 | fluent_form_label: 11 | en: 12 | en: Title (English) 13 | fr: Titre (anglais) 14 | fr: 15 | en: Title (French) 16 | fr: Titre (français) 17 | fluent_help_text: 18 | en: 19 | en: The English name by which the dataset is known 20 | fr: Nom anglais désignant le jeu de données 21 | fr: 22 | en: The French name by which the dataset is known 23 | fr: Nom français désignant le jeu de données 24 | preset: fluent_core_translated 25 | required: true 26 | 27 | - field_name: name 28 | label: URL 29 | preset: dataset_slug 30 | 31 | - field_name: owner_org 32 | label: Organization 33 | preset: dataset_organization 34 | 35 | - field_name: notes_translated 36 | label: 37 | en: Description 38 | fr: Description 39 | fluent_form_label: 40 | en: 41 | en: Description (English) 42 | fr: Description (anglais) 43 | fr: 44 | en: Description (French) 45 | fr: Description (français) 46 | fluent_help_text: 47 | en: 48 | en: "An account of the dataset, in English. A description may include but is not limited to: an abstract, a table of contents, or a free-text account of the resource." 49 | fr: Description du jeu de données, en anglais. La description peut comprendre un résumé, une table des matières ou un texte libre. 50 | fr: 51 | en: "An account of the dataset, in French. A description may include but is not limited to: an abstract, a table of contents, or a free-text account of the resource." 52 | fr: Description du jeu de données, en anglais. La description peut comprendre un résumé, une table des matières ou un texte libre. 53 | 54 | preset: fluent_core_translated 55 | 56 | - field_name: keywords 57 | label: 58 | en: Keywords 59 | fr: Mots-clés 60 | preset: fluent_tags 61 | required: true 62 | 63 | resource_fields: 64 | - field_name: url 65 | label: URL 66 | preset: resource_url_upload 67 | 68 | - field_name: name_translated 69 | label: 70 | en: Title 71 | fr: Titre 72 | fluent_form_label: 73 | en: 74 | en: Title (English) 75 | fr: Titre (anglais) 76 | fr: 77 | en: Title (French) 78 | fr: Titre (français) 79 | fluent_help_text: 80 | en: 81 | en: An English name given to the resource. 82 | fr: Nom anglais attribué à la ressource 83 | fr: 84 | en: A French name given to the resource. 85 | fr: Nom français attribué à la ressource 86 | preset: fluent_core_translated 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ckanext-fluent 2 | 3 | This extension provides a way to store and return multilingual 4 | fields in CKAN datasets, resources, organizations and groups. 5 | 6 | The [ckanext-scheming](https://github.com/ckan/ckanext-scheming) 7 | extension is required for this extension to work. 8 | 9 | ## Installation 10 | 11 | ``` 12 | git clone https://github.com/ckan/ckanext-fluent.git 13 | cd ckanext-fluent 14 | python setup.py develop 15 | pip install -r requirements.txt 16 | ``` 17 | 18 | Add the `fluent` plugin to your ckan.plugins configuration 19 | settings and use ckanext-scheming or a custom form plugin to 20 | use the provided validators to store multilingual text in 21 | extra fields. 22 | 23 | The easiest way to use fluent multilingual text fields is with 24 | [ckanext-scheming](https://github.com/open-data/ckanext-scheming/). 25 | Add `ckanext.fluent:presets.json` to your scheming.presets 26 | configuration settings: 27 | 28 | ```ini 29 | scheming.presets = ckanext.scheming:presets.json 30 | ckanext.fluent:presets.json 31 | ``` 32 | 33 | An [example schema](https://github.com/ckan/ckanext-fluent/blob/master/ckanext/fluent/fluent_scheming.yaml) 34 | is may be used with the `scheming_datasets` plugin and this schema setting: 35 | 36 | ```ini 37 | scheming.dataset_schemas=ckanext.fluent:fluent_scheming.yaml 38 | ``` 39 | 40 | This example schema registers a new `fluent-test` dataset type. Visit 41 | `/fluent-test/new` on your ckan site to create a new dataset with this schema. 42 | 43 | ## `fluent_text` fields 44 | 45 | A fluent multilingual text field in a scheming schema: 46 | 47 | ```json 48 | { 49 | "field_name": "books", 50 | "preset": "fluent_text", 51 | "label": { 52 | "en": "Books", 53 | "fr": "Livres" 54 | }, 55 | "form_languages": ["en", "fr"] 56 | } 57 | ``` 58 | 59 | This new extra field "books" will appear as multiple fields in the 60 | dataset form, one for each language specified in `form_languages`. 61 | 62 | ![Example of fluent_text form snippet](docs/multilingual-form.png) 63 | 64 | When the dataset is accessed from the API the language values appear 65 | and are updated as an object, eg.: 66 | 67 | ```json 68 | { 69 | "...": "...", 70 | "books": { 71 | "en": "Franklin", 72 | "fr": "Benjamin" 73 | }, 74 | "...": "..." 75 | } 76 | ``` 77 | 78 | ## `fluent_tags` fields 79 | 80 | Example multilingual tag field: 81 | 82 | ```json 83 | { 84 | "field_name": "keywords", 85 | "label": { 86 | "en": "Keywords", 87 | "fr": "Mots-clés", 88 | }, 89 | "preset": "fluent_tags" 90 | } 91 | ``` 92 | 93 | Note: this preset is not supported for use on the core `tags` field. 94 | 95 | This new extra field "keywords" will appear as multiple fields in the 96 | dataset form, one for each language specified in `form_languages`. 97 | 98 | ![Example of fluent_tags form snippet](docs/multilingual-tags.png) 99 | 100 | When the dataset is accessed from the API the language values appear 101 | and are updated as an object with list values, eg.: 102 | 103 | ```json 104 | { 105 | "...": "...", 106 | "keywords": { 107 | "en": ["what"], 108 | "fr": ["quoi"] 109 | }, 110 | "...": "..." 111 | } 112 | ``` 113 | 114 | ## `fluent_core_translated` fields 115 | 116 | Fluent should not be directly used on ckan core fields such as `title` and `notes`. 117 | To use fluent to translate core fields, you should use a field with the `_translated` 118 | suffix appended to the core field name (e.g. `title_translated`) and use the `fluent_core_translated` 119 | preset. By doing so, the translated version of the field is stored in the field with the 120 | `_translated` suffix while the core field displays the value for the site's default language. 121 | 122 | ```json 123 | 124 | { 125 | "field_name": "title_translated", 126 | "preset": "fluent_core_translated", 127 | "label": { 128 | "en": "Franklin", 129 | "fr": "Benjamin" 130 | } 131 | } 132 | ``` 133 | -------------------------------------------------------------------------------- /ckanext/fluent/helpers.py: -------------------------------------------------------------------------------- 1 | from ckan.lib.i18n import get_available_locales 2 | 3 | from ckan.plugins.toolkit import ( 4 | h, chained_helper, get_action, ObjectNotFound, NotAuthorized 5 | ) 6 | 7 | 8 | def fluent_form_languages(field=None, entity_type=None, object_type=None, 9 | schema=None): 10 | """ 11 | Return a list of language codes for this form (or form field) 12 | 13 | 1. return field['form_languages'] if it is defined 14 | 2. return schema['form_languages'] if it is defined 15 | 3. get schema from entity_type + object_type then 16 | return schema['form_languages'] if they are defined 17 | 4. return languages from site configuration 18 | """ 19 | if field and 'form_languages' in field: 20 | return field['form_languages'] 21 | if schema and 'form_languages' in schema: 22 | return schema['form_languages'] 23 | if entity_type and object_type: 24 | schema = h.scheming_get_schema(entity_type, object_type) 25 | if schema and 'form_languages' in schema: 26 | return schema['form_languages'] 27 | 28 | langs = [] 29 | for l in get_available_locales(): 30 | if l.language not in langs: 31 | langs.append(l.language) 32 | return langs 33 | 34 | 35 | def fluent_alternate_languages(field=None, schema=None): 36 | """ 37 | Return a dict of alternates acceptable as replacements for 38 | required languages, as given in the field or schema. 39 | 40 | e.g. {'en': ['en-GB']} 41 | """ 42 | if field and 'alternate_languages' in field: 43 | return field['alternate_languages'] 44 | if schema and 'alternate_languages' in schema: 45 | return schema['alternate_languages'] 46 | return {} 47 | 48 | 49 | def fluent_form_label(field, lang): 50 | """ 51 | Return a label for the input field for the given language 52 | 53 | If the field has a fluent_form_label defined the label will 54 | be taken from there. If a matching label can't be found 55 | this helper will return the language code in uppercase and 56 | the standard label or field name. 57 | """ 58 | form_label = field.get('fluent_form_label', {}) 59 | 60 | if lang in form_label: 61 | return h.scheming_language_text(form_label[lang]) 62 | 63 | return lang.upper() + ' ' + h.scheming_language_text( 64 | field.get('label', field['field_name']) 65 | ) 66 | 67 | 68 | @chained_helper 69 | def scheming_missing_required_fields( 70 | next_helper, pages, data=None, package_id=None): 71 | """ 72 | """ 73 | if package_id: 74 | try: 75 | data = get_action('package_show')({}, {"id": package_id}) 76 | except (ObjectNotFound, NotAuthorized): 77 | pass 78 | if data is None: 79 | data = {} 80 | missing = next_helper(pages, data, package_id) 81 | if not data.get('type'): 82 | return missing 83 | 84 | schema = h.scheming_get_dataset_schema(data['type']) 85 | for page_missing, page in zip(missing, pages): 86 | for f in page['fields']: 87 | if ( 88 | f['field_name'] not in data 89 | or f['field_name'] in page_missing 90 | or 'validators' not in f 91 | or not any(v.startswith('fluent_') for v in f['validators'].split()) 92 | or not f.get('required') 93 | ): 94 | continue 95 | required_langs = fluent_form_languages(f, schema=schema) 96 | alternate_langs = fluent_alternate_languages(f, schema=schema) 97 | value = data[f['field_name']] 98 | for lang in required_langs: 99 | if value.get(lang) or any( 100 | value.get(l) for l in alternate_langs.get(lang, []) 101 | ): 102 | continue 103 | page_missing.append(f['field_name']) 104 | break 105 | return missing 106 | -------------------------------------------------------------------------------- /ckanext/fluent/validators.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | import six 4 | 5 | from ckan.plugins.toolkit import missing, get_validator, Invalid, config, _ 6 | 7 | from ckanext.fluent.helpers import ( 8 | fluent_form_languages, fluent_alternate_languages) 9 | from ckanext.scheming.helpers import scheming_language_text 10 | from ckanext.scheming.validation import ( 11 | scheming_validator, validators_from_string) 12 | 13 | 14 | # loose definition of BCP47-like strings 15 | BCP_47_LANGUAGE = u'^[a-z]{2,8}(-[0-9a-zA-Z]{1,8})*$' 16 | 17 | LANG_SUFFIX = '_translated' 18 | 19 | tag_length_validator = get_validator('tag_length_validator') 20 | tag_name_validator = get_validator('tag_name_validator') 21 | 22 | 23 | @scheming_validator 24 | def fluent_core_translated_output(field, schema): 25 | assert field['field_name'].endswith(LANG_SUFFIX), 'Output validator "fluent_core_translated" must only used on a field that ends with "_translated"' 26 | 27 | def validator(key, data, errors, context): 28 | """ 29 | Return a value for a core field using a multilingual dict. 30 | """ 31 | data[key] = fluent_text_output(data[key]) 32 | 33 | k = key[-1] 34 | new_key = key[:-1] + (k[:-len(LANG_SUFFIX)],) 35 | 36 | if new_key in data: 37 | data[new_key] = scheming_language_text(data[key], config.get('ckan.locale_default', 'en')) 38 | 39 | return validator 40 | 41 | 42 | @scheming_validator 43 | def fluent_text(field, schema): 44 | """ 45 | Accept multilingual text input in the following forms 46 | and convert to a json string for storage: 47 | 48 | 1. a multilingual dict, eg. 49 | 50 | {"en": "Text", "fr": "texte"} 51 | 52 | 2. a JSON encoded version of a multilingual dict, for 53 | compatibility with old ways of loading data, eg. 54 | 55 | '{"en": "Text", "fr": "texte"}' 56 | 57 | 3. separate fields per language (for form submissions): 58 | 59 | fieldname-en = "Text" 60 | fieldname-fr = "texte" 61 | 62 | When using this validator in a ckanext-scheming schema setting 63 | "required" to true will make all form languages required to 64 | pass validation. 65 | """ 66 | # combining scheming required checks and fluent field processing 67 | # into a single validator makes this validator more complicated, 68 | # but should be easier for fluent users and eliminates quite a 69 | # bit of duplication in handling the different types of input 70 | required_langs = [] 71 | alternate_langs = {} 72 | if field and field.get('required'): 73 | required_langs = fluent_form_languages(field, schema=schema) 74 | alternate_langs = fluent_alternate_languages(field, schema=schema) 75 | 76 | def validator(key, data, errors, context): 77 | # just in case there was an error before our validator, 78 | # bail out here because our errors won't be useful 79 | if errors[key]: 80 | return 81 | 82 | enforce_required = True 83 | if not schema.get('draft_fields_required', True): 84 | if data.get(('state',), '').startswith('draft'): 85 | enforce_required = False 86 | 87 | value = data[key] 88 | 89 | prefix = key[-1] + '-' 90 | extras = data.get(key[:-1] + ('__extras',), {}) 91 | # 1 or 2. dict or JSON encoded string 92 | # only if no separate field values present 93 | if value is not missing and not any(n.startswith(prefix) for n in extras): 94 | if isinstance(value, six.string_types): 95 | try: 96 | value = json.loads(value) 97 | except ValueError: 98 | errors[key].append(_('Failed to decode JSON string')) 99 | return 100 | except UnicodeDecodeError: 101 | errors[key].append(_('Invalid encoding for JSON string')) 102 | return 103 | if not isinstance(value, dict): 104 | errors[key].append(_('expecting JSON object')) 105 | return 106 | 107 | for lang, text in value.items(): 108 | try: 109 | m = re.match(BCP_47_LANGUAGE, lang) 110 | except TypeError: 111 | errors[key].append(_('invalid type for language code: %r') 112 | % lang) 113 | continue 114 | if not m: 115 | errors[key].append(_('invalid language code: "%s"') % lang) 116 | continue 117 | if not isinstance(text, six.string_types): 118 | errors[key].append(_('invalid type for "%s" value') % lang) 119 | continue 120 | if isinstance(text, str): 121 | try: 122 | value[lang] = text if six.PY3 else text.decode( 123 | 'utf-8') 124 | except UnicodeDecodeError: 125 | errors[key]. append(_('invalid encoding for "%s" value') 126 | % lang) 127 | 128 | if enforce_required: 129 | for lang in required_langs: 130 | if value.get(lang) or any( 131 | value.get(l) for l in alternate_langs.get(lang, [])): 132 | continue 133 | errors[key].append(_('Required language "%s" missing') % lang) 134 | 135 | if not errors[key]: 136 | data[key] = json.dumps(value, ensure_ascii=False) 137 | return 138 | 139 | # 3. separate fields 140 | output = {} 141 | 142 | for name, text in extras.items(): 143 | if not name.startswith(prefix): 144 | continue 145 | lang = name.split('-', 1)[1] 146 | m = re.match(BCP_47_LANGUAGE, lang) 147 | if not m: 148 | errors[name] = [_('invalid language code: "%s"') % lang] 149 | output = None 150 | continue 151 | 152 | if output is not None: 153 | output[lang] = text 154 | 155 | if enforce_required: 156 | for lang in required_langs: 157 | if extras.get(prefix + lang) or any( 158 | extras.get(prefix + l) for l in alternate_langs.get(lang, [])): 159 | continue 160 | errors[key[:-1] + (key[-1] + '-' + lang,)] = [_('Missing value')] 161 | output = None 162 | 163 | if output is None: 164 | return 165 | 166 | for lang in output: 167 | del extras[prefix + lang] 168 | data[key] = json.dumps(output, ensure_ascii=False) 169 | 170 | return validator 171 | 172 | 173 | def fluent_text_output(value): 174 | """ 175 | Return stored json representation as a multilingual dict, if 176 | value is already a dict just pass it through. 177 | """ 178 | if isinstance(value, dict): 179 | return value 180 | try: 181 | return json.loads(value) 182 | except ValueError: 183 | # plain string in the db, assume default locale 184 | return {config.get('ckan.locale_default', 'en'): value} 185 | 186 | 187 | @scheming_validator 188 | def fluent_tags(field, schema): 189 | """ 190 | Accept multilingual lists of tags in the following forms 191 | and convert to a json string for storage. 192 | 193 | 1. a multilingual dict of lists of tag strings, eg. 194 | 195 | {"en": ["big", "large"], "fr": ["grande"]} 196 | 197 | 2. separate fields per language with comma-separated values 198 | (for form submissions) 199 | 200 | fieldname-en = "big,large" 201 | fieldname-fr = "grande" 202 | 203 | Validation of each tag is performed with validators 204 | tag_length_validator and tag_name_validator. When using 205 | ckanext-scheming these may be overridden with the 206 | "tag_validators" field value 207 | """ 208 | # XXX this validator is too long and should be broken up 209 | 210 | required_langs = [] 211 | alternate_langs = {} 212 | if field and field.get('required'): 213 | required_langs = fluent_form_languages(field, schema=schema) 214 | alternate_langs = fluent_alternate_languages(field, schema=schema) 215 | 216 | tag_validators = [tag_length_validator, tag_name_validator] 217 | if field and 'tag_validators' in field: 218 | tag_validators = validators_from_string( 219 | field['tag_validators'], field, schema) 220 | 221 | def validator(key, data, errors, context): 222 | if errors[key]: 223 | return 224 | 225 | enforce_required = True 226 | if not schema.get('draft_fields_required', True): 227 | if data.get(('state',), '').startswith('draft'): 228 | enforce_required = False 229 | 230 | value = data[key] 231 | prefix = key[-1] + '-' 232 | extras = data.get(key[:-1] + ('__extras',), {}) 233 | # 1. dict of lists of tag strings 234 | # only if no separate field values present 235 | if value is not missing and not any(n.startswith(prefix) for n in extras): 236 | if not isinstance(value, dict): 237 | errors[key].append(_('expecting JSON object')) 238 | return 239 | 240 | for lang, keys in value.items(): 241 | try: 242 | m = re.match(BCP_47_LANGUAGE, lang) 243 | except TypeError: 244 | errors[key].append(_('invalid type for language code: %r') 245 | % lang) 246 | continue 247 | if not m: 248 | errors[key].append(_('invalid language code: "%s"') % lang) 249 | continue 250 | if not isinstance(keys, list): 251 | errors[key].append(_('invalid type for "%s" value') % lang) 252 | continue 253 | out = [] 254 | for i, v in enumerate(keys): 255 | if not isinstance(v, six.string_types): 256 | errors[key].append( 257 | _('invalid type for "{lang}" value item {num}').format( 258 | lang=lang, num=i)) 259 | continue 260 | 261 | if isinstance(v, str): 262 | try: 263 | out.append(v if six.PY3 else v.decode( 264 | 'utf-8')) 265 | except UnicodeDecodeError: 266 | errors[key]. append(_( 267 | 'expected UTF-8 encoding for ' 268 | '"{lang}" value item {num}').format( 269 | lang=lang, num=i)) 270 | else: 271 | out.append(v) 272 | 273 | tags = [] 274 | errs = [] 275 | for tag in out: 276 | newtag, tagerrs = _validate_single_tag(tag, tag_validators) 277 | errs.extend(tagerrs) 278 | tags.append(newtag) 279 | if errs: 280 | errors[key].extend(errs) 281 | value[lang] = tags 282 | 283 | if enforce_required: 284 | for lang in required_langs: 285 | if value.get(lang) or any( 286 | value.get(l) for l in alternate_langs.get(lang, [])): 287 | continue 288 | errors[key].append(_('Required language "%s" missing') % lang) 289 | 290 | if not errors[key]: 291 | data[key] = json.dumps(value) 292 | return 293 | 294 | # 2. separate fields 295 | output = {} 296 | 297 | for name, text in extras.items(): 298 | if not name.startswith(prefix): 299 | continue 300 | lang = name.split('-', 1)[1] 301 | m = re.match(BCP_47_LANGUAGE, lang) 302 | if not m: 303 | errors[name] = [_('invalid language code: "%s"') % lang] 304 | output = None 305 | continue 306 | 307 | if not isinstance(text, six.string_types): 308 | errors[name].append(_('invalid type')) 309 | continue 310 | 311 | if isinstance(text, str): 312 | try: 313 | text = text if six.PY3 else text.decode( 314 | 'utf-8') 315 | except UnicodeDecodeError: 316 | errors[name].append(_('expected UTF-8 encoding')) 317 | continue 318 | 319 | if output is not None and text: 320 | tags = [] 321 | errs = [] 322 | for tag in text.split(','): 323 | newtag, tagerrs = _validate_single_tag(tag, tag_validators) 324 | errs.extend(tagerrs) 325 | tags.append(newtag) 326 | output[lang] = tags 327 | if errs: 328 | errors[key[:-1] + (name,)] = errs 329 | 330 | if enforce_required: 331 | for lang in required_langs: 332 | if extras.get(prefix + lang) or any( 333 | extras.get(prefix + l) for l in alternate_langs.get(lang, [])): 334 | continue 335 | errors[key[:-1] + (key[-1] + '-' + lang,)] = [_('Missing value')] 336 | output = None 337 | 338 | if output is None: 339 | return 340 | 341 | for lang in output: 342 | del extras[prefix + lang] 343 | data[key] = json.dumps(output) 344 | 345 | return validator 346 | 347 | 348 | def fluent_tags_output(value): 349 | """ 350 | Return stored json representation as a multilingual dict, if 351 | value is already a dict just pass it through. 352 | """ 353 | if isinstance(value, dict): 354 | return value 355 | return json.loads(value) 356 | 357 | 358 | def _validate_single_tag(name, validators): 359 | """ 360 | Return (new_name, errors_list) for validators in the form 361 | validator(value, context) 362 | """ 363 | errors = [] 364 | for v in validators: 365 | try: 366 | name = v(name, {}) 367 | except Invalid as e: 368 | errors.append(e.error) 369 | return name, errors 370 | --------------------------------------------------------------------------------