├── .env ├── .github └── workflows │ ├── deploy.yml │ └── test.yml ├── .gitignore ├── ADR.md ├── Dockerfile ├── MANIFEST.in ├── Makefile ├── README.md ├── ckanext ├── __init__.py └── usmetadata │ ├── __init__.py │ ├── blueprint.py │ ├── db_utils.py │ ├── fanstatic │ ├── add_subagency.js │ ├── dataset_edit_form.js │ ├── dataset_url.js │ ├── redactions.js │ ├── resource_form.css │ ├── resource_form.js │ └── webassets.yml │ ├── helper.py │ ├── media_types.json │ ├── plugin.py │ ├── public │ ├── base │ │ └── images │ │ │ └── inventory-logo.png │ ├── images │ │ └── icons │ │ │ └── ckan.ico │ ├── partial_redaction.jpg │ ├── redacted_icon.png │ └── redaction_clear.png │ ├── resource_formats.json │ ├── templates │ ├── base.html │ ├── footer.html │ ├── organization │ │ ├── activity_stream.html │ │ ├── member_new.html │ │ └── read_base.html │ ├── package │ │ ├── base.html │ │ ├── read.html │ │ ├── read_base.html │ │ ├── resource_read.html │ │ └── snippets │ │ │ ├── expanded_common_core_fields.html │ │ │ ├── info.html │ │ │ ├── metadata_info.html │ │ │ ├── package_basic_fields.html │ │ │ ├── package_form.html │ │ │ ├── package_metadata_fields.html │ │ │ ├── required_common_core_fields.html │ │ │ ├── required_if_applicable_common_core_fields.html │ │ │ ├── resource_form.html │ │ │ ├── resource_item.html │ │ │ ├── resources.html │ │ │ └── stages.html │ └── snippets │ │ ├── activity_item.html │ │ ├── license.html │ │ ├── organization.html │ │ └── package_item.html │ └── tests │ ├── test_expanded_metadata.py │ ├── test_usmetadata.py │ └── unit_test.py ├── dev-requirements.txt ├── docker-compose.yml ├── requirements.txt ├── setup.cfg ├── setup.py ├── test.ini └── test.sh /.env: -------------------------------------------------------------------------------- 1 | # DB image settings 2 | POSTGRES_PASSWORD=ckan 3 | DATASTORE_READONLY_PASSWORD=datastore 4 | 5 | # Basic 6 | CKAN_SITE_ID=default 7 | CKAN_SITE_URL=http://ckan:5000 8 | CKAN__SITE_LOGO=/base/images/inventory-logo.png 9 | CKAN_PORT=5000 10 | CKAN_SYSADMIN_NAME=admin 11 | CKAN_SYSADMIN_PASSWORD=password 12 | CKAN_SYSADMIN_EMAIL=your_email@example.com 13 | TZ=UTC 14 | 15 | # Database connections (TODO: avoid duplication) 16 | CKAN_SQLALCHEMY_URL=postgresql://ckan_default:pass@db/ckan_test 17 | CKAN_DATASTORE_WRITE_URL=postgresql://ckan_default:pass@db/datastore_test 18 | CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore 19 | 20 | # Test database connections 21 | TEST_CKAN_SQLALCHEMY_URL=postgres://ckan_default:pass@db/ckan_test 22 | TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan_default:pass@db/datastore_test 23 | TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test 24 | 25 | # Other services connections 26 | CKAN_SOLR_URL=http://solr:8983/solr/ckan 27 | CKAN_REDIS_URL=redis://redis:6379/1 28 | CKAN_DATAPUSHER_URL=http://datapusher:8800 29 | CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 30 | 31 | TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan 32 | TEST_CKAN_REDIS_URL=redis://redis:6379/1 33 | 34 | # Core settings 35 | CKAN__STORAGE_PATH=/var/lib/ckan 36 | 37 | CKAN_SMTP_SERVER=smtp.corporateict.domain:25 38 | CKAN_SMTP_STARTTLS=True 39 | CKAN_SMTP_USER=user 40 | CKAN_SMTP_PASSWORD=pass 41 | CKAN_SMTP_MAIL_FROM=ckan@localhost 42 | 43 | CKAN___BROKER_BACKEND=redis 44 | CKAN___BROKER_HOST=redis://redis/1 45 | CKAN___CELERY_RESULT_BACKEND=redis 46 | CKAN___REDIS_HOST=redis 47 | CKAN___REDIS_PORT=6379 48 | CKAN___REDIS_DB=0 49 | CKAN___REDIS_CONNECT_RETRY=True 50 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI 2 | on: 3 | pull_request: 4 | branches: [main] 5 | types: [closed] 6 | workflow_dispatch: 7 | inputs: 8 | version_no: 9 | description: 'Release Version:' 10 | required: true 11 | 12 | jobs: 13 | deploy: 14 | name: Publish to PyPI 15 | runs-on: ubuntu-latest 16 | if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' 17 | steps: 18 | - name: checkout 19 | uses: actions/checkout@v3 20 | - name: Update setup.py if manual release 21 | if: github.event_name == 'workflow_dispatch' 22 | run: | 23 | sed -i "s/version='[0-9]\{1,2\}.[0-9]\{1,4\}.[0-9]\{1,4\}',/version='${{github.event.inputs.version_no}}',/g" setup.py 24 | - name: Create packages 25 | run: | 26 | python setup.py sdist 27 | python setup.py bdist_wheel 28 | - name: pypi-publish 29 | uses: pypa/gh-action-pypi-publish@v1.5.1 30 | with: 31 | user: __token__ 32 | password: ${{ secrets.PYPI_API_TOKEN }} 33 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [push] 3 | 4 | jobs: 5 | lint-test: 6 | name: Lint + Test 7 | uses: gsa/data.gov/.github/workflows/ckan-test.yml@main 8 | with: 9 | ext_name: usmetadata 10 | plugins: usmetadata 11 | secrets: inherit 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ckanext/usmetadata/tests/__init__.py 2 | ckanext_usmetadata.egg-info/ 3 | *.pyc 4 | .coverage 5 | 6 | # OS generated files # 7 | ###################### 8 | *.DS_Store 9 | ._* 10 | .Spotlight-V100 11 | .Trashes 12 | *ehthumbs.db 13 | *Thumbs.db 14 | *.swp 15 | *.bak 16 | -------------------------------------------------------------------------------- /ADR.md: -------------------------------------------------------------------------------- 1 | 2 | ADRs for ckanext-usmetadata 3 | =========================== 4 | 5 | ## 1. Reduce complexity of schema logic 6 | 7 | Date: 2021-08-03 8 | 9 | ### Status 10 | 11 | Unknown 12 | 13 | ### Context 14 | 15 | The existing schema definitions for ckanext-usmetadata were complex and it's effect was not completely known. 16 | 17 | ### Decision 18 | 19 | To expedite the upgrade to PY3, proper validation was made more lenient. There are a bunch of unit tests in 20 | unit_test.py and the ones that were failing or otherwise broken were commented out to be worked on in the 21 | future. 22 | 23 | ### Consequences 24 | 25 | - Until the schema is updated properly, users may be able to input data that is not exportable in the 26 | data.json file. Due to the limited number of users and very little new users, this risk was deemed acceptable. 27 | 28 | ## 2. Update test suite 29 | 30 | Date: 2021-08-03 31 | 32 | ### Status 33 | 34 | Accepted 35 | 36 | ### Context 37 | 38 | The old tests were written for an older version of ckan (possibly 2.5) using nosetests and relying on 39 | outdated frameworks. 40 | 41 | ### Decision 42 | 43 | A few sources were researched to properly upgrade the tests: 44 | - https://docs.python.org/3/library/unittest.html 45 | - https://docs.ckan.org/en/2.9/extensions/testing-extensions.html 46 | - https://flask.palletsprojects.com/en/2.0.x/reqcontext/ 47 | - https://github.com/ckan/ckan/issues/4247 48 | 49 | ### Consequences 50 | 51 | - non-known. 52 | 53 | ## 3. Write custom validators 54 | 55 | Date: 2021-08-03 56 | 57 | ### Status 58 | 59 | Accepted 60 | 61 | ### Context 62 | 63 | There were two validators that were imported from [formencode](https://github.com/formencode/formencode), 64 | Regex and UnicodeString. In PY2+CKAN2.8, these data types had no issues with the sql code. However, 65 | in PY3+CKAN2.9, sql didn't know how to adapt these types. 66 | 67 | ### Decision 68 | 69 | Custom vaildator functions were written to handle the functionality that the imported validators provided. 70 | The following link was used as a reference, 71 | - https://docs.ckan.org/en/2.9/extensions/adding-custom-fields.html#custom-validators 72 | The validators were not registered with the plugin because the place where they were defined was already 73 | importing the plugin, so there would have been a circular dependency. Either way, the validators work as 74 | standalone functions that get called. 75 | 76 | ### Consequences 77 | 78 | - There are probably unknown consequences. 79 | 80 | 81 | ## 4. Update docker test environment 82 | 83 | Date: 2021-08-06 84 | 85 | ### Status 86 | 87 | Not Implemented 88 | 89 | ### Context 90 | 91 | Currently, a custom Dockerfile is used to install pip requirements and also the desired working extension. 92 | The CKAN 2.8 and 2.9 docker dev images, [openknowledge/ckan-dev](https://github.com/okfn/docker-ckan/blob/ 93 | master/ckan-dev/2.8/setup/start_ckan_development.sh), support installing these things as part of the startup 94 | of the container. 95 | 96 | An example of updating to not use the dockerfile is seen in [ckanext-dcat_usmetadata](https://github.com/ 97 | GSA/ckanext-dcat_usmetadata/commit/8df5e938d750e26caddd3688b40b696991a5e0ad). While there is still a 98 | Dockerfile, it only installed base linux packages and doesn't handle anything with the extension. Since 99 | this extension does not need any additional linux packages, the docker-compose.yml would directly call 100 | `image: openknowledge/ckan-dev:${CKAN_VERSION}`. 101 | 102 | ### Decision 103 | 104 | This change was not implemented yet because no further development was necessary since the prototype of 105 | the development workflow was completed. There is a bit of residual py3 bugfixes/cleanup that will be 106 | done at some point before deployment. It was thought that this could wait until then. 107 | 108 | ### Consequences 109 | 110 | - No tangible consequencees. 111 | - Just slightly different development pipelines. 112 | 113 | ## 5. Remove resource view 114 | 115 | Date: 2023-06-28 116 | 117 | ### Status 118 | 119 | Implemented 120 | 121 | ### Context 122 | 123 | To fix a broken layout, a call to CKAN core's resource view was removed. This allowed for a preview of the asset in CKAN. 124 | 125 | ### Decision 126 | 127 | As it was not working, and wouldn't result in a loss of necessary functionality it was decided to remove this block. 128 | 129 | 130 | ### Consquences 131 | 132 | - Cleaner layout on resource pages 133 | - No preview of resources 134 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG CKAN_VERSION=2.10.01 2 | FROM openknowledge/ckan-dev:${CKAN_VERSION} 3 | ARG CKAN_VERSION 4 | 5 | RUN pip install --upgrade pip 6 | 7 | COPY . $APP_DIR/ 8 | 9 | RUN pip install -r $APP_DIR/requirements.txt -r $APP_DIR/dev-requirements.txt -e $APP_DIR/. 10 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include requirements.txt 3 | recursive-include ckanext/usmetadata *.html *.json *.js *.less *.css *.yml *.png 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CKAN_VERSION ?= 2.10.01 2 | COMPOSE_FILE ?= docker-compose.yml 3 | 4 | build: ## Build the docker containers 5 | CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) build 6 | 7 | lint: ## Lint the code 8 | @# our linting only runs with python3 9 | @# TODO use CKAN_VERSION make variable once 2.8 is deprecated 10 | CKAN_VERSION=2.9 docker-compose -f docker-compose.yml run --rm app flake8 . --count --show-source --statistics --exclude ckan,nose 11 | 12 | clean: ## Clean workspace and containers 13 | find . -name *.pyc -delete 14 | CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) down -v --remove-orphan 15 | 16 | test: ## Run tests in a new container 17 | CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) run --rm app /srv/app/test.sh 18 | 19 | up: ## Start the containers 20 | CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) up 21 | 22 | 23 | .DEFAULT_GOAL := help 24 | .PHONY: build clean help lint test up 25 | 26 | # Output documentation for top-level targets 27 | # Thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html 28 | help: ## This help 29 | @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-10s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ckanext-usmetadata 2 | 3 | ![Github Actions](https://github.com/GSA/ckanext-usmetadata/actions/workflows/test.yml/badge.svg) 4 | [![PyPI version](https://badge.fury.io/py/ckanext-usmetadata.svg)](https://badge.fury.io/py/ckanext-usmetadata) 5 | 6 | This CKAN Extension expands CKAN to offer a number of custom fields related to the [DCAT-US Schema](https://resources.data.gov/schemas/dcat-us/v1.1/) 7 | 8 | ### Installation 9 | 10 | Add `ckanext-usmetadata` to your requirements.txt, and then pip install 11 | 12 | Then in your CKAN .ini file, add `usmetadata` 13 | to your ckan.plugins line: 14 | 15 | ckan.plugins = (other plugins here...) usmetadata 16 | 17 | ### Requirements 18 | 19 | This extension is compatible with these versions of CKAN. 20 | 21 | CKAN version | Compatibility 22 | ------------ | ------------- 23 | <=2.9 | no 24 | 2.10 | yes 25 | 26 | ### Development 27 | 28 | You may also install by cloning the git repo, then running ''python setup.py develop'' from the root of your source 29 | directory, which will install an egg link so that you can modify the code and see results [localhost:5000](http://localhost:5000/). 30 | 31 | Clean up any containers and volumes. 32 | 33 | $ make down 34 | 35 | Open a shell to run commands in the container. 36 | 37 | $ docker-compose exec ckan bash 38 | 39 | If you're unfamiliar with docker-compose, see our 40 | [cheatsheet](https://github.com/GSA/datagov-deploy/wiki/Docker-Best-Practices#cheatsheet) 41 | and the [official docs](https://docs.docker.com/compose/reference/). 42 | 43 | For additional make targets, see the help. 44 | 45 | $ make help 46 | 47 | ### Testing 48 | 49 | They follow the guidelines for [testing CKAN 50 | extensions](https://docs.ckan.org/en/2.9/extensions/testing-extensions.html#testing-extensions). 51 | 52 | To run the extension tests, start the containers with `make up`, then: 53 | 54 | $ make test 55 | 56 | Lint the code. 57 | 58 | $ make lint 59 | 60 | ### Matrix builds 61 | 62 | In order to support multiple versions of CKAN, or even upgrade to new versions 63 | of CKAN, we support development and testing through the `CKAN_VERSION` 64 | environment variable. 65 | 66 | $ make CKAN_VERSION=2.9 test 67 | 68 | ## Credit / Copying 69 | 70 | Credit to the original owner of the repo. Everything here is built on top of the original foundation. 71 | -------------------------------------------------------------------------------- /ckanext/__init__.py: -------------------------------------------------------------------------------- 1 | # this is a namespace package 2 | try: 3 | import pkg_resources 4 | pkg_resources.declare_namespace(__name__) 5 | except ImportError: 6 | import pkgutil 7 | __path__ = pkgutil.extend_path(__path__, __name__) 8 | -------------------------------------------------------------------------------- /ckanext/usmetadata/__init__.py: -------------------------------------------------------------------------------- 1 | # this is a namespace package 2 | try: 3 | import pkg_resources 4 | pkg_resources.declare_namespace(__name__) 5 | except ImportError: 6 | import pkgutil 7 | __path__ = pkgutil.extend_path(__path__, __name__) 8 | -------------------------------------------------------------------------------- /ckanext/usmetadata/blueprint.py: -------------------------------------------------------------------------------- 1 | import cgi 2 | import datetime 3 | from flask import Blueprint 4 | from flask import redirect 5 | from flask.wrappers import Response as response 6 | from logging import getLogger 7 | import re 8 | import requests 9 | 10 | from ckan.common import _, json, g 11 | import ckan.lib.base as base 12 | import ckan.lib.helpers as h 13 | import ckan.lib.dictization.model_dictize as model_dictize 14 | import ckan.lib.navl.dictization_functions as dict_fns 15 | import ckan.lib.plugins 16 | import ckan.logic as logic 17 | import ckan.model as model 18 | from ckan.plugins.toolkit import c, config, request, requires_ckan_version 19 | from ckanext.usmetadata import helper as local_helper 20 | 21 | 22 | requires_ckan_version("2.9") 23 | 24 | datapusher = Blueprint('usmetadata', __name__) 25 | log = getLogger(__name__) 26 | 27 | render = base.render 28 | abort = base.abort 29 | 30 | NotFound = logic.NotFound 31 | NotAuthorized = logic.NotAuthorized 32 | ValidationError = logic.ValidationError 33 | check_access = logic.check_access 34 | get_action = logic.get_action 35 | tuplize_dict = logic.tuplize_dict 36 | clean_dict = logic.clean_dict 37 | parse_params = logic.parse_params 38 | flatten_to_string_key = logic.flatten_to_string_key 39 | lookup_package_plugin = ckan.lib.plugins.lookup_package_plugin 40 | 41 | 42 | URL_REGEX = re.compile( 43 | r'^(?:http|ftp)s?://' # http:// or https:// or ftp:// or ftps:// 44 | r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... 45 | r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip 46 | r'(?::\d+)?' # optional port 47 | r'(?:/?|[/?]\S+)$', re.IGNORECASE) 48 | 49 | IANA_MIME_REGEX = re.compile(r"^[-\w]+/[-\w]+(\.[-\w]+)*([+][-\w]+)?$") 50 | 51 | TEMPORAL_REGEX_1 = re.compile( 52 | r'^([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?' 53 | r'|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]' 54 | r'\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?(\/)([\+-]?\d{4}' 55 | r'(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|' 56 | r'(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]' 57 | r'\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$' 58 | ) 59 | 60 | TEMPORAL_REGEX_2 = re.compile( 61 | r'^(R\d*\/)?([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\4([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])' 62 | r'(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)' 63 | r'([\.,]\d+(?!:))?)?(\18[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?(\/)' 64 | r'P(?:\d+(?:\.\d+)?Y)?(?:\d+(?:\.\d+)?M)?(?:\d+(?:\.\d+)?W)?(?:\d+(?:\.\d+)?D)?(?:T(?:\d+(?:\.\d+)?H)?' 65 | r'(?:\d+(?:\.\d+)?M)?(?:\d+(?:\.\d+)?S)?)?$' 66 | ) 67 | 68 | TEMPORAL_REGEX_3 = re.compile( 69 | r'^(R\d*\/)?P(?:\d+(?:\.\d+)?Y)?(?:\d+(?:\.\d+)?M)?(?:\d+(?:\.\d+)?W)?(?:\d+(?:\.\d+)?D)?(?:T(?:\d+' 70 | r'(?:\.\d+)?H)?(?:\d+(?:\.\d+)?M)?(?:\d+(?:\.\d+)?S)?)?\/([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])' 71 | r'(\4([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))' 72 | r'([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\18[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])' 73 | r'([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$' 74 | ) 75 | 76 | LANGUAGE_REGEX = re.compile( 77 | r'^(((([A-Za-z]{2,3}(-([A-Za-z]{3}(-[A-Za-z]{3}){0,2}))?)|[A-Za-z]{4}|[A-Za-z]{5,8})(-([A-Za-z]{4}))?' 78 | r'(-([A-Za-z]{2}|[0-9]{3}))?(-([A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(-([0-9A-WY-Za-wy-z](-[A-Za-z0-9]{2,8})+))*' 79 | r'(-(x(-[A-Za-z0-9]{1,8})+))?)|(x(-[A-Za-z0-9]{1,8})+)|' 80 | r'((en-GB-oed|i-ami|i-bnn|i-default|i-enochian|i-hak|i-klingon|i-lux|i-mingo' 81 | r'|i-navajo|i-pwn|i-tao|i-tay|i-tsu|sgn-BE-FR|sgn-BE-NL|sgn-CH-DE)|' 82 | r'(art-lojban|cel-gaulish|no-bok|no-nyn|zh-guoyu|zh-hakka|zh-min|zh-min-nan|zh-xiang)))$' 83 | ) 84 | 85 | PRIMARY_IT_INVESTMENT_UII_REGEX = re.compile(r"^[0-9]{3}-[0-9]{9}$") 86 | 87 | ISSUED_REGEX = re.compile( 88 | r'^([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?' 89 | r'|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]' 90 | r'\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$' 91 | ) 92 | 93 | REDACTED_REGEX = re.compile( 94 | r'^(\[\[REDACTED).*?(\]\])$' 95 | ) 96 | 97 | 98 | def get_package_type(self, id): 99 | """ 100 | Copied from https://github.com/ckan/ckan/blob/2.8/ckan/controllers/package.py#L866-L874 101 | Probably depreciated code.. 102 | 103 | Given the id of a package this method will return the type of the 104 | package, or 'dataset' if no type is currently set 105 | """ 106 | pkg = model.Package.get(id) 107 | if pkg: 108 | return pkg.type or 'dataset' 109 | return None 110 | 111 | 112 | def setup_template_variables(context, data_dict, package_type=None): 113 | """ 114 | Copied from https://github.com/ckan/ckan/blob/2.9/ckan/views/dataset.py#L51-L54 115 | Doesn't exist in CKAN 2.8, so I don't know what happened here.. 116 | """ 117 | return lookup_package_plugin(package_type).setup_template_variables( 118 | context, data_dict 119 | ) 120 | 121 | 122 | def get_package_info_usmetadata(id, context, errors, error_summary): 123 | data = get_action('package_show')(context, {'id': id}) 124 | data_dict = get_action('package_show')(context, {'id': id}) 125 | data_dict['id'] = id 126 | data_dict['state'] = 'active' 127 | context['allow_state_change'] = True 128 | try: 129 | get_action('package_update')(context, data_dict) 130 | except ValidationError as e: 131 | errors = e.error_dict 132 | error_summary = e.error_summary 133 | # ## TODO: Find out where 'new_metadata' is defined 134 | # return new_metadata(id, data, errors, error_summary) 135 | return 136 | except NotAuthorized: 137 | abort(401, _('Unauthorized to update dataset')) 138 | redirect(h.url_for(controller='package', 139 | action='read', id=id)) 140 | errors = errors or {} 141 | error_summary = error_summary or {} 142 | return {'data': data, 'errors': errors, 'error_summary': error_summary, 'pkg_name': id} 143 | 144 | 145 | def map_old_keys(error_summary): 146 | replace = { 147 | 'Format': 'Media Type' 148 | } 149 | for old_key, new_key in list(replace.items()): 150 | if old_key in list(error_summary.keys()): 151 | error_summary[new_key] = error_summary[old_key] 152 | del error_summary[old_key] 153 | return error_summary 154 | 155 | 156 | def resource_form(package_type): 157 | # backwards compatibility with plugins not inheriting from 158 | # DefaultDatasetPlugin and not implmenting resource_form 159 | plugin = lookup_package_plugin(package_type) 160 | if hasattr(plugin, 'resource_form'): 161 | result = plugin.resource_form() 162 | if result is not None: 163 | return result 164 | return lookup_package_plugin().resource_form() 165 | 166 | 167 | def new_resource_usmetadata(id, data=None, errors=None, error_summary=None): 168 | ''' FIXME: This is a temporary action to allow styling of the 169 | forms. ''' 170 | if request.method == 'POST' and not data: 171 | save_action = request.params.get('save') 172 | data = data or clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(request.POST)))) 173 | # we don't want to include save as it is part of the form 174 | del data['save'] 175 | resource_id = data['id'] 176 | del data['id'] 177 | 178 | context = {'model': model, 'session': model.Session, 179 | 'user': c.user or c.author, 'auth_user_obj': c.userobj} 180 | 181 | # see if we have any data that we are trying to save 182 | data_provided = False 183 | for key, value in data.items(): 184 | if ((value or isinstance(value, cgi.FieldStorage)) and key != 'resource_type'): 185 | data_provided = True 186 | break 187 | 188 | if not data_provided and save_action != "go-dataset-complete": 189 | if save_action == 'go-dataset': 190 | # go to final stage of adddataset 191 | redirect(h.url_for(controller='package', 192 | action='edit', id=id)) 193 | # see if we have added any resources 194 | try: 195 | data_dict = get_action('package_show')(context, {'id': id}) 196 | except NotAuthorized: 197 | abort(401, _('Unauthorized to update dataset')) 198 | except NotFound: 199 | abort(404, 200 | _('The dataset {id} could not be found.').format(id=id)) 201 | if str.lower(config.get('ckan.package.resource_required', 'true')) == 'true' and not len( 202 | data_dict['resources']): 203 | # no data so keep on page 204 | msg = _('You must add at least one data resource') 205 | # On new templates do not use flash message 206 | if g.legacy_templates: 207 | h.flash_error(msg) 208 | redirect(h.url_for(controller='package', 209 | action='new_resource', id=id)) 210 | else: 211 | errors = {} 212 | error_summary = {_('Error'): msg} 213 | return new_resource_usmetadata(id, data, errors, error_summary) 214 | # we have a resource so let them add metadata 215 | # redirect(h.url_for(controller='package', 216 | # action='new_metadata', id=id)) 217 | extra_vars = get_package_info_usmetadata(id, context, errors, error_summary) 218 | package_type = get_package_type(id) 219 | setup_template_variables(context, {}, package_type=package_type) 220 | return render('package/new_package_metadata.html', extra_vars=extra_vars) 221 | 222 | data['package_id'] = id 223 | try: 224 | if resource_id: 225 | data['id'] = resource_id 226 | get_action('resource_update')(context, data) 227 | else: 228 | get_action('resource_create')(context, data) 229 | except ValidationError as e: 230 | errors = e.error_dict 231 | # error_summary = e.error_summary 232 | error_summary = map_old_keys(e.error_summary) 233 | # return new_resource(id, data, errors, error_summary) 234 | return new_resource_usmetadata(id, data, errors, error_summary) 235 | 236 | except NotAuthorized: 237 | abort(401, _('Unauthorized to create a resource')) 238 | except NotFound: 239 | abort(404, _('The dataset {id} could not be found.' 240 | ).format(id=id)) 241 | if save_action == 'go-metadata': 242 | # go to final stage of add dataset 243 | # redirect(h.url_for(controller='package', 244 | # action='new_metadata', id=id)) 245 | # Github Issue # 129. Removing last stage of dataset creation. 246 | extra_vars = get_package_info_usmetadata(id, context, errors, error_summary) 247 | package_type = get_package_type(id) 248 | setup_template_variables(context, {}, package_type=package_type) 249 | return render('package/new_package_metadata.html', extra_vars=extra_vars) 250 | elif save_action == 'go-dataset': 251 | # go to first stage of add dataset 252 | redirect(h.url_for(controller='package', 253 | action='edit', id=id)) 254 | elif save_action == 'go-dataset-complete': 255 | # go to first stage of add dataset 256 | redirect(h.url_for(controller='package', 257 | action='read', id=id)) 258 | else: 259 | # add more resources 260 | redirect(h.url_for(controller='package', 261 | action='new_resource', id=id)) 262 | 263 | # get resources for sidebar 264 | context = {'model': model, 'session': model.Session, 265 | 'user': c.user or c.author, 'auth_user_obj': c.userobj} 266 | try: 267 | pkg_dict = get_action('package_show')(context, {'id': id}) 268 | except NotFound: 269 | abort(404, _('The dataset {id} could not be found.').format(id=id)) 270 | try: 271 | check_access( 272 | 'resource_create', context, {"package_id": pkg_dict["id"]}) 273 | except NotAuthorized: 274 | abort(401, _('Unauthorized to create a resource for this package')) 275 | 276 | package_type = pkg_dict['type'] or 'dataset' 277 | 278 | errors = errors or {} 279 | error_summary = error_summary or {} 280 | extra_vars = {'data': data, 'errors': errors, 'error_summary': error_summary, 'action': 'new', 281 | 'resource_form_snippet': resource_form(package_type), 'dataset_type': package_type, 282 | 'pkg_name': id} 283 | # get resources for sidebar 284 | context = {'model': model, 'session': model.Session, 285 | 'user': c.user or c.author, 'auth_user_obj': c.userobj} 286 | try: 287 | pkg_dict = get_action('package_show')(context, {'id': id}) 288 | except NotFound: 289 | abort(404, _('The dataset {id} could not be found.').format(id=id)) 290 | # required for nav menu 291 | extra_vars['pkg_dict'] = pkg_dict 292 | template = 'package/new_resource_not_draft.html' 293 | if pkg_dict['state'] == 'draft': 294 | extra_vars['stage'] = ['complete', 'active'] 295 | template = 'package/new_resource.html' 296 | elif pkg_dict['state'] == 'draft-complete': 297 | extra_vars['stage'] = ['complete', 'active', 'complete'] 298 | template = 'package/new_resource.html' 299 | return render(template, extra_vars=extra_vars) 300 | 301 | 302 | # AJAX validator 303 | # class DatasetValidator(BaseController): 304 | # """Controller to validate resource""" 305 | 306 | 307 | def dv_check_if_unique(unique_id, owner_org, pkg_name): 308 | packages = dv_get_packages(owner_org) 309 | for package in packages: 310 | for extra in package['extras']: 311 | if extra['key'] == 'unique_id' and extra['value'] == unique_id and pkg_name != package['id']: 312 | return package['name'] 313 | return False 314 | 315 | 316 | def dv_get_packages(owner_org): 317 | # Build the data.json file. 318 | packages = dv_get_all_group_packages(group_id=owner_org) 319 | # get packages for sub-agencies. 320 | sub_agency = model.Group.get(owner_org) 321 | if 'sub-agencies' in list(sub_agency.extras.col.keys()) \ 322 | and sub_agency.extras.col['sub-agencies'].state == 'active': 323 | sub_agencies = sub_agency.extras.col['sub-agencies'].value 324 | sub_agencies_list = sub_agencies.split(",") 325 | for sub in sub_agencies_list: 326 | sub_packages = dv_get_all_group_packages(group_id=sub) 327 | for sub_package in sub_packages: 328 | packages.append(sub_package) 329 | 330 | return packages 331 | 332 | 333 | def dv_get_all_group_packages(group_id): 334 | """ 335 | Gets all of the group packages, public or private, returning them as a list of CKAN's dictized packages. 336 | """ 337 | result = [] 338 | for pkg_rev in model.Group.get(group_id).packages(with_private=True, context={'user_is_admin': True}): 339 | result.append(model_dictize.package_dictize(pkg_rev, {'model': model})) 340 | 341 | return result 342 | 343 | 344 | def dv_validate_dataset(): 345 | try: 346 | pkg_name = request.params.get('pkg_name', False) 347 | owner_org = request.params.get('owner_org', False) 348 | unique_id = request.params.get('unique_id', False) 349 | rights = request.params.get('rights', False) 350 | license_url = request.params.get('license_url', False) 351 | temporal = request.params.get('temporal', False) 352 | described_by = request.params.get('described_by', False) 353 | described_by_type = request.params.get('described_by_type', False) 354 | conforms_to = request.params.get('conforms_to', False) 355 | landing_page = request.params.get('landing_page', False) 356 | language = request.params.get('language', False) 357 | investment_uii = request.params.get('investment_uii', False) 358 | references = request.params.get('references', False) 359 | issued = request.params.get('issued', False) 360 | system_of_records = request.params.get('system_of_records', False) 361 | 362 | errors = {} 363 | warnings = {} 364 | 365 | matching_package = dv_check_if_unique(unique_id, owner_org, pkg_name) 366 | if unique_id and matching_package: 367 | errors['unique_id'] = 'Already being used by ' + request.host_url + '/dataset/' \ 368 | + matching_package 369 | if rights and len(rights) > 255: 370 | errors['access-level-comment'] = 'The length of the string exceeds limit of 255 chars' 371 | 372 | dv_check_url(license_url, errors, warnings, 'license-new', True, True) 373 | dv_check_url(described_by, errors, warnings, 'data_dictionary', True, True) 374 | dv_check_url(conforms_to, errors, warnings, 'conforms_to', True, True) 375 | dv_check_url(landing_page, errors, warnings, 'homepage_url', True, True) 376 | dv_check_url(system_of_records, errors, warnings, 'system_of_records') 377 | 378 | if described_by_type and not IANA_MIME_REGEX.match(described_by_type) \ 379 | and not REDACTED_REGEX.match(described_by_type): 380 | errors['data_dictionary_type'] = 'The value is not valid IANA MIME Media type' 381 | 382 | if temporal and not REDACTED_REGEX.match(temporal): 383 | if "/" not in temporal: 384 | errors['temporal'] = 'Invalid Temporal Format. Missing slash' 385 | elif not TEMPORAL_REGEX_1.match(temporal) \ 386 | and not TEMPORAL_REGEX_2.match(temporal) \ 387 | and not TEMPORAL_REGEX_3.match(temporal): 388 | errors['temporal'] = 'Invalid Temporal Format' 389 | 390 | if language: # and not REDACTED_REGEX.match(language): 391 | language = language.split(',') 392 | for s in language: 393 | s = s.strip() 394 | if not LANGUAGE_REGEX.match(s): 395 | errors['language'] = 'Invalid Language Format: ' + str(s) 396 | 397 | if investment_uii and not REDACTED_REGEX.match(investment_uii): 398 | if not PRIMARY_IT_INVESTMENT_UII_REGEX.match(investment_uii): 399 | errors['primary-it-investment-uii'] = 'Invalid Format. Must be `023-000000001` format' 400 | 401 | if references and not REDACTED_REGEX.match(references): 402 | references = references.split(',') 403 | for s in references: 404 | url = s.strip() 405 | if not URL_REGEX.match(url) and not REDACTED_REGEX.match(url): 406 | errors['related_documents'] = 'One of urls is invalid: ' + url 407 | 408 | if issued and not REDACTED_REGEX.match(issued): 409 | if not ISSUED_REGEX.match(issued): 410 | errors['release_date'] = 'Invalid Format' 411 | 412 | if errors: 413 | return json.dumps({'ResultSet': {'Invalid': errors, 'Warnings': warnings}}) 414 | return json.dumps({'ResultSet': {'Success': errors, 'Warnings': warnings}}) 415 | except Exception as ex: 416 | log.error('validate_resource exception: %s ', ex) 417 | return json.dumps({'ResultSet': {'Error': 'Unknown error'}}) 418 | 419 | 420 | def dv_check_url(url, errors, warnings, error_key, skip_empty=True, allow_redacted=False): 421 | if skip_empty and not url: 422 | return 423 | url = url.strip() 424 | if allow_redacted and REDACTED_REGEX.match(url): 425 | return 426 | if not URL_REGEX.match(url): 427 | errors[error_key] = 'Invalid URL format' 428 | return 429 | # else: 430 | # try: 431 | # r = requests.head(url, verify=False) 432 | # if r.status_code > 399: 433 | # r = requests.get(url, verify=False) 434 | # if r.status_code > 399: 435 | # warnings[error_key] = 'URL returns status ' + str(r.status_code) + ' (' + str(r.reason) + ')' 436 | # except Exception as ex: 437 | # log.error('check_url exception: %s ', ex) 438 | # warnings[error_key] = 'Could not check url' 439 | 440 | 441 | # AJAX validator 442 | # class ResourceValidator(BaseController): 443 | # """Controller to validate resource""" 444 | 445 | 446 | def rv_validate_resource(): 447 | try: 448 | url = request.params.get('url', False) 449 | resource_type = request.params.get('resource_type', False) 450 | described_by = request.params.get('describedBy', False) 451 | described_by_type = request.params.get('describedByType', False) 452 | conforms_to = request.params.get('conformsTo', False) 453 | media_type = request.params.get('format', False) 454 | 455 | errors = {} 456 | warnings = {} 457 | 458 | # if media_type and not REDACTED_REGEX.match(media_type) \ 459 | # and not IANA_MIME_REGEX.match(media_type): 460 | # if media_type and not IANA_MIME_REGEX.match(media_type): 461 | # errors['format'] = 'The value is not valid IANA MIME Media type' 462 | # elif not media_type and resource_type in ['file', 'upload']: 463 | # if url or resource_type == 'upload': 464 | # errors['format'] = 'The value is required for this type of resource' 465 | 466 | lower_types = [mtype.lower() for mtype in local_helper.media_types] 467 | if media_type and media_type.lower() not in lower_types: 468 | errors['format'] = 'The value is not valid format' 469 | elif not media_type and resource_type in ['file', 'upload']: 470 | if url or resource_type == 'upload': 471 | errors['format'] = 'The value is required for this type of resource' 472 | 473 | rv_check_url(described_by, errors, warnings, 'describedBy', True, True) 474 | rv_check_url(conforms_to, errors, warnings, 'conformsTo', True, True) 475 | 476 | if described_by_type and not REDACTED_REGEX.match(described_by_type.strip()) \ 477 | and not IANA_MIME_REGEX.match(described_by_type.strip()): 478 | errors['describedByType'] = 'The value is not valid IANA MIME Media type' 479 | 480 | if errors: 481 | return json.dumps({'ResultSet': {'Invalid': errors, 'Warnings': warnings}}) 482 | return json.dumps({'ResultSet': {'Success': errors, 'Warnings': warnings}}) 483 | except Exception as ex: 484 | log.error('validate_resource exception: %s ', ex) 485 | return json.dumps({'ResultSet': {'Error': 'Unknown error'}}) 486 | 487 | 488 | def rv_check_url(url, errors, warnings, error_key, skip_empty=True, allow_redacted=False): 489 | if skip_empty and not url: 490 | return 491 | url = url.strip() 492 | if allow_redacted and REDACTED_REGEX.match(url): 493 | return 494 | if not URL_REGEX.match(url): 495 | errors[error_key] = 'Invalid URL format' 496 | else: 497 | try: 498 | r = requests.head(url, verify=False) 499 | if r.status_code > 399: 500 | r = requests.get(url, verify=False) 501 | if r.status_code > 399: 502 | warnings[error_key] = 'URL returns status ' + str(r.status_code) + ' (' + str(r.reason) + ')' 503 | except Exception as ex: 504 | log.error('check_url exception: %s ', ex) 505 | warnings[error_key] = 'Could not check url' 506 | 507 | 508 | # class CloneController(BaseController): 509 | # """Controller to clone dataset metadata""" 510 | 511 | 512 | def cc_clone_dataset_metadata(id): 513 | context = {'model': model, 'session': model.Session, 514 | 'user': c.user or c.author, 'auth_user_obj': c.userobj} 515 | pkg_dict = get_action('package_show')(context, {'id': id}) 516 | 517 | # udpate name and title 518 | pkg_dict['title'] = "Clone of " + pkg_dict['title'] 519 | 520 | # name can not be more than 100 characters 521 | pkg_dict['name'] = pkg_dict['name'][:85] + "-" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") 522 | 523 | pkg_dict['state'] = 'draft' 524 | pkg_dict['tag_string'] = [''] 525 | # remove id from original dataset 526 | if 'id' in pkg_dict: 527 | del pkg_dict['id'] 528 | 529 | # remove resource for now. 530 | if 'resources' in pkg_dict: 531 | del pkg_dict['resources'] 532 | 533 | # extras have to on top level. otherwise validation fails 534 | temp = {} 535 | for extra in pkg_dict['extras']: 536 | temp[extra['key']] = extra['value'] 537 | 538 | del pkg_dict['extras'] 539 | pkg_dict['extras'] = [] 540 | for key, value in temp.items(): 541 | if key != 'title': 542 | pkg_dict[key] = value 543 | 544 | # somehow package is getting added to context. If we dont remove it current dataset gets updated 545 | if 'package' in context: 546 | del context['package'] 547 | 548 | # disabling validation 549 | context['cloning'] = True 550 | 551 | # create new package 552 | pkg_dict_new = get_action('package_create')(context, pkg_dict) 553 | 554 | # redirect to draft edit 555 | redirect(h.url_for(controller='package', action='edit', id=pkg_dict_new['name'])) 556 | 557 | 558 | # class CurlController(BaseController): 559 | # """Controller to obtain info by url""" 560 | 561 | 562 | def cuc_get_content_type(): 563 | # set content type (charset required or pylons throws an error) 564 | try: 565 | url = request.params.get('url', '') 566 | 567 | if REDACTED_REGEX.match(url): 568 | return json.dumps({'ResultSet': { 569 | 'CType': False, 570 | 'Status': 'OK', 571 | 'Redacted': True, 572 | 'Reason': '[[REDACTED]]' 573 | }}) 574 | 575 | if not URL_REGEX.match(url): 576 | return json.dumps({'ResultSet': {'Error': 'Invalid URL', 'InvalidFormat': 'True', 'Red': 'True'}}) 577 | 578 | r = requests.head(url, verify=False) 579 | method = 'HEAD' 580 | if r.status_code > 399 or r.headers.get('content-type') is None: 581 | r = requests.get(url, verify=False) 582 | method = 'GET' 583 | if r.status_code > 399 or r.headers.get('content-type') is None: 584 | # return json.dumps({'ResultSet': {'Error': 'Returned status: ' + str(r.status_code)}}) 585 | return json.dumps({'ResultSet': { 586 | 'CType': False, 587 | 'Status': r.status_code, 588 | 'Reason': r.reason, 589 | 'Method': method}}) 590 | content_type = r.headers.get('content-type') 591 | content_type = content_type.split(';', 1) 592 | unified_format = h.unified_resource_format(content_type[0]) 593 | return json.dumps({'ResultSet': { 594 | 'CType': unified_format, 595 | 'Status': r.status_code, 596 | 'Reason': r.reason, 597 | 'Method': method}}) 598 | except Exception as ex: 599 | log.error('get_content_type exception: %s ', ex) 600 | return json.dumps({'ResultSet': {'Error': 'unknown error'}}) 601 | # return json.dumps({'ResultSet': {'Error': type(e).__name__}}) 602 | 603 | 604 | # class MediaController(BaseController): 605 | # """Controller to return the acceptable media types as JSON, powering the front end""" 606 | 607 | 608 | def mc_get_media_types(): 609 | # set content type (charset required or pylons throws an error) 610 | q = request.params.get('incomplete', '').lower() 611 | 612 | response.content_type = 'application/json; charset=UTF-8' 613 | 614 | retval = [] 615 | 616 | if q in local_helper.media_types_dict: 617 | retval.append(local_helper.media_types_dict[q][1]) 618 | 619 | media_keys = list(local_helper.media_types_dict.keys()) 620 | for media_type in media_keys: 621 | if q in media_type.lower() and local_helper.media_types_dict[media_type][1] not in retval: 622 | retval.append(local_helper.media_types_dict[media_type][1]) 623 | if len(retval) >= 50: 624 | break 625 | 626 | return json.dumps({'ResultSet': {'Result': retval}}) 627 | 628 | # class LicenseURLController(BaseController): 629 | # """Controller to return the acceptable media types as JSON, powering the front end""" 630 | 631 | 632 | def lc_get_license_url(): 633 | # set content type (charset required or pylons throws an error) 634 | 635 | response.content_type = 'application/json; charset=UTF-8' 636 | 637 | retval = [] 638 | 639 | for key in local_helper.license_options: 640 | retval.append(key) 641 | 642 | return json.dumps({'ResultSet': {'Result': retval}}) 643 | 644 | 645 | datapusher.add_url_rule('/dataset/new_resource/', 646 | view_func=new_resource_usmetadata) 647 | datapusher.add_url_rule('/api/2/util/resource/license_url_autocomplete', 648 | view_func=lc_get_license_url) 649 | datapusher.add_url_rule('/dataset//clone', 650 | view_func=cc_clone_dataset_metadata) 651 | 652 | datapusher.add_url_rule('/api/2/util/resource/media_autocomplete', 653 | view_func=mc_get_media_types) 654 | datapusher.add_url_rule('/api/2/util/resource/content_type', 655 | view_func=cuc_get_content_type) 656 | datapusher.add_url_rule('/api/2/util/resource/validate_resource', 657 | view_func=rv_validate_resource) 658 | datapusher.add_url_rule('/api/2/util/resource/validate_dataset', 659 | view_func=dv_validate_dataset) 660 | -------------------------------------------------------------------------------- /ckanext/usmetadata/db_utils.py: -------------------------------------------------------------------------------- 1 | __author__ = 'ykhadilkar' 2 | 3 | import ckan.model as model 4 | 5 | cached_tables = {} 6 | 7 | 8 | def get_organization_title(dataset_id): 9 | query = "select id, title from package where package.id = '" + dataset_id + "'" 10 | connection = model.Session.connection() 11 | res = connection.execute(query).fetchone() 12 | return res[1] 13 | 14 | 15 | def get_parent_organizations(c): 16 | items = {} 17 | 18 | if not c.userobj: 19 | return items 20 | 21 | if c.userobj.sysadmin: 22 | query = "select package_id, title from package_extra , package " \ 23 | "where package_extra.key = 'is_parent' and package_extra.value = 'true' " \ 24 | "and package.id = package_extra.package_id and package.state = 'active' " 25 | else: 26 | userGroupsIds = c.userobj.get_group_ids() 27 | ids = [] 28 | 29 | for id in userGroupsIds: 30 | ids.append(id) 31 | 32 | # Ugly hack - If user has access to only one organization then SQL query blows up because IN statement ends up with 33 | # dangling comma at the end. Adding dumy id should fix that. 34 | if (len(ids) == 0): 35 | ids.append("null") 36 | ids.append("dummy-id") 37 | elif (len(ids) == 1): 38 | ids.append("dummy-id") 39 | query = "select package_id, title from package_extra , package " \ 40 | "where package_extra.key = 'is_parent' and package_extra.value = 'true' " \ 41 | "and package.id = package_extra.package_id and package.state = 'active' " \ 42 | "and package.owner_org in " + str(tuple(ids)) 43 | 44 | connection = model.Session.connection() 45 | res = connection.execute(query).fetchall() 46 | for result in res: 47 | items[result[0]] = result[1] 48 | 49 | return items 50 | -------------------------------------------------------------------------------- /ckanext/usmetadata/fanstatic/add_subagency.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by ykhadilkar on 11/19/14. 3 | */ 4 | $(document).ready(function(){ 5 | for( var i=1; i<6; i++){ 6 | if($('#field-publisher_'+i).val() === ""){ 7 | $('.field-publisher_'+i).hide() 8 | } 9 | } 10 | 11 | $("#add-sub-agency").click(function(){ 12 | for( var i=1; i<6; i++) { 13 | if (!$('.field-publisher_'+i).is(":visible")) { 14 | $('.field-publisher_'+i).show(); 15 | break; 16 | } 17 | } 18 | }); 19 | }); -------------------------------------------------------------------------------- /ckanext/usmetadata/fanstatic/dataset_edit_form.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var DatasetForm = new function () { 4 | var obj = this; 5 | this.form_is_valid = true; 6 | 7 | this.validate_dataset = function () { 8 | $.getJSON( 9 | '/api/2/util/resource/validate_dataset', 10 | { 11 | 'pkg_name': $('[name="pkg_name"]').val(), 12 | 'owner_org': $('#field-organizations').val(), 13 | 'unique_id': $('#field-unique_id').val(), 14 | 'rights': $('#field-access-level-comment').val(), 15 | 'license_url': $('#field-license-new').val(), 16 | 'temporal': $('#field-temporal').val(), 17 | 'described_by': $('#field-data_dictionary').val(), 18 | 'described_by_type': $('#field-data_dictionary_type').val(), 19 | 'conforms_to': $('#field-conforms_to').val(), 20 | 'landing_page': $('#field-homepage_url').val(), 21 | 'language': $('#field-language').val(), 22 | 'investment_uii': $('#field-primary-it-investment-uii').val(), 23 | 'references': $('#field-related_documents').val(), 24 | 'issued': $('#field-release_date').val(), 25 | 'system_of_records': $('#field-system_of_records').val() 26 | }, 27 | function (result) { 28 | $('input').next('p.bad').remove(); 29 | $('input').next('p.warning').remove(); 30 | $('input').parent().prev('label').removeClass('bad'); 31 | 32 | if (typeof(result.ResultSet['Warnings']) !== "undefined") { 33 | var WarningObj = result.ResultSet['Warnings']; 34 | for (var warningObjKey in WarningObj) { 35 | if (WarningObj.hasOwnProperty(warningObjKey)) { 36 | $('#field-' + warningObjKey).after('

Warning: ' 37 | + WarningObj[warningObjKey] + '

'); 38 | } 39 | } 40 | } 41 | 42 | if (typeof(result.ResultSet['Invalid']) !== "undefined") { 43 | var InvalidObj = result.ResultSet['Invalid']; 44 | for (var invalidObjKey in InvalidObj) { 45 | if (InvalidObj.hasOwnProperty(invalidObjKey)) { 46 | $('#field-' + invalidObjKey).after('

' 47 | + InvalidObj[invalidObjKey] + '

'); 48 | $('#field-' + invalidObjKey).parent().prev('label').addClass('bad'); 49 | } 50 | } 51 | obj.form_is_valid = false; 52 | } else { 53 | obj.form_is_valid = true; 54 | } 55 | } 56 | ); 57 | }; 58 | 59 | this.bootstrap = function () { 60 | $("#field-is_parent").change(function () { 61 | if ('true' === $("#field-is_parent").val()) { 62 | $(".control-group-dataset-parent").hide(); 63 | $("#field-parent_dataset").val(""); 64 | } else { 65 | $(".control-group-dataset-parent").show(); 66 | 67 | } 68 | }).change(); 69 | 70 | $('#field-organizations') 71 | .add('#field-unique_id') 72 | .add('#field-access-level-comment') 73 | .add('#field-license-new') 74 | .add('#field-temporal') 75 | .add('#field-data_dictionary') 76 | .add('#field-data_dictionary_type') 77 | .add('#field-conforms_to') 78 | .add('#field-homepage_url') 79 | .add('#field-language') 80 | .add('#field-primary-it-investment-uii') 81 | .add('#field-related_documents') 82 | .add('#field-release_date') 83 | .add('#field-system_of_records') 84 | .change(this.validate_dataset); 85 | 86 | this.validate_dataset(); 87 | 88 | $('form.dataset-form').submit(function (event) { 89 | if (!obj.form_is_valid) { 90 | event.preventDefault(); 91 | } 92 | }); 93 | 94 | $('#field-spatial').parents('div.control-group').addClass('exempt-allowed'); 95 | $('#field-temporal').parents('div.control-group').addClass('exempt-allowed'); 96 | $('#field-title').parents('div.control-group').addClass('exempt-allowed'); 97 | $('#field-notes').parents('div.control-group').addClass('exempt-allowed'); 98 | $('#field-modified').parents('div.control-group').addClass('exempt-allowed'); 99 | $('#field-tags').parents('div.control-group').addClass('exempt-allowed'); 100 | 101 | RedactionControl.append_redacted_icons(); 102 | RedactionControl.preload_redacted_inputs(); 103 | //$('.exemption_reason').renderEyes(); 104 | this.reload_redacted_controls(); 105 | $(':input[name="public_access_level"]').change(this.reload_redacted_controls); 106 | }; 107 | 108 | this.reload_redacted_controls = function () { 109 | // https://resources.data.gov/schemas/dcat-us/v1.1/#accessLevel 110 | var level = $(':input[name="public_access_level"]').val(); 111 | if ('public' === level) { 112 | $('.redacted-icon').add('.redacted-marker').add('.exemption_reason').hide(); 113 | return; 114 | } 115 | RedactionControl.show_redacted_controls(); 116 | }; 117 | }(); 118 | 119 | $().ready(function () { 120 | if ($('form.dataset-form').length && $(':input[name="pkg_name"]').length) { 121 | DatasetForm.bootstrap(); 122 | } 123 | }); 124 | 125 | //$.fn.extend({ 126 | // renderEyes: function () { 127 | // if ($(this).val()) { 128 | // $(this).parents('.control-group').children('.redacted-icon').removeClass('icon-eye-open'); 129 | // $(this).parents('.control-group').children('.redacted-icon').addClass('icon-eye-close'); 130 | // } else { 131 | // $(this).parents('.control-group').children('.redacted-icon').removeClass('icon-eye-close'); 132 | // $(this).parents('.control-group').children('.redacted-icon').addClass('icon-eye-open'); 133 | // } 134 | // } 135 | //}); 136 | 137 | 138 | -------------------------------------------------------------------------------- /ckanext/usmetadata/fanstatic/dataset_url.js: -------------------------------------------------------------------------------- 1 | this.ckan.module('usmetadata-slug-preview-slug', function (_) { 2 | return { 3 | options: { 4 | prefix: '', 5 | placeholder: '', 6 | i18n: { 7 | url: _('URL'), 8 | edit: _('Edit') 9 | } 10 | }, 11 | 12 | initialize: function () { 13 | var sandbox = this.sandbox; 14 | var options = this.options; 15 | var el = this.el; 16 | var _ = sandbox.translate; 17 | 18 | var slug = el.slug(); 19 | var parent = slug.parents('.control-group'); 20 | var preview; 21 | 22 | if (!(parent.length)) { 23 | return; 24 | } 25 | 26 | // Leave the slug field visible 27 | if (!parent.hasClass('error')) { 28 | preview = parent.slugPreview({ 29 | prefix: options.prefix, 30 | placeholder: options.placeholder, 31 | i18n: { 32 | 'URL': this.i18n('url'), 33 | 'Edit': this.i18n('edit') 34 | } 35 | }); 36 | 37 | // If the user manually enters text into the input we cancel the slug 38 | // listeners so that we don't clobber the slug when the title next changes. 39 | slug.keypress(function () { 40 | if (event.charCode) { 41 | sandbox.publish('slug-preview-modified', preview[0]); 42 | } 43 | }); 44 | 45 | sandbox.publish('slug-preview-created', preview[0]); 46 | } 47 | 48 | // Watch for updates to the target field and update the hidden slug field 49 | // triggering the "change" event manually. 50 | sandbox.subscribe('slug-target-changed', function (value) { 51 | slug.val(value).trigger('change'); 52 | }); 53 | } 54 | }; 55 | }); 56 | -------------------------------------------------------------------------------- /ckanext/usmetadata/fanstatic/redactions.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | // https://resources.data.gov/schemas/dcat-us/v1.1/#accessLevel 4 | var RedactionControl = new function () { 5 | var obj = this; 6 | 7 | // excluding these fields from partial redactions since they are not supported yet by POD schema v1.1 8 | this.excluded_partial_redactions = [ 9 | 'bureau_code', 10 | 'category', 11 | 'conformsTo', 12 | 'conforms_to', 13 | 'contact_email', 14 | 'data_dictionary', 15 | 'data_dictionary_type', 16 | 'describedBy', 17 | 'homepage_url', 18 | 'language', 19 | 'license_new', 20 | 'modified', 21 | 'primary_it_investment_uii', 22 | 'program_code', 23 | 'publisher', 24 | 'related_documents', 25 | 'release_date', 26 | 'system_of_records', 27 | 'tag_string', 28 | 'temporal', 29 | 'url' 30 | ]; 31 | 32 | this.exempt_reasons = [ 33 | { 34 | 'value': 'B3', 35 | 'short': 'B3 - Specifically exempted from disclosure by statute provided …', 36 | 'full': "Specifically exempted from disclosure by statute (other than FOIA), provided that such " + 37 | "statute (A) requires that the matters be withheld from the public in such a manner as to leave no" + 38 | " discretion on the issue, or (B) establishes particular criteria for withholding or refers to" + 39 | " particular types of matters to be withheld." 40 | }, 41 | { 42 | 'value': 'B4', 43 | 'short': 'B4 - Trade secrets and commercial or financial information obtained from …', 44 | 'full': "Trade secrets and commercial or financial information obtained from a person" + 45 | " and privileged or confidential." 46 | }, 47 | { 48 | 'value': 'B5', 49 | 'short': 'B5 - Inter-agency or intra-agency memorandums or letters which …', 50 | 'full': "Inter-agency or intra-agency memorandums or letters which would not be available by law " + 51 | "to a party other than an agency in litigation with the agency." 52 | }, 53 | { 54 | 'value': 'B6', 55 | 'short': 'B6 - Personnel and medical files and similar files the disclosure of which …', 56 | 'full': "Personnel and medical files and similar files the disclosure of which would constitute" + 57 | " a clearly unwarranted invasion of personal privacy." 58 | } 59 | ]; 60 | 61 | this.render_redacted_input = function (key, val) { 62 | val = typeof val !== 'undefined' ? val : false; 63 | 64 | var currentInput = $(':input[name="' + key + '"]'); 65 | var controlsDiv = currentInput.parents('.controls'); 66 | if (!controlsDiv.length) { 67 | return; 68 | } 69 | 70 | var reason_select = $(document.createElement('select')) 71 | .attr('name', "redacted_" + key) 72 | .attr('rel', key) 73 | .addClass('exemption_reason'); 74 | 75 | 76 | $(document.createElement('option')) 77 | .attr('value', '') 78 | .text('Select FOIA Exemption Reason for Redaction') 79 | .appendTo(reason_select); 80 | 81 | for (var index in this.exempt_reasons) { 82 | var reason = this.exempt_reasons[index]; 83 | var options = { 84 | value: reason.value, alt: reason.full, title: reason.full, 85 | text: reason.short 86 | }; 87 | if (reason.value === val) { 88 | options['selected'] = 'selected'; 89 | } 90 | $("