├── .coveragerc ├── .flake8 ├── .github └── workflows │ └── test.yaml ├── .gitignore ├── .gitmodules ├── .isort.cfg ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── ckanext ├── __init__.py └── blob_storage │ ├── __init__.py │ ├── actions.py │ ├── authz.py │ ├── blueprints.py │ ├── cli.py │ ├── download_handler.py │ ├── fanstatic │ ├── css │ │ └── main.3c074c8b.chunk.css │ └── js │ │ ├── 2.60dfd475.chunk.js │ │ ├── main.c8e9dac3.chunk.js │ │ └── runtime-main.c155da57.js │ ├── helpers.py │ ├── i18n │ └── .gitignore │ ├── interfaces.py │ ├── plugin.py │ ├── public │ └── .gitignore │ ├── templates │ ├── blob_storage │ │ └── snippets │ │ │ └── upload_module.html │ └── package │ │ ├── new_resource.html │ │ ├── new_resource_not_draft.html │ │ ├── resource_edit.html │ │ └── snippets │ │ └── resource_form.html │ ├── tests │ ├── __init__.py │ ├── test_actions.py │ ├── test_authz.py │ ├── test_blueprint.py │ ├── test_helpers.py │ ├── test_plugin.py │ └── test_validators.py │ └── validators.py ├── dev-requirements.in ├── dev-requirements.py2.txt ├── dev-requirements.py3.txt ├── docker-compose.yml ├── docker ├── giftless.yaml └── nginx.conf ├── requirements.in ├── requirements.py2.txt ├── requirements.py3.txt ├── setup.cfg ├── setup.py ├── test.ini └── yarn.lock /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = 3 | */site-packages/* 4 | */python?.?/* 5 | ckan/* 6 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | max-complexity = 13 4 | exclude = .venv*,venv*,.git,__pycache__,.tox,.eggs,*.egg 5 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-20.04 13 | 14 | services: 15 | postgres: 16 | image: postgres:11-alpine 17 | env: 18 | POSTGRES_PASSWORD: ckan 19 | POSTGRES_USER: ckan 20 | POSTGRES_DB: ckan 21 | options: >- 22 | --health-cmd pg_isready 23 | --health-interval 5s 24 | --health-timeout 5s 25 | --health-retries 5 26 | ports: 27 | - 5432:5432 28 | 29 | redis: 30 | image: redis 31 | options: >- 32 | --health-cmd "redis-cli ping" 33 | --health-interval 5s 34 | --health-timeout 5s 35 | --health-retries 5 36 | ports: 37 | - 6379:6379 38 | 39 | solr: 40 | image: ckan/solr 41 | env: 42 | CKAN_SOLR_PASSWORD: ckan 43 | ports: 44 | - 8983:8983 45 | 46 | strategy: 47 | matrix: 48 | python-version: [ 2.7, 3.7 ] 49 | ckan-version: [ 2.8, 2.9 ] 50 | exclude: 51 | - python-version: 3.7 52 | ckan-version: 2.8 53 | env: 54 | CKAN_PATH: ./ckan 55 | VIRTUAL_ENV: "1" # let's fake a virtual environment, why not 56 | 57 | steps: 58 | - uses: actions/checkout@v2 59 | - name: Install Python 60 | uses: actions/setup-python@v2 61 | with: 62 | python-version: ${{ matrix.python-version }} 63 | - name: Set up the test environment 64 | run: | 65 | make ckan-install CKAN_VERSION=${{ matrix.ckan-version }} 66 | make create-test-db 67 | make dev-setup 68 | - name: Run tests 69 | run: make test 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .ropeproject 2 | bower_components 3 | .vscode/ 4 | node_modules 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | env/ 16 | /.venv*/ 17 | /.python-version 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | sdist/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .cache 41 | .pytest_cache 42 | nosetests.xml 43 | coverage.xml 44 | 45 | # Sphinx documentation 46 | docs/_build/ 47 | 48 | # Editors and IDEs 49 | /.idea 50 | 51 | # Make sentinel files 52 | /.make-status/ 53 | 54 | # Development environment extras 55 | /ckan 56 | /.env 57 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "datapub"] 2 | path = datapub 3 | url = https://github.com/datopian/datapub.git 4 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | line_length = 120 3 | known_first_party = ckanext.blob_storage 4 | known_third_party = ckan 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Datopian / Viderum, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | include requirements.txt 4 | recursive-include ckanext/blob_storage *.html *.json *.js *.less *.css *.mo 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for ckanext-blob-storage 2 | 3 | PACKAGE_DIR := ckanext/blob_storage 4 | PACKAGE_NAME := ckanext.blob_storage 5 | 6 | SHELL := bash 7 | PYTHON := python 8 | PIP := pip 9 | PIP_COMPILE := pip-compile 10 | PYTEST := pytest 11 | PASTER := paster 12 | DOCKER_COMPOSE := docker-compose 13 | PSQL := psql 14 | GIT := git 15 | 16 | # Find GNU sed in path (on OS X gsed should be preferred) 17 | SED := $(shell which gsed sed | head -n1) 18 | 19 | # The `ckan` command line only exists in newer versions of CKAN 20 | CKAN_CLI := $(shell which ckan | head -n1) 21 | 22 | TEST_INI_PATH := ./test.ini 23 | TEST_PATH := 24 | SENTINELS := .make-status 25 | 26 | PYTHON_VERSION := $(shell $(PYTHON) -c 'import sys; print(sys.version_info[0])') 27 | 28 | PACKAGE_TAG_PREFIX := "v" 29 | PACKAGE_TAG_SUFFIX := "" 30 | PACKAGE_VERSION := $(shell $(PYTHON) -c 'import $(PACKAGE_NAME) as p; print(p.__version__)') 31 | 32 | # CKAN environment variables 33 | CKAN_PATH := ckan 34 | CKAN_REPO_URL := https://github.com/ckan/ckan.git 35 | CKAN_VERSION := ckan-2.8.3 36 | CKAN_CONFIG_FILE := $(CKAN_PATH)/development.ini 37 | CKAN_SITE_URL := http://localhost:5000 38 | POSTGRES_USER := ckan 39 | POSTGRES_PASSWORD := ckan 40 | POSTGRES_DB := ckan 41 | POSTGRES_HOST := 127.0.0.1 42 | CKAN_SOLR_PASSWORD := ckan 43 | DATASTORE_DB_NAME := datastore 44 | DATASTORE_DB_RO_USER := datastore_ro 45 | DATASTORE_DB_RO_PASSWORD := datastore_ro 46 | CKAN_LOAD_PLUGINS := stats text_view image_view recline_view datastore authz_service blob_storage 47 | 48 | CKAN_CONFIG_VALUES := \ 49 | ckan.site_url=$(CKAN_SITE_URL) \ 50 | sqlalchemy.url=postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST)/$(POSTGRES_DB) \ 51 | ckan.datastore.write_url=postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST)/$(DATASTORE_DB_NAME) \ 52 | ckan.datastore.read_url=postgresql://$(DATASTORE_DB_RO_USER):$(DATASTORE_DB_RO_PASSWORD)@$(POSTGRES_HOST)/$(DATASTORE_DB_NAME) \ 53 | ckan.plugins='$(CKAN_LOAD_PLUGINS)' \ 54 | ckan.storage_path='%(here)s/storage' \ 55 | solr_url=http://127.0.0.1:8983/solr/ckan \ 56 | ckanext.blob_storage.storage_service_url=http://localhost:9419 \ 57 | ckanext.blob_storage.storage_namespace=my-ckan-ns \ 58 | ckanext.authz_service.jwt_algorithm=HS256 \ 59 | ckanext.authz_service.jwt_private_key=this-is-a-test-only-key \ 60 | ckanext.authz_service.jwt_include_user_email=true 61 | 62 | CKAN_TEST_CONFIG_VALUES := \ 63 | sqlalchemy.url=postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST)/$(POSTGRES_DB)_test \ 64 | ckan.datastore.write_url=postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST)/$(DATASTORE_DB_NAME)_test \ 65 | ckan.datastore.read_url=postgresql://$(DATASTORE_DB_RO_USER):$(DATASTORE_DB_RO_PASSWORD)@$(POSTGRES_HOST)/$(DATASTORE_DB_NAME)_test 66 | 67 | ifdef WITH_COVERAGE 68 | COVERAGE_ARG := --cov=$(PACKAGE_NAME) 69 | else 70 | COVERAGE_ARG := 71 | endif 72 | 73 | 74 | dev-requirements.%.txt: dev-requirements.in 75 | $(PIP_COMPILE) --no-index dev-requirements.in -o $@ 76 | 77 | requirements.%.txt: requirements.in 78 | $(PIP_COMPILE) --no-index requirements.in -o $@ 79 | 80 | ## Update requirements files for the current Python version 81 | requirements: $(SENTINELS)/requirements 82 | .PHONEY: requirements 83 | 84 | ## Install this extension to the current Python environment 85 | install: $(SENTINELS)/install 86 | .PHONY: install 87 | 88 | ## Set up the extension for development in the current Python environment 89 | develop: $(SENTINELS)/develop 90 | .PHONEY: develop 91 | 92 | ## Run all tests 93 | test: $(SENTINELS)/tests-passed 94 | .PHONY: test 95 | 96 | ## Install the right version of CKAN into the virtual environment 97 | ckan-install: $(SENTINELS)/ckan-installed 98 | @echo "Current CKAN version: $(shell cat $(SENTINELS)/ckan-version)" 99 | .PHONY: ckan-install 100 | 101 | ## Run CKAN in the local virtual environment 102 | ckan-start: $(SENTINELS)/ckan-installed $(SENTINELS)/install-dev $(CKAN_CONFIG_FILE) | _check_virtualenv 103 | ifdef CKAN_CLI 104 | $(CKAN_CLI) -c $(CKAN_CONFIG_FILE) db init 105 | $(CKAN_CLI) -c $(CKAN_CONFIG_FILE) server -r 106 | else 107 | $(PASTER) --plugin=ckan db init -c $(CKAN_CONFIG_FILE) 108 | $(PASTER) --plugin=ckan serve --reload --monitor-restart $(CKAN_CONFIG_FILE) 109 | endif 110 | .PHONY: ckan-start 111 | 112 | ## Create a version tag 113 | version-tag: 114 | @echo "Creating tag: $(PACKAGE_TAG_PREFIX)$(PACKAGE_VERSION)$(PACKAGE_TAG_SUFFIX)" 115 | $(GIT) tag "$(PACKAGE_TAG_PREFIX)$(PACKAGE_VERSION)$(PACKAGE_TAG_SUFFIX)" 116 | $(GIT) push --tags 117 | .PHONY: version-tag 118 | 119 | $(CKAN_PATH): 120 | $(GIT) clone $(CKAN_REPO_URL) $@ 121 | 122 | $(CKAN_CONFIG_FILE): $(SENTINELS)/ckan-installed $(SENTINELS)/develop | _check_virtualenv 123 | ifdef CKAN_CLI 124 | $(CKAN_CLI) generate config $(CKAN_CONFIG_FILE) 125 | $(CKAN_CLI) config-tool $(CKAN_CONFIG_FILE) -s DEFAULT debug=true 126 | $(CKAN_CLI) config-tool $(CKAN_CONFIG_FILE) $(CKAN_CONFIG_VALUES) 127 | else 128 | $(PASTER) make-config --no-interactive ckan $(CKAN_CONFIG_FILE) 129 | $(PASTER) --plugin=ckan config-tool $(CKAN_CONFIG_FILE) -s DEFAULT debug=true 130 | $(PASTER) --plugin=ckan config-tool $(CKAN_CONFIG_FILE) $(CKAN_CONFIG_VALUES) 131 | endif 132 | 133 | .env: 134 | @___POSTGRES_USER=$(POSTGRES_USER) \ 135 | ___POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \ 136 | ___POSTGRES_DB=$(POSTGRES_DB) \ 137 | ___CKAN_SOLR_PASSWORD=$(CKAN_SOLR_PASSWORD) \ 138 | ___DATASTORE_DB_NAME=$(DATASTORE_DB_NAME) \ 139 | ___DATASTORE_DB_USER=$(POSTGRES_USER) \ 140 | ___DATASTORE_DB_RO_USER=$(DATASTORE_DB_RO_USER) \ 141 | ___DATASTORE_DB_RO_PASSWORD=$(DATASTORE_DB_RO_PASSWORD) \ 142 | env | grep ^___ | $(SED) 's/^___//' > .env 143 | @cat .env 144 | 145 | ## Create the database for test running 146 | create-test-db: 147 | @echo " \ 148 | CREATE ROLE $(DATASTORE_DB_RO_USER) NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$(DATASTORE_DB_RO_PASSWORD)'; \ 149 | CREATE DATABASE $(DATASTORE_DB_NAME)_test OWNER $(POSTGRES_USER) ENCODING 'utf-8'; \ 150 | CREATE DATABASE $(POSTGRES_DB)_test OWNER $(POSTGRES_USER) ENCODING 'utf-8'; \ 151 | GRANT ALL PRIVILEGES ON DATABASE $(DATASTORE_DB_NAME)_test TO $(POSTGRES_USER); \ 152 | GRANT ALL PRIVILEGES ON DATABASE $(POSTGRES_DB)_test TO $(POSTGRES_USER); \ 153 | " | PGPASSWORD=$(POSTGRES_PASSWORD) $(PSQL) -h $(POSTGRES_HOST) --username "$(POSTGRES_USER)" 154 | .PHONY: create-test-db 155 | 156 | ## Start all Docker services 157 | docker-up: .env 158 | $(DOCKER_COMPOSE) up -d 159 | @until $(DOCKER_COMPOSE) exec db pg_isready -U $(POSTGRES_USER); do sleep 1; done 160 | @sleep 2 161 | @echo " \ 162 | CREATE ROLE $(DATASTORE_DB_RO_USER) NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$(DATASTORE_DB_RO_PASSWORD)'; \ 163 | CREATE DATABASE $(DATASTORE_DB_NAME) OWNER $(POSTGRES_USER) ENCODING 'utf-8'; \ 164 | CREATE DATABASE $(DATASTORE_DB_NAME)_test OWNER $(POSTGRES_USER) ENCODING 'utf-8'; \ 165 | CREATE DATABASE $(POSTGRES_DB)_test OWNER $(POSTGRES_USER) ENCODING 'utf-8'; \ 166 | GRANT ALL PRIVILEGES ON DATABASE $(DATASTORE_DB_NAME) TO $(POSTGRES_USER); \ 167 | GRANT ALL PRIVILEGES ON DATABASE $(DATASTORE_DB_NAME)_test TO $(POSTGRES_USER); \ 168 | GRANT ALL PRIVILEGES ON DATABASE $(POSTGRES_DB)_test TO $(POSTGRES_USER); \ 169 | " | $(DOCKER_COMPOSE) exec -T db psql --username "$(POSTGRES_USER)" 170 | .PHONY: docker-up 171 | 172 | ## Stop all Docker services 173 | docker-down: .env 174 | $(DOCKER_COMPOSE) down 175 | .PHONY: docker-down 176 | 177 | ## Initialize the development environment 178 | dev-setup: _check_virtualenv $(SENTINELS)/ckan-installed $(CKAN_PATH)/who.ini $(CKAN_CONFIG_FILE) $(SENTINELS)/develop 179 | .PHONY: setup 180 | 181 | ## Start a full development environment 182 | dev-start: dev-setup docker-up ckan-start 183 | .PHONY: start-dev 184 | 185 | # Private targets 186 | 187 | _check_virtualenv: 188 | @if [ -z "$(VIRTUAL_ENV)" ]; then \ 189 | echo "You are not in a virtual environment - activate your virtual environment first"; \ 190 | exit 1; \ 191 | fi 192 | .PHONY: _check_virtualenv 193 | 194 | $(SENTINELS): 195 | mkdir -p $@ 196 | 197 | $(SENTINELS)/ckan-version: $(CKAN_PATH) | _check_virtualenv $(SENTINELS) 198 | $(GIT) -C $(CKAN_PATH) remote update 199 | $(GIT) -C $(CKAN_PATH) checkout $(CKAN_VERSION) 200 | if [ -e $(CKAN_PATH)/requirement-setuptools.txt ]; then $(PIP) install -r $(CKAN_PATH)/requirement-setuptools.txt; fi 201 | if [[ "$(PYTHON_VERSION)" == "2" && -e $(CKAN_PATH)/requirements-py2.txt ]]; then \ 202 | $(PIP) install -r $(CKAN_PATH)/requirements-py2.txt; \ 203 | else \ 204 | $(PIP) install -r $(CKAN_PATH)/requirements.txt; \ 205 | fi 206 | $(PIP) install -r $(CKAN_PATH)/dev-requirements.txt 207 | $(PIP) install -e $(CKAN_PATH) 208 | echo "$(CKAN_VERSION)" > $@ 209 | 210 | $(SENTINELS)/ckan-installed: $(SENTINELS)/ckan-version | $(SENTINELS) 211 | @if [ "$(shell cat $(SENTINELS)/ckan-version)" != "$(CKAN_VERSION)" ]; then \ 212 | echo "Switching to CKAN $(CKAN_VERSION)"; \ 213 | rm $(SENTINELS)/ckan-version; \ 214 | $(MAKE) $(SENTINELS)/ckan-version; \ 215 | fi 216 | @touch $@ 217 | 218 | $(SENTINELS)/test.ini: $(TEST_INI_PATH) $(CKAN_PATH) $(CKAN_PATH)/test-core.ini | $(SENTINELS) 219 | $(SED) "s@use = config:.*@use = config:$(CKAN_PATH)/test-core.ini@" -i $(TEST_INI_PATH) 220 | ifdef CKAN_CLI 221 | $(CKAN_CLI) config-tool $(CKAN_PATH)/test-core.ini $(CKAN_CONFIG_VALUES) $(CKAN_TEST_CONFIG_VALUES) 222 | else 223 | $(PASTER) --plugin=ckan config-tool $(CKAN_PATH)/test-core.ini $(CKAN_CONFIG_VALUES) $(CKAN_TEST_CONFIG_VALUES) 224 | endif 225 | @touch $@ 226 | 227 | $(SENTINELS)/requirements: requirements.py$(PYTHON_VERSION).txt dev-requirements.py$(PYTHON_VERSION).txt | $(SENTINELS) 228 | @touch $@ 229 | 230 | $(SENTINELS)/install: requirements.py$(PYTHON_VERSION).txt | $(SENTINELS) 231 | $(PIP) install -r requirements.py$(PYTHON_VERSION).txt 232 | @touch $@ 233 | 234 | $(SENTINELS)/install-dev: requirements.py$(PYTHON_VERSION).txt | $(SENTINELS) 235 | $(PIP) install -r dev-requirements.py$(PYTHON_VERSION).txt 236 | $(PIP) install -e . 237 | @touch $@ 238 | 239 | $(SENTINELS)/develop: $(SENTINELS)/requirements $(SENTINELS)/install $(SENTINELS)/install-dev setup.py | $(SENTINELS) 240 | @touch $@ 241 | 242 | $(SENTINELS)/test-setup: $(SENTINELS)/develop $(SENTINELS)/test.ini 243 | ifdef CKAN_CLI 244 | $(CKAN_CLI) -c $(TEST_INI_PATH) db init 245 | else 246 | $(PASTER) --plugin=ckan db init -c $(TEST_INI_PATH) 247 | endif 248 | @touch $@ 249 | 250 | $(SENTINELS)/tests-passed: $(SENTINELS)/test-setup $(shell find $(PACKAGE_DIR) -type f) .flake8 .isort.cfg | $(SENTINELS) 251 | $(PYTEST) $(COVERAGE_ARG) \ 252 | --flake8 \ 253 | --isort \ 254 | --ckan-ini=$(TEST_INI_PATH) \ 255 | --doctest-modules \ 256 | --ignore $(PACKAGE_DIR)/cli.py \ 257 | -s \ 258 | $(PACKAGE_DIR)/$(TEST_PATH) 259 | @touch $@ 260 | 261 | # Help related variables and targets 262 | 263 | GREEN := $(shell tput -Txterm setaf 2) 264 | YELLOW := $(shell tput -Txterm setaf 3) 265 | WHITE := $(shell tput -Txterm setaf 7) 266 | RESET := $(shell tput -Txterm sgr0) 267 | TARGET_MAX_CHAR_NUM := 15 268 | 269 | ## Show help 270 | help: 271 | @echo '' 272 | @echo 'Usage:' 273 | @echo ' ${YELLOW}make${RESET} ${GREEN}${RESET}' 274 | @echo '' 275 | @echo 'Targets:' 276 | @awk '/^[a-zA-Z\-\_0-9]+:/ { \ 277 | helpMessage = match(lastLine, /^## (.*)/); \ 278 | if (helpMessage) { \ 279 | helpCommand = substr($$1, 0, index($$1, ":")-1); \ 280 | helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \ 281 | printf " ${YELLOW}%-$(TARGET_MAX_CHAR_NUM)s${RESET} ${GREEN}%s${RESET}\n", helpCommand, helpMessage; \ 282 | } \ 283 | } \ 284 | { lastLine = $$0 }' $(MAKEFILE_LIST) 285 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ckanext-blob-storage 2 | ======================== 3 | [![Build Status](https://travis-ci.org/datopian/ckanext-blob-storage.svg?branch=master)](https://travis-ci.org/datopian/ckanext-blob-storage) 4 | ![Tests](https://github.com/datopian/ckanext-authz-service/workflows/test/badge.svg?branch=master) 5 | [![Coverage Status](https://coveralls.io/repos/github/datopian/ckanext-blob-storage/badge.svg?branch=master)](https://coveralls.io/github/datopian/ckanext-blob-storage?branch=master) 6 | 7 | **Move CKAN resource storage management to an external micro-service** 8 | 9 | `ckanext-blob-storage` replace's CKAN's default local blob storage functionality with pluggable storage layer supporting cloud and local. It supports direct to cloud file uploading following the design in https://tech.datopian.com/blob-storage/#ckan-v3 10 | 11 | The design is pluggable so one can use all the major storage backends as well as local, cloud based (e.g. S3, Azure Blobs, GCP, etc.) or any other storage. In addition, the service allows clients (typically browsers) to upload and download files directly to storage without passing them through CKAN, which can greatly improve file access efficiency. 12 | 13 | Authentication and authorization to the blob storage management service is done via JWT tokens provided by 14 | [`ckanext-authz-service`](https://github.com/datopian/ckanext-authz-service). 15 | 16 | Internally, the blob storage management service is in fact a Git LFS server 17 | implementation, which means access via 3rd party Git based tools is also 18 | potentially possible. 19 | 20 | Configuration settings 21 | ---------------------- 22 | 23 | `ckanext.blob_storage.storage_service_url = 'https://...'` 24 | 25 | Set the URL of the blob storage microservice (the Git LFS server). This 26 | must be a URL accessible to browsers connecting to the service. 27 | 28 | `ckanext.blob_storage.storage_namespace = my-ckan-instance` 29 | 30 | Set the in-storage namespace used for this CKAN instance. This is useful if 31 | multiple CKAN instances are using the same storage microservice instance, and 32 | you need to seperate permission scopes between them. 33 | 34 | If not specified, `ckan` will be used as the default namespace. 35 | 36 | Required resource fields 37 | ------------------------ 38 | 39 | There are a few resource fields that are required for `ckanext-blob-storage` to 40 | operate. API / SDK users needs to set them on the requests to create new 41 | resources. 42 | 43 | The required fields are: 44 | * `url`: the file name, without path (required by vanilla CKAN not just by blob storage) 45 | * `url_type`: set to "upload" for uploaded files 46 | * `sha256`: the SHA256 of the file 47 | * `size`: the size of the file in bytes 48 | * `lfs_prefix`: the LFS server path of where the file has been stored by Giftless. 49 | Something like **org/dataset** or **storage_namespace/dataset_id**. 50 | 51 | If `sha256`, `size` or `lfs_prefix` are missing for uploads 52 | (`'url_type == 'upload'`), the API call will return a ValidationError: 53 | 54 | ``` 55 | { 56 | "help": "http://ckan:5000/api/3/action/help_show?name=resource_create", 57 | "success": false, 58 | "error": { 59 | "__type": "Validation Error", 60 | "url_type": [ 61 | "Resource's sha256 field cannot be missing for uploads.", 62 | "Resource's size field cannot be missing for uploads.", 63 | "Resource's lfs_prefix field cannot be missing for uploads." 64 | ] 65 | } 66 | } 67 | ``` 68 | 69 | Requirements 70 | ------------ 71 | * This extension works with CKAN 2.8.x and CKAN 2.9.x. 72 | * `ckanext-authz-service` must be installed and enabled 73 | * A working and configured Git LFS server accessible to the browser. We 74 | recommend usign [Giftless](https://github.com/datopian/giftless) but other 75 | implementations may be configured to work as well. 76 | 77 | Installation 78 | ------------ 79 | 80 | To install ckanext-blob-storage: 81 | 82 | 1. Activate your CKAN virtual environment, for example: 83 | ``` 84 | . /usr/lib/ckan/default/bin/activate 85 | ``` 86 | 87 | 2. Install the ckanext-blob-storage Python package into your virtual environment: 88 | ``` 89 | pip install ckanext-blob-storage 90 | ``` 91 | 92 | 3. Add `blob_storage` to the `ckan.plugins` setting in your CKAN 93 | config file (by default the config file is located at 94 | `/etc/ckan/default/production.ini`). 95 | 96 | 4. Restart CKAN. For example if you've deployed CKAN with Apache on Ubuntu: 97 | ``` 98 | sudo service apache2 reload 99 | ``` 100 | 101 | Developer installation 102 | ---------------------- 103 | 104 | To install `ckanext-blob-storage` for development, do the following: 105 | 106 | 1. Pull the project code from Github 107 | ``` 108 | git clone https://github.com/datopian/ckanext-blob-storage.git 109 | cd ckanext-blob-storage 110 | ``` 111 | 2. Create a Python 2.7 virtual environment (The flag `-p py27` is used to ensure that you are using the right Python version when create the virtualenv). 112 | ``` 113 | virtualenv .venv27 -p py27 114 | source .venv27/bin/activate 115 | ``` 116 | 117 | 3. Run the following command to bootstrap the entire environment 118 | ``` 119 | make dev-start 120 | ``` 121 | 122 | This will pull and install CKAN and all it's dependencies into your virtual 123 | environment, create all necessary configuration files, launch external services 124 | using Docker Compose and start the CKAN development server. 125 | 126 | You can create an user using the web interface at [`localhost:5000`](http://localhost:5000/) but the user will not be an _admin_ with permissions to create organizations or datasets. If you need to turn your user in an _admin_, make sure the virtual environment is still active and use this command, replacing the `` with the user name you created: 127 | 128 | ``` 129 | paster --plugin=ckan sysadmin -c ckan/development.ini add 130 | ``` 131 | 132 | You can repeat the last command at any time to start developing again. 133 | 134 | Type `make help` to get a like of user commands useful to managing the local 135 | environment. 136 | 137 | Update DataPub (resource editor) app 138 | ------------------------------------ 139 | 140 | 1. Init submodule for the resource editor app 141 | ``` 142 | git submodule init 143 | git submodule update 144 | ``` 145 | 146 | 2. Build the resource editor app 147 | ``` 148 | cd datapub 149 | yarn 150 | yarn build 151 | ``` 152 | 153 | 3. Replace bundles in `fanstatic` directory 154 | ``` 155 | rm ckanext/blob_storage/fanstatic/js/* 156 | cp datapub/build/static/js/*.js ckanext/blob_storage/fanstatic/js/ 157 | ``` 158 | 159 | If you also want to re-use stylesheets: 160 | 161 | ``` 162 | rm ckanext/blob_storage/fanstatic/css/* 163 | cp datapub/build/static/css/*.css ckanext/blob_storage/fanstatic/css/ 164 | ``` 165 | 166 | 4. Now, make sure to update the resources in `templates/blob_storage/snippets/upload_module.html` 167 | 168 | ``` 169 | {% resource 'blob-storage/css/main.{hash}.chunk.css' %} 170 | 171 | {% resource 'blob-storage/js/runtime-main.{hash}.js' %} 172 | {% resource 'blob-storage/js/2.{hash}.chunk.js' %} 173 | {% resource 'blob-storage/js/main.{hash}.chunk.js' %} 174 | ``` 175 | 176 | Installing with Docker 177 | ---------------------- 178 | 179 | Unlike other CKAN extensions, blob storage needs node modules to be installed 180 | and build in order to work properly. You will need to install node and npm. 181 | Below is how your Dockerfile might look like 182 | 183 | ``` 184 | RUN apt-get -q -y install \ 185 | python-pip \ 186 | curl \ 187 | git-core 188 | 189 | RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && apt-get install nodejs && npm version 190 | 191 | # Install ckanext-blob-storage 192 | RUN git clone --branch ${CKANEXT_BLOB_STORAGE_VERSION} https://github.com/datopian/ckanext-blob-storage 193 | RUN pip install --no-cache-dir -r "ckanext-blob-storage/requirements.py2.txt" 194 | RUN pip install -e ckanext-blob-storage 195 | 196 | # Install other extensions 197 | ... 198 | ``` 199 | 200 | __NOTE:__ We assume that you have Giftless server running with configuration as 201 | in [giftless.yaml][giftless] and nginx is configured as in [nginx.conf][nginx] 202 | 203 | 204 | 205 | ### Working with `requirements.txt` files 206 | 207 | #### tl;dr 208 | 209 | * You *do not* touch `*requirements.*.txt` files directly. We use 210 | [`pip-tools`][1] and custom `make` targets to manage these files. 211 | * Use `make develop` to install the right development time requirements into your 212 | current virtual environment 213 | * Use `make install` to install the right runtime requirements into your current 214 | virtual environment 215 | * To add requirements, edit `requirements.in` or `dev-requirements.in` and run 216 | `make requirements`. This will recompile the requirements file(s) **for your 217 | current Python version**. You may need to do this for the other Python version 218 | by switching to a different Python virtual environment before committing your 219 | changes. 220 | 221 | #### More background 222 | This project manages requirements in a relatively complex way, in order to 223 | seamlessly support Python 2.7 and 3.x. 224 | 225 | For this reason, you will see 4 requirements files in the project root: 226 | 227 | * `requirements.py2.txt` - Python 2 runtime requirements 228 | * `requirements.py3.txt` - Python 3 runtime requirements 229 | * `dev-requirements.py2.txt` - Python 2 development requirements 230 | * `dev-requirements.py3.txt` - Python 3 development requirements 231 | 232 | These are generated using the `pip-compile` command (a part of `pip-tools`) 233 | from the corresponding `requirements.in` and `dev-requirements.in` files. 234 | 235 | To understand why `pip-compile` is used, read the `pip-tools` manual. In 236 | short, this allows us to pin dependencies of dependencies, thus resolving 237 | potential deployment conflicts, without the headache of managing the specific 238 | version of each Nth-level dependency. 239 | 240 | In order to support both Python 2.7 and 3.x, which tend to require slightly 241 | different dependencies, we use `requirements.in` files to generate 242 | major-version specific requirements files. These, in turn, should be used 243 | when installing the package. 244 | 245 | In order to simplify things, the `make` targets specified above will automate 246 | the process *for the current Python version*. 247 | 248 | #### Adding Requirements 249 | 250 | Requirements are managed in `.in` files - these are the only files that 251 | should be edited directly. 252 | 253 | Take care to specify a version for each requirement, to the level required 254 | to maintain future compatibility, but not to specify an *exact* version 255 | unless necessary. 256 | 257 | For example, the following are good `requirements.in` lines: 258 | 259 | pyjwt[crypto]==1.7.* 260 | pyyaml==5.* 261 | pytz 262 | 263 | This allows these packages to be upgraded to a minor version, without the risk 264 | of breaking compatibility. 265 | 266 | Note that `pytz` is specified with no version on purpose, as we want it updated 267 | to the latest possible version on each new rebuild. 268 | 269 | Developers wanting to add new requirements (runtime or development time), 270 | should take special care to update the `requirements.txt` files for all 271 | supported Python versions by running `make requirements` on different 272 | virtual environment, after updating the relevant `.in` file. 273 | 274 | #### Applying Patch-level upgrades to requirements 275 | 276 | You can delete `*requirements.*.txt` and run `make requirements`. 277 | 278 | TODO: we can probably do this in a better way - create a `make` target 279 | for this. 280 | 281 | 282 | Tests 283 | ----- 284 | 285 | To run the tests, do: 286 | 287 | make test 288 | 289 | To run the tests and produce a coverage report, first make sure you have 290 | coverage installed in your virtualenv (``pip install coverage``) then run: 291 | 292 | make coverage 293 | 294 | Releasing a new version of ckanext-blob-storage 295 | ------------------------------------------------ 296 | 297 | ckanext-blob-storage should be available on PyPI as https://pypi.org/project/ckanext-blob-storage. 298 | To publish a new version to PyPI follow these steps: 299 | 300 | 1. Update the version number in the `setup.py` file. 301 | See [PEP 440](http://legacy.python.org/dev/peps/pep-0440/#public-version-identifiers) 302 | for how to choose version numbers. 303 | 304 | 2. Make sure you have the latest version of necessary packages: 305 | ``` 306 | pip install --upgrade setuptools wheel twine 307 | ``` 308 | 309 | 3. Create a source and binary distributions of the new version: 310 | ``` 311 | python setup.py sdist bdist_wheel && twine check dist/* 312 | ``` 313 | 314 | Fix any errors you get. 315 | 316 | 4. Upload the source distribution to PyPI: 317 | ``` 318 | twine upload dist/* 319 | ``` 320 | 321 | 5. Commit any outstanding changes: 322 | ``` 323 | git commit -a 324 | ``` 325 | 326 | 6. Tag the new release of the project on GitHub with the version number from 327 | the ``setup.py`` file. For example if the version number in ``setup.py`` is 328 | 0.0.1 then do: 329 | ``` 330 | git tag 0.0.1 331 | git push --tags 332 | ``` 333 | 334 | 335 | [1]: https://pypi.org/project/pip-tools/ 336 | [giftless]: docker/giftless.yaml 337 | [nginx]: docker/nginx.conf -------------------------------------------------------------------------------- /ckanext/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | # this is a namespace package 4 | try: 5 | import pkg_resources 6 | pkg_resources.declare_namespace(__name__) 7 | except ImportError: 8 | import pkgutil 9 | __path__ = pkgutil.extend_path(__path__, __name__) 10 | -------------------------------------------------------------------------------- /ckanext/blob_storage/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.8.2' 2 | -------------------------------------------------------------------------------- /ckanext/blob_storage/actions.py: -------------------------------------------------------------------------------- 1 | """Blob Storage API actions 2 | """ 3 | import ast 4 | import logging 5 | from typing import Any, Dict, Optional 6 | 7 | from ckan.plugins import toolkit 8 | from giftless_client import LfsClient 9 | from giftless_client.exc import LfsError 10 | from six import ensure_text 11 | 12 | from . import helpers 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | @toolkit.side_effect_free 18 | def get_resource_download_spec(context, data_dict): 19 | """Get a signed URL from LFS server to download a resource 20 | """ 21 | resource = _get_resource(context, data_dict) 22 | activity_id = data_dict.get('activity_id') 23 | inline = toolkit.asbool(data_dict.get('inline')) 24 | 25 | for k in ('lfs_prefix', 'sha256', 'size'): 26 | if k not in resource: 27 | return {} 28 | 29 | return get_lfs_download_spec(context, resource, inline=inline, activity_id=activity_id) 30 | 31 | 32 | def get_lfs_download_spec(context, # type: Dict[str, Any] 33 | resource, # type: Dict[str, Any] 34 | sha256=None, # type: Optional[str] 35 | size=None, # type: Optional[int] 36 | filename=None, # type: Optional[str] 37 | storage_prefix=None, # type: Optional[str] 38 | inline=False, # type: Optional[bool] 39 | activity_id=None # type: Optional[str] 40 | ): # type: (...) -> Dict[str, Any] 41 | """Get the LFS download spec (URL and headers) for a resource 42 | 43 | This function allows overriding the expected sha256 and size for situations where 44 | an additional file is associated with a CKAN resource and we want to override it. 45 | In these cases, we will use the parent resource for authorization checks and 46 | sha256 and size to request an object from the LFS server. You should *only* use 47 | these override arguments if you know what you are doing, as allowing client side 48 | code to override the sha256 and size could lead to potential security issues. 49 | """ 50 | if storage_prefix is None: 51 | storage_prefix = resource['lfs_prefix'] 52 | if size is None: 53 | size = resource['size'] 54 | if sha256 is None: 55 | sha256 = resource['sha256'] 56 | if filename is None: 57 | filename = helpers.resource_filename(resource) 58 | 59 | package = toolkit.get_action('package_show')(context, {'id': resource['package_id']}) 60 | authz_token = get_download_authz_token( 61 | context, 62 | package['organization']['name'], 63 | package['name'], 64 | resource['id'], 65 | activity_id=activity_id) 66 | client = context.get('download_lfs_client', LfsClient(helpers.server_url(), authz_token)) 67 | 68 | resources = [{"oid": sha256, "size": size, "x-filename": filename}] 69 | 70 | if inline: 71 | resources[0]["x-disposition"] = "inline" 72 | 73 | object_spec = _get_resource_download_lfs_objects(client, storage_prefix, resources)[0] 74 | 75 | assert object_spec['oid'] == sha256 76 | assert object_spec['size'] == size 77 | 78 | if 'error' in object_spec: 79 | raise toolkit.ObjectNotFound('Object error [{}]: {}'.format(object_spec['error'].get('message', '[no message]'), 80 | object_spec['error'].get('code', 'unknown'))) 81 | 82 | return object_spec['actions']['download'] 83 | 84 | 85 | @toolkit.side_effect_free 86 | def resource_schema_show(context, data_dict): 87 | """Get a resource schema as a dictionary instead of string 88 | """ 89 | resource = _get_resource(context, data_dict) 90 | 91 | if resource.get('schema', False): 92 | try: 93 | return ast.literal_eval(resource['schema']) 94 | except ValueError: 95 | return resource['schema'] 96 | return {} 97 | 98 | 99 | @toolkit.side_effect_free 100 | def resource_sample_show(context, data_dict): 101 | """Get a resource sample as a list of dictionaries instead of string 102 | """ 103 | resource = _get_resource(context, data_dict) 104 | 105 | if resource.get('sample', False): 106 | try: 107 | return ast.literal_eval(resource['sample']) 108 | except ValueError: 109 | return resource['sample'] 110 | return {} 111 | 112 | 113 | def _get_resource_download_lfs_objects(client, lfs_prefix, resources): 114 | """Get LFS download operation response objects for a given resource list 115 | """ 116 | log.debug("Requesting download spec from LFS server for %s", resources) 117 | try: 118 | batch_response = client.batch(lfs_prefix, 'download', resources) 119 | except LfsError as e: 120 | if e.status_code == 404: 121 | raise toolkit.ObjectNotFound("The requested resource does not exist") 122 | elif e.status_code == 422: 123 | raise toolkit.ObjectNotFound("Object parameters mismatch") 124 | elif e.status_code == 403: 125 | raise toolkit.ObjectNotFound("Request was denied by the LFS server") 126 | else: 127 | raise 128 | 129 | return batch_response['objects'] 130 | 131 | 132 | def get_download_authz_token(context, org_name, package_name, resource_id, activity_id=None): 133 | # type: (Dict[str, Any], str, str, str, str) -> str 134 | """Get an authorization token for getting the download URL from LFS 135 | """ 136 | authorize = toolkit.get_action('authz_authorize') 137 | if not authorize: 138 | raise RuntimeError("Cannot find authz_authorize; Is ckanext-authz-service installed?") 139 | 140 | scope = helpers.resource_authz_scope( 141 | package_name, 142 | org_name=org_name, 143 | actions='read', 144 | resource_id=resource_id, 145 | activity_id=activity_id 146 | ) 147 | log.debug("Requesting authorization token for scope: %s", scope) 148 | authz_result = authorize(context, {"scopes": [scope]}) 149 | if not authz_result or not authz_result.get('token', False): 150 | raise RuntimeError("Failed to get authorization token for LFS server") 151 | log.debug("Granted scopes: %s", authz_result['granted_scopes']) 152 | 153 | if len(authz_result['granted_scopes']) == 0: 154 | raise toolkit.NotAuthorized("You are not authorized to download this resource") 155 | 156 | return ensure_text(authz_result['token']) 157 | 158 | 159 | def _get_resource(context, data_dict): 160 | """Get resource by ID 161 | """ 162 | if 'resource' in data_dict: 163 | return data_dict['resource'] 164 | return toolkit.get_action('resource_show')(context, {'id': data_dict['id']}) 165 | -------------------------------------------------------------------------------- /ckanext/blob_storage/authz.py: -------------------------------------------------------------------------------- 1 | """Authorization related helpers 2 | """ 3 | import logging 4 | 5 | from ckan.plugins import toolkit 6 | 7 | from ckanext.authz_service.authz_binding import resource as resource_authz 8 | from ckanext.authz_service.authz_binding.common import get_user_context 9 | from ckanext.authz_service.authzzie import Scope 10 | 11 | from . import helpers 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | def check_object_permissions(id, dataset_id=None, organization_id=None, context=None): 17 | """Check object (resource in storage) permissions 18 | 19 | This wrap's ckanext-authz-service's default logic for checking resource 20 | by checking for global-prefix/dataset-uuid/* style object scopes. 21 | """ 22 | if not context: 23 | context = get_user_context() 24 | # support for resource_id/activity_id 25 | id = id.split('/')[0] 26 | if dataset_id and organization_id and organization_id == helpers.storage_namespace(): 27 | log.debug("Requesting authorization for object: %s/%s in namespace %s", dataset_id, id, organization_id) 28 | dataset = toolkit.get_action('package_show')(context, {'id': dataset_id}) 29 | dataset_id = dataset['name'] 30 | try: 31 | organization_id = dataset['organization']['name'] 32 | except (KeyError, TypeError): 33 | organization_id = None # Dataset has no organization 34 | log.debug("Real resource path is res:%s/%s/%s", organization_id, dataset_id, id) 35 | 36 | return resource_authz.check_resource_permissions(id, dataset_id, organization_id, context=context) 37 | 38 | 39 | def object_id_parser(*args, **kwargs): 40 | """Object (resource in storage) ID parser 41 | """ 42 | return resource_authz.resource_id_parser(*args, **kwargs) 43 | 44 | 45 | def normalize_object_scope(_, granted_scope): 46 | # type: (Scope, Scope) -> Scope 47 | """Normalize resource scope by trimming out the org/dataset part and only leaving the sha256 48 | 49 | This helps us deal with cases where a resource is moved (i.e. the dataset is renamed, or the 50 | organization is renamed, or the dataset is moved to a different organization) 51 | """ 52 | if not granted_scope: 53 | return granted_scope 54 | 55 | entity_ref_parts = granted_scope.entity_ref.split('/') 56 | 57 | # let's check that we have a full scope 58 | if len(entity_ref_parts) < 3: 59 | return granted_scope 60 | for part in entity_ref_parts: 61 | if part is None or part in {'', '*'}: 62 | return granted_scope 63 | 64 | if len(entity_ref_parts) > 3: 65 | activity_id = entity_ref_parts[3] 66 | else: 67 | activity_id = None 68 | 69 | storage_id = _get_resource_storage_id(organization_id=entity_ref_parts[0], 70 | dataset_id=entity_ref_parts[1], 71 | resource_id=entity_ref_parts[2], 72 | activity_id=activity_id) 73 | return Scope(granted_scope.entity_type, storage_id, granted_scope.actions, granted_scope.subscope) 74 | 75 | 76 | def _get_resource_storage_id(organization_id, dataset_id, resource_id, activity_id): 77 | # type: (str, str, str, str) -> str 78 | """Get the exact ID of the resource in storage, as opposed to it's ID in CKAN 79 | 80 | A resource in CKAN is identified by //. However, 81 | the and can change if the dataset is renamed or moved or the 82 | organization is renamed. For this reason, we replace the original CKAN ID with a 83 | "static" ID composed if /. in turn is the 84 | / that the dataset had *when it was originally uploaded*, and 85 | does not change over time. 86 | """ 87 | context = get_user_context() 88 | if activity_id and toolkit.check_ckan_version(min_version='2.9'): 89 | activity = toolkit.get_action(u'activity_show')( 90 | context, {u'id': activity_id, u'include_data': True}) 91 | dataset = activity['data']['package'] 92 | else: 93 | dataset = toolkit.get_action('package_show')(context, {'id': dataset_id}) 94 | 95 | resource = None 96 | for res in dataset['resources']: 97 | if res['id'] == resource_id: 98 | resource = res 99 | break 100 | 101 | if not resource: 102 | toolkit.ObjectNotFound("Resource not found.") 103 | 104 | if resource.get('sha256') and resource.get('lfs_prefix'): 105 | return '{}/{}'.format(resource['lfs_prefix'], resource['sha256']) 106 | 107 | return '{}/{}/{}'.format(organization_id, dataset_id, resource_id) 108 | -------------------------------------------------------------------------------- /ckanext/blob_storage/blueprints.py: -------------------------------------------------------------------------------- 1 | """ckanext-blob-storage Flask blueprints 2 | """ 3 | from ckan.plugins import toolkit 4 | from flask import Blueprint, request 5 | 6 | from .download_handler import call_download_handlers, call_pre_download_handlers, get_context 7 | 8 | blueprint = Blueprint( 9 | 'blob_storage', 10 | __name__, 11 | ) 12 | 13 | 14 | def download(id, resource_id, filename=None): 15 | """Download resource blueprint 16 | 17 | This calls all registered download handlers in order, until 18 | a response is returned to the user 19 | """ 20 | context = get_context() 21 | resource = None 22 | 23 | try: 24 | resource = toolkit.get_action('resource_show')(context, {'id': resource_id}) 25 | if id != resource['package_id']: 26 | return toolkit.abort(404, toolkit._('Resource not found belonging to package')) 27 | package = toolkit.get_action('package_show')(context, {'id': id}) 28 | except toolkit.ObjectNotFound: 29 | return toolkit.abort(404, toolkit._('Resource not found')) 30 | except toolkit.NotAuthorized: 31 | return toolkit.abort(401, toolkit._('Not authorized to read resource {0}'.format(id))) 32 | 33 | activity_id = request.args.get('activity_id') 34 | inline = toolkit.asbool(request.args.get('preview')) 35 | 36 | if activity_id and toolkit.check_ckan_version(min_version='2.9'): 37 | try: 38 | activity = toolkit.get_action(u'activity_show')( 39 | context, {u'id': activity_id, u'include_data': True}) 40 | activity_dataset = activity['data']['package'] 41 | assert activity_dataset['id'] == id 42 | activity_resources = activity_dataset['resources'] 43 | for r in activity_resources: 44 | if r['id'] == resource_id: 45 | resource = r 46 | package = activity_dataset 47 | break 48 | except toolkit.NotFound: 49 | toolkit.abort(404, toolkit._(u'Activity not found')) 50 | 51 | try: 52 | resource = call_pre_download_handlers(resource, package, activity_id=activity_id) 53 | return call_download_handlers(resource, package, filename, inline, activity_id=activity_id) 54 | except toolkit.ObjectNotFound: 55 | return toolkit.abort(404, toolkit._('Resource not found')) 56 | except toolkit.NotAuthorized: 57 | return toolkit.abort(401, toolkit._('Not authorized to read resource {0}'.format(resource_id))) 58 | 59 | 60 | blueprint.add_url_rule(u'/dataset//resource//download', view_func=download) 61 | blueprint.add_url_rule(u'/dataset//resource//download/', view_func=download) 62 | -------------------------------------------------------------------------------- /ckanext/blob_storage/cli.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import logging 3 | import os 4 | import shutil 5 | import tempfile 6 | import time 7 | from contextlib import contextmanager 8 | from typing import Any, Dict, Generator, Tuple 9 | 10 | import requests 11 | from ckan.lib.cli import CkanCommand 12 | from ckan.lib.helpers import _get_auto_flask_context # noqa we need this for Flask request context 13 | from ckan.model import Resource, Session, User 14 | from ckan.plugins import toolkit 15 | from flask import Response 16 | from giftless_client import LfsClient 17 | from giftless_client.types import ObjectAttributes 18 | from six import binary_type, string_types 19 | from sqlalchemy.orm import load_only 20 | from sqlalchemy.orm.attributes import flag_modified 21 | from werkzeug.wsgi import FileWrapper 22 | 23 | from ckanext.blob_storage import helpers 24 | from ckanext.blob_storage.download_handler import call_download_handlers 25 | 26 | 27 | def _log(): 28 | return logging.getLogger(__name__) 29 | 30 | 31 | class MigrateResourcesCommand(CkanCommand): 32 | """Migrate all non-migrated resources to external blob storage 33 | """ 34 | summary = __doc__.split('\n')[0] 35 | usage = __doc__ 36 | min_args = 0 37 | 38 | _user = None 39 | _max_failures = 3 40 | _retry_delay = 3 41 | 42 | def command(self): 43 | self._load_config() 44 | self._user = User.get(self.site_user['name']) 45 | with app_context() as context: 46 | context.g.user = self.site_user['name'] 47 | context.g.userobj = self._user 48 | self.migrate_all_resources() 49 | 50 | def migrate_all_resources(self): 51 | """Do the actual migration 52 | """ 53 | migrated = 0 54 | for resource_obj in get_unmigrated_resources(): 55 | _log().info("Starting to migrate resource %s [%s]", resource_obj.id, resource_obj.name) 56 | failed = 0 57 | while failed < self._max_failures: 58 | try: 59 | self.migrate_resource(resource_obj) 60 | _log().info("Finished migrating resource %s", resource_obj.id) 61 | migrated += 1 62 | break 63 | except Exception: 64 | _log().exception("Failed to migrate resource %s, retrying...", resource_obj.id) 65 | failed += 1 66 | time.sleep(self._retry_delay) 67 | else: 68 | _log().error("Skipping resource %s [%s] after %d failures", resource_obj.id, resource_obj.name, failed) 69 | 70 | _log().info("Finished migrating %d resources", migrated) 71 | 72 | def migrate_resource(self, resource_obj): 73 | # type: (Resource) -> None 74 | dataset, resource_dict = get_resource_dataset(resource_obj) 75 | resource_name = helpers.resource_filename(resource_dict) 76 | 77 | with download_resource(resource_dict, dataset) as resource_file: 78 | _log().debug("Starting to upload file: %s", resource_file) 79 | lfs_namespace = helpers.storage_namespace() 80 | props = self.upload_resource(resource_file, dataset['id'], lfs_namespace, resource_name) 81 | props['lfs_prefix'] = '{}/{}'.format(lfs_namespace, dataset['id']) 82 | props['sha256'] = props.pop('oid') 83 | _log().debug("Upload complete; sha256=%s, size=%d", props['sha256'], props['size']) 84 | 85 | update_storage_props(resource_obj, props) 86 | 87 | def upload_resource(self, resource_file, dataset_id, lfs_namespace, filename): 88 | # type: (str, str, str, str) -> ObjectAttributes 89 | """Upload a resource file to new storage using LFS server 90 | """ 91 | token = self.get_upload_authz_token(dataset_id) 92 | lfs_client = LfsClient(helpers.server_url(), token) 93 | with open(resource_file, 'rb') as f: 94 | props = lfs_client.upload(f, lfs_namespace, dataset_id, filename=filename) 95 | 96 | # Only return standard object attributes 97 | return {k: v for k, v in props.items() if k[0:2] != 'x-'} 98 | 99 | def get_upload_authz_token(self, dataset_id): 100 | # type: (str) -> str 101 | """Get an authorization token to upload the file to LFS 102 | """ 103 | authorize = toolkit.get_action('authz_authorize') 104 | if not authorize: 105 | raise RuntimeError("Cannot find authz_authorize; Is ckanext-authz-service installed?") 106 | 107 | context = {'ignore_auth': True, 'auth_user_obj': self._user} 108 | scope = helpers.resource_authz_scope(dataset_id, actions='write') 109 | authz_result = authorize(context, {"scopes": [scope]}) 110 | 111 | if not authz_result or not authz_result.get('token', False): 112 | raise RuntimeError("Failed to get authorization token for LFS server") 113 | 114 | if len(authz_result['granted_scopes']) == 0: 115 | raise toolkit.NotAuthorized("You are not authorized to upload resources") 116 | 117 | return authz_result['token'] 118 | 119 | 120 | def update_storage_props(resource, lfs_props): 121 | # type: (Resource, Dict[str, Any]) -> None 122 | """Update the resource with new storage properties 123 | """ 124 | resource.extras['lfs_prefix'] = lfs_props['lfs_prefix'] 125 | resource.extras['sha256'] = lfs_props['sha256'] 126 | resource.size = lfs_props['size'] 127 | flag_modified(resource, 'extras') 128 | 129 | 130 | @contextmanager 131 | def download_resource(resource, dataset): 132 | # type: (Dict[str, Any], Dict[str, Any]) -> str 133 | """Download the resource to a local file and provide the file name 134 | 135 | This is a context manager that will delete the local file once context is closed 136 | """ 137 | resource_file = tempfile.mktemp(prefix='ckan-blob-migration-') 138 | try: 139 | response = call_download_handlers(resource, dataset) 140 | if response.status_code == 200: 141 | _save_downloaded_response_data(response, resource_file) 142 | elif response.status_code in {301, 302}: 143 | _save_redirected_response_data(response, resource_file) 144 | else: 145 | raise RuntimeError("Unexpected download response code: {}".format(response.status_code)) 146 | yield resource_file 147 | finally: 148 | try: 149 | os.unlink(resource_file) 150 | except OSError as e: 151 | if e.errno != errno.ENOENT: 152 | raise 153 | 154 | 155 | def _save_downloaded_response_data(response, file_name): 156 | # type: (Response, str) -> None 157 | """Get an HTTP response object with open file containing a resource and save the data locally 158 | to a temporary file 159 | """ 160 | with open(file_name, 'wb') as f: 161 | if isinstance(response.response, (string_types, binary_type)): 162 | _log().debug("Response contains inline string data, saving to %s", file_name) 163 | f.write(response.response) 164 | elif isinstance(response.response, FileWrapper): 165 | _log().debug("Response is a werkzeug.wsgi.FileWrapper, copying to %s", file_name) 166 | for chunk in response.response: 167 | f.write(chunk) 168 | elif hasattr(response.response, 'read'): # assume an open stream / file 169 | _log().debug("Response contains an open file object, copying to %s", file_name) 170 | shutil.copyfileobj(response.response, f) 171 | else: 172 | raise ValueError("Don't know how to handle response type: {}".format(type(response.response))) 173 | 174 | 175 | def _save_redirected_response_data(response, file_name): 176 | # type: (Response, str) -> None 177 | """Download the URL of a remote resource we got redirected to, and save it locally 178 | 179 | Return the local file name 180 | """ 181 | resource_url = response.headers['Location'] 182 | _log().debug("Resource is at %s, downloading ...", resource_url) 183 | with requests.get(resource_url, stream=True) as source, open(file_name, 'wb') as dest: 184 | source.raise_for_status() 185 | _log().debug("Resource downloading, HTTP status code is %d, Content-type is %s", 186 | source.status_code, 187 | source.headers.get('Content-type', 'unknown')) 188 | for chunk in source.iter_content(chunk_size=1024 * 16): 189 | dest.write(chunk) 190 | _log().debug("Remote resource downloaded to %s", file_name) 191 | 192 | 193 | def get_resource_dataset(resource_obj): 194 | # type: (Resource) -> Tuple[Dict[str, Any], Dict[str, Any]] 195 | """Fetch the CKAN dataset dictionary for a DB-fetched resource 196 | """ 197 | context = {"ignore_auth": True, "use_cache": False} 198 | dataset = toolkit.get_action('package_show')(context, {"id": resource_obj.package_id}) 199 | resource = [r for r in dataset['resources'] if r['id'] == resource_obj.id][0] 200 | 201 | return dataset, resource 202 | 203 | 204 | def get_unmigrated_resources(): 205 | # type: () -> Generator[Resource, None, None] 206 | """Generator of un-migrated resource 207 | 208 | This works by fetching one resource at a time using SELECT FOR UPDATE SKIP LOCKED. 209 | Once the resource has been migrated to the new storage, it will be unlocked. This 210 | allows running multiple migrator scripts in parallel, without any conflicts and 211 | with small chance of re-doing any work. 212 | 213 | While a specific resource is being migrated, it will be locked for modification 214 | on the DB level. Users can still read the resource without any effect. 215 | """ 216 | session = Session() 217 | session.revisioning_disabled = True 218 | 219 | # Start from inspecting all uploaded, undeleted resources 220 | all_resources = session.query(Resource).filter( 221 | Resource.url_type == 'upload', 222 | Resource.state != 'deleted', 223 | ).order_by( 224 | Resource.created 225 | ).options(load_only("id", "extras", "package_id")) 226 | 227 | for resource in all_resources: 228 | if not _needs_migration(resource): 229 | _log().debug("Skipping resource %s as it was already migrated", resource.id) 230 | continue 231 | 232 | with db_transaction(session): 233 | locked_resource = session.query(Resource).filter(Resource.id == resource.id).\ 234 | with_for_update(skip_locked=True).one_or_none() 235 | 236 | if locked_resource is None: 237 | _log().debug("Skipping resource %s as it is locked (being migrated?)", resource.id) 238 | continue 239 | 240 | # let's double check as the resource might have been migrated by another process by now 241 | if not _needs_migration(locked_resource): 242 | continue 243 | 244 | yield locked_resource 245 | 246 | 247 | def _needs_migration(resource): 248 | # type: (Resource) -> bool 249 | """Check the attributes of a resource to see if it was migrated 250 | """ 251 | if not (resource.extras.get('lfs_prefix') and resource.extras.get('sha256')): 252 | return True 253 | 254 | expected_prefix = '/'.join([helpers.storage_namespace(), resource.package_id]) 255 | return resource.extras.get('lfs_prefix') != expected_prefix 256 | 257 | 258 | @contextmanager 259 | def db_transaction(session): 260 | try: 261 | yield session 262 | except Exception: 263 | session.rollback() 264 | raise 265 | else: 266 | session.commit() 267 | 268 | 269 | @contextmanager 270 | def app_context(): 271 | context = _get_auto_flask_context() 272 | try: 273 | context.push() 274 | yield context 275 | finally: 276 | context.pop() 277 | -------------------------------------------------------------------------------- /ckanext/blob_storage/download_handler.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import os 3 | 4 | from ckan import model, plugins 5 | from ckan.lib import uploader 6 | from ckan.plugins import toolkit as tk 7 | from flask import send_file 8 | 9 | from .interfaces import IResourceDownloadHandler 10 | 11 | 12 | def get_context(): 13 | """Get a default context dict 14 | """ 15 | return { 16 | 'model': model, 17 | 'user': tk.c.user, 18 | 'auth_user_obj': tk.c.userobj, 19 | } 20 | 21 | 22 | def call_pre_download_handlers(resource, package, activity_id=None): 23 | """Call all registered plugins pre-download callback 24 | """ 25 | for plugin in plugins.PluginImplementations(IResourceDownloadHandler): 26 | if not hasattr(plugin, 'pre_resource_download'): 27 | continue 28 | new_resource = plugin.pre_resource_download(resource, package, activity_id=None) 29 | if new_resource: 30 | resource = new_resource 31 | 32 | return resource 33 | 34 | 35 | def call_download_handlers(resource, package, filename=None, inline=False, activity_id=None): 36 | """Call all registered plugins download handlers 37 | """ 38 | for plugin in plugins.PluginImplementations(IResourceDownloadHandler): 39 | if not hasattr(plugin, 'resource_download'): 40 | continue 41 | 42 | if _handler_supports_extra_arg(plugin.resource_download): 43 | response = plugin.resource_download(resource, package, filename, inline, activity_id) 44 | else: 45 | response = plugin.resource_download(resource, package, filename) 46 | 47 | if response: 48 | return response 49 | 50 | return fallback_download_method(resource) 51 | 52 | 53 | def download_handler(resource, _, filename=None, inline=False, activity_id=None): 54 | """Get the download URL from LFS server and redirect the user there 55 | """ 56 | if resource.get('url_type') != 'upload' or not resource.get('lfs_prefix'): 57 | return None 58 | context = get_context() 59 | data_dict = {'resource': resource, 60 | 'filename': filename, 61 | 'inline': inline, 62 | 'activity_id': activity_id} 63 | 64 | resource_download_spec = tk.get_action('get_resource_download_spec')(context, data_dict) 65 | href = resource_download_spec.get('href') 66 | 67 | if href: 68 | return tk.redirect_to(href) 69 | else: 70 | return tk.abort(404, tk._('No download is available')) 71 | 72 | 73 | def fallback_download_method(resource): 74 | """Fall back to the built in CKAN download method 75 | """ 76 | if resource.get('url_type') == 'upload': 77 | upload = uploader.get_resource_uploader(resource) 78 | filepath = upload.get_path(resource[u'id']) 79 | if os.path.exists(filepath): 80 | return send_file(filepath) 81 | else: 82 | return tk.abort(404, tk._('File not found')) 83 | elif u'url' not in resource: 84 | return tk.abort(404, tk._('No download is available')) 85 | 86 | return tk.redirect_to(resource[u'url']) 87 | 88 | 89 | def _handler_supports_extra_arg(handler_function): 90 | try: 91 | # Python 3 92 | args = inspect.getfullargspec(handler_function).args 93 | except AttributeError: 94 | # Python 2 95 | args = inspect.getargspec(handler_function).args 96 | 97 | return 'inline' in args and 'activity_id' in args 98 | -------------------------------------------------------------------------------- /ckanext/blob_storage/fanstatic/css/main.3c074c8b.chunk.css: -------------------------------------------------------------------------------- 1 | .metadata-form{display:grid;grid-template-columns:1fr 1fr;grid-column-gap:20px;-webkit-column-gap:20px;column-gap:20px}.metadata-name{color:#1a3b5d;text-align:left;grid-area:name}.metadata-label{font-size:14px;margin-bottom:5px;margin-left:5px;text-align:left;font-weight:500;color:#1a3b5d;width:100%;display:block;-webkit-user-select:none;-ms-user-select:none;user-select:none}.metadata-input{margin-bottom:20px}.metadata-input__input{width:100%;height:40px;border-radius:5px;box-shadow:none;border:1px solid #ced6e0;transition:all .3s ease-in-out;font-size:18px;padding:5px 15px;background:none;color:#1a3b5d;font-family:"Source Sans Pro",sans-serif;box-sizing:border-box}.btn{margin-top:17px;padding:10px;width:50%}.table-container{margin-top:2.6rem;margin-bottom:2rem;display:flex}.table-schema-info_container{overflow:overlay}.table-schema-info_container::-webkit-scrollbar-track{-webkit-box-shadow:inset 0 0 6px rgba(0,0,0,.2);border-radius:10px;background-color:#fff}.table-schema-info_container::-webkit-scrollbar{width:8px;height:8px;background-color:#fff}.table-schema-info_container::-webkit-scrollbar-thumb{border-radius:10px;-webkit-box-shadow:inset 0 0 6px rgba(0,0,0,.2);background-color:#e3e9ed}.table-schema-info_table{width:auto!important}.table-schema-help{vertical-align:middle}.table-schema-help_row{padding:1.3rem}.table-thead-tr{height:50px;border-radius:8px}.table-thead-th{background:#f1f1f1;color:#363844;font-weight:700;text-align:center;vertical-align:middle;position:-webkit-sticky;position:sticky;top:0;z-index:10}.table-tbody-help-tr{color:#363844;height:40px}.table-tbody-help-td{vertical-align:middle;font-weight:700}.table-tbody-help-td-empty{height:50px}.table-tbody-td{padding:5px;border:1px solid #f1f1f1;background:#fff;max-width:100px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:14px}.table-tbody-input{border:none;padding:0 10px;height:40px}.table-tbody-select{position:relative;display:inline-block;vertical-align:middle;background-color:#fafafa;width:100%;border:none;padding:10px;height:40px}.table-btn-save{padding:10px;width:90%;background-color:#4fa8fd;border:none;border-radius:3px;color:#fff;font-weight:500;cursor:pointer;outline:0}.table-btn-save:disabled{background:#e3e9ed!important;cursor:default}.svg{display:block;margin:20px auto;max-width:100%}.svg-circle,.svg-circle-bg{fill:none}.svg-circle-text{font-size:.68rem;text-anchor:middle;margin-top:30px;fill:#3f4656;font-weight:700}.time-remaining{font-size:.55rem;float:left;margin-top:-13px;color:#3f4656}.upload-area__drop{position:relative;text-align:center;display:flex;justify-content:center;margin-top:20px;height:220px}.upload-area__drop__input{grid-area:drop;padding:1rem;position:relative;display:grid;grid-template-columns:auto auto;align-items:center;min-height:200px;border:1px dashed #3f4c6b;margin:0 auto;width:95%;border-radius:8px;cursor:pointer;outline:0;box-sizing:border-box;z-index:10;color:transparent}.upload-area__drop__input::-webkit-file-upload-button{flex-shrink:0;font-size:.65rem;color:#fff;border-radius:3px;padding:8px 15px;text-transform:uppercase;text-align:center;border:none;outline:0;margin:0 auto;visibility:hidden}.upload-area__drop__icon{width:80px;position:absolute;top:20%;left:40%;cursor:pointer;fill:#e3ebff}.upload-area__drop__text{color:#789;opacity:.7;font-weight:400;position:absolute;top:55%;font-size:.78rem;cursor:pointer}.upload-area__url{text-align:left;width:95%;margin:0 auto}.upload-area__url__label{display:block;padding:4px}.upload-area__url__input{width:100%;height:40px;border-radius:5px;box-shadow:none;border:1px solid #ced6e0;transition:all .3s ease-in-out;font-size:18px;padding:5px 15px;background:none;color:#1a3b5d;font-family:"Source Sans Pro",sans-serif;box-sizing:border-box;font-size:1rem}.choose-btn{padding:10px;width:90%;background-color:#4fa8fd;border:none;border-radius:3px;color:#fff;font-weight:500;cursor:pointer;outline:0;font-weight:700}.choose-text{font-size:1rem;font-weight:300}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI","Roboto","Oxygen","Ubuntu","Cantarell","Fira Sans","Droid Sans","Helvetica Neue",sans-serif;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}code{font-family:source-code-pro,Menlo,Monaco,Consolas,"Courier New",monospace}.App{text-align:center;min-height:100vh;color:#222}.upload-wrapper{width:100%;min-width:360px;max-width:960px;margin:0 auto;background-color:#fff;padding:32px 24px;border-radius:12px;min-height:60%;display:grid;grid-template-columns:1fr 1fr;grid-template-rows:auto 300px auto auto;grid-template-areas:"header header" "uploadArea uploadArea" "editArea editArea";grid-row-gap:20px;row-gap:20px;grid-column-gap:20px;-webkit-column-gap:20px;column-gap:20px}.upload-header{grid-area:header;border-bottom:1px solid #e3ebff}.upload-header__title{color:#3f4656;font-weight:400}.upload-area{grid-area:uploadArea;display:grid;grid-template-columns:40% auto;grid-template-rows:1fr 1fr;grid-template-areas:"drop info";align-items:center}.upload-area__info{grid-area:info;padding-top:20px}.upload-list{padding:0}.list-item{display:flex}.upload-list-item{margin:0 auto;background-color:#fff;padding:8px 24px;list-style:none;max-width:480px;width:85%;border-top-left-radius:8px;border-bottom-left-radius:8px;color:#222;display:flex;align-items:center;justify-content:space-between;box-shadow:0 4px 11px 3px rgba(18,22,33,.05)}.upload-file-name{max-width:19ch;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:#3f4656}.upload-file-size{text-align:left;font-size:.75rem;color:#789}.upload-message{color:#3f4656}.upload-switcher{grid-area:switcher}.upload-edit-area{grid-area:editArea}.btn{max-width:220px;margin:2rem;border-radius:5px;padding:10px 25px;height:45px;font-size:1.2rem;color:#fff;background-color:#00a7e1;outline:none;border:none;cursor:pointer;box-shadow:0 4px 11px 3px rgba(18,22,33,.05)}.btn:disabled{background:#e3e9ed!important;cursor:default}.btn:hover{background-color:#009ace}.btn-delete{margin:2rem;border-radius:5px;padding:10px 25px;height:45px;font-size:1.2rem;color:#fff;outline:none;border:none;cursor:pointer;box-shadow:0 4px 11px 3px rgba(18,22,33,.05);background-color:#f11301}.btn-delete :disabled{background:#e3e9ed!important;cursor:default}.btn-delete:hover{background-color:#c70e00}.resource-edit-actions{display:flex;justify-content:space-between} 2 | /*# sourceMappingURL=main.3c074c8b.chunk.css.map */ -------------------------------------------------------------------------------- /ckanext/blob_storage/fanstatic/js/main.c8e9dac3.chunk.js: -------------------------------------------------------------------------------- 1 | (this.webpackJsonpdatapub=this.webpackJsonpdatapub||[]).push([[0],{232:function(e,a){},297:function(e){e.exports=JSON.parse('[{"value":"utf_8","label":"UTF-8"},{"value":"iso_8859_1","label":"ISO-8859-1"},{"value":"windows_1251","label":"Windows-1251"},{"value":"windows_1252","label":"Windows-1252"},{"value":"shift_jis","label":"Shift JIS"},{"value":"gb2312","label":"GB2312"},{"value":"euc_kr","label":"EUC-KR"},{"value":"iso_8859_2","label":"ISO-8859-2"},{"value":"windows_1250","label":"Windows-1250"},{"value":"euc_jp","label":"EUC-JP"},{"value":"gbk","label":"GBK"},{"value":"big5","label":"Big5"},{"value":"iso_8859_15","label":"ISO-8859-15"},{"value":"windows_1256","label":"Windows-1256"},{"value":"iso_8859_9","label":"ISO-8859-9"}]')},298:function(e){e.exports=JSON.parse('[["PPTX","Powerpoint OOXML Presentation","application/vnd.openxmlformats-officedocument.presentationml.presentation",[]],["EXE","Windows Executable Program","application/x-msdownload",[]],["DOC","Word Document","application/msword",[]],["KML","KML File","application/vnd.google-earth.kml+xml",[]],["XLS","Excel Document","application/vnd.ms-excel",["Excel","application/msexcel","application/x-msexcel","application/x-ms-excel","application/x-excel","application/x-dos_ms_excel","application/xls","application/x-xls"]],["WCS","Web Coverage Service","wcs",[]],["JS","JavaScript","application/x-javascript",[]],["MDB","Access Database","application/x-msaccess",[]],["NetCDF","NetCDF File","application/netcdf",[]],["ArcGIS Map Service","ArcGIS Map Service","ArcGIS Map Service",["arcgis map service"]],["TSV","Tab Separated Values File","text/tab-separated-values",["text/tsv"]],["WFS","Web Feature Service",null,[]],["WMTS","Web Map Tile Service",null,[]],["ArcGIS Online Map","ArcGIS Online Map","ArcGIS Online Map",["web map application"]],["Perl","Perl Script","text/x-perl",[]],["KMZ","KMZ File","application/vnd.google-earth.kmz+xml",["application/vnd.google-earth.kmz"]],["OWL","Web Ontology Language","application/owl+xml",[]],["N3","N3 Triples","application/x-n3",[]],["ZIP","Zip File","application/zip",["zip","http://purl.org/NET/mediatypes/application/zip"]],["GZ","Gzip File","application/gzip",["application/x-gzip"]],["QGIS","QGIS File","application/x-qgis",[]],["ODS","OpenDocument Spreadsheet","application/vnd.oasis.opendocument.spreadsheet",[]],["ODT","OpenDocument Text","application/vnd.oasis.opendocument.text",[]],["JSON","JavaScript Object Notation","application/json",[]],["BMP","Bitmap Image File","image/x-ms-bmp",[]],["HTML","Web Page","text/html",["htm","http://purl.org/net/mediatypes/text/html"]],["RAR","RAR Compressed File","application/rar",[]],["TIFF","TIFF Image File","image/tiff",[]],["ODB","OpenDocument Database","application/vnd.oasis.opendocument.database",[]],["TXT","Text File","text/plain",[]],["DCR","Adobe Shockwave format","application/x-director",[]],["ODF","OpenDocument Math Formula","application/vnd.oasis.opendocument.formula",[]],["ODG","OpenDocument Image","application/vnd.oasis.opendocument.graphics",[]],["XML","XML File","application/xml",["text/xml","http://purl.org/net/mediatypes/application/xml"]],["XLSX","Excel OOXML Spreadsheet","application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",[]],["DOCX","Word OOXML Document","application/vnd.openxmlformats-officedocument.wordprocessingml.document",[]],["BIN","Binary Data","application/octet-stream",["bin"]],["XSLT","Extensible Stylesheet Language Transformations","application/xslt+xml",[]],["WMS","Web Mapping Service","WMS",["wms"]],["SVG","SVG vector image","image/svg+xml",["svg"]],["PPT","Powerpoint Presentation","application/vnd.ms-powerpoint",[]],["ODP","OpenDocument Presentation","application/vnd.oasis.opendocument.presentation",[]],["JPEG","JPG Image File","image/jpeg",["jpeg","jpg"]],["SPARQL","SPARQL end-point","application/sparql-results+xml",[]],["GIF","GIF Image File","image/gif",[]],["RDF","RDF","application/rdf+xml",["rdf/xml"]],["E00"," ARC/INFO interchange file format","application/x-e00",[]],["PDF","PDF File","application/pdf",[]],["CSV","Comma Separated Values File","text/csv",["text/comma-separated-values"]],["ODC","OpenDocument Chart","application/vnd.oasis.opendocument.chart",[]],["Atom Feed","Atom Feed","application/atom+xml",[]],["MrSID","MrSID","image/x-mrsid",[]],["ArcGIS Map Preview","ArcGIS Map Preview","ArcGIS Map Preview",["arcgis map preview"]],["XYZ","XYZ Chemical File","chemical/x-xyz",[]],["MOP","MOPAC Input format","chemical/x-mopac-input",[]],["Esri REST","Esri Rest API Endpoint","Esri REST",["arcgis_rest"]],["dBase","dBase Database","application/x-dbf",["dbf"]],["MXD","ESRI ArcGIS project file","application/x-mxd",[]],["TAR","TAR Compressed File","application/x-tar",[]],["PNG","PNG Image File","image/png",[]],["RSS","RSS feed","application/rss+xml",[]],["GeoJSON","Geographic JavaScript Object Notation","application/geo+json",["geojson"]],["SHP","Shapefile",null,["esri shapefile"]],["TORRENT","Torrent","application/x-bittorrent",["bittorrent"]],["ICS","iCalendar","text/calendar",["ifb","iCal"]],["GTFS","General Transit Feed Specification",null,[]],["XLSB","Excel OOXML Spreadsheet - Binary Workbook","application/vnd.ms-excel.sheet.binary.macroEnabled.12",[]],["XLSM","Excel OOXML Spreadsheet - Macro-Enabled","application/vnd.ms-excel.sheet.macroEnabled.12",[]],["XLTM","Excel OOXML Spreadsheet - Macro-Enabled Template","application/vnd.ms-excel.template.macroEnabled.12",[]],["XLAM","Excel OOXML Spreadsheet - Macro-Enabled Add-In","application/vnd.ms-excel.addin.macroEnabled.12",[]],["DOTX","Word OOXML - Template","application/vnd.openxmlformats-officedocument.wordprocessingml.template",[]],["DOCM","Word OOXML - Macro Enabled","application/vnd.ms-word.document.macroEnabled.12",[]],["DOTM","Word OOXML Template - Macro Enabled","application/vnd.ms-word.template.macroEnabled.12",[]],["XLTX","Excel OOXML Spreadsheet - Template","application/vnd.openxmlformats-officedocument.spreadsheetml.template",[]],["POTX","PowerPoint OOXML Presentation - Template","application/vnd.openxmlformats-officedocument.presentationml.template",[]],["PPSX","PowerPoint Open XML - Slide Show","application/vnd.openxmlformats-officedocument.presentationml.slideshow",[]],["PPAM","PowerPoint Open XML - Macro-Enabled Add-In","application/vnd.ms-powerpoint.addin.macroEnabled.12",[]],["PPTM","PowerPoint Open XML - Macro-Enabled","application/vnd.ms-powerpoint.presentation.macroEnabled.12",[]],["POTM","PowerPoint Open XML - Macro-Enabled Template","application/vnd.ms-powerpoint.template.macroEnabled.12",[]],["PPSM","PowerPoint Open XML - Macro-Enabled Slide Show","application/vnd.ms-powerpoint.slideshow.macroEnabled.12",[]]]')},300:function(e){e.exports=JSON.parse('{"type":["string","number","integer","boolean","object","array","date","time","datetime","year","yearmonth","duration","geopoint","geojson","any"]}')},301:function(e,a,t){e.exports=t(675)},336:function(e,a){},338:function(e,a){},347:function(e,a){},349:function(e,a){},375:function(e,a){},376:function(e,a){},381:function(e,a){},383:function(e,a){},407:function(e,a){},449:function(e,a,t){},451:function(e,a,t){},491:function(e,a){},494:function(e,a){},495:function(e,a){},502:function(e,a){},504:function(e,a){},602:function(e,a,t){var n={"./geojson.json":274,"./table-schema.json":603,"./topojson.json":275};function r(e){var a=l(e);return t(a)}function l(e){if(!t.o(n,e)){var a=new Error("Cannot find module '"+e+"'");throw a.code="MODULE_NOT_FOUND",a}return n[e]}r.keys=function(){return Object.keys(n)},r.resolve=l,e.exports=r,r.id=602},669:function(e,a,t){},670:function(e,a,t){},671:function(e,a,t){e.exports=t.p+"static/media/computing-cloud.ec5708b6.svg"},672:function(e,a,t){},673:function(e,a,t){},674:function(e,a,t){},675:function(e,a,t){"use strict";t.r(a),t.d(a,"ResourceEditor",(function(){return X}));var n=t(32),r=t(0),l=t.n(r),c=t(153),o=t.n(c),i=t(25),s=t(3),p=t.n(s),d=t(33),m=t(92),u=t(93),f=t(94),h=t(96),b=t(95),g=t(295),v=t(90),E=t.n(v),x=t(296),S=(t(449),t(297)),O=t(298),w=[{label:"Access Restriction",name:"restricted",input_type:"select",values:['{"level": "public"}','{"level": "private"}'],options:["Public","Private"]}],N=function(e){var a=e.metadata,t=e.handleChange;return l.a.createElement(l.a.Fragment,null,l.a.createElement("h3",{className:"metadata-name"},a.path),l.a.createElement("div",{className:"metadata-form"},l.a.createElement("div",{className:"metadata-input"},l.a.createElement("label",{className:"metadata-label",htmlFor:"title"},"Title:"),l.a.createElement("input",{className:"metadata-input__input",type:"text",name:"title",id:"title",value:a.title||a.name,onChange:t})),l.a.createElement("div",{className:"metadata-input"},l.a.createElement("label",{className:"metadata-label",htmlFor:"description"},"Description:"),l.a.createElement("input",{className:"metadata-input__input",type:"text",name:"description",id:"description",value:a.description||"",onChange:t})),l.a.createElement("div",{className:"metadata-input"},l.a.createElement("label",{className:"metadata-label",htmlFor:"encoding"},"Encoding:"),l.a.createElement("select",{className:"metadata-input__input",name:"encoding",id:"encoding",value:a.encoding||"",onChange:t,required:!0},l.a.createElement("option",{value:"",disabled:!0},"Select..."),S.map((function(e){return l.a.createElement("option",{key:"format-".concat(e.value),value:e.value},e.label)})))),l.a.createElement("div",{className:"metadata-input"},l.a.createElement("label",{className:"metadata-label",htmlFor:"format"},"Format:"),l.a.createElement("select",{className:"metadata-input__input",name:"format",id:"format",value:(a.format||"").toLowerCase(),onChange:t,required:!0},l.a.createElement("option",{value:"",disabled:!0},"Select..."),O.map((function(e){return l.a.createElement("option",{key:"format-".concat(e[0]),value:e[0].toLowerCase()},e[0])})))),w&&w.map((function(e){return l.a.createElement("div",{key:"metadata-custom-".concat(e.name),className:"metadata-input"},l.a.createElement("label",{className:"metadata-label",htmlFor:"format"},e.label,":"),l.a.createElement("select",{className:"metadata-input__input",name:e.name,id:e.name,value:a[e.name]||"",onChange:t,required:!0},l.a.createElement("option",{value:"",disabled:!0},"Select..."),e.options.map((function(a,t){return l.a.createElement("option",{key:"".concat(e.name,"-").concat(t),value:e.values[t]},a)}))))}))))},_=t(155),M=t(299),y=t(300),P=(t(451),function(e){var a=Object(r.useState)(e.schema),t=Object(n.a)(a,2),c=t[0],o=t[1],s=l.a.useMemo((function(){return Object(_.a)(e.data)}),[c]),p=c.fields.map((function(e,a){return{Header:e.name?e.name:"column_".concat(a+1),accessor:e.name?e.name:"column_".concat(a+1)}})),d=l.a.useMemo((function(){return Object(_.a)(p)}),[c]),m=Object(M.useTable)({columns:d,data:s}),u=m.getTableProps,f=m.getTableBodyProps,h=m.headerGroups,b=m.rows,g=m.prepareRow,v=function(e,a,t){var n=Object(i.a)({},c);n.fields[t][a]=e.target.value,o(n)};Object(r.useEffect)((function(){o(e.schema)}),[e.schema]);var E=function(e){return"type"===e?c.fields.map((function(a,t){return l.a.createElement("td",{key:"schema-type-field-".concat(e,"-").concat(t)},l.a.createElement("select",{className:"table-tbody-select",value:a[e]||"",onChange:function(a){return v(a,e,t)}},y.type.map((function(e,a){return l.a.createElement("option",{key:"schema-type-field-option-".concat(e,"-").concat(a),value:e},e)}))))})):c.fields.map((function(a,t){return l.a.createElement("td",{key:"schema-field-".concat(e,"-").concat(t)},l.a.createElement("input",{className:"table-tbody-input",type:"text",value:a[e],onChange:function(a){return v(a,e,t)}}))}))};return l.a.createElement(l.a.Fragment,null,l.a.createElement("div",{className:"table-container"},l.a.createElement("table",{className:"table-schema-help"},l.a.createElement("tbody",null,l.a.createElement("tr",{className:"table-tbody-help-tr"},l.a.createElement("td",{className:"table-tbody-help-td-empty"})),l.a.createElement("tr",{className:"table-tbody-help-tr"},l.a.createElement("td",{className:"table-tbody-help-td"},"Title")),l.a.createElement("tr",{className:"table-tbody-help-tr"},l.a.createElement("td",{className:"table-tbody-help-td"},"Description")),l.a.createElement("tr",{className:"table-tbody-help-tr"},l.a.createElement("td",{className:"table-tbody-help-td"},"Type")),l.a.createElement("tr",{className:"table-tbody-help-tr"},l.a.createElement("td",{className:"table-tbody-help-td"},"Format")))),l.a.createElement("div",{className:"table-schema-info_container"},l.a.createElement("table",Object.assign({className:"table-schema-info_table"},u()),l.a.createElement("thead",null,h.map((function(e){return l.a.createElement("tr",Object.assign({className:"table-thead-tr"},e.getHeaderGroupProps()),e.headers.map((function(e){return l.a.createElement("th",Object.assign({className:"table-thead-th"},e.getHeaderProps()),e.render("Header"))})))}))),l.a.createElement("tbody",f(),l.a.createElement("tr",{className:"table-tbody-tr-help"},E("title")),l.a.createElement("tr",{className:"table-tbody-tr-help"},E("description")),l.a.createElement("tr",null,E("type")),l.a.createElement("tr",null,E("format")),b.map((function(e){return g(e),l.a.createElement("tr",e.getRowProps(),e.cells.map((function(e){return l.a.createElement("td",Object.assign({},e.getCellProps(),{className:"table-tbody-td"}),e.render("Cell"))})))})))))))}),I=t(154),k=t(152),C=t.n(k),F=(t(669),function(e){var a=Object(r.useState)(0),t=Object(n.a)(a,2),c=t[0],o=t[1],i=Object(r.useRef)(null),s=e.size,p=e.progress,d=e.strokeWidth,m=e.circleOneStroke,u=e.circleTwoStroke,f=e.timeRemaining,h=s/2,b=s/2-d/2,g=2*Math.PI*b;return Object(r.useEffect)((function(){o((100-p)/100*g),i.current.style="transition: stroke-dashoffset 850ms ease-in-out"}),[o,p,g,c]),l.a.createElement(l.a.Fragment,null,l.a.createElement("svg",{className:"svg",width:s,height:s},l.a.createElement("circle",{className:"svg-circle-bg",stroke:m,cx:h,cy:h,r:b,strokeWidth:d}),l.a.createElement("circle",{className:"svg-circle",ref:i,stroke:u,cx:h,cy:h,r:b,strokeWidth:d,strokeDasharray:g,strokeDashoffset:c}),l.a.createElement("text",{x:"".concat(h),y:"".concat(h+2),className:"svg-circle-text"},p,"%")),f>0&&l.a.createElement("span",{className:"time-remaining"},f>60?"".concat(Math.floor(f/60)," minute").concat(Math.floor(f/60)>1?"s":""):"".concat(Math.floor(f)," second").concat(f>1?"s":"")," left"))}),j=function(e){var a=arguments.length>1&&void 0!==arguments[1]?arguments[1]:1;if(0===e)return"0 Bytes";var t=1e3,n=a<0?0:a,r=["Bytes","KB","MB","GB","TB","PB","EB","ZB","YB"],l=Math.floor(Math.log(e)/Math.log(t));return parseFloat((e/Math.pow(t,l)).toFixed(n))+" "+r[l]},T=(t(670),t(671),function(e){var a=e.onChangeHandler;return l.a.createElement("div",{className:"upload-area__drop"},l.a.createElement("input",{className:"upload-area__drop__input",type:"file",name:"file",onChange:a}),l.a.createElement("img",{className:"upload-area__drop__icon",src:"https://www.shareicon.net/data/256x256/2015/09/05/96087_cloud_512x512.png",alt:"upload-icon"}),l.a.createElement("span",{className:"upload-area__drop__text"},"Drag and drop your files",l.a.createElement("br",null),"or ",l.a.createElement("br",null),"click to select"))}),D=(t(672),function(e){var a=e.onChangeUrl;return l.a.createElement("div",{className:"upload-area__url"},l.a.createElement("label",{className:"upload-area__url__label",htmlFor:"input-url"},"URL:"),l.a.createElement("input",{className:"upload-area__url__input",id:"input-url",type:"url",name:"input-url",onBlur:a,onKeyDown:function(e){return function(e){13===e.keyCode&&e.target.blur()}(e)},placeholder:"https://www.data.com/sample.csv"}))}),R=(t(673),function(e){var a=e.onChangeUrl,t=e.onChangeHandler,c=Object(r.useState)(!1),o=Object(n.a)(c,2),i=o[0],s=o[1];return l.a.createElement("div",{className:"upload-choose"},i?l.a.createElement(l.a.Fragment,null,"file"===i&&l.a.createElement(T,{onChangeHandler:t}),"url"===i&&l.a.createElement(D,{onChangeUrl:a})):l.a.createElement("div",null,l.a.createElement("button",{className:"choose-btn",onClick:function(){return s("file")}},"Choose a file to Upload "),l.a.createElement("p",{className:"choose-text"},"OR"),l.a.createElement("button",{className:"choose-btn",onClick:function(){return s("url")}},"Link a file already online")))}),L=function(e){Object(h.a)(t,e);var a=Object(b.a)(t);function t(e){var n;return Object(m.a)(this,t),(n=a.call(this,e)).onChangeHandler=function(){var e=Object(d.a)(p.a.mark((function e(a){var t,r,l,c,o,i;return p.a.wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(t=n.state,r=t.formattedSize,l=t.selectedFile,!(a.target.files.length>0)){e.next=23;break}return l=a.target.files[0],c=I.open(l),e.prev=4,e.next=7,c.rows({size:20,keyed:!0});case 7:return o=e.sent,e.next=10,C()(o);case 10:return c.descriptor.sample=e.sent,e.next=13,c.addSchema();case 13:e.next=18;break;case 15:e.prev=15,e.t0=e.catch(4),console.error(e.t0);case 18:return r=j(c.size),e.next=21,c.hashSha256();case 21:i=e.sent,n.props.metadataHandler(Object.assign(c.descriptor,{hash:i}));case 23:n.setState({selectedFile:l,loaded:0,success:!1,fileExists:!1,error:!1,formattedSize:r}),n.onClickHandler();case 25:case"end":return e.stop()}}),e,null,[[4,15]])})));return function(a){return e.apply(this,arguments)}}(),n.onUploadProgress=function(e){n.onTimeRemaining(e.loaded),n.setState({loaded:e.loaded/e.total*100})},n.onTimeRemaining=function(e){var a=e/(((new Date).getTime()-n.state.start)/1e3)/1024,t=(n.state.fileSize-e)/a;n.setState({timeRemaining:t/1e3})},n.onClickHandler=Object(d.a)(p.a.mark((function e(){var a,t,r,l;return p.a.wrap((function(e){for(;;)switch(e.prev=e.next){case 0:a=(new Date).getTime(),t=n.state.selectedFile,r=n.props.client,l=I.open(t),n.setState({fileSize:l.size,start:a,loading:!0}),n.props.handleUploadStatus({loading:!0,error:!1,success:!1}),r.pushBlob(l,n.onUploadProgress).then((function(e){n.setState({success:e.success,loading:!1,fileExists:e.fileExists,loaded:100}),n.props.handleUploadStatus({loading:!1,success:e.success})})).catch((function(e){n.setState({error:!0,loading:!1}),n.props.handleUploadStatus({loading:!1,success:!1,error:!0})}));case 7:case"end":return e.stop()}}),e)}))),n.state={datasetId:e.datasetId,selectedFile:null,fileSize:0,formattedSize:"0 KB",start:"",loaded:0,success:!1,error:!1,fileExists:!1,loading:!1,timeRemaining:0},n}return Object(u.a)(t,[{key:"render",value:function(){var e=this.state,a=e.success,t=e.fileExists,n=e.error,r=e.timeRemaining,c=e.selectedFile,o=e.formattedSize;return l.a.createElement("div",{className:"upload-area"},l.a.createElement(R,{onChangeHandler:this.onChangeHandler,onChangeUrl:function(e){return console.log("Get url:",e.target.value)}}),l.a.createElement("div",{className:"upload-area__info"},c&&l.a.createElement(l.a.Fragment,null,l.a.createElement("ul",{className:"upload-list"},l.a.createElement("li",{className:"list-item"},l.a.createElement("div",{className:"upload-list-item"},l.a.createElement("div",null,l.a.createElement("p",{className:"upload-file-name"},c.name),l.a.createElement("p",{className:"upload-file-size"},o)),l.a.createElement("div",null,l.a.createElement(F,{progress:Math.round(this.state.loaded),size:50,strokeWidth:5,circleOneStroke:"#d9edfe",circleTwoStroke:"#7ea9e1",timeRemaining:r}))))),l.a.createElement("h2",{className:"upload-message"},a&&!t&&!n&&"File uploaded successfully",t&&"File uploaded successfully",n&&"Upload failed"))))}}]),t}(l.a.Component),X=(t(674),function(e){Object(h.a)(t,e);var a=Object(b.a)(t);function t(e){var n;return Object(m.a)(this,t),(n=a.call(this,e)).handleChangeMetadata=function(e){var a=e.target,t=a.value,r=a.name,l=n.state.resource;l[r]=t,n.setState({resource:l})},n.handleSubmitMetadata=Object(d.a)(p.a.mark((function e(){var a,t,r,l,c;return p.a.wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return a=n.state,t=a.resource,r=a.client,e.next=3,n.createResource(t);case 3:return!0,e.next=7,r.action("package_show",{id:n.state.datasetId});case 7:if(l=e.sent,"draft"!=(c=l.result).state){e.next=13;break}return c.state="active",e.next=13,r.action("package_update",c);case 13:return e.abrupt("return",window.location.href="/dataset/".concat(n.state.datasetId));case 14:case"end":return e.stop()}}),e)}))),n.createResource=function(){var e=Object(d.a)(p.a.mark((function e(a){var t,r,l,c,o,s,d,m,u;return p.a.wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(t=n.state.client,r=n.props.config,l=r.organizationId,c=r.datasetId,o=r.resourceId,s=E.a.resourceFrictionlessToCkan(a),d=Object(i.a)({},s.sample),delete s.sample,m=s.bq_table_name?s.bq_table_name:Object(x.v4)(),u=Object(i.a)(Object(i.a)({},s),{},{package_id:n.state.datasetId,name:a.name||a.title,sha256:a.hash,size:a.size,lfs_prefix:"".concat(l,"/").concat(c),url:a.name,url_type:"upload",bq_table_name:(p=m,p.replace(/-/g,"")),sample:d}),!o){e.next=13;break}return u=Object(i.a)(Object(i.a)({},u),{},{id:o}),e.next=12,t.action("resource_update",u);case 12:return e.abrupt("return",window.location.href="/dataset/".concat(c));case 13:return e.next=15,t.action("resource_create",u).then((function(e){n.onChangeResourceId(e.result.id)}));case 15:case"end":return e.stop()}var p}),e)})));return function(a){return e.apply(this,arguments)}}(),n.deleteResource=Object(d.a)(p.a.mark((function e(){var a,t,r,l;return p.a.wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(a=n.state,t=a.resource,r=a.client,l=a.datasetId,!window.confirm("Are you sure to delete this resource?")){e.next=5;break}return e.next=4,r.action("resource_delete",{id:t.id});case 4:return e.abrupt("return",window.location.href="/dataset/".concat(l));case 5:case"end":return e.stop()}}),e)}))),n.handleUploadStatus=function(e){var a=n.state.ui,t=Object(i.a)(Object(i.a)({},a),{},{success:e.success,error:e.error,loading:e.loading});n.setState({ui:t})},n.onChangeResourceId=function(e){n.setState({resourceId:e})},n.state={datasetId:n.props.config.datasetId,resourceId:"",resource:n.props.resource||{},ui:{fileOrLink:"",uploadComplete:void 0,success:!1,error:!1,loading:!1},client:null,isResourceEdit:!1},n.metadataHandler=n.metadataHandler.bind(Object(f.a)(n)),n}return Object(u.a)(t,[{key:"componentDidMount",value:function(){var e=Object(d.a)(p.a.mark((function e(){var a,t,n,r,l,c,o,i,s,d,m,u,f,h;return p.a.wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(a=this.props.config,t=a.authToken,n=a.api,r=a.lfs,l=a.organizationId,c=a.datasetId,o=a.resourceId,i=new g.Client("".concat(t),"".concat(l),"".concat(c),"".concat(n),"".concat(r)),!o){e.next=17;break}return e.next=6,i.action("resource_show",{id:o});case 6:return s=e.sent,e.next=9,i.action("resource_schema_show",{id:o});case 9:return d=e.sent,e.next=12,i.action("resource_sample_show",{id:o});case 12:m=e.sent,u=s.result,f=[];try{for(h in m.result)f.push(m.result[h]);u.sample=f,u.schema=d.result}catch(p){console.error(p),u.schema={fields:[]},u.sample=[]}return e.abrupt("return",this.setState({client:i,resourceId:o,resource:u,isResourceEdit:!0}));case 17:this.setState({client:i});case 18:case"end":return e.stop()}}),e,this)})));return function(){return e.apply(this,arguments)}}()},{key:"metadataHandler",value:function(e){this.setState({resource:e})}},{key:"render",value:function(){var e=this,a=this.state.ui,t=(a.loading,a.success);return l.a.createElement("div",{className:"App"},l.a.createElement("form",{className:"upload-wrapper",onSubmit:function(a){return a.preventDefault(),e.state.isResourceEdit?e.createResource(e.state.resource):e.handleSubmitMetadata()}},l.a.createElement("div",{className:"upload-header"},l.a.createElement("h2",{className:"upload-header__title"},"Resource Editor")),l.a.createElement(L,{client:this.state.client,resource:this.state.resource,metadataHandler:this.metadataHandler,datasetId:this.state.datasetId,handleUploadStatus:this.handleUploadStatus,onChangeResourceId:this.onChangeResourceId}),l.a.createElement("div",{className:"upload-edit-area"},l.a.createElement(N,{metadata:this.state.resource,handleChange:this.handleChangeMetadata}),this.state.resource.schema&&l.a.createElement(P,{schema:this.state.resource.schema,data:this.state.resource.sample||[]}),this.state.isResourceEdit?l.a.createElement("div",{className:"resource-edit-actions"},l.a.createElement("button",{type:"button",className:"btn btn-delete",onClick:this.deleteResource},"Delete"),l.a.createElement("button",{className:"btn"},"Update")):l.a.createElement("button",{disabled:!t,className:"btn"},"Save and Publish"))))}}]),t}(l.a.Component));X.defaultProps={config:{authToken:"be270cae-1c77-4853-b8c1-30b6cf5e9878",api:"http://localhost:5000",lfs:"http://localhost:5001",organizationId:"myorg",datasetId:"data-test-2"}};var A=X;var G=document.getElementById("ResourceEditor");if(G){var z={datasetId:G.getAttribute("data-dataset-id"),api:G.getAttribute("data-api"),lfs:G.getAttribute("data-lfs"),authToken:G.getAttribute("data-auth-token"),organizationId:G.getAttribute("data-organization-id"),resourceId:G.getAttribute("data-resource-id")};o.a.render(l.a.createElement(l.a.StrictMode,null,l.a.createElement(A,{config:z,resource:G.getAttribute("data-resource")})),G)}}},[[301,1,2]]]); 2 | //# sourceMappingURL=main.c8e9dac3.chunk.js.map -------------------------------------------------------------------------------- /ckanext/blob_storage/fanstatic/js/runtime-main.c155da57.js: -------------------------------------------------------------------------------- 1 | !function(e){function t(t){for(var n,a,l=t[0],p=t[1],f=t[2],c=0,s=[];c str 15 | """Get the resource storage prefix for a package name 16 | """ 17 | if org_name is None: 18 | org_name = storage_namespace() 19 | return '{}/{}'.format(org_name, package_name) 20 | 21 | 22 | def resource_authz_scope(package_name, actions=None, org_name=None, resource_id=None, activity_id=None): 23 | # type: (str, Optional[str], Optional[str], Optional[str], Optional[str]) -> str 24 | """Get the authorization scope for package resources 25 | """ 26 | if actions is None: 27 | actions = 'read,write' 28 | if resource_id is None: 29 | resource_id = '*' 30 | scope = 'obj:{}/{}:{}'.format( 31 | resource_storage_prefix(package_name, org_name), 32 | _resource_version(resource_id, activity_id), 33 | actions 34 | ) 35 | return scope 36 | 37 | 38 | def _resource_version(resource_id, activity_id): 39 | result = resource_id 40 | if activity_id: 41 | result += "/{}".format(activity_id) 42 | return result 43 | 44 | 45 | def server_url(): 46 | # type: () -> Optional[str] 47 | """Get the configured server URL 48 | """ 49 | url = toolkit.config.get(SERVER_URL_CONF_KEY) 50 | if not url: 51 | raise ValueError("Configuration option '{}' is not set".format( 52 | SERVER_URL_CONF_KEY)) 53 | if url[-1] == '/': 54 | url = url[0:-1] 55 | return url 56 | 57 | 58 | def storage_namespace(): 59 | """Get the storage namespace for this CKAN instance 60 | """ 61 | ns = toolkit.config.get(STORAGE_NAMESPACE_CONF_KEY) 62 | if ns: 63 | return ns 64 | return 'ckan' 65 | 66 | 67 | def organization_name_for_package(package): 68 | # type: (Dict[str, Any]) -> Optional[str] 69 | """Get the organization name for a known, fetched package dict 70 | """ 71 | context = {'ignore_auth': True} 72 | org = package.get('organization') 73 | if not org and package.get('owner_org'): 74 | org = toolkit.get_action('organization_show')(context, {'id': package['owner_org']}) 75 | if org: 76 | return org.get('name') 77 | return None 78 | 79 | 80 | def resource_filename(resource): 81 | """Get original file name from resource 82 | """ 83 | if 'url' not in resource: 84 | return resource['name'] 85 | 86 | if resource['url'][0:6] in {'http:/', 'https:'}: 87 | url_path = urlparse(resource['url']).path 88 | return path.basename(url_path) 89 | return resource['url'] 90 | -------------------------------------------------------------------------------- /ckanext/blob_storage/i18n/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datopian/ckanext-blob-storage/d264f86a532de1c1f879dd9f93582d8dc0877d1e/ckanext/blob_storage/i18n/.gitignore -------------------------------------------------------------------------------- /ckanext/blob_storage/interfaces.py: -------------------------------------------------------------------------------- 1 | """CKAN plugin interface 2 | """ 3 | from typing import Any, Dict, Optional 4 | 5 | from ckan.plugins import Interface 6 | 7 | 8 | class IResourceDownloadHandler(Interface): 9 | """A CKAN plugin interface for registering resource download handler plugins 10 | """ 11 | 12 | def pre_resource_download(self, resource, package, activity_id=None): 13 | # type: (Dict[str, Any], Dict[str, Any], Optional[str]) -> Optional[Dict[str, Any]] 14 | """Pre-resource download function 15 | 16 | Called before resource download handlers are called, accepting the 17 | resource and package as dict. Can be used to perform special 18 | authorization checks or somehow modify the resource data before 19 | other download handlers are called. 20 | 21 | Should either return a new / modified ``resource`` dictionary, or 22 | ``None`` in which case the original ``resource`` will remain in use. 23 | 24 | It can also raise exceptions, such as 25 | :exc:`ckan.plugins.toolkit.NotAuthorized` to abort the download process 26 | with a specific error. 27 | """ 28 | pass 29 | 30 | def resource_download(self, resource, package, filename=None, inline=False, activity_id=None): 31 | # type: (Dict[str, Any], Dict[str, Any], Optional[str], Optional[bool], Optional[str]) -> Any 32 | """Download a resource 33 | 34 | Called to download a resource, with the resource, package and filename 35 | (if specified in the download request URL) already provided, and after 36 | some basic authorization checks such as ``resource_show`` have been 37 | performed. Additionally if the resource needs to be displayed inline 38 | (ie not served as an attachment) the `inline=True` argument can be 39 | passed. 40 | 41 | All resource download handlers will be called in plugin load order by 42 | the Blueprint view function responsible for file downloads. The first 43 | plugin to return a non-empty value will stop the loop, and the value 44 | (assumed to be a response object) will be returned from the view 45 | function. 46 | 47 | This can be a response containing the resource data or a redirection 48 | response or even an error response. 49 | 50 | Download handlers can also raise exceptions such as 51 | :exc:`ckan.plugins.toolkit.ObjectNotFound` to break the cycle and 52 | return to the user with an error. 53 | 54 | Returning ``None`` or any other empty value will cause the next handler 55 | to be called. 56 | 57 | Eventually, if all registered handlers have been called and no response 58 | has been provided (and no exception has been raised), the default CKAN 59 | download behavior will be executed. 60 | """ 61 | pass 62 | -------------------------------------------------------------------------------- /ckanext/blob_storage/plugin.py: -------------------------------------------------------------------------------- 1 | import ckan.plugins as plugins 2 | import ckan.plugins.toolkit as toolkit 3 | 4 | from ckanext.authz_service.authzzie import Authzzie 5 | from ckanext.authz_service.interfaces import IAuthorizationBindings 6 | 7 | from . import actions, authz, helpers, validators 8 | from .blueprints import blueprint 9 | from .download_handler import download_handler 10 | from .interfaces import IResourceDownloadHandler 11 | 12 | 13 | class BlobStoragePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm): 14 | plugins.implements(plugins.IConfigurer) 15 | plugins.implements(plugins.ITemplateHelpers) 16 | plugins.implements(plugins.IBlueprint) 17 | plugins.implements(plugins.IActions) 18 | plugins.implements(IAuthorizationBindings) 19 | plugins.implements(IResourceDownloadHandler, inherit=True) 20 | plugins.implements(plugins.IValidators) 21 | plugins.implements(plugins.IDatasetForm) 22 | 23 | # IDatasetForm 24 | def create_package_schema(self): 25 | # let's grab the default schema in our plugin 26 | schema = super(BlobStoragePlugin, self).create_package_schema() 27 | 28 | schema['resources'].update({ 29 | 'url_type': [ 30 | toolkit.get_validator('ignore_missing'), 31 | toolkit.get_validator('upload_has_sha256'), 32 | toolkit.get_validator('upload_has_size'), 33 | toolkit.get_validator('upload_has_lfs_prefix') 34 | ], 35 | 'sha256': [ 36 | toolkit.get_validator('ignore_missing'), 37 | toolkit.get_validator('valid_sha256') 38 | ], 39 | 'size': [ 40 | toolkit.get_validator('ignore_missing'), 41 | toolkit.get_validator('is_positive_integer'), 42 | ], 43 | 'lfs_prefix': [ 44 | toolkit.get_validator('ignore_missing'), 45 | toolkit.get_validator('valid_lfs_prefix'), 46 | ] 47 | }) 48 | 49 | return schema 50 | 51 | def update_package_schema(self): 52 | schema = super(BlobStoragePlugin, self).update_package_schema() 53 | 54 | schema['resources'].update({ 55 | 'url_type': [ 56 | toolkit.get_validator('ignore_missing'), 57 | toolkit.get_validator('upload_has_sha256'), 58 | toolkit.get_validator('upload_has_size'), 59 | toolkit.get_validator('upload_has_lfs_prefix') 60 | ], 61 | 'sha256': [ 62 | toolkit.get_validator('ignore_missing'), 63 | toolkit.get_validator('valid_sha256') 64 | ], 65 | 'size': [ 66 | toolkit.get_validator('ignore_missing'), 67 | toolkit.get_validator('is_positive_integer'), 68 | ], 69 | 'lfs_prefix': [ 70 | toolkit.get_validator('ignore_missing'), 71 | toolkit.get_validator('valid_lfs_prefix'), 72 | ] 73 | }) 74 | 75 | return schema 76 | 77 | def is_fallback(self): 78 | # Return True to register this plugin as the default handler for 79 | # package types not handled by any other IDatasetForm plugin. 80 | return True 81 | 82 | def package_types(self): 83 | # This plugin doesn't handle any special package types, it just 84 | # registers itself as the default (above). 85 | return [] 86 | 87 | # IValidators 88 | 89 | def get_validators(self): 90 | return { 91 | u'upload_has_sha256': validators.upload_has_sha256, 92 | u'upload_has_size': validators.upload_has_size, 93 | u'upload_has_lfs_prefix': validators.upload_has_lfs_prefix, 94 | u'valid_sha256': validators.valid_sha256, 95 | u'valid_lfs_prefix': validators.valid_lfs_prefix, 96 | } 97 | 98 | # IConfigurer 99 | 100 | def update_config(self, config): 101 | toolkit.add_template_directory(config, 'templates') 102 | toolkit.add_public_directory(config, 'public') 103 | toolkit.add_resource('fanstatic', 'blob-storage') 104 | 105 | # ITemplateHelpers 106 | 107 | def get_helpers(self): 108 | return {'blob_storage_server_url': helpers.server_url, 109 | 'blob_storage_storage_namespace': helpers.storage_namespace} 110 | 111 | # IBlueprint 112 | 113 | def get_blueprint(self): 114 | return blueprint 115 | 116 | # IActions 117 | 118 | def get_actions(self): 119 | return { 120 | 'get_resource_download_spec': actions.get_resource_download_spec, 121 | 'resource_schema_show': actions.resource_schema_show, 122 | 'resource_sample_show': actions.resource_sample_show 123 | } 124 | 125 | # IAuthorizationBindings 126 | 127 | def register_authz_bindings(self, authorizer): 128 | # type: (Authzzie) -> None 129 | """Authorization Bindings 130 | 131 | This aliases CKANs Resource entity and actions to scopes understood by 132 | Giftless' JWT authorization scheme 133 | """ 134 | # Register object authorization bindings 135 | authorizer.register_entity_ref_parser('obj', authz.object_id_parser) 136 | authorizer.register_authorizer('obj', authz.check_object_permissions, 137 | actions={'update', 'read'}, 138 | subscopes=(None, 'data', 'metadata')) 139 | authorizer.register_action_alias('write', 'update', 'obj') 140 | authorizer.register_scope_normalizer('obj', authz.normalize_object_scope) 141 | 142 | # IResourceDownloadHandler 143 | 144 | def resource_download(self, resource, package, filename=None, inline=False, activity_id=None): 145 | return download_handler(resource, package, filename, inline, activity_id) 146 | -------------------------------------------------------------------------------- /ckanext/blob_storage/public/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datopian/ckanext-blob-storage/d264f86a532de1c1f879dd9f93582d8dc0877d1e/ckanext/blob_storage/public/.gitignore -------------------------------------------------------------------------------- /ckanext/blob_storage/templates/blob_storage/snippets/upload_module.html: -------------------------------------------------------------------------------- 1 | {% resource 'blob-storage/css/main.3c074c8b.chunk.css' %} 2 | 3 | {% resource 'blob-storage/js/runtime-main.c155da57.js' %} 4 | {% resource 'blob-storage/js/2.60dfd475.chunk.js' %} 5 | {% resource 'blob-storage/js/main.c8e9dac3.chunk.js' %} 6 | 7 |
14 |
15 | -------------------------------------------------------------------------------- /ckanext/blob_storage/templates/package/new_resource.html: -------------------------------------------------------------------------------- 1 | {% ckan_extends %} 2 | 3 | {% set package_id = pkg_dict.id %} 4 | 5 | {% block form %} 6 | {% snippet 'blob_storage/snippets/upload_module.html', pkg_id=package_id, api_key=c.userobj.apikey, base_url=g.site_url, parent=super %} 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /ckanext/blob_storage/templates/package/new_resource_not_draft.html: -------------------------------------------------------------------------------- 1 | {% ckan_extends %} 2 | 3 | {% set package_id = pkg_dict.id %} 4 | 5 | {% block form %} 6 | {% snippet 'blob_storage/snippets/upload_module.html', pkg_id=package_id, api_key=c.userobj.apikey, base_url=g.site_url, parent=super %} 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /ckanext/blob_storage/templates/package/resource_edit.html: -------------------------------------------------------------------------------- 1 | {% ckan_extends %} 2 | 3 | {% block form %} 4 | {% snippet 'blob_storage/snippets/upload_module.html', resource_id=res.id, pkg_id=pkg.id, parent=super %} 5 | {% endblock %} 6 | -------------------------------------------------------------------------------- /ckanext/blob_storage/templates/package/snippets/resource_form.html: -------------------------------------------------------------------------------- 1 | {% ckan_extends %} 2 | 3 | {% block basic_fields_url %} 4 | {{ super() }} 5 | {% block upload_progress_bar %} 6 | 9 | {% endblock %} 10 | {% endblock %} 11 | -------------------------------------------------------------------------------- /ckanext/blob_storage/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from typing import Any, Dict 3 | 4 | from ckan import model 5 | from ckan.tests import helpers 6 | from mock import patch 7 | 8 | 9 | class FunctionalTestBase(helpers.FunctionalTestBase): 10 | 11 | _load_plugins = ['blob_storage'] 12 | 13 | 14 | @contextmanager 15 | def user_context(user): 16 | # type: (Dict[str, Any]) -> Dict[str, Any] 17 | """Context manager that creates a CKAN context dict for a user, then 18 | both patches our `get_user_context` function to return it and also 19 | yields the context for use inside tests 20 | """ 21 | userobj = model.User.get(user['name']) 22 | context = {"model": model, 23 | "user": user['name'], 24 | "auth_user_obj": userobj, 25 | "userobj": userobj} 26 | 27 | def mock_context(): 28 | return context 29 | 30 | with patch('ckanext.blob_storage.authz.get_user_context', mock_context): 31 | yield context 32 | 33 | 34 | @contextmanager 35 | def temporary_file(content): 36 | # type: (str) -> str 37 | """Context manager that creates a temporary file with specified content 38 | and yields its name. Once the context is exited the file is deleted. 39 | """ 40 | import tempfile 41 | file = tempfile.NamedTemporaryFile() 42 | file.write(content) 43 | file.flush() 44 | yield file.name 45 | -------------------------------------------------------------------------------- /ckanext/blob_storage/tests/test_actions.py: -------------------------------------------------------------------------------- 1 | import ckan.plugins.toolkit as toolkit 2 | import pytest 3 | from ckan.tests import factories 4 | 5 | 6 | @pytest.mark.usefixtures("clean_db") 7 | def test_validation_error_if_not_sha256(): 8 | with pytest.raises(toolkit.ValidationError): 9 | factories.Dataset( 10 | resources=[ 11 | { 12 | 'url': '/my/file.csv', 13 | 'url_type': 'upload', 14 | 'size': 12345, 15 | 'lfs_prefix': 'lfs/prefix' 16 | } 17 | ] 18 | ) 19 | 20 | 21 | @pytest.mark.usefixtures("clean_db") 22 | def test_validation_error_if_not_size_on_uploads(): 23 | with pytest.raises(toolkit.ValidationError): 24 | factories.Dataset( 25 | resources=[ 26 | { 27 | 'url': '/my/file.csv', 28 | 'url_type': 'upload', 29 | 'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 30 | 'lfs_prefix': 'lfs/prefix' 31 | } 32 | ] 33 | ) 34 | 35 | 36 | @pytest.mark.usefixtures("clean_db") 37 | def test_validation_error_if_not_lfs_prefix_on_uploads(): 38 | with pytest.raises(toolkit.ValidationError): 39 | factories.Dataset( 40 | resources=[ 41 | { 42 | 'url': '/my/file.csv', 43 | 'url_type': 'upload', 44 | 'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 45 | 'size': 123456 46 | } 47 | ] 48 | ) 49 | 50 | 51 | @pytest.mark.usefixtures("clean_db") 52 | def test_no_validation_error_if_not_upload(): 53 | factories.Dataset( 54 | resources=[{'url': 'https://www.example.com', 'url_type': ''}] 55 | ) 56 | 57 | 58 | @pytest.mark.usefixtures("clean_db") 59 | def test_no_validation_error_if_all_fields_are_set(): 60 | dataset = factories.Dataset( 61 | resources=[ 62 | { 63 | 'url': '/my/file.csv', 64 | 'url_type': 'upload', 65 | 'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 66 | 'size': 12345, 67 | 'lfs_prefix': 'lfs/prefix' 68 | } 69 | ] 70 | ) 71 | 72 | assert dataset['resources'][0]['sha256'] == 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919' 73 | assert dataset['resources'][0]['size'] == 12345 74 | assert dataset['resources'][0]['lfs_prefix'] == 'lfs/prefix' 75 | 76 | 77 | @pytest.mark.usefixtures("clean_db") 78 | def test_validation_error_if_wrong_sha256(): 79 | with pytest.raises(toolkit.ValidationError): 80 | factories.Dataset( 81 | resources=[ 82 | { 83 | 'url': '/my/file.csv', 84 | 'url_type': 'upload', 85 | 'sha256': 'wrong_sha256', 86 | 'size': 123456, 87 | 'lfs_prefix': 'lfs/prefix' 88 | } 89 | ] 90 | ) 91 | 92 | 93 | @pytest.mark.usefixtures("clean_db") 94 | def test_validation_error_if_size_not_positive_integer(): 95 | with pytest.raises(toolkit.ValidationError): 96 | factories.Dataset( 97 | resources=[ 98 | { 99 | 'url': '/my/file.csv', 100 | 'url_type': 'upload', 101 | 'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 102 | 'size': -12, 103 | 'lfs_prefix': 'lfs/prefix' 104 | } 105 | ] 106 | ) 107 | 108 | with pytest.raises(toolkit.ValidationError): 109 | factories.Dataset( 110 | resources=[ 111 | { 112 | 'url': '/my/file.csv', 113 | 'url_type': 'upload', 114 | 'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 115 | 'size': 0, 116 | 'lfs_prefix': 'lfs/prefix' 117 | } 118 | ] 119 | ) 120 | 121 | 122 | @pytest.mark.usefixtures("clean_db") 123 | def test_validation_error_if_empty_lfs_prefix(): 124 | with pytest.raises(toolkit.ValidationError): 125 | factories.Dataset( 126 | resources=[ 127 | { 128 | 'url': '/my/file.csv', 129 | 'url_type': 'upload', 130 | 'sha256': 'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 131 | 'size': 123456, 132 | 'lfs_prefix': '' 133 | } 134 | ] 135 | ) 136 | -------------------------------------------------------------------------------- /ckanext/blob_storage/tests/test_authz.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from ckan.plugins import toolkit 3 | from ckan.tests import factories, helpers 4 | 5 | from ckanext.authz_service.authzzie import Scope 6 | from ckanext.blob_storage import authz 7 | from ckanext.blob_storage.tests import user_context 8 | 9 | 10 | def test_normalize_object_scope(): 11 | scope = Scope('foo', 'bar', {'read'}) 12 | normalized_scope = authz.normalize_object_scope(None, scope) 13 | assert 'foo:bar:read' == str(normalized_scope) 14 | 15 | 16 | @pytest.mark.usefixtures('clean_db', 'reset_db', 'with_request_context') 17 | def test_normalize_object_scope_with_lfs(): 18 | sysadmin = factories.Sysadmin() 19 | org = factories.Organization() 20 | dataset = factories.Dataset(owner_org=org['id']) 21 | resource = factories.Resource( 22 | url='/my/file.csv', 23 | url_type='upload', 24 | sha256='cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 25 | size=123456, 26 | lfs_prefix='lfs_prefix', 27 | package_id=dataset['id'] 28 | ) 29 | 30 | scope_str = 'obj:{}/{}/{}:read'.format(org['name'], dataset['name'], resource['id']) 31 | scope = Scope.from_string(scope_str) 32 | 33 | with user_context(sysadmin): 34 | normalized_scope = authz.normalize_object_scope(None, scope) 35 | 36 | expected_scpope = 'obj:{}/{}:read'.format( 37 | resource['lfs_prefix'], 38 | resource['sha256'] 39 | ) 40 | assert expected_scpope == str(normalized_scope) 41 | 42 | 43 | @pytest.mark.usefixtures('clean_db', 'reset_db', 'with_request_context') 44 | def test_normalize_object_scope_with_activity_id(): 45 | if not toolkit.check_ckan_version(min_version="2.9"): 46 | pytest.skip("activity_id feature only available in CKAN 2.9+") 47 | sysadmin = factories.Sysadmin() 48 | org = factories.Organization() 49 | dataset = factories.Dataset(owner_org=org['id']) 50 | resource = factories.Resource( 51 | url='/my/file.csv', 52 | url_type='upload', 53 | sha256='cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 54 | size=123456, 55 | lfs_prefix='lfs_prefix', 56 | package_id=dataset['id'] 57 | ) 58 | resource_2 = helpers.call_action( 59 | 'resource_patch', 60 | id=resource['id'], 61 | url='/my/file-2.csv', 62 | sha256='dd71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 63 | size=2345678, 64 | ) 65 | 66 | # Building scope without activity_id 67 | scope_str = 'obj:{}/{}/{}:read'.format(org['name'], dataset['name'], resource['id']) 68 | scope = Scope.from_string(scope_str) 69 | 70 | with user_context(sysadmin): 71 | normalized_scope = authz.normalize_object_scope(None, scope) 72 | 73 | # Expected scope should have the current lfs metadata (lfs_prefix and sha256) 74 | expected_scope = 'obj:{}/{}:read'.format( 75 | resource_2['lfs_prefix'], 76 | resource_2['sha256'] 77 | ) 78 | assert expected_scope == str(normalized_scope) 79 | 80 | # Editing the resource so the latest version is different from resource_2 81 | helpers.call_action( 82 | 'resource_patch', 83 | id=resource['id'], 84 | url='/my/file-3.csv', 85 | sha256='ee71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 86 | size=3456789, 87 | ) 88 | 89 | resource_2_activity_id = helpers.call_action( 90 | 'package_activity_list', 91 | id=dataset['id'] 92 | )[-1]['id'] 93 | 94 | # Building scope with activity_id of resource_2 95 | scope_str = 'obj:{}/{}/{}/{}:read'.format( 96 | org['name'], 97 | dataset['name'], 98 | resource['id'], 99 | resource_2_activity_id 100 | ) 101 | scope = Scope.from_string(scope_str) 102 | 103 | with user_context(sysadmin): 104 | normalized_scope = authz.normalize_object_scope(None, scope) 105 | 106 | assert expected_scope == str(normalized_scope) 107 | -------------------------------------------------------------------------------- /ckanext/blob_storage/tests/test_blueprint.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | from ckan.plugins import toolkit 4 | from ckan.tests import factories 5 | 6 | 7 | @pytest.mark.usefixtures('clean_db') 8 | def test_preview_arg(app): 9 | 10 | org = factories.Organization() 11 | dataset = factories.Dataset(owner_org=org['id']) 12 | resource = factories.Resource( 13 | package_id=dataset['id'], 14 | url_type='upload', 15 | sha256='cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919', 16 | size=12, 17 | lfs_prefix='lfs/prefix' 18 | ) 19 | 20 | with mock.patch('ckanext.blob_storage.blueprints.call_download_handlers') as m: 21 | 22 | m.return_value = '' 23 | 24 | url = toolkit.url_for( 25 | 'blob_storage.download', 26 | id=dataset['id'], 27 | resource_id=resource['id'], 28 | preview=1 29 | ) 30 | 31 | app.get(url) 32 | 33 | args = m.call_args 34 | 35 | assert args[0][0]['id'] == resource['id'] # resource 36 | assert args[0][1]['id'] == dataset['id'] # dataset 37 | assert args[0][2] is None # filename 38 | assert args[0][3] is True # inline 39 | 40 | url = toolkit.url_for( 41 | 'blob_storage.download', 42 | id=dataset['id'], 43 | resource_id=resource['id'], 44 | filename='test.csv', 45 | preview=1 46 | ) 47 | 48 | app.get(url) 49 | 50 | args = m.call_args 51 | 52 | assert args[0][2] == 'test.csv' # filename 53 | assert args[0][3] is True # inline 54 | 55 | url = toolkit.url_for( 56 | 'blob_storage.download', 57 | id=dataset['id'], 58 | resource_id=resource['id'], 59 | filename='test.csv', 60 | ) 61 | 62 | app.get(url) 63 | 64 | args = m.call_args 65 | 66 | assert args[0][3] is False # inline 67 | -------------------------------------------------------------------------------- /ckanext/blob_storage/tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from ckan.tests import factories 3 | 4 | from ckanext.blob_storage import helpers 5 | 6 | 7 | @pytest.mark.ckan_config('ckanext.blob_storage.storage_service_url', 'https://foo.example.com/lfs') 8 | def test_server_url_from_config(): 9 | assert 'https://foo.example.com/lfs' == helpers.server_url() 10 | 11 | 12 | @pytest.mark.ckan_config('ckanext.blob_storage.storage_service_url', 'https://foo.example.com/lfs/') 13 | def test_server_url_strip_trailing_slash(): 14 | assert 'https://foo.example.com/lfs' == helpers.server_url() 15 | 16 | 17 | def test_resource_storage_prefix_explicit_org(): 18 | prefix = helpers.resource_storage_prefix('mypackage', 'myorg') 19 | assert 'myorg/mypackage' == prefix 20 | 21 | 22 | @pytest.mark.usefixtures('clean_db', 'reset_db') 23 | @pytest.mark.ckan_config('ckanext.blob_storage.storage_namespace', 'some-namespace') 24 | def test_resource_storage_prefix_unspecified_org(): 25 | prefix = helpers.resource_storage_prefix('mypackage') 26 | assert 'some-namespace/mypackage' == prefix 27 | 28 | 29 | @pytest.mark.usefixtures('clean_db', 'reset_db') 30 | @pytest.mark.ckan_config('ckanext.blob_storage.storage_namespace', 'some-namespace') 31 | def test_resource_storage_prefix_no_org(): 32 | factories.Dataset(name='mypackage') 33 | prefix = helpers.resource_storage_prefix('mypackage') 34 | assert 'some-namespace/mypackage' == prefix 35 | 36 | 37 | def test_resource_authz_scope_default_actions(): 38 | scope = helpers.resource_authz_scope('mypackage', org_name='myorg') 39 | assert 'obj:myorg/mypackage/*:read,write' == scope 40 | 41 | 42 | def test_resource_authz_scope_custom_actions(): 43 | scope = helpers.resource_authz_scope('mypackage', actions='read', org_name='myorg') 44 | assert 'obj:myorg/mypackage/*:read' == scope 45 | 46 | 47 | @pytest.mark.ckan_config('ckanext.blob_storage.storage_namespace', 'some-namespace') 48 | def test_resource_authz_scope_default_namespace(): 49 | scope = helpers.resource_authz_scope('mypackage') 50 | assert 'obj:some-namespace/mypackage/*:read,write' == scope 51 | 52 | 53 | @pytest.mark.ckan_config('ckanext.blob_storage.storage_namespace', None) 54 | def test_resource_authz_scope_no_configured_namespace(): 55 | scope = helpers.resource_authz_scope('mypackage') 56 | assert 'obj:ckan/mypackage/*:read,write' == scope 57 | 58 | 59 | @pytest.mark.ckan_config('ckanext.blob_storage.storage_namespace', None) 60 | def test_resource_authz_scope_with_activity_id(): 61 | scope = helpers.resource_authz_scope('mypackage', activity_id='activity-id') 62 | assert 'obj:ckan/mypackage/*/activity-id:read,write' == scope 63 | scope = helpers.resource_authz_scope('mypackage', resource_id='resource-id', activity_id='activity-id') 64 | assert 'obj:ckan/mypackage/resource-id/activity-id:read,write' == scope 65 | -------------------------------------------------------------------------------- /ckanext/blob_storage/tests/test_plugin.py: -------------------------------------------------------------------------------- 1 | """Tests for plugin.py 2 | """ 3 | import ckanext.blob_storage.plugin as plugin 4 | 5 | 6 | def test_plugin(): 7 | p = plugin.BlobStoragePlugin() 8 | assert p 9 | -------------------------------------------------------------------------------- /ckanext/blob_storage/tests/test_validators.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from ckan.plugins.toolkit import Invalid 3 | 4 | from ckanext.blob_storage import validators 5 | 6 | 7 | def test_upload_has_sha256(): 8 | key = ('resources', 0, 'url_type') 9 | flattened_data = { 10 | (u'resources', 0, u'url_type'): u'upload', 11 | (u'resources', 0, u'url'): u'/my/file.csv', 12 | (u'resources', 0, u'sha256'): u'cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919' 13 | } 14 | validators.upload_has_sha256(key, flattened_data, {}, {}) 15 | 16 | with pytest.raises(Invalid): 17 | flattened_data = { 18 | (u'resources', 0, u'url_type'): u'upload', 19 | (u'resources', 0, u'url'): u'/my/file.csv', 20 | } 21 | validators.upload_has_sha256(key, flattened_data, {}, {}) 22 | 23 | 24 | def test_upload_has_size(): 25 | key = ('resources', 0, 'url_type') 26 | flattened_data = { 27 | (u'resources', 0, u'url_type'): u'upload', 28 | (u'resources', 0, u'url'): u'/my/file.csv', 29 | (u'resources', 0, u'size'): 123456 30 | } 31 | validators.upload_has_size(key, flattened_data, {}, {}) 32 | 33 | with pytest.raises(Invalid): 34 | flattened_data = { 35 | (u'resources', 0, u'url_type'): u'upload', 36 | (u'resources', 0, u'url'): u'/my/file.csv', 37 | } 38 | validators.upload_has_size(key, flattened_data, {}, {}) 39 | 40 | 41 | def test_upload_has_lfs_prefix(): 42 | key = ('resources', 0, 'url_type') 43 | flattened_data = { 44 | (u'resources', 0, u'url_type'): u'upload', 45 | (u'resources', 0, u'url'): u'/my/file.csv', 46 | (u'resources', 0, u'lfs_prefix'): u'lfs/prefix' 47 | } 48 | validators.upload_has_lfs_prefix(key, flattened_data, {}, {}) 49 | 50 | with pytest.raises(Invalid): 51 | flattened_data = { 52 | (u'resources', 0, u'url_type'): u'upload', 53 | (u'resources', 0, u'url'): u'/my/file.csv' 54 | } 55 | validators.upload_has_lfs_prefix(key, flattened_data, {}, {}) 56 | 57 | 58 | def test_valid_sha256(): 59 | validators.valid_sha256('cc71500070cf26cd6e8eab7c9eec3a937be957d144f445ad24003157e2bd0919') 60 | 61 | with pytest.raises(Invalid): 62 | validators.valid_sha256('wrong_sha256') 63 | 64 | 65 | def test_valid_lfs_prefix(): 66 | validators.valid_lfs_prefix("lfs/prefix") 67 | 68 | with pytest.raises(Invalid): 69 | validators.valid_lfs_prefix("") 70 | 71 | 72 | def test_sha256_doesnt_raise_if_not_upload(): 73 | key = ('resources', 0, 'url_type') 74 | flattened_data = { 75 | (u'resources', 0, u'url_type'): u'', 76 | (u'resources', 0, u'url'): u'https://www.google.com', 77 | } 78 | validators.upload_has_sha256(key, flattened_data, {}, {}) 79 | 80 | 81 | def test_size_doesnt_raise_if_not_upload(): 82 | key = ('resources', 0, 'url_type') 83 | flattened_data = { 84 | (u'resources', 0, u'url_type'): u'', 85 | (u'resources', 0, u'url'): u'https://www.google.com', 86 | } 87 | validators.upload_has_size(key, flattened_data, {}, {}) 88 | 89 | 90 | def test_lfs_prefix_doesnt_raise_if_not_upload(): 91 | key = ('resources', 0, 'url_type') 92 | flattened_data = { 93 | (u'resources', 0, u'url_type'): u'', 94 | (u'resources', 0, u'url'): u'https://www.google.com', 95 | } 96 | validators.upload_has_lfs_prefix(key, flattened_data, {}, {}) 97 | -------------------------------------------------------------------------------- /ckanext/blob_storage/validators.py: -------------------------------------------------------------------------------- 1 | from ckan.plugins.toolkit import Invalid 2 | 3 | 4 | def upload_has_sha256(key, flattened_data, errors, context): 5 | if flattened_data[key] == 'upload': 6 | if (key[0], key[1], 'sha256') not in flattened_data: 7 | raise Invalid("Resource's sha256 field cannot be missing for uploads.") 8 | 9 | 10 | def valid_sha256(value): 11 | if not _is_hex_str(value, 64): 12 | raise Invalid("Resource's sha256 is not a valid hex-only string.") 13 | return value 14 | 15 | 16 | def upload_has_size(key, flattened_data, errors, context): 17 | if flattened_data[key] == 'upload': 18 | if (key[0], key[1], 'size') not in flattened_data: 19 | raise Invalid("Resource's size field cannot be missing for uploads.") 20 | 21 | 22 | def upload_has_lfs_prefix(key, flattened_data, errors, context): 23 | if flattened_data[key] == 'upload': 24 | if (key[0], key[1], 'lfs_prefix') not in flattened_data: 25 | raise Invalid("Resource's lfs_prefix field cannot be missing for uploads.") 26 | 27 | 28 | def valid_lfs_prefix(value): 29 | if value == "": 30 | raise Invalid("Resource's lfs_prefix field cannot be empty.") 31 | return value 32 | 33 | 34 | def _is_hex_str(value, chars=40): 35 | # type: (str, int) -> bool 36 | """Check if a string is a hex-only string of exactly :param:`chars` characters length. 37 | This is useful to verify that a string contains a valid SHA, MD5 or UUID-like value. 38 | >>> _is_hex_str('0f1128046248f83dc9b9ab187e16fad0ff596128f1524d05a9a77c4ad932f10a', 64) 39 | True 40 | >>> _is_hex_str('0f1128046248f83dc9b9ab187e16fad0ff596128f1524d05a9a77c4ad932f10a', 32) 41 | False 42 | >>> _is_hex_str('0f1128046248f83dc9b9ab187e1xfad0ff596128f1524d05a9a77c4ad932f10a', 64) 43 | False 44 | >>> _is_hex_str('ef42bab1191da272f13935f78c401e3de0c11afb') 45 | True 46 | >>> _is_hex_str('ef42bab1191da272f13935f78c401e3de0c11afb'.upper()) 47 | True 48 | >>> _is_hex_str('ef42bab1191da272f13935f78c401e3de0c11afb', 64) 49 | False 50 | >>> _is_hex_str('ef42bab1191da272f13935.78c401e3de0c11afb') 51 | False 52 | """ 53 | if len(value) != chars: 54 | return False 55 | try: 56 | int(value, 16) 57 | except ValueError: 58 | return False 59 | return True 60 | -------------------------------------------------------------------------------- /dev-requirements.in: -------------------------------------------------------------------------------- 1 | pip-tools==4.5.* 2 | pytest==4.6.* 3 | pytest-ckan==0.0.* 4 | pytest-cov==2.8.* 5 | pytest-flake8==1.0.* 6 | pytest-isort==0.3.* 7 | -------------------------------------------------------------------------------- /dev-requirements.py2.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --no-index --output-file=dev-requirements.py2.txt dev-requirements.in 6 | # 7 | atomicwrites==1.3.0 # via pytest 8 | attrs==19.3.0 # via pytest 9 | backports.functools-lru-cache==1.6.1 # via isort 10 | click==7.1.1 # via pip-tools 11 | configparser==4.0.2 # via flake8, importlib-metadata 12 | contextlib2==0.6.0.post1 # via importlib-metadata 13 | coverage==4.5.4 # via pytest-cov 14 | entrypoints==0.3 # via flake8 15 | enum34==1.1.10 # via flake8 16 | flake8==3.7.9 # via pytest-flake8 17 | funcsigs==1.0.2 # via pytest 18 | functools32==3.2.3.post2 # via flake8 19 | futures==3.3.0 # via isort 20 | importlib-metadata==1.6.0 # via pluggy, pytest 21 | isort==4.3.21 # via pytest-isort 22 | mccabe==0.6.1 # via flake8 23 | more-itertools==5.0.0 # via pytest 24 | packaging==20.3 # via pytest 25 | pathlib2==2.3.5 # via importlib-metadata, pytest 26 | pip-tools==4.5.1 # via -r dev-requirements.in 27 | pluggy==0.13.1 # via pytest 28 | py==1.8.1 # via pytest 29 | pycodestyle==2.5.0 # via flake8 30 | pyflakes==2.1.1 # via flake8 31 | pyparsing==2.4.7 # via packaging 32 | pytest-ckan==0.0.12 # via -r dev-requirements.in 33 | pytest-cov==2.8.1 # via -r dev-requirements.in 34 | pytest-flake8==1.0.5 # via -r dev-requirements.in 35 | pytest-isort==0.3.1 # via -r dev-requirements.in 36 | pytest==4.6.9 # via -r dev-requirements.in, pytest-ckan, pytest-cov, pytest-flake8, pytest-isort 37 | scandir==1.10.0 # via pathlib2 38 | six==1.14.0 # via more-itertools, packaging, pathlib2, pip-tools, pytest 39 | typing==3.7.4.1 # via flake8 40 | wcwidth==0.1.9 # via pytest 41 | zipp==1.2.0 # via importlib-metadata 42 | -------------------------------------------------------------------------------- /dev-requirements.py3.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --no-index --output-file=dev-requirements.py3.txt dev-requirements.in 6 | # 7 | atomicwrites==1.3.0 # via pytest 8 | attrs==19.3.0 # via pytest 9 | click==7.1.1 # via pip-tools 10 | coverage==4.5.4 # via pytest-cov 11 | entrypoints==0.3 # via flake8 12 | flake8==3.7.9 # via pytest-flake8 13 | importlib-metadata==1.6.0 # via pluggy, pytest 14 | isort==4.3.21 # via pytest-isort 15 | mccabe==0.6.1 # via flake8 16 | more-itertools==8.2.0 # via pytest 17 | packaging==20.3 # via pytest 18 | pip-tools==4.5.1 # via -r dev-requirements.in 19 | pluggy==0.13.1 # via pytest 20 | py==1.8.1 # via pytest 21 | pycodestyle==2.5.0 # via flake8 22 | pyflakes==2.1.1 # via flake8 23 | pyparsing==2.4.7 # via packaging 24 | pytest-ckan==0.0.12 # via -r dev-requirements.in 25 | pytest-cov==2.8.1 # via -r dev-requirements.in 26 | pytest-flake8==1.0.5 # via -r dev-requirements.in 27 | pytest-isort==0.3.1 # via -r dev-requirements.in 28 | pytest==4.6.9 # via -r dev-requirements.in, pytest-ckan, pytest-cov, pytest-flake8, pytest-isort 29 | six==1.14.0 # via packaging, pip-tools, pytest 30 | wcwidth==0.1.9 # via pytest 31 | zipp==3.1.0 # via importlib-metadata 32 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Docker Compose file for ckanext-blob-storage 2 | # ------------------------------------------------ 3 | # The purpose of this docker-compose file is to simplify setting up a 4 | # development environment for this CKAN extension; It defines Docker based 5 | # services for all external CKAN dependencies (DB, Solr, Redis) but not for 6 | # CKAN itself - you should probably run CKAN in a local virtual environment 7 | # to simplify debugging. 8 | # 9 | # Most likely, you do not want to use this file directly with `docker-compose` 10 | # but use the provided Make targets to manage things. 11 | 12 | version: "3" 13 | 14 | volumes: 15 | db_data: 16 | 17 | services: 18 | 19 | db: 20 | image: postgres:9-alpine 21 | ports: 22 | - "5432:5432" 23 | environment: 24 | - POSTGRES_USER=${POSTGRES_USER} 25 | - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} 26 | - POSTGRES_DB=${POSTGRES_DB} 27 | volumes: 28 | - db_data:/var/lib/postgresql/data 29 | 30 | solr: 31 | image: ckan/solr 32 | environment: 33 | - CKAN_SOLR_PASSWORD=${CKAN_SOLR_PASSWORD} 34 | ports: 35 | - 8983:8983 36 | 37 | redis: 38 | image: redis:latest 39 | ports: 40 | - 6379:6379 41 | 42 | giftless: 43 | image: datopian/giftless 44 | command: ["--http", "0.0.0.0:9419", "-M", "-T", "--threads", "2", "-p", "2", "--manage-script-name", "--callable", "app"] 45 | environment: 46 | - GIFTLESS_CONFIG_FILE=/etc/giftless.yaml 47 | - GIFTLESS_DEBUG=true 48 | volumes: 49 | - ./docker/giftless.yaml:/etc/giftless.yaml:ro 50 | - ../giftless:/app # Use local giftless 51 | ports: 52 | - "9419:9419" 53 | 54 | # nginx: 55 | # image: nginx:1.13 56 | # volumes: 57 | # - ./docker/nginx.conf:/etc/nginx/conf.d/default.conf 58 | # ports: 59 | # - "9419:80" 60 | -------------------------------------------------------------------------------- /docker/giftless.yaml: -------------------------------------------------------------------------------- 1 | # Giftless configuration file for local development purposes 2 | --- 3 | # JWT based authentication and authorization from CKAN 4 | AUTH_PROVIDERS: 5 | - factory: giftless.auth.jwt:factory 6 | options: 7 | algorithm: HS256 8 | private_key: this-is-a-test-only-key 9 | # - giftless.auth.allow_anon:read_only 10 | 11 | # In a local environment we'll use the default storage adapter 12 | TRANSFER_ADAPTERS: {} 13 | 14 | # Set private key for internal pre-signed requests (e.g. requests to verify) 15 | PRE_AUTHORIZED_ACTION_PROVIDER: 16 | options: 17 | private_key: another-test-only-key 18 | 19 | # Enable CORS requests from localhost:5000 20 | MIDDLEWARE: 21 | - class: wsgi_cors_middleware:CorsMiddleware 22 | kwargs: 23 | origin: http://localhost:5000 24 | headers: ['Content-type', 'Accept', 'Authorization'] 25 | methods: ['GET', 'POST', 'PUT'] -------------------------------------------------------------------------------- /docker/nginx.conf: -------------------------------------------------------------------------------- 1 | # Nginx configuration file for development purposes 2 | 3 | server { 4 | listen 80 default_server; 5 | server_name _; # catch all 6 | charset utf-8; 7 | client_max_body_size 1G; 8 | 9 | location / { 10 | if ($request_method = OPTIONS ) { 11 | add_header "Access-Control-Allow-Origin" *; 12 | add_header "Access-Control-Allow-Methods" "GET, POST, PUT, OPTIONS, HEAD"; 13 | add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; 14 | return 204; 15 | } 16 | 17 | # Pass to WSGI 18 | include uwsgi_params; 19 | uwsgi_pass giftless:5000; 20 | 21 | # CORS handling 22 | if ($request_method ~* "(GET|POST|PUT|HEAD)") { 23 | add_header "Access-Control-Allow-Origin" * always; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | requests==2.*,>=2.23.0 2 | six==1.14.* 3 | typing==3.7.* 4 | giftless-client==0.1.* 5 | -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 6 | git+https://github.com/datopian/ckanext-authz-service.git@v0.1.5#egg=ckanext-authz-service 7 | -------------------------------------------------------------------------------- /requirements.py2.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --no-index --output-file=requirements.py2.txt requirements.in 6 | # 7 | certifi==2020.4.5.1 8 | # via requests 9 | cffi==1.14.0 10 | # via cryptography 11 | chardet==3.0.4 12 | # via requests 13 | git+https://github.com/datopian/ckanext-authz-service.git@v0.1.5#egg=ckanext-authz-service 14 | # via -r requirements.in 15 | click==7.1.2 16 | # via giftless-client 17 | cryptography==2.9.2 18 | # via pyjwt 19 | enum34==1.1.10 20 | # via cryptography 21 | giftless-client==0.1.0 22 | # via -r requirements.in 23 | idna==2.9 24 | # via requests 25 | ipaddress==1.0.23 26 | # via cryptography 27 | pycparser==2.20 28 | # via cffi 29 | pyjwt[crypto]==1.7.1 30 | # via -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 31 | python-dateutil==2.8.1 32 | # via giftless-client 33 | pytz==2020.1 34 | # via -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 35 | pyyaml==5.3.1 36 | # via -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 37 | requests==2.25.1 38 | # via 39 | # -r requirements.in 40 | # giftless-client 41 | six==1.14.0 42 | # via 43 | # -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 44 | # -r requirements.in 45 | # cryptography 46 | # python-dateutil 47 | typing-extensions==3.7.4.2 48 | # via 49 | # -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 50 | # giftless-client 51 | typing==3.7.4.1 52 | # via 53 | # -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 54 | # -r requirements.in 55 | # typing-extensions 56 | urllib3==1.25.9 57 | # via requests 58 | -------------------------------------------------------------------------------- /requirements.py3.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --no-index --output-file=requirements.py3.txt requirements.in 6 | # 7 | certifi==2020.4.5.1 8 | # via requests 9 | cffi==1.14.0 10 | # via cryptography 11 | chardet==3.0.4 12 | # via requests 13 | git+https://github.com/datopian/ckanext-authz-service.git@v0.1.5#egg=ckanext-authz-service 14 | # via -r requirements.in 15 | click==7.1.2 16 | # via giftless-client 17 | cryptography==2.8 18 | # via pyjwt 19 | giftless-client==0.1.0 20 | # via -r requirements.in 21 | idna==2.9 22 | # via requests 23 | pycparser==2.20 24 | # via cffi 25 | pyjwt[crypto]==1.7.1 26 | # via -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 27 | python-dateutil==2.8.1 28 | # via giftless-client 29 | pytz==2019.3 30 | # via -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 31 | pyyaml==5.3.1 32 | # via -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 33 | requests==2.25.1 34 | # via 35 | # -r requirements.in 36 | # giftless-client 37 | six==1.14.0 38 | # via 39 | # -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 40 | # -r requirements.in 41 | # cryptography 42 | # python-dateutil 43 | typing-extensions==3.7.4.1 44 | # via 45 | # -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 46 | # giftless-client 47 | typing==3.7.4.1 48 | # via 49 | # -r https://raw.githubusercontent.com/datopian/ckanext-authz-service/v0.1.5/requirements.in 50 | # -r requirements.in 51 | urllib3==1.25.9 52 | # via requests 53 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [extract_messages] 2 | keywords = translate isPlural 3 | add_comments = TRANSLATORS: 4 | output_file = ckanext/blob_storage/i18n/ckanext-blob_storage.pot 5 | width = 80 6 | 7 | [init_catalog] 8 | domain = ckanext-blob_storage 9 | input_file = ckanext/blob_storage/i18n/ckanext-blob_storage.pot 10 | output_dir = ckanext/blob_storage/i18n 11 | 12 | [update_catalog] 13 | domain = ckanext-blob_storage 14 | input_file = ckanext/blob_storage/i18n/ckanext-blob_storage.pot 15 | output_dir = ckanext/blob_storage/i18n 16 | previous = true 17 | 18 | [compile_catalog] 19 | domain = ckanext-blob_storage 20 | directory = ckanext/blob_storage/i18n 21 | statistics = true 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from setuptools import setup, find_packages # Always prefer setuptools over distutils 3 | from codecs import open # To use a consistent encoding 4 | from os import path 5 | 6 | import ckanext.blob_storage 7 | 8 | here = path.abspath(path.dirname(__file__)) 9 | 10 | # Get the long description from the relevant file 11 | with open(path.join(here, 'README.md'), encoding='utf-8') as f: 12 | long_description = f.read() 13 | 14 | setup( 15 | name='''ckanext-blob-storage''', 16 | 17 | # Versions should comply with PEP440. For a discussion on single-sourcing 18 | # the version across setup.py and the project code, see 19 | # http://packaging.python.org/en/latest/tutorial.html#version 20 | version=ckanext.blob_storage.__version__, 21 | 22 | description='''Store CKAN data files using an external Git LFS based storage microservice''', 23 | long_description=long_description, 24 | long_description_content_type='text/markdown', 25 | 26 | # The project's main homepage. 27 | url='https://github.com/datopian/ckanext-blob-storage', 28 | 29 | # Author details 30 | author='''Shahar Evron''', 31 | author_email='''shahar.evron@datopian.com''', 32 | 33 | # Choose your license 34 | license='MIT', 35 | 36 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 37 | classifiers=[ 38 | # How mature is this project? Common values are 39 | # 3 - Alpha 40 | # 4 - Beta 41 | # 5 - Production/Stable 42 | 'Development Status :: 4 - Beta', 43 | 44 | # Pick your license as you wish (should match "license" above) 45 | 'License :: OSI Approved :: MIT', 46 | 47 | # Specify the Python versions you support here. In particular, ensure 48 | # that you indicate whether you support Python 2, Python 3 or both. 49 | 'Programming Language :: Python :: 2.7', 50 | 'Programming Language :: Python :: 3', 51 | ], 52 | 53 | 54 | # What does your project relate to? 55 | keywords='''CKAN authorization jwt authz microservices integration''', 56 | 57 | # You can just specify the packages manually here if your project is 58 | # simple. Or you can use find_packages(). 59 | packages=find_packages(exclude=['contrib', 'docs', 'tests*']), 60 | namespace_packages=['ckanext'], 61 | 62 | install_requires=[ 63 | 'six', 64 | 'typing', 65 | 'ckanext-authz-service', 66 | ], 67 | 68 | # If there are data files included in your packages that need to be 69 | # installed, specify them here. If using Python 2.6 or less, then these 70 | # have to be included in MANIFEST.in as well. 71 | include_package_data=True, 72 | package_data={}, 73 | 74 | # Although 'package_data' is the preferred approach, in some case you may 75 | # need to place data files outside of your packages. 76 | # see http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files 77 | # In this case, 'data_file' will be installed into '/my_data' 78 | data_files=[], 79 | 80 | # To provide executable scripts, use entry points in preference to the 81 | # "scripts" keyword. Entry points provide cross-platform support and allow 82 | # pip to create the appropriate form of executable for the target platform. 83 | entry_points=''' 84 | [ckan.plugins] 85 | blob_storage=ckanext.blob_storage.plugin:BlobStoragePlugin 86 | 87 | [babel.extractors] 88 | ckan = ckan.lib.extract:extract_ckan 89 | 90 | [paste.paster_command] 91 | migrate-resources = ckanext.blob_storage.cli:MigrateResourcesCommand 92 | ''', 93 | 94 | # If you are changing from the default layout of your extension, you may 95 | # have to change the message extractors, you can read more about babel 96 | # message extraction at 97 | # http://babel.pocoo.org/docs/messages/#extraction-method-mapping-and-configuration 98 | message_extractors={ 99 | 'ckanext': [ 100 | ('**.py', 'python', None), 101 | ('**.js', 'javascript', None), 102 | ('**/templates/**.html', 'ckan', None), 103 | ], 104 | } 105 | ) 106 | -------------------------------------------------------------------------------- /test.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | debug = false 3 | smtp_server = localhost 4 | error_email_from = paste@localhost 5 | 6 | [server:main] 7 | use = egg:Paste#http 8 | host = 0.0.0.0 9 | port = 5000 10 | 11 | [app:main] 12 | use = config:ckan/test-core.ini 13 | 14 | # ckanext-blob-storage settings for testing purposes 15 | 16 | 17 | 18 | # Logging configuration 19 | [loggers] 20 | keys = root, ckan, sqlalchemy 21 | 22 | [handlers] 23 | keys = console 24 | 25 | [formatters] 26 | keys = generic 27 | 28 | [logger_root] 29 | level = WARN 30 | handlers = console 31 | 32 | [logger_ckan] 33 | qualname = ckan 34 | handlers = 35 | level = INFO 36 | 37 | [logger_sqlalchemy] 38 | handlers = 39 | qualname = sqlalchemy.engine 40 | level = WARN 41 | 42 | [handler_console] 43 | class = StreamHandler 44 | args = (sys.stdout,) 45 | level = NOTSET 46 | formatter = generic 47 | 48 | [formatter_generic] 49 | format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s 50 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | --------------------------------------------------------------------------------