├── .github └── FUNDING.yml ├── .gitignore ├── COPYING ├── COPYING.fr ├── README.md ├── ckanext ├── __init__.py └── cloudstorage │ ├── __init__.py │ ├── cli.py │ ├── controller.py │ ├── fanstatic │ └── scripts │ │ ├── cloudstorage-multipart-upload.js │ │ ├── resource.config │ │ └── vendor │ │ ├── file-upload.js │ │ └── jquery-widget.js │ ├── helpers.py │ ├── logic │ ├── __init__.py │ ├── action │ │ ├── __init__.py │ │ └── multipart.py │ └── auth │ │ ├── __init__.py │ │ └── multipart.py │ ├── model.py │ ├── plugin.py │ ├── storage.py │ └── templates │ ├── cloudstorage │ └── snippets │ │ └── multipart_module.html │ ├── package │ ├── new_resource.html │ ├── new_resource_not_draft.html │ └── resource_edit.html │ └── page.html └── setup.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: TkTech 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with a single custom sponsorship URL 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | # generic 3 | *.pyc 4 | *.swp 5 | *.swo 6 | .DS_Store 7 | ckan.egg-info/* 8 | sandbox/* 9 | dist 10 | 11 | # other 12 | pip-requirements-local.txt 13 | migrate/* 14 | build/* 15 | tmp/* 16 | solr_runtime/* 17 | fl_notes.txt 18 | *.ini 19 | .noseids 20 | *~ 21 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Government of Canada 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /COPYING.fr: -------------------------------------------------------------------------------- 1 | Licence MIT 2 | 3 | (c) Droit d'auteur – Gouvernement du Canada, 2017 4 | 5 | La présente autorise toute personne d'obtenir gratuitement une copie du 6 | présent logiciel et des documents connexes (le « logiciel »), de traiter le 7 | logiciel sans restriction, y compris, mais sans s'y limiter, les droits 8 | d'utiliser, de copier, de modifier, de fusionner, de publier, de distribuer, 9 | d'accorder une sous licence et de vendre des copies dudit logiciel, et de 10 | permettre aux personnes auxquelles le logiciel est fourni de le faire, selon 11 | les conditions suivantes : 12 | 13 | L'avis de droit d'auteur ci dessus et le présent avis de permission seront 14 | inclus dans toutes les copies et les sections importantes du logiciel. 15 | 16 | LE LOGICIEL EST FOURNI « TEL QUEL », SANS AUCUNE GARANTIE, EXPRESSE OU 17 | IMPLICITE, Y COMPRIS, MAIS SANS S'Y LIMITER, LA GARANTIE DE QUALITÉ MARCHANDE, 18 | L'ADAPTATION À UN USAGE PARTICULIER ET L'ABSENCE DE CONTREFAÇON. EN AUCUN CAS 19 | LES AUTEURS OU LES DÉTENTEURS DU DROIT D'AUTEUR NE SERONT TENUS RESPONSABLES 20 | DE TOUTE DEMANDE, DOMMAGE OU BRIS DE CONTRAT, DÉLIT CIVIL OU TOUT AUTRE 21 | MANQUEMENT LIÉ AU LOGICIEL, À SON UTILISATION OU À D'AUTRES ÉCHANGES LIÉS AU 22 | LOGICIEL. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ckanext-cloudstorage 2 | 3 | Implements support for using S3, Azure, or any of 15 different storage 4 | providers supported by [libcloud][] to [CKAN][]. 5 | 6 | # Setup 7 | 8 | After installing `ckanext-cloudstorage`, add it to your list of plugins in 9 | your `.ini`: 10 | 11 | ckan.plugins = stats cloudstorage 12 | 13 | If you haven't already, setup [CKAN file storage][ckanstorage] or the file 14 | upload button will not appear. 15 | 16 | Every driver takes two options, regardless of which one you use. Both 17 | the name of the driver and the name of the container/bucket are 18 | case-sensitive: 19 | 20 | ckanext.cloudstorage.driver = AZURE_BLOBS 21 | ckanext.cloudstorage.container_name = demo 22 | 23 | You can find a list of driver names [here][storage] (see the `Provider 24 | Constant` column.) 25 | 26 | Each driver takes its own setup options. See the [libcloud][] documentation. 27 | These options are passed in using `driver_options`, which is a Python dict. 28 | For most drivers, this is all you need: 29 | 30 | ckanext.cloudstorage.driver_options = {"key": "", "secret": ""} 31 | 32 | # Support 33 | 34 | Most libcloud-based providers should work out of the box, but only those listed 35 | below have been tested: 36 | 37 | | Provider | Uploads | Downloads | Secure URLs (private resources) | 38 | | --- | --- | --- | --- | 39 | | Azure | YES | YES | YES (if `azure-storage` is installed) | 40 | | AWS S3 | YES | YES | YES (if `boto` is installed) | 41 | | Rackspace | YES | YES | No | 42 | 43 | # What are "Secure URLs"? 44 | 45 | "Secure URLs" are a method of preventing access to private resources. By 46 | default, anyone that figures out the URL to your resource on your storage 47 | provider can download it. Secure URLs allow you to disable public access and 48 | instead let ckanext-cloudstorage generate temporary, one-use URLs to download 49 | the resource. This means that the normal CKAN-provided access restrictions can 50 | apply to resources with no further effort on your part, but still get all the 51 | benefits of your CDN/blob storage. 52 | 53 | ckanext.cloudstorage.use_secure_urls = 1 54 | 55 | This option also enables multipart uploads, but you need to create database tables 56 | first. Run next command from extension folder: 57 | `paster cloudstorage initdb -c /etc/ckan/default/production.ini ` 58 | 59 | With that feature you can use `cloudstorage_clean_multipart` action, which is available 60 | only for sysadmins. After executing, all unfinished multipart uploads, older than 7 days, 61 | will be aborted. You can configure this lifetime, example: 62 | 63 | ckanext.cloudstorage.max_multipart_lifetime = 7 64 | 65 | # Migrating From FileStorage 66 | 67 | If you already have resources that have been uploaded and saved using CKAN's 68 | built-in FileStorage, cloudstorage provides an easy migration command. 69 | Simply setup cloudstorage as explained above, enable the plugin, and run the 70 | migrate command. Provide the path to your resources on-disk (the 71 | `ckan.storage_path` setting in your CKAN `.ini` + `/resources`), and 72 | cloudstorage will take care of the rest. Ex: 73 | 74 | paster cloudstorage migrate -c ../ckan/development.ini 75 | 76 | # Notes 77 | 78 | 1. You should disable public listing on the cloud service provider you're 79 | using, if supported. 80 | 2. Currently, only resources are supported. This means that things like group 81 | and organization images still use CKAN's local file storage. 82 | 83 | # FAQ 84 | 85 | - *DataViews aren't showing my data!* - did you setup CORS rules properly on 86 | your hosting service? ckanext-cloudstorage can try to fix them for you automatically, 87 | run: 88 | 89 | paster cloudstorage fix-cors -c= 90 | 91 | - *Help! I can't seem to get it working!* - send me a mail! tk@tkte.ch 92 | 93 | [libcloud]: https://libcloud.apache.org/ 94 | [ckan]: http://ckan.org/ 95 | [storage]: https://libcloud.readthedocs.io/en/latest/storage/supported_providers.html 96 | [ckanstorage]: http://docs.ckan.org/en/latest/maintaining/filestore.html#setup-file-uploads 97 | -------------------------------------------------------------------------------- /ckanext/__init__.py: -------------------------------------------------------------------------------- 1 | # this is a namespace package 2 | try: 3 | import pkg_resources 4 | pkg_resources.declare_namespace(__name__) 5 | except ImportError: 6 | import pkgutil 7 | __path__ = pkgutil.extend_path(__path__, __name__) 8 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TkTech/ckanext-cloudstorage/ae83a6714505ba81417343989b2be2e7cabae854/ckanext/cloudstorage/__init__.py -------------------------------------------------------------------------------- /ckanext/cloudstorage/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import os.path 5 | import cgi 6 | import tempfile 7 | 8 | from docopt import docopt 9 | from ckan.lib.cli import CkanCommand 10 | 11 | from ckanapi import LocalCKAN 12 | from ckanext.cloudstorage.storage import ( 13 | CloudStorage, 14 | ResourceCloudStorage 15 | ) 16 | from ckanext.cloudstorage.model import ( 17 | create_tables, 18 | drop_tables 19 | ) 20 | from ckan.logic import NotFound 21 | 22 | USAGE = """ckanext-cloudstorage 23 | 24 | Commands: 25 | - fix-cors Update CORS rules where possible. 26 | - migrate Upload local storage to the remote. 27 | - initdb Reinitalize database tables. 28 | 29 | Usage: 30 | cloudstorage fix-cors ... [--c=] 31 | cloudstorage migrate [] [--c=] 32 | cloudstorage initdb [--c=] 33 | 34 | Options: 35 | -c= The CKAN configuration file. 36 | """ 37 | 38 | 39 | class FakeFileStorage(cgi.FieldStorage): 40 | def __init__(self, fp, filename): 41 | self.file = fp 42 | self.filename = filename 43 | 44 | 45 | class PasterCommand(CkanCommand): 46 | summary = 'ckanext-cloudstorage maintence utilities.' 47 | usage = USAGE 48 | 49 | def command(self): 50 | self._load_config() 51 | args = docopt(USAGE, argv=self.args) 52 | 53 | if args['fix-cors']: 54 | _fix_cors(args) 55 | elif args['migrate']: 56 | _migrate(args) 57 | elif args['initdb']: 58 | _initdb() 59 | 60 | 61 | def _migrate(args): 62 | path = args[''] 63 | single_id = args[''] 64 | if not os.path.isdir(path): 65 | print('The storage directory cannot be found.') 66 | return 67 | 68 | lc = LocalCKAN() 69 | resources = {} 70 | failed = [] 71 | 72 | # The resource folder is stuctured like so on disk: 73 | # - storage/ 74 | # - ... 75 | # - resources/ 76 | # - <3 letter prefix> 77 | # - <3 letter prefix> 78 | # - 79 | # ... 80 | # ... 81 | # ... 82 | for root, dirs, files in os.walk(path): 83 | # Only the bottom level of the tree actually contains any files. We 84 | # don't care at all about the overall structure. 85 | if not files: 86 | continue 87 | 88 | split_root = root.split('/') 89 | resource_id = split_root[-2] + split_root[-1] 90 | 91 | for file_ in files: 92 | ckan_res_id = resource_id + file_ 93 | if single_id and ckan_res_id != single_id: 94 | continue 95 | 96 | resources[ckan_res_id] = os.path.join( 97 | root, 98 | file_ 99 | ) 100 | 101 | for i, resource in enumerate(resources.iteritems(), 1): 102 | resource_id, file_path = resource 103 | print('[{i}/{count}] Working on {id}'.format( 104 | i=i, 105 | count=len(resources), 106 | id=resource_id 107 | )) 108 | 109 | try: 110 | resource = lc.action.resource_show(id=resource_id) 111 | except NotFound: 112 | print(u'\tResource not found') 113 | continue 114 | if resource['url_type'] != 'upload': 115 | print(u'\t`url_type` is not `upload`. Skip') 116 | continue 117 | 118 | with open(file_path, 'rb') as fin: 119 | resource['upload'] = FakeFileStorage( 120 | fin, 121 | resource['url'].split('/')[-1] 122 | ) 123 | try: 124 | uploader = ResourceCloudStorage(resource) 125 | uploader.upload(resource['id']) 126 | except Exception as e: 127 | failed.append(resource_id) 128 | print(u'\tError of type {0} during upload: {1}'.format(type(e), e)) 129 | 130 | if failed: 131 | log_file = tempfile.NamedTemporaryFile(delete=False) 132 | log_file.file.writelines(failed) 133 | print(u'ID of all failed uploads are saved to `{0}`'.format(log_file.name)) 134 | 135 | 136 | def _fix_cors(args): 137 | cs = CloudStorage() 138 | 139 | if cs.can_use_advanced_azure: 140 | from azure.storage import blob as azure_blob 141 | from azure.storage import CorsRule 142 | 143 | blob_service = azure_blob.BlockBlobService( 144 | cs.driver_options['key'], 145 | cs.driver_options['secret'] 146 | ) 147 | 148 | blob_service.set_blob_service_properties( 149 | cors=[ 150 | CorsRule( 151 | allowed_origins=args[''], 152 | allowed_methods=['GET'] 153 | ) 154 | ] 155 | ) 156 | print('Done!') 157 | else: 158 | print( 159 | 'The driver {driver_name} being used does not currently' 160 | ' support updating CORS rules through' 161 | ' cloudstorage.'.format( 162 | driver_name=cs.driver_name 163 | ) 164 | ) 165 | 166 | 167 | def _initdb(): 168 | drop_tables() 169 | create_tables() 170 | print("DB tables are reinitialized") 171 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/controller.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os.path 4 | 5 | from pylons import c 6 | from pylons.i18n import _ 7 | 8 | from ckan import logic, model 9 | from ckan.lib import base, uploader 10 | import ckan.lib.helpers as h 11 | 12 | 13 | class StorageController(base.BaseController): 14 | def resource_download(self, id, resource_id, filename=None): 15 | context = { 16 | 'model': model, 17 | 'session': model.Session, 18 | 'user': c.user or c.author, 19 | 'auth_user_obj': c.userobj 20 | } 21 | 22 | try: 23 | resource = logic.get_action('resource_show')( 24 | context, 25 | { 26 | 'id': resource_id 27 | } 28 | ) 29 | except logic.NotFound: 30 | base.abort(404, _('Resource not found')) 31 | except logic.NotAuthorized: 32 | base.abort(401, _('Unauthorized to read resource {0}'.format(id))) 33 | 34 | # This isn't a file upload, so either redirect to the source 35 | # (if available) or error out. 36 | if resource.get('url_type') != 'upload': 37 | url = resource.get('url') 38 | if not url: 39 | base.abort(404, _('No download is available')) 40 | h.redirect_to(url) 41 | 42 | if filename is None: 43 | # No filename was provided so we'll try to get one from the url. 44 | filename = os.path.basename(resource['url']) 45 | 46 | upload = uploader.get_resource_uploader(resource) 47 | 48 | # if the client requests with a Content-Type header (e.g. Text preview) 49 | # we have to add the header to the signature 50 | try: 51 | content_type = getattr(c.pylons.request, "content_type", None) 52 | except AttributeError: 53 | content_type = None 54 | uploaded_url = upload.get_url_from_filename(resource['id'], filename, 55 | content_type=content_type) 56 | 57 | # The uploaded file is missing for some reason, such as the 58 | # provider being down. 59 | if uploaded_url is None: 60 | base.abort(404, _('No download is available')) 61 | 62 | h.redirect_to(uploaded_url) 63 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/fanstatic/scripts/cloudstorage-multipart-upload.js: -------------------------------------------------------------------------------- 1 | ckan.module('cloudstorage-multipart-upload', function($, _) { 2 | 'use strict'; 3 | 4 | return { 5 | options: { 6 | cloud: 'S3', 7 | i18n: { 8 | resource_create: _('Resource has been created.'), 9 | resource_update: _('Resource has been updated.'), 10 | undefined_upload_id: _('Undefined uploadId.'), 11 | upload_completed: _('Upload completed. You will be redirected in few seconds...'), 12 | unable_to_finish: _('Unable to finish multipart upload') 13 | } 14 | }, 15 | 16 | _partNumber: 1, 17 | 18 | _uploadId: null, 19 | _packageId: null, 20 | _resourceId: null, 21 | _uploadSize: null, 22 | _uploadName: null, 23 | _uploadedParts: null, 24 | _clickedBtn: null, 25 | _redirect_url: null, 26 | 27 | initialize: function() { 28 | $.proxyAll(this, /_on/); 29 | this.options.packageId = this.options.packageId.slice(1); 30 | this._form = this.$('form'); 31 | // this._origin = $('#field-image-upload'); 32 | // this._file = this._origin.clone() 33 | this._file = $('#field-image-upload'); 34 | this._url = $('#field-image-url'); 35 | this._save = $('[name=save]'); 36 | this._id = $('input[name=id]'); 37 | this._progress = $('
', { 38 | class: 'hide controls progress progress-striped active' 39 | }); 40 | this._bar = $('
', {class:'bar'}); 41 | this._progress.append(this._bar); 42 | this._progress.insertAfter(this._url.parent().parent()); 43 | this._resumeBtn = $('', {class: 'hide btn btn-info controls'}).insertAfter( 44 | this._progress).text('Resume Upload'); 45 | 46 | var self = this; 47 | 48 | this._file.fileupload({ 49 | url: this.sandbox.client.url('/api/action/cloudstorage_upload_multipart'), 50 | maxChunkSize: 5 * 1024 * 1024, 51 | replaceFileInput: false, 52 | formData: this._onGenerateAdditionalData, 53 | submit: this._onUploadFileSubmit, 54 | chunkdone: this._onChunkUploaded, 55 | add: this._onFileUploadAdd, 56 | progressall: this._onFileUploadProgress, 57 | done: this._onFinishUpload, 58 | fail: this._onUploadFail, 59 | always: this._onAnyEndedUpload 60 | }); 61 | 62 | this._save.on('click', this._onSaveClick); 63 | 64 | this._onCheckExistingMultipart('choose'); 65 | }, 66 | 67 | _onChunkUploaded: function () { 68 | this._uploadedParts = this._partNumber++; 69 | }, 70 | 71 | _onCheckExistingMultipart: function (operation) { 72 | var self = this; 73 | var id = this._id.val(); 74 | if (!id) return; 75 | this.sandbox.client.call( 76 | 'POST', 77 | 'cloudstorage_check_multipart', 78 | {id: id}, 79 | function (data) { 80 | if (!data.result) return; 81 | var upload = data.result.upload; 82 | 83 | var name = upload.name.slice(upload.name.lastIndexOf('/')+1); 84 | self._uploadId = upload.id; 85 | self._uploadSize = upload.size; 86 | self._uploadedParts = upload.parts; 87 | self._uploadName = upload.original_name; 88 | self._partNumber = self._uploadedParts + 1; 89 | 90 | 91 | var current_chunk_size = self._file.fileupload('option', 'maxChunkSize'); 92 | var uploaded_bytes = current_chunk_size * upload.parts; 93 | self._file.fileupload('option', 'uploadedBytes', uploaded_bytes); 94 | 95 | self.sandbox.notify( 96 | 'Incomplete upload', 97 | 'File: ' + upload.original_name + 98 | '; Size: ' + self._uploadSize, 99 | 'warning'); 100 | self._onEnableResumeBtn(operation); 101 | }, 102 | function (error) { 103 | console.log(error); 104 | setTimeout(function() { 105 | self._onCheckExistingMultipart(operation); 106 | }, 2000); 107 | } 108 | 109 | ); 110 | }, 111 | 112 | _onEnableResumeBtn: function (operation) { 113 | var self = this; 114 | this.$('.btn-remove-url').remove(); 115 | if (operation === 'choose'){ 116 | self._onDisableSave(true); 117 | 118 | } 119 | this._resumeBtn 120 | .off('click') 121 | .on('click', function (event) { 122 | switch (operation) { 123 | case 'resume': 124 | self._save.trigger('click'); 125 | self._onDisableResumeBtn(); 126 | break; 127 | case 'choose': 128 | default: 129 | self._file.trigger('click'); 130 | break; 131 | } 132 | }) 133 | .show(); 134 | }, 135 | 136 | _onDisableResumeBtn: function () { 137 | this._resumeBtn.hide(); 138 | }, 139 | 140 | _onUploadFail: function (e, data) { 141 | this._onHandleError('Upload fail'); 142 | this._onCheckExistingMultipart('resume'); 143 | }, 144 | 145 | _onUploadFileSubmit: function (event, data) { 146 | if (!this._uploadId) { 147 | this._onDisableSave(false); 148 | this.sandbox.notify( 149 | 'Upload error', 150 | this.i18n('undefined_upload_id'), 151 | 'error' 152 | ); 153 | return false; 154 | } 155 | 156 | this._setProgressType('info', this._progress); 157 | this._progress.show('slow'); 158 | }, 159 | 160 | _onGenerateAdditionalData: function (form) { 161 | return [ 162 | { 163 | name: 'partNumber', 164 | value: this._partNumber 165 | }, 166 | { 167 | name: 'uploadId', 168 | value: this._uploadId 169 | }, 170 | { 171 | name: 'id', 172 | value: this._resourceId 173 | } 174 | 175 | ]; 176 | }, 177 | 178 | _onAnyEndedUpload: function () { 179 | this._partNumber = 1; 180 | }, 181 | 182 | _countChunkSize: function (size, chunk) { 183 | while (size / chunk > 10000) chunk *= 2; 184 | return chunk; 185 | }, 186 | 187 | _onFileUploadAdd: function (event, data) { 188 | this._setProgress(0, this._bar); 189 | var file = data.files[0]; 190 | var target = $(event.target); 191 | 192 | var chunkSize = this._countChunkSize(file.size, target.fileupload('option', 'maxChunkSize')); 193 | 194 | if (this._uploadName && this._uploadSize && this._uploadedParts !== null) { 195 | if (this._uploadSize !== file.size || this._uploadName !== file.name){ 196 | this._file.val(''); 197 | this._onCleanUpload(); 198 | this.sandbox.notify( 199 | 'Mismatch file', 200 | 'You are trying to upload wrong file. Cancel previous upload first.', 201 | 'error' 202 | ); 203 | event.preventDefault(); 204 | throw 'Wrong file'; 205 | } 206 | 207 | 208 | var loaded = chunkSize * this._uploadedParts; 209 | 210 | // target.fileupload('option', 'uploadedBytes', loaded); 211 | this._onFileUploadProgress(event, { 212 | total: file.size, 213 | loaded: loaded 214 | }); 215 | 216 | this._progress.show('slow'); 217 | this._onDisableResumeBtn(); 218 | this._save.trigger('click'); 219 | 220 | if (loaded >= file.size){ 221 | this._onFinishUpload(); 222 | } 223 | 224 | } 225 | 226 | 227 | target.fileupload('option', 'maxChunkSize', chunkSize); 228 | 229 | this.el.off('multipartstarted.cloudstorage'); 230 | this.el.on('multipartstarted.cloudstorage', function () { 231 | data.submit(); 232 | }); 233 | }, 234 | 235 | _onFileUploadProgress: function (event, data) { 236 | var progress = 100 / (data.total / data.loaded); 237 | this._setProgress(progress, this._bar); 238 | }, 239 | 240 | _onSaveClick: function(event, pass) { 241 | if (pass || !window.FileList || !this._file || !this._file.val()) { 242 | return; 243 | } 244 | event.preventDefault(); 245 | var dataset_id = this.options.packageId; 246 | this._clickedBtn = $(event.target).attr('value'); 247 | if (this._clickedBtn == 'go-dataset') { 248 | this._onDisableSave(false); 249 | this._redirect_url = this.sandbox.url( 250 | '/dataset/edit/' + 251 | dataset_id); 252 | window.location = this._redirect_url; 253 | } else { 254 | try{ 255 | this._onDisableSave(true); 256 | this._onSaveForm(); 257 | } catch(error){ 258 | console.log(error); 259 | this._onDisableSave(false); 260 | } 261 | } 262 | 263 | // this._form.trigger('submit', true); 264 | }, 265 | 266 | _onSaveForm: function() { 267 | var file = this._file[0].files[0]; 268 | var self = this; 269 | var formData = this._form.serializeArray().reduce( 270 | function (result, item) { 271 | result[item.name] = item.value; 272 | return result; 273 | }, {}); 274 | 275 | formData.multipart_name = file.name; 276 | formData.url = file.name; 277 | formData.package_id = this.options.packageId; 278 | formData.size = file.size; 279 | formData.url_type = 'upload'; 280 | var action = formData.id ? 'resource_update' : 'resource_create'; 281 | var url = this._form.attr('action') || window.location.href; 282 | this.sandbox.client.call( 283 | 'POST', 284 | action, 285 | formData, 286 | function (data) { 287 | var result = data.result; 288 | self._packageId = result.package_id; 289 | self._resourceId = result.id; 290 | 291 | self._id.val(result.id); 292 | self.sandbox.notify( 293 | result.id, 294 | self.i18n(action, {id: result.id}), 295 | 'success' 296 | ); 297 | self._onPerformUpload(file); 298 | }, 299 | function (err, st, msg) { 300 | self.sandbox.notify( 301 | 'Error', 302 | msg, 303 | 'error' 304 | ); 305 | self._onHandleError('Unable to save resource'); 306 | } 307 | ); 308 | 309 | }, 310 | 311 | 312 | _onPerformUpload: function(file) { 313 | var id = this._id.val(); 314 | var self = this; 315 | if (this._uploadId === null) 316 | this._onPrepareUpload(file, id).then( 317 | function (data) { 318 | self._uploadId = data.result.id; 319 | self.el.trigger('multipartstarted.cloudstorage'); 320 | }, 321 | function (err) { 322 | console.log(err); 323 | self._onHandleError('Unable to initiate multipart upload'); 324 | } 325 | ); 326 | else 327 | this.el.trigger('multipartstarted.cloudstorage'); 328 | 329 | }, 330 | 331 | _onPrepareUpload: function(file, id) { 332 | 333 | return $.ajax({ 334 | method: 'POST', 335 | url: this.sandbox.client.url('/api/action/cloudstorage_initiate_multipart'), 336 | data: JSON.stringify({ 337 | id: id, 338 | name: encodeURIComponent(file.name), 339 | size: file.size 340 | }) 341 | }); 342 | 343 | }, 344 | 345 | _onAbortUpload: function(id) { 346 | var self = this; 347 | this.sandbox.client.call( 348 | 'POST', 349 | 'cloudstorage_abort_multipart', 350 | { 351 | id: id 352 | }, 353 | function (data) { 354 | console.log(data); 355 | }, 356 | function (err) { 357 | console.log(err); 358 | self._onHandleError('Unable to abort multipart upload'); 359 | } 360 | ); 361 | 362 | }, 363 | 364 | _onFinishUpload: function() { 365 | var self = this; 366 | var data_dict = { 367 | 'uploadId': this._uploadId, 368 | 'id': this._resourceId, 369 | 'save_action': this._clickedBtn 370 | } 371 | this.sandbox.client.call( 372 | 'POST', 373 | 'cloudstorage_finish_multipart', 374 | data_dict, 375 | function (data) { 376 | 377 | self._progress.hide('fast'); 378 | self._onDisableSave(false); 379 | 380 | if (self._resourceId && self._packageId){ 381 | self.sandbox.notify( 382 | 'Success', 383 | self.i18n('upload_completed'), 384 | 'success' 385 | ); 386 | // self._form.remove(); 387 | if (self._clickedBtn == 'again') { 388 | this._redirect_url = self.sandbox.url( 389 | '/dataset/new_resource/' + 390 | self._packageId 391 | ); 392 | } else { 393 | this._redirect_url = self.sandbox.url( 394 | '/dataset/' + 395 | self._packageId 396 | ); 397 | } 398 | self._form.attr('action', this._redirect_url); 399 | self._form.attr('method', 'GET'); 400 | self.$('[name]').attr('name', null); 401 | setTimeout(function(){ 402 | self._form.submit(); 403 | }, 3000); 404 | 405 | } 406 | }, 407 | function (err) { 408 | console.log(err); 409 | self._onHandleError(self.i18n('unable_to_finish')); 410 | } 411 | ); 412 | this._setProgressType('success', this._progress); 413 | }, 414 | 415 | _onDisableSave: function (value) { 416 | this._save.attr('disabled', value); 417 | }, 418 | 419 | _setProgress: function (progress, bar) { 420 | bar.css('width', progress + '%'); 421 | }, 422 | 423 | _setProgressType: function (type, progress) { 424 | progress 425 | .removeClass('progress-success progress-danger progress-info') 426 | .addClass('progress-' + type); 427 | }, 428 | 429 | _onHandleError: function (msg) { 430 | this.sandbox.notify( 431 | 'Error', 432 | msg, 433 | 'error' 434 | ); 435 | this._onDisableSave(false); 436 | }, 437 | 438 | _onCleanUpload: function () { 439 | this.$('.btn-remove-url').trigger('click'); 440 | } 441 | 442 | }; 443 | }); 444 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/fanstatic/scripts/resource.config: -------------------------------------------------------------------------------- 1 | [depends] 2 | 3 | main = base/main 4 | 5 | [groups] 6 | 7 | main = 8 | vendor/jquery-widget.js 9 | vendor/file-upload.js 10 | cloudstorage-multipart-upload.js 11 | 12 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/fanstatic/scripts/vendor/file-upload.js: -------------------------------------------------------------------------------- 1 | /* 2 | * jQuery File Upload Plugin 3 | * https://github.com/blueimp/jQuery-File-Upload 4 | * 5 | * Copyright 2010, Sebastian Tschan 6 | * https://blueimp.net 7 | * 8 | * Licensed under the MIT license: 9 | * http://www.opensource.org/licenses/MIT 10 | */ 11 | 12 | /* jshint nomen:false */ 13 | /* global define, require, window, document, location, Blob, FormData */ 14 | 15 | ;(function (factory) { 16 | 'use strict'; 17 | if (typeof define === 'function' && define.amd) { 18 | // Register as an anonymous AMD module: 19 | define([ 20 | 'jquery', 21 | 'jquery.ui.widget' 22 | ], factory); 23 | } else if (typeof exports === 'object') { 24 | // Node/CommonJS: 25 | factory( 26 | require('jquery'), 27 | require('./vendor/jquery.ui.widget') 28 | ); 29 | } else { 30 | // Browser globals: 31 | factory(window.jQuery); 32 | } 33 | }(function ($) { 34 | 'use strict'; 35 | 36 | // Detect file input support, based on 37 | // http://viljamis.com/blog/2012/file-upload-support-on-mobile/ 38 | $.support.fileInput = !(new RegExp( 39 | // Handle devices which give false positives for the feature detection: 40 | '(Android (1\\.[0156]|2\\.[01]))' + 41 | '|(Windows Phone (OS 7|8\\.0))|(XBLWP)|(ZuneWP)|(WPDesktop)' + 42 | '|(w(eb)?OSBrowser)|(webOS)' + 43 | '|(Kindle/(1\\.0|2\\.[05]|3\\.0))' 44 | ).test(window.navigator.userAgent) || 45 | // Feature detection for all other devices: 46 | $('').prop('disabled')); 47 | 48 | // The FileReader API is not actually used, but works as feature detection, 49 | // as some Safari versions (5?) support XHR file uploads via the FormData API, 50 | // but not non-multipart XHR file uploads. 51 | // window.XMLHttpRequestUpload is not available on IE10, so we check for 52 | // window.ProgressEvent instead to detect XHR2 file upload capability: 53 | $.support.xhrFileUpload = !!(window.ProgressEvent && window.FileReader); 54 | $.support.xhrFormDataFileUpload = !!window.FormData; 55 | 56 | // Detect support for Blob slicing (required for chunked uploads): 57 | $.support.blobSlice = window.Blob && (Blob.prototype.slice || 58 | Blob.prototype.webkitSlice || Blob.prototype.mozSlice); 59 | 60 | // Helper function to create drag handlers for dragover/dragenter/dragleave: 61 | function getDragHandler(type) { 62 | var isDragOver = type === 'dragover'; 63 | return function (e) { 64 | e.dataTransfer = e.originalEvent && e.originalEvent.dataTransfer; 65 | var dataTransfer = e.dataTransfer; 66 | if (dataTransfer && $.inArray('Files', dataTransfer.types) !== -1 && 67 | this._trigger( 68 | type, 69 | $.Event(type, {delegatedEvent: e}) 70 | ) !== false) { 71 | e.preventDefault(); 72 | if (isDragOver) { 73 | dataTransfer.dropEffect = 'copy'; 74 | } 75 | } 76 | }; 77 | } 78 | 79 | // The fileupload widget listens for change events on file input fields defined 80 | // via fileInput setting and paste or drop events of the given dropZone. 81 | // In addition to the default jQuery Widget methods, the fileupload widget 82 | // exposes the "add" and "send" methods, to add or directly send files using 83 | // the fileupload API. 84 | // By default, files added via file input selection, paste, drag & drop or 85 | // "add" method are uploaded immediately, but it is possible to override 86 | // the "add" callback option to queue file uploads. 87 | $.widget('blueimp.fileupload', { 88 | 89 | options: { 90 | // The drop target element(s), by the default the complete document. 91 | // Set to null to disable drag & drop support: 92 | dropZone: $(document), 93 | // The paste target element(s), by the default undefined. 94 | // Set to a DOM node or jQuery object to enable file pasting: 95 | pasteZone: undefined, 96 | // The file input field(s), that are listened to for change events. 97 | // If undefined, it is set to the file input fields inside 98 | // of the widget element on plugin initialization. 99 | // Set to null to disable the change listener. 100 | fileInput: undefined, 101 | // By default, the file input field is replaced with a clone after 102 | // each input field change event. This is required for iframe transport 103 | // queues and allows change events to be fired for the same file 104 | // selection, but can be disabled by setting the following option to false: 105 | replaceFileInput: true, 106 | // The parameter name for the file form data (the request argument name). 107 | // If undefined or empty, the name property of the file input field is 108 | // used, or "files[]" if the file input name property is also empty, 109 | // can be a string or an array of strings: 110 | paramName: undefined, 111 | // By default, each file of a selection is uploaded using an individual 112 | // request for XHR type uploads. Set to false to upload file 113 | // selections in one request each: 114 | singleFileUploads: true, 115 | // To limit the number of files uploaded with one XHR request, 116 | // set the following option to an integer greater than 0: 117 | limitMultiFileUploads: undefined, 118 | // The following option limits the number of files uploaded with one 119 | // XHR request to keep the request size under or equal to the defined 120 | // limit in bytes: 121 | limitMultiFileUploadSize: undefined, 122 | // Multipart file uploads add a number of bytes to each uploaded file, 123 | // therefore the following option adds an overhead for each file used 124 | // in the limitMultiFileUploadSize configuration: 125 | limitMultiFileUploadSizeOverhead: 512, 126 | // Set the following option to true to issue all file upload requests 127 | // in a sequential order: 128 | sequentialUploads: false, 129 | // To limit the number of concurrent uploads, 130 | // set the following option to an integer greater than 0: 131 | limitConcurrentUploads: undefined, 132 | // Set the following option to true to force iframe transport uploads: 133 | forceIframeTransport: false, 134 | // Set the following option to the location of a redirect url on the 135 | // origin server, for cross-domain iframe transport uploads: 136 | redirect: undefined, 137 | // The parameter name for the redirect url, sent as part of the form 138 | // data and set to 'redirect' if this option is empty: 139 | redirectParamName: undefined, 140 | // Set the following option to the location of a postMessage window, 141 | // to enable postMessage transport uploads: 142 | postMessage: undefined, 143 | // By default, XHR file uploads are sent as multipart/form-data. 144 | // The iframe transport is always using multipart/form-data. 145 | // Set to false to enable non-multipart XHR uploads: 146 | multipart: true, 147 | // To upload large files in smaller chunks, set the following option 148 | // to a preferred maximum chunk size. If set to 0, null or undefined, 149 | // or the browser does not support the required Blob API, files will 150 | // be uploaded as a whole. 151 | maxChunkSize: undefined, 152 | // When a non-multipart upload or a chunked multipart upload has been 153 | // aborted, this option can be used to resume the upload by setting 154 | // it to the size of the already uploaded bytes. This option is most 155 | // useful when modifying the options object inside of the "add" or 156 | // "send" callbacks, as the options are cloned for each file upload. 157 | uploadedBytes: undefined, 158 | // By default, failed (abort or error) file uploads are removed from the 159 | // global progress calculation. Set the following option to false to 160 | // prevent recalculating the global progress data: 161 | recalculateProgress: true, 162 | // Interval in milliseconds to calculate and trigger progress events: 163 | progressInterval: 100, 164 | // Interval in milliseconds to calculate progress bitrate: 165 | bitrateInterval: 500, 166 | // By default, uploads are started automatically when adding files: 167 | autoUpload: true, 168 | 169 | // Error and info messages: 170 | messages: { 171 | uploadedBytes: 'Uploaded bytes exceed file size' 172 | }, 173 | 174 | // Translation function, gets the message key to be translated 175 | // and an object with context specific data as arguments: 176 | i18n: function (message, context) { 177 | message = this.messages[message] || message.toString(); 178 | if (context) { 179 | $.each(context, function (key, value) { 180 | message = message.replace('{' + key + '}', value); 181 | }); 182 | } 183 | return message; 184 | }, 185 | 186 | // Additional form data to be sent along with the file uploads can be set 187 | // using this option, which accepts an array of objects with name and 188 | // value properties, a function returning such an array, a FormData 189 | // object (for XHR file uploads), or a simple object. 190 | // The form of the first fileInput is given as parameter to the function: 191 | formData: function (form) { 192 | return form.serializeArray(); 193 | }, 194 | 195 | // The add callback is invoked as soon as files are added to the fileupload 196 | // widget (via file input selection, drag & drop, paste or add API call). 197 | // If the singleFileUploads option is enabled, this callback will be 198 | // called once for each file in the selection for XHR file uploads, else 199 | // once for each file selection. 200 | // 201 | // The upload starts when the submit method is invoked on the data parameter. 202 | // The data object contains a files property holding the added files 203 | // and allows you to override plugin options as well as define ajax settings. 204 | // 205 | // Listeners for this callback can also be bound the following way: 206 | // .bind('fileuploadadd', func); 207 | // 208 | // data.submit() returns a Promise object and allows to attach additional 209 | // handlers using jQuery's Deferred callbacks: 210 | // data.submit().done(func).fail(func).always(func); 211 | add: function (e, data) { 212 | if (e.isDefaultPrevented()) { 213 | return false; 214 | } 215 | if (data.autoUpload || (data.autoUpload !== false && 216 | $(this).fileupload('option', 'autoUpload'))) { 217 | data.process().done(function () { 218 | data.submit(); 219 | }); 220 | } 221 | }, 222 | 223 | // Other callbacks: 224 | 225 | // Callback for the submit event of each file upload: 226 | // submit: function (e, data) {}, // .bind('fileuploadsubmit', func); 227 | 228 | // Callback for the start of each file upload request: 229 | // send: function (e, data) {}, // .bind('fileuploadsend', func); 230 | 231 | // Callback for successful uploads: 232 | // done: function (e, data) {}, // .bind('fileuploaddone', func); 233 | 234 | // Callback for failed (abort or error) uploads: 235 | // fail: function (e, data) {}, // .bind('fileuploadfail', func); 236 | 237 | // Callback for completed (success, abort or error) requests: 238 | // always: function (e, data) {}, // .bind('fileuploadalways', func); 239 | 240 | // Callback for upload progress events: 241 | // progress: function (e, data) {}, // .bind('fileuploadprogress', func); 242 | 243 | // Callback for global upload progress events: 244 | // progressall: function (e, data) {}, // .bind('fileuploadprogressall', func); 245 | 246 | // Callback for uploads start, equivalent to the global ajaxStart event: 247 | // start: function (e) {}, // .bind('fileuploadstart', func); 248 | 249 | // Callback for uploads stop, equivalent to the global ajaxStop event: 250 | // stop: function (e) {}, // .bind('fileuploadstop', func); 251 | 252 | // Callback for change events of the fileInput(s): 253 | // change: function (e, data) {}, // .bind('fileuploadchange', func); 254 | 255 | // Callback for paste events to the pasteZone(s): 256 | // paste: function (e, data) {}, // .bind('fileuploadpaste', func); 257 | 258 | // Callback for drop events of the dropZone(s): 259 | // drop: function (e, data) {}, // .bind('fileuploaddrop', func); 260 | 261 | // Callback for dragover events of the dropZone(s): 262 | // dragover: function (e) {}, // .bind('fileuploaddragover', func); 263 | 264 | // Callback for the start of each chunk upload request: 265 | // chunksend: function (e, data) {}, // .bind('fileuploadchunksend', func); 266 | 267 | // Callback for successful chunk uploads: 268 | // chunkdone: function (e, data) {}, // .bind('fileuploadchunkdone', func); 269 | 270 | // Callback for failed (abort or error) chunk uploads: 271 | // chunkfail: function (e, data) {}, // .bind('fileuploadchunkfail', func); 272 | 273 | // Callback for completed (success, abort or error) chunk upload requests: 274 | // chunkalways: function (e, data) {}, // .bind('fileuploadchunkalways', func); 275 | 276 | // The plugin options are used as settings object for the ajax calls. 277 | // The following are jQuery ajax settings required for the file uploads: 278 | processData: false, 279 | contentType: false, 280 | cache: false, 281 | timeout: 0 282 | }, 283 | 284 | // A list of options that require reinitializing event listeners and/or 285 | // special initialization code: 286 | _specialOptions: [ 287 | 'fileInput', 288 | 'dropZone', 289 | 'pasteZone', 290 | 'multipart', 291 | 'forceIframeTransport' 292 | ], 293 | 294 | _blobSlice: $.support.blobSlice && function () { 295 | var slice = this.slice || this.webkitSlice || this.mozSlice; 296 | return slice.apply(this, arguments); 297 | }, 298 | 299 | _BitrateTimer: function () { 300 | this.timestamp = ((Date.now) ? Date.now() : (new Date()).getTime()); 301 | this.loaded = 0; 302 | this.bitrate = 0; 303 | this.getBitrate = function (now, loaded, interval) { 304 | var timeDiff = now - this.timestamp; 305 | if (!this.bitrate || !interval || timeDiff > interval) { 306 | this.bitrate = (loaded - this.loaded) * (1000 / timeDiff) * 8; 307 | this.loaded = loaded; 308 | this.timestamp = now; 309 | } 310 | return this.bitrate; 311 | }; 312 | }, 313 | 314 | _isXHRUpload: function (options) { 315 | return !options.forceIframeTransport && 316 | ((!options.multipart && $.support.xhrFileUpload) || 317 | $.support.xhrFormDataFileUpload); 318 | }, 319 | 320 | _getFormData: function (options) { 321 | var formData; 322 | if ($.type(options.formData) === 'function') { 323 | return options.formData(options.form); 324 | } 325 | if ($.isArray(options.formData)) { 326 | return options.formData; 327 | } 328 | if ($.type(options.formData) === 'object') { 329 | formData = []; 330 | $.each(options.formData, function (name, value) { 331 | formData.push({name: name, value: value}); 332 | }); 333 | return formData; 334 | } 335 | return []; 336 | }, 337 | 338 | _getTotal: function (files) { 339 | var total = 0; 340 | $.each(files, function (index, file) { 341 | total += file.size || 1; 342 | }); 343 | return total; 344 | }, 345 | 346 | _initProgressObject: function (obj) { 347 | var progress = { 348 | loaded: 0, 349 | total: 0, 350 | bitrate: 0 351 | }; 352 | if (obj._progress) { 353 | $.extend(obj._progress, progress); 354 | } else { 355 | obj._progress = progress; 356 | } 357 | }, 358 | 359 | _initResponseObject: function (obj) { 360 | var prop; 361 | if (obj._response) { 362 | for (prop in obj._response) { 363 | if (obj._response.hasOwnProperty(prop)) { 364 | delete obj._response[prop]; 365 | } 366 | } 367 | } else { 368 | obj._response = {}; 369 | } 370 | }, 371 | 372 | _onProgress: function (e, data) { 373 | if (e.lengthComputable) { 374 | var now = ((Date.now) ? Date.now() : (new Date()).getTime()), 375 | loaded; 376 | if (data._time && data.progressInterval && 377 | (now - data._time < data.progressInterval) && 378 | e.loaded !== e.total) { 379 | return; 380 | } 381 | data._time = now; 382 | loaded = Math.floor( 383 | e.loaded / e.total * (data.chunkSize || data._progress.total) 384 | ) + (data.uploadedBytes || 0); 385 | // Add the difference from the previously loaded state 386 | // to the global loaded counter: 387 | this._progress.loaded += (loaded - data._progress.loaded); 388 | this._progress.bitrate = this._bitrateTimer.getBitrate( 389 | now, 390 | this._progress.loaded, 391 | data.bitrateInterval 392 | ); 393 | data._progress.loaded = data.loaded = loaded; 394 | data._progress.bitrate = data.bitrate = data._bitrateTimer.getBitrate( 395 | now, 396 | loaded, 397 | data.bitrateInterval 398 | ); 399 | // Trigger a custom progress event with a total data property set 400 | // to the file size(s) of the current upload and a loaded data 401 | // property calculated accordingly: 402 | this._trigger( 403 | 'progress', 404 | $.Event('progress', {delegatedEvent: e}), 405 | data 406 | ); 407 | // Trigger a global progress event for all current file uploads, 408 | // including ajax calls queued for sequential file uploads: 409 | this._trigger( 410 | 'progressall', 411 | $.Event('progressall', {delegatedEvent: e}), 412 | this._progress 413 | ); 414 | } 415 | }, 416 | 417 | _initProgressListener: function (options) { 418 | var that = this, 419 | xhr = options.xhr ? options.xhr() : $.ajaxSettings.xhr(); 420 | // Accesss to the native XHR object is required to add event listeners 421 | // for the upload progress event: 422 | if (xhr.upload) { 423 | $(xhr.upload).bind('progress', function (e) { 424 | var oe = e.originalEvent; 425 | // Make sure the progress event properties get copied over: 426 | e.lengthComputable = oe.lengthComputable; 427 | e.loaded = oe.loaded; 428 | e.total = oe.total; 429 | that._onProgress(e, options); 430 | }); 431 | options.xhr = function () { 432 | return xhr; 433 | }; 434 | } 435 | }, 436 | 437 | _isInstanceOf: function (type, obj) { 438 | // Cross-frame instanceof check 439 | return Object.prototype.toString.call(obj) === '[object ' + type + ']'; 440 | }, 441 | 442 | _initXHRData: function (options) { 443 | var that = this, 444 | formData, 445 | file = options.files[0], 446 | // Ignore non-multipart setting if not supported: 447 | multipart = options.multipart || !$.support.xhrFileUpload, 448 | paramName = $.type(options.paramName) === 'array' ? 449 | options.paramName[0] : options.paramName; 450 | options.headers = $.extend({}, options.headers); 451 | if (options.contentRange) { 452 | options.headers['Content-Range'] = options.contentRange; 453 | } 454 | if (!multipart || options.blob || !this._isInstanceOf('File', file)) { 455 | options.headers['Content-Disposition'] = 'attachment; filename="' + 456 | encodeURI(file.name) + '"'; 457 | } 458 | if (!multipart) { 459 | options.contentType = file.type || 'application/octet-stream'; 460 | options.data = options.blob || file; 461 | } else if ($.support.xhrFormDataFileUpload) { 462 | if (options.postMessage) { 463 | // window.postMessage does not allow sending FormData 464 | // objects, so we just add the File/Blob objects to 465 | // the formData array and let the postMessage window 466 | // create the FormData object out of this array: 467 | formData = this._getFormData(options); 468 | if (options.blob) { 469 | formData.push({ 470 | name: paramName, 471 | value: options.blob 472 | }); 473 | } else { 474 | $.each(options.files, function (index, file) { 475 | formData.push({ 476 | name: ($.type(options.paramName) === 'array' && 477 | options.paramName[index]) || paramName, 478 | value: file 479 | }); 480 | }); 481 | } 482 | } else { 483 | if (that._isInstanceOf('FormData', options.formData)) { 484 | formData = options.formData; 485 | } else { 486 | formData = new FormData(); 487 | $.each(this._getFormData(options), function (index, field) { 488 | formData.append(field.name, field.value); 489 | }); 490 | } 491 | if (options.blob) { 492 | formData.append(paramName, options.blob, file.name); 493 | } else { 494 | $.each(options.files, function (index, file) { 495 | // This check allows the tests to run with 496 | // dummy objects: 497 | if (that._isInstanceOf('File', file) || 498 | that._isInstanceOf('Blob', file)) { 499 | formData.append( 500 | ($.type(options.paramName) === 'array' && 501 | options.paramName[index]) || paramName, 502 | file, 503 | file.uploadName || file.name 504 | ); 505 | } 506 | }); 507 | } 508 | } 509 | options.data = formData; 510 | } 511 | // Blob reference is not needed anymore, free memory: 512 | options.blob = null; 513 | }, 514 | 515 | _initIframeSettings: function (options) { 516 | var targetHost = $('').prop('href', options.url).prop('host'); 517 | // Setting the dataType to iframe enables the iframe transport: 518 | options.dataType = 'iframe ' + (options.dataType || ''); 519 | // The iframe transport accepts a serialized array as form data: 520 | options.formData = this._getFormData(options); 521 | // Add redirect url to form data on cross-domain uploads: 522 | if (options.redirect && targetHost && targetHost !== location.host) { 523 | options.formData.push({ 524 | name: options.redirectParamName || 'redirect', 525 | value: options.redirect 526 | }); 527 | } 528 | }, 529 | 530 | _initDataSettings: function (options) { 531 | if (this._isXHRUpload(options)) { 532 | if (!this._chunkedUpload(options, true)) { 533 | if (!options.data) { 534 | this._initXHRData(options); 535 | } 536 | this._initProgressListener(options); 537 | } 538 | if (options.postMessage) { 539 | // Setting the dataType to postmessage enables the 540 | // postMessage transport: 541 | options.dataType = 'postmessage ' + (options.dataType || ''); 542 | } 543 | } else { 544 | this._initIframeSettings(options); 545 | } 546 | }, 547 | 548 | _getParamName: function (options) { 549 | var fileInput = $(options.fileInput), 550 | paramName = options.paramName; 551 | if (!paramName) { 552 | paramName = []; 553 | fileInput.each(function () { 554 | var input = $(this), 555 | name = input.prop('name') || 'files[]', 556 | i = (input.prop('files') || [1]).length; 557 | while (i) { 558 | paramName.push(name); 559 | i -= 1; 560 | } 561 | }); 562 | if (!paramName.length) { 563 | paramName = [fileInput.prop('name') || 'files[]']; 564 | } 565 | } else if (!$.isArray(paramName)) { 566 | paramName = [paramName]; 567 | } 568 | return paramName; 569 | }, 570 | 571 | _initFormSettings: function (options) { 572 | // Retrieve missing options from the input field and the 573 | // associated form, if available: 574 | if (!options.form || !options.form.length) { 575 | options.form = $(options.fileInput.prop('form')); 576 | // If the given file input doesn't have an associated form, 577 | // use the default widget file input's form: 578 | if (!options.form.length) { 579 | options.form = $(this.options.fileInput.prop('form')); 580 | } 581 | } 582 | options.paramName = this._getParamName(options); 583 | if (!options.url) { 584 | options.url = options.form.prop('action') || location.href; 585 | } 586 | // The HTTP request method must be "POST" or "PUT": 587 | options.type = (options.type || 588 | ($.type(options.form.prop('method')) === 'string' && 589 | options.form.prop('method')) || '' 590 | ).toUpperCase(); 591 | if (options.type !== 'POST' && options.type !== 'PUT' && 592 | options.type !== 'PATCH') { 593 | options.type = 'POST'; 594 | } 595 | if (!options.formAcceptCharset) { 596 | options.formAcceptCharset = options.form.attr('accept-charset'); 597 | } 598 | }, 599 | 600 | _getAJAXSettings: function (data) { 601 | var options = $.extend({}, this.options, data); 602 | this._initFormSettings(options); 603 | this._initDataSettings(options); 604 | return options; 605 | }, 606 | 607 | // jQuery 1.6 doesn't provide .state(), 608 | // while jQuery 1.8+ removed .isRejected() and .isResolved(): 609 | _getDeferredState: function (deferred) { 610 | if (deferred.state) { 611 | return deferred.state(); 612 | } 613 | if (deferred.isResolved()) { 614 | return 'resolved'; 615 | } 616 | if (deferred.isRejected()) { 617 | return 'rejected'; 618 | } 619 | return 'pending'; 620 | }, 621 | 622 | // Maps jqXHR callbacks to the equivalent 623 | // methods of the given Promise object: 624 | _enhancePromise: function (promise) { 625 | promise.success = promise.done; 626 | promise.error = promise.fail; 627 | promise.complete = promise.always; 628 | return promise; 629 | }, 630 | 631 | // Creates and returns a Promise object enhanced with 632 | // the jqXHR methods abort, success, error and complete: 633 | _getXHRPromise: function (resolveOrReject, context, args) { 634 | var dfd = $.Deferred(), 635 | promise = dfd.promise(); 636 | context = context || this.options.context || promise; 637 | if (resolveOrReject === true) { 638 | dfd.resolveWith(context, args); 639 | } else if (resolveOrReject === false) { 640 | dfd.rejectWith(context, args); 641 | } 642 | promise.abort = dfd.promise; 643 | return this._enhancePromise(promise); 644 | }, 645 | 646 | // Adds convenience methods to the data callback argument: 647 | _addConvenienceMethods: function (e, data) { 648 | var that = this, 649 | getPromise = function (args) { 650 | return $.Deferred().resolveWith(that, args).promise(); 651 | }; 652 | data.process = function (resolveFunc, rejectFunc) { 653 | if (resolveFunc || rejectFunc) { 654 | data._processQueue = this._processQueue = 655 | (this._processQueue || getPromise([this])).then( 656 | function () { 657 | if (data.errorThrown) { 658 | return $.Deferred() 659 | .rejectWith(that, [data]).promise(); 660 | } 661 | return getPromise(arguments); 662 | } 663 | ).then(resolveFunc, rejectFunc); 664 | } 665 | return this._processQueue || getPromise([this]); 666 | }; 667 | data.submit = function () { 668 | if (this.state() !== 'pending') { 669 | data.jqXHR = this.jqXHR = 670 | (that._trigger( 671 | 'submit', 672 | $.Event('submit', {delegatedEvent: e}), 673 | this 674 | ) !== false) && that._onSend(e, this); 675 | } 676 | return this.jqXHR || that._getXHRPromise(); 677 | }; 678 | data.abort = function () { 679 | if (this.jqXHR) { 680 | return this.jqXHR.abort(); 681 | } 682 | this.errorThrown = 'abort'; 683 | that._trigger('fail', null, this); 684 | return that._getXHRPromise(false); 685 | }; 686 | data.state = function () { 687 | if (this.jqXHR) { 688 | return that._getDeferredState(this.jqXHR); 689 | } 690 | if (this._processQueue) { 691 | return that._getDeferredState(this._processQueue); 692 | } 693 | }; 694 | data.processing = function () { 695 | return !this.jqXHR && this._processQueue && that 696 | ._getDeferredState(this._processQueue) === 'pending'; 697 | }; 698 | data.progress = function () { 699 | return this._progress; 700 | }; 701 | data.response = function () { 702 | return this._response; 703 | }; 704 | }, 705 | 706 | // Parses the Range header from the server response 707 | // and returns the uploaded bytes: 708 | _getUploadedBytes: function (jqXHR) { 709 | var range = jqXHR.getResponseHeader('Range'), 710 | parts = range && range.split('-'), 711 | upperBytesPos = parts && parts.length > 1 && 712 | parseInt(parts[1], 10); 713 | return upperBytesPos && upperBytesPos + 1; 714 | }, 715 | 716 | // Uploads a file in multiple, sequential requests 717 | // by splitting the file up in multiple blob chunks. 718 | // If the second parameter is true, only tests if the file 719 | // should be uploaded in chunks, but does not invoke any 720 | // upload requests: 721 | _chunkedUpload: function (options, testOnly) { 722 | options.uploadedBytes = options.uploadedBytes || 0; 723 | var that = this, 724 | file = options.files[0], 725 | fs = file.size, 726 | ub = options.uploadedBytes, 727 | mcs = options.maxChunkSize || fs, 728 | slice = this._blobSlice, 729 | dfd = $.Deferred(), 730 | promise = dfd.promise(), 731 | jqXHR, 732 | upload; 733 | if (!(this._isXHRUpload(options) && slice && (ub || mcs < fs)) || 734 | options.data) { 735 | return false; 736 | } 737 | if (testOnly) { 738 | return true; 739 | } 740 | if (ub >= fs) { 741 | file.error = options.i18n('uploadedBytes'); 742 | return this._getXHRPromise( 743 | false, 744 | options.context, 745 | [null, 'error', file.error] 746 | ); 747 | } 748 | // The chunk upload method: 749 | upload = function () { 750 | // Clone the options object for each chunk upload: 751 | var o = $.extend({}, options), 752 | currentLoaded = o._progress.loaded; 753 | o.blob = slice.call( 754 | file, 755 | ub, 756 | ub + mcs, 757 | file.type 758 | ); 759 | // Store the current chunk size, as the blob itself 760 | // will be dereferenced after data processing: 761 | o.chunkSize = o.blob.size; 762 | // Expose the chunk bytes position range: 763 | o.contentRange = 'bytes ' + ub + '-' + 764 | (ub + o.chunkSize - 1) + '/' + fs; 765 | // Process the upload data (the blob and potential form data): 766 | that._initXHRData(o); 767 | // Add progress listeners for this chunk upload: 768 | that._initProgressListener(o); 769 | jqXHR = ((that._trigger('chunksend', null, o) !== false && $.ajax(o)) || 770 | that._getXHRPromise(false, o.context)) 771 | .done(function (result, textStatus, jqXHR) { 772 | ub = that._getUploadedBytes(jqXHR) || 773 | (ub + o.chunkSize); 774 | // Create a progress event if no final progress event 775 | // with loaded equaling total has been triggered 776 | // for this chunk: 777 | if (currentLoaded + o.chunkSize - o._progress.loaded) { 778 | that._onProgress($.Event('progress', { 779 | lengthComputable: true, 780 | loaded: ub - o.uploadedBytes, 781 | total: ub - o.uploadedBytes 782 | }), o); 783 | } 784 | options.uploadedBytes = o.uploadedBytes = ub; 785 | o.result = result; 786 | o.textStatus = textStatus; 787 | o.jqXHR = jqXHR; 788 | that._trigger('chunkdone', null, o); 789 | that._trigger('chunkalways', null, o); 790 | if (ub < fs) { 791 | // File upload not yet complete, 792 | // continue with the next chunk: 793 | upload(); 794 | } else { 795 | dfd.resolveWith( 796 | o.context, 797 | [result, textStatus, jqXHR] 798 | ); 799 | } 800 | }) 801 | .fail(function (jqXHR, textStatus, errorThrown) { 802 | o.jqXHR = jqXHR; 803 | o.textStatus = textStatus; 804 | o.errorThrown = errorThrown; 805 | that._trigger('chunkfail', null, o); 806 | that._trigger('chunkalways', null, o); 807 | dfd.rejectWith( 808 | o.context, 809 | [jqXHR, textStatus, errorThrown] 810 | ); 811 | }); 812 | }; 813 | this._enhancePromise(promise); 814 | promise.abort = function () { 815 | return jqXHR.abort(); 816 | }; 817 | upload(); 818 | return promise; 819 | }, 820 | 821 | _beforeSend: function (e, data) { 822 | if (this._active === 0) { 823 | // the start callback is triggered when an upload starts 824 | // and no other uploads are currently running, 825 | // equivalent to the global ajaxStart event: 826 | this._trigger('start'); 827 | // Set timer for global bitrate progress calculation: 828 | this._bitrateTimer = new this._BitrateTimer(); 829 | // Reset the global progress values: 830 | this._progress.loaded = this._progress.total = 0; 831 | this._progress.bitrate = 0; 832 | } 833 | // Make sure the container objects for the .response() and 834 | // .progress() methods on the data object are available 835 | // and reset to their initial state: 836 | this._initResponseObject(data); 837 | this._initProgressObject(data); 838 | data._progress.loaded = data.loaded = data.uploadedBytes || 0; 839 | data._progress.total = data.total = this._getTotal(data.files) || 1; 840 | data._progress.bitrate = data.bitrate = 0; 841 | this._active += 1; 842 | // Initialize the global progress values: 843 | this._progress.loaded += data.loaded; 844 | this._progress.total += data.total; 845 | }, 846 | 847 | _onDone: function (result, textStatus, jqXHR, options) { 848 | var total = options._progress.total, 849 | response = options._response; 850 | if (options._progress.loaded < total) { 851 | // Create a progress event if no final progress event 852 | // with loaded equaling total has been triggered: 853 | this._onProgress($.Event('progress', { 854 | lengthComputable: true, 855 | loaded: total, 856 | total: total 857 | }), options); 858 | } 859 | response.result = options.result = result; 860 | response.textStatus = options.textStatus = textStatus; 861 | response.jqXHR = options.jqXHR = jqXHR; 862 | this._trigger('done', null, options); 863 | }, 864 | 865 | _onFail: function (jqXHR, textStatus, errorThrown, options) { 866 | var response = options._response; 867 | if (options.recalculateProgress) { 868 | // Remove the failed (error or abort) file upload from 869 | // the global progress calculation: 870 | this._progress.loaded -= options._progress.loaded; 871 | this._progress.total -= options._progress.total; 872 | } 873 | response.jqXHR = options.jqXHR = jqXHR; 874 | response.textStatus = options.textStatus = textStatus; 875 | response.errorThrown = options.errorThrown = errorThrown; 876 | this._trigger('fail', null, options); 877 | }, 878 | 879 | _onAlways: function (jqXHRorResult, textStatus, jqXHRorError, options) { 880 | // jqXHRorResult, textStatus and jqXHRorError are added to the 881 | // options object via done and fail callbacks 882 | this._trigger('always', null, options); 883 | }, 884 | 885 | _onSend: function (e, data) { 886 | if (!data.submit) { 887 | this._addConvenienceMethods(e, data); 888 | } 889 | var that = this, 890 | jqXHR, 891 | aborted, 892 | slot, 893 | pipe, 894 | options = that._getAJAXSettings(data), 895 | send = function () { 896 | that._sending += 1; 897 | // Set timer for bitrate progress calculation: 898 | options._bitrateTimer = new that._BitrateTimer(); 899 | jqXHR = jqXHR || ( 900 | ((aborted || that._trigger( 901 | 'send', 902 | $.Event('send', {delegatedEvent: e}), 903 | options 904 | ) === false) && 905 | that._getXHRPromise(false, options.context, aborted)) || 906 | that._chunkedUpload(options) || $.ajax(options) 907 | ).done(function (result, textStatus, jqXHR) { 908 | that._onDone(result, textStatus, jqXHR, options); 909 | }).fail(function (jqXHR, textStatus, errorThrown) { 910 | that._onFail(jqXHR, textStatus, errorThrown, options); 911 | }).always(function (jqXHRorResult, textStatus, jqXHRorError) { 912 | that._onAlways( 913 | jqXHRorResult, 914 | textStatus, 915 | jqXHRorError, 916 | options 917 | ); 918 | that._sending -= 1; 919 | that._active -= 1; 920 | if (options.limitConcurrentUploads && 921 | options.limitConcurrentUploads > that._sending) { 922 | // Start the next queued upload, 923 | // that has not been aborted: 924 | var nextSlot = that._slots.shift(); 925 | while (nextSlot) { 926 | if (that._getDeferredState(nextSlot) === 'pending') { 927 | nextSlot.resolve(); 928 | break; 929 | } 930 | nextSlot = that._slots.shift(); 931 | } 932 | } 933 | if (that._active === 0) { 934 | // The stop callback is triggered when all uploads have 935 | // been completed, equivalent to the global ajaxStop event: 936 | that._trigger('stop'); 937 | } 938 | }); 939 | return jqXHR; 940 | }; 941 | this._beforeSend(e, options); 942 | if (this.options.sequentialUploads || 943 | (this.options.limitConcurrentUploads && 944 | this.options.limitConcurrentUploads <= this._sending)) { 945 | if (this.options.limitConcurrentUploads > 1) { 946 | slot = $.Deferred(); 947 | this._slots.push(slot); 948 | pipe = slot.then(send); 949 | } else { 950 | this._sequence = this._sequence.then(send, send); 951 | pipe = this._sequence; 952 | } 953 | // Return the piped Promise object, enhanced with an abort method, 954 | // which is delegated to the jqXHR object of the current upload, 955 | // and jqXHR callbacks mapped to the equivalent Promise methods: 956 | pipe.abort = function () { 957 | aborted = [undefined, 'abort', 'abort']; 958 | if (!jqXHR) { 959 | if (slot) { 960 | slot.rejectWith(options.context, aborted); 961 | } 962 | return send(); 963 | } 964 | return jqXHR.abort(); 965 | }; 966 | return this._enhancePromise(pipe); 967 | } 968 | return send(); 969 | }, 970 | 971 | _onAdd: function (e, data) { 972 | var that = this, 973 | result = true, 974 | options = $.extend({}, this.options, data), 975 | files = data.files, 976 | filesLength = files.length, 977 | limit = options.limitMultiFileUploads, 978 | limitSize = options.limitMultiFileUploadSize, 979 | overhead = options.limitMultiFileUploadSizeOverhead, 980 | batchSize = 0, 981 | paramName = this._getParamName(options), 982 | paramNameSet, 983 | paramNameSlice, 984 | fileSet, 985 | i, 986 | j = 0; 987 | if (!filesLength) { 988 | return false; 989 | } 990 | if (limitSize && files[0].size === undefined) { 991 | limitSize = undefined; 992 | } 993 | if (!(options.singleFileUploads || limit || limitSize) || 994 | !this._isXHRUpload(options)) { 995 | fileSet = [files]; 996 | paramNameSet = [paramName]; 997 | } else if (!(options.singleFileUploads || limitSize) && limit) { 998 | fileSet = []; 999 | paramNameSet = []; 1000 | for (i = 0; i < filesLength; i += limit) { 1001 | fileSet.push(files.slice(i, i + limit)); 1002 | paramNameSlice = paramName.slice(i, i + limit); 1003 | if (!paramNameSlice.length) { 1004 | paramNameSlice = paramName; 1005 | } 1006 | paramNameSet.push(paramNameSlice); 1007 | } 1008 | } else if (!options.singleFileUploads && limitSize) { 1009 | fileSet = []; 1010 | paramNameSet = []; 1011 | for (i = 0; i < filesLength; i = i + 1) { 1012 | batchSize += files[i].size + overhead; 1013 | if (i + 1 === filesLength || 1014 | ((batchSize + files[i + 1].size + overhead) > limitSize) || 1015 | (limit && i + 1 - j >= limit)) { 1016 | fileSet.push(files.slice(j, i + 1)); 1017 | paramNameSlice = paramName.slice(j, i + 1); 1018 | if (!paramNameSlice.length) { 1019 | paramNameSlice = paramName; 1020 | } 1021 | paramNameSet.push(paramNameSlice); 1022 | j = i + 1; 1023 | batchSize = 0; 1024 | } 1025 | } 1026 | } else { 1027 | paramNameSet = paramName; 1028 | } 1029 | data.originalFiles = files; 1030 | $.each(fileSet || files, function (index, element) { 1031 | var newData = $.extend({}, data); 1032 | newData.files = fileSet ? element : [element]; 1033 | newData.paramName = paramNameSet[index]; 1034 | that._initResponseObject(newData); 1035 | that._initProgressObject(newData); 1036 | that._addConvenienceMethods(e, newData); 1037 | result = that._trigger( 1038 | 'add', 1039 | $.Event('add', {delegatedEvent: e}), 1040 | newData 1041 | ); 1042 | return result; 1043 | }); 1044 | return result; 1045 | }, 1046 | 1047 | _replaceFileInput: function (data) { 1048 | var input = data.fileInput, 1049 | inputClone = input.clone(true), 1050 | restoreFocus = input.is(document.activeElement); 1051 | // Add a reference for the new cloned file input to the data argument: 1052 | data.fileInputClone = inputClone; 1053 | $('
').append(inputClone)[0].reset(); 1054 | // Detaching allows to insert the fileInput on another form 1055 | // without loosing the file input value: 1056 | input.after(inputClone).detach(); 1057 | // If the fileInput had focus before it was detached, 1058 | // restore focus to the inputClone. 1059 | if (restoreFocus) { 1060 | inputClone.focus(); 1061 | } 1062 | // Avoid memory leaks with the detached file input: 1063 | $.cleanData(input.unbind('remove')); 1064 | // Replace the original file input element in the fileInput 1065 | // elements set with the clone, which has been copied including 1066 | // event handlers: 1067 | this.options.fileInput = this.options.fileInput.map(function (i, el) { 1068 | if (el === input[0]) { 1069 | return inputClone[0]; 1070 | } 1071 | return el; 1072 | }); 1073 | // If the widget has been initialized on the file input itself, 1074 | // override this.element with the file input clone: 1075 | if (input[0] === this.element[0]) { 1076 | this.element = inputClone; 1077 | } 1078 | }, 1079 | 1080 | _handleFileTreeEntry: function (entry, path) { 1081 | var that = this, 1082 | dfd = $.Deferred(), 1083 | errorHandler = function (e) { 1084 | if (e && !e.entry) { 1085 | e.entry = entry; 1086 | } 1087 | // Since $.when returns immediately if one 1088 | // Deferred is rejected, we use resolve instead. 1089 | // This allows valid files and invalid items 1090 | // to be returned together in one set: 1091 | dfd.resolve([e]); 1092 | }, 1093 | successHandler = function (entries) { 1094 | that._handleFileTreeEntries( 1095 | entries, 1096 | path + entry.name + '/' 1097 | ).done(function (files) { 1098 | dfd.resolve(files); 1099 | }).fail(errorHandler); 1100 | }, 1101 | readEntries = function () { 1102 | dirReader.readEntries(function (results) { 1103 | if (!results.length) { 1104 | successHandler(entries); 1105 | } else { 1106 | entries = entries.concat(results); 1107 | readEntries(); 1108 | } 1109 | }, errorHandler); 1110 | }, 1111 | dirReader, entries = []; 1112 | path = path || ''; 1113 | if (entry.isFile) { 1114 | if (entry._file) { 1115 | // Workaround for Chrome bug #149735 1116 | entry._file.relativePath = path; 1117 | dfd.resolve(entry._file); 1118 | } else { 1119 | entry.file(function (file) { 1120 | file.relativePath = path; 1121 | dfd.resolve(file); 1122 | }, errorHandler); 1123 | } 1124 | } else if (entry.isDirectory) { 1125 | dirReader = entry.createReader(); 1126 | readEntries(); 1127 | } else { 1128 | // Return an empy list for file system items 1129 | // other than files or directories: 1130 | dfd.resolve([]); 1131 | } 1132 | return dfd.promise(); 1133 | }, 1134 | 1135 | _handleFileTreeEntries: function (entries, path) { 1136 | var that = this; 1137 | return $.when.apply( 1138 | $, 1139 | $.map(entries, function (entry) { 1140 | return that._handleFileTreeEntry(entry, path); 1141 | }) 1142 | ).then(function () { 1143 | return Array.prototype.concat.apply( 1144 | [], 1145 | arguments 1146 | ); 1147 | }); 1148 | }, 1149 | 1150 | _getDroppedFiles: function (dataTransfer) { 1151 | dataTransfer = dataTransfer || {}; 1152 | var items = dataTransfer.items; 1153 | if (items && items.length && (items[0].webkitGetAsEntry || 1154 | items[0].getAsEntry)) { 1155 | return this._handleFileTreeEntries( 1156 | $.map(items, function (item) { 1157 | var entry; 1158 | if (item.webkitGetAsEntry) { 1159 | entry = item.webkitGetAsEntry(); 1160 | if (entry) { 1161 | // Workaround for Chrome bug #149735: 1162 | entry._file = item.getAsFile(); 1163 | } 1164 | return entry; 1165 | } 1166 | return item.getAsEntry(); 1167 | }) 1168 | ); 1169 | } 1170 | return $.Deferred().resolve( 1171 | $.makeArray(dataTransfer.files) 1172 | ).promise(); 1173 | }, 1174 | 1175 | _getSingleFileInputFiles: function (fileInput) { 1176 | fileInput = $(fileInput); 1177 | var entries = fileInput.prop('webkitEntries') || 1178 | fileInput.prop('entries'), 1179 | files, 1180 | value; 1181 | if (entries && entries.length) { 1182 | return this._handleFileTreeEntries(entries); 1183 | } 1184 | files = $.makeArray(fileInput.prop('files')); 1185 | if (!files.length) { 1186 | value = fileInput.prop('value'); 1187 | if (!value) { 1188 | return $.Deferred().resolve([]).promise(); 1189 | } 1190 | // If the files property is not available, the browser does not 1191 | // support the File API and we add a pseudo File object with 1192 | // the input value as name with path information removed: 1193 | files = [{name: value.replace(/^.*\\/, '')}]; 1194 | } else if (files[0].name === undefined && files[0].fileName) { 1195 | // File normalization for Safari 4 and Firefox 3: 1196 | $.each(files, function (index, file) { 1197 | file.name = file.fileName; 1198 | file.size = file.fileSize; 1199 | }); 1200 | } 1201 | return $.Deferred().resolve(files).promise(); 1202 | }, 1203 | 1204 | _getFileInputFiles: function (fileInput) { 1205 | if (!(fileInput instanceof $) || fileInput.length === 1) { 1206 | return this._getSingleFileInputFiles(fileInput); 1207 | } 1208 | return $.when.apply( 1209 | $, 1210 | $.map(fileInput, this._getSingleFileInputFiles) 1211 | ).then(function () { 1212 | return Array.prototype.concat.apply( 1213 | [], 1214 | arguments 1215 | ); 1216 | }); 1217 | }, 1218 | 1219 | _onChange: function (e) { 1220 | var that = this, 1221 | data = { 1222 | fileInput: $(e.target), 1223 | form: $(e.target.form) 1224 | }; 1225 | this._getFileInputFiles(data.fileInput).always(function (files) { 1226 | data.files = files; 1227 | if (that.options.replaceFileInput) { 1228 | that._replaceFileInput(data); 1229 | } 1230 | if (that._trigger( 1231 | 'change', 1232 | $.Event('change', {delegatedEvent: e}), 1233 | data 1234 | ) !== false) { 1235 | that._onAdd(e, data); 1236 | } 1237 | }); 1238 | }, 1239 | 1240 | _onPaste: function (e) { 1241 | var items = e.originalEvent && e.originalEvent.clipboardData && 1242 | e.originalEvent.clipboardData.items, 1243 | data = {files: []}; 1244 | if (items && items.length) { 1245 | $.each(items, function (index, item) { 1246 | var file = item.getAsFile && item.getAsFile(); 1247 | if (file) { 1248 | data.files.push(file); 1249 | } 1250 | }); 1251 | if (this._trigger( 1252 | 'paste', 1253 | $.Event('paste', {delegatedEvent: e}), 1254 | data 1255 | ) !== false) { 1256 | this._onAdd(e, data); 1257 | } 1258 | } 1259 | }, 1260 | 1261 | _onDrop: function (e) { 1262 | e.dataTransfer = e.originalEvent && e.originalEvent.dataTransfer; 1263 | var that = this, 1264 | dataTransfer = e.dataTransfer, 1265 | data = {}; 1266 | if (dataTransfer && dataTransfer.files && dataTransfer.files.length) { 1267 | e.preventDefault(); 1268 | this._getDroppedFiles(dataTransfer).always(function (files) { 1269 | data.files = files; 1270 | if (that._trigger( 1271 | 'drop', 1272 | $.Event('drop', {delegatedEvent: e}), 1273 | data 1274 | ) !== false) { 1275 | that._onAdd(e, data); 1276 | } 1277 | }); 1278 | } 1279 | }, 1280 | 1281 | _onDragOver: getDragHandler('dragover'), 1282 | 1283 | _onDragEnter: getDragHandler('dragenter'), 1284 | 1285 | _onDragLeave: getDragHandler('dragleave'), 1286 | 1287 | _initEventHandlers: function () { 1288 | if (this._isXHRUpload(this.options)) { 1289 | this._on(this.options.dropZone, { 1290 | dragover: this._onDragOver, 1291 | drop: this._onDrop, 1292 | // event.preventDefault() on dragenter is required for IE10+: 1293 | dragenter: this._onDragEnter, 1294 | // dragleave is not required, but added for completeness: 1295 | dragleave: this._onDragLeave 1296 | }); 1297 | this._on(this.options.pasteZone, { 1298 | paste: this._onPaste 1299 | }); 1300 | } 1301 | if ($.support.fileInput) { 1302 | this._on(this.options.fileInput, { 1303 | change: this._onChange 1304 | }); 1305 | } 1306 | }, 1307 | 1308 | _destroyEventHandlers: function () { 1309 | this._off(this.options.dropZone, 'dragenter dragleave dragover drop'); 1310 | this._off(this.options.pasteZone, 'paste'); 1311 | this._off(this.options.fileInput, 'change'); 1312 | }, 1313 | 1314 | _setOption: function (key, value) { 1315 | var reinit = $.inArray(key, this._specialOptions) !== -1; 1316 | if (reinit) { 1317 | this._destroyEventHandlers(); 1318 | } 1319 | this._super(key, value); 1320 | if (reinit) { 1321 | this._initSpecialOptions(); 1322 | this._initEventHandlers(); 1323 | } 1324 | }, 1325 | 1326 | _initSpecialOptions: function () { 1327 | var options = this.options; 1328 | if (options.fileInput === undefined) { 1329 | options.fileInput = this.element.is('input[type="file"]') ? 1330 | this.element : this.element.find('input[type="file"]'); 1331 | } else if (!(options.fileInput instanceof $)) { 1332 | options.fileInput = $(options.fileInput); 1333 | } 1334 | if (!(options.dropZone instanceof $)) { 1335 | options.dropZone = $(options.dropZone); 1336 | } 1337 | if (!(options.pasteZone instanceof $)) { 1338 | options.pasteZone = $(options.pasteZone); 1339 | } 1340 | }, 1341 | 1342 | _getRegExp: function (str) { 1343 | var parts = str.split('/'), 1344 | modifiers = parts.pop(); 1345 | parts.shift(); 1346 | return new RegExp(parts.join('/'), modifiers); 1347 | }, 1348 | 1349 | _isRegExpOption: function (key, value) { 1350 | return key !== 'url' && $.type(value) === 'string' && 1351 | /^\/.*\/[igm]{0,3}$/.test(value); 1352 | }, 1353 | 1354 | _initDataAttributes: function () { 1355 | var that = this, 1356 | options = this.options, 1357 | data = this.element.data(); 1358 | // Initialize options set via HTML5 data-attributes: 1359 | $.each( 1360 | this.element[0].attributes, 1361 | function (index, attr) { 1362 | var key = attr.name.toLowerCase(), 1363 | value; 1364 | if (/^data-/.test(key)) { 1365 | // Convert hyphen-ated key to camelCase: 1366 | key = key.slice(5).replace(/-[a-z]/g, function (str) { 1367 | return str.charAt(1).toUpperCase(); 1368 | }); 1369 | value = data[key]; 1370 | if (that._isRegExpOption(key, value)) { 1371 | value = that._getRegExp(value); 1372 | } 1373 | options[key] = value; 1374 | } 1375 | } 1376 | ); 1377 | }, 1378 | 1379 | _create: function () { 1380 | this._initDataAttributes(); 1381 | this._initSpecialOptions(); 1382 | this._slots = []; 1383 | this._sequence = this._getXHRPromise(true); 1384 | this._sending = this._active = 0; 1385 | this._initProgressObject(this); 1386 | this._initEventHandlers(); 1387 | }, 1388 | 1389 | // This method is exposed to the widget API and allows to query 1390 | // the number of active uploads: 1391 | active: function () { 1392 | return this._active; 1393 | }, 1394 | 1395 | // This method is exposed to the widget API and allows to query 1396 | // the widget upload progress. 1397 | // It returns an object with loaded, total and bitrate properties 1398 | // for the running uploads: 1399 | progress: function () { 1400 | return this._progress; 1401 | }, 1402 | 1403 | // This method is exposed to the widget API and allows adding files 1404 | // using the fileupload API. The data parameter accepts an object which 1405 | // must have a files property and can contain additional options: 1406 | // .fileupload('add', {files: filesList}); 1407 | add: function (data) { 1408 | var that = this; 1409 | if (!data || this.options.disabled) { 1410 | return; 1411 | } 1412 | if (data.fileInput && !data.files) { 1413 | this._getFileInputFiles(data.fileInput).always(function (files) { 1414 | data.files = files; 1415 | that._onAdd(null, data); 1416 | }); 1417 | } else { 1418 | data.files = $.makeArray(data.files); 1419 | this._onAdd(null, data); 1420 | } 1421 | }, 1422 | 1423 | // This method is exposed to the widget API and allows sending files 1424 | // using the fileupload API. The data parameter accepts an object which 1425 | // must have a files or fileInput property and can contain additional options: 1426 | // .fileupload('send', {files: filesList}); 1427 | // The method returns a Promise object for the file upload call. 1428 | send: function (data) { 1429 | if (data && !this.options.disabled) { 1430 | if (data.fileInput && !data.files) { 1431 | var that = this, 1432 | dfd = $.Deferred(), 1433 | promise = dfd.promise(), 1434 | jqXHR, 1435 | aborted; 1436 | promise.abort = function () { 1437 | aborted = true; 1438 | if (jqXHR) { 1439 | return jqXHR.abort(); 1440 | } 1441 | dfd.reject(null, 'abort', 'abort'); 1442 | return promise; 1443 | }; 1444 | this._getFileInputFiles(data.fileInput).always( 1445 | function (files) { 1446 | if (aborted) { 1447 | return; 1448 | } 1449 | if (!files.length) { 1450 | dfd.reject(); 1451 | return; 1452 | } 1453 | data.files = files; 1454 | jqXHR = that._onSend(null, data); 1455 | jqXHR.then( 1456 | function (result, textStatus, jqXHR) { 1457 | dfd.resolve(result, textStatus, jqXHR); 1458 | }, 1459 | function (jqXHR, textStatus, errorThrown) { 1460 | dfd.reject(jqXHR, textStatus, errorThrown); 1461 | } 1462 | ); 1463 | } 1464 | ); 1465 | return this._enhancePromise(promise); 1466 | } 1467 | data.files = $.makeArray(data.files); 1468 | if (data.files.length) { 1469 | return this._onSend(null, data); 1470 | } 1471 | } 1472 | return this._getXHRPromise(false, data && data.context); 1473 | } 1474 | 1475 | }); 1476 | 1477 | })); 1478 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/fanstatic/scripts/vendor/jquery-widget.js: -------------------------------------------------------------------------------- 1 | /*! jQuery UI - v1.11.4+CommonJS - 2015-08-28 2 | * http://jqueryui.com 3 | * Includes: widget.js 4 | * Copyright 2015 jQuery Foundation and other contributors; Licensed MIT */ 5 | 6 | (function( factory ) { 7 | if ( typeof define === "function" && define.amd ) { 8 | 9 | // AMD. Register as an anonymous module. 10 | define([ "jquery" ], factory ); 11 | 12 | } else if ( typeof exports === "object" ) { 13 | 14 | // Node/CommonJS 15 | factory( require( "jquery" ) ); 16 | 17 | } else { 18 | 19 | // Browser globals 20 | factory( jQuery ); 21 | } 22 | }(function( $ ) { 23 | /*! 24 | * jQuery UI Widget 1.11.4 25 | * http://jqueryui.com 26 | * 27 | * Copyright jQuery Foundation and other contributors 28 | * Released under the MIT license. 29 | * http://jquery.org/license 30 | * 31 | * http://api.jqueryui.com/jQuery.widget/ 32 | */ 33 | 34 | 35 | var widget_uuid = 0, 36 | widget_slice = Array.prototype.slice; 37 | 38 | $.cleanData = (function( orig ) { 39 | return function( elems ) { 40 | var events, elem, i; 41 | for ( i = 0; (elem = elems[i]) != null; i++ ) { 42 | try { 43 | 44 | // Only trigger remove when necessary to save time 45 | events = $._data( elem, "events" ); 46 | if ( events && events.remove ) { 47 | $( elem ).triggerHandler( "remove" ); 48 | } 49 | 50 | // http://bugs.jquery.com/ticket/8235 51 | } catch ( e ) {} 52 | } 53 | orig( elems ); 54 | }; 55 | })( $.cleanData ); 56 | 57 | $.widget = function( name, base, prototype ) { 58 | var fullName, existingConstructor, constructor, basePrototype, 59 | // proxiedPrototype allows the provided prototype to remain unmodified 60 | // so that it can be used as a mixin for multiple widgets (#8876) 61 | proxiedPrototype = {}, 62 | namespace = name.split( "." )[ 0 ]; 63 | 64 | name = name.split( "." )[ 1 ]; 65 | fullName = namespace + "-" + name; 66 | 67 | if ( !prototype ) { 68 | prototype = base; 69 | base = $.Widget; 70 | } 71 | 72 | // create selector for plugin 73 | $.expr[ ":" ][ fullName.toLowerCase() ] = function( elem ) { 74 | return !!$.data( elem, fullName ); 75 | }; 76 | 77 | $[ namespace ] = $[ namespace ] || {}; 78 | existingConstructor = $[ namespace ][ name ]; 79 | constructor = $[ namespace ][ name ] = function( options, element ) { 80 | // allow instantiation without "new" keyword 81 | if ( !this._createWidget ) { 82 | return new constructor( options, element ); 83 | } 84 | 85 | // allow instantiation without initializing for simple inheritance 86 | // must use "new" keyword (the code above always passes args) 87 | if ( arguments.length ) { 88 | this._createWidget( options, element ); 89 | } 90 | }; 91 | // extend with the existing constructor to carry over any static properties 92 | $.extend( constructor, existingConstructor, { 93 | version: prototype.version, 94 | // copy the object used to create the prototype in case we need to 95 | // redefine the widget later 96 | _proto: $.extend( {}, prototype ), 97 | // track widgets that inherit from this widget in case this widget is 98 | // redefined after a widget inherits from it 99 | _childConstructors: [] 100 | }); 101 | 102 | basePrototype = new base(); 103 | // we need to make the options hash a property directly on the new instance 104 | // otherwise we'll modify the options hash on the prototype that we're 105 | // inheriting from 106 | basePrototype.options = $.widget.extend( {}, basePrototype.options ); 107 | $.each( prototype, function( prop, value ) { 108 | if ( !$.isFunction( value ) ) { 109 | proxiedPrototype[ prop ] = value; 110 | return; 111 | } 112 | proxiedPrototype[ prop ] = (function() { 113 | var _super = function() { 114 | return base.prototype[ prop ].apply( this, arguments ); 115 | }, 116 | _superApply = function( args ) { 117 | return base.prototype[ prop ].apply( this, args ); 118 | }; 119 | return function() { 120 | var __super = this._super, 121 | __superApply = this._superApply, 122 | returnValue; 123 | 124 | this._super = _super; 125 | this._superApply = _superApply; 126 | 127 | returnValue = value.apply( this, arguments ); 128 | 129 | this._super = __super; 130 | this._superApply = __superApply; 131 | 132 | return returnValue; 133 | }; 134 | })(); 135 | }); 136 | constructor.prototype = $.widget.extend( basePrototype, { 137 | // TODO: remove support for widgetEventPrefix 138 | // always use the name + a colon as the prefix, e.g., draggable:start 139 | // don't prefix for widgets that aren't DOM-based 140 | widgetEventPrefix: existingConstructor ? (basePrototype.widgetEventPrefix || name) : name 141 | }, proxiedPrototype, { 142 | constructor: constructor, 143 | namespace: namespace, 144 | widgetName: name, 145 | widgetFullName: fullName 146 | }); 147 | 148 | // If this widget is being redefined then we need to find all widgets that 149 | // are inheriting from it and redefine all of them so that they inherit from 150 | // the new version of this widget. We're essentially trying to replace one 151 | // level in the prototype chain. 152 | if ( existingConstructor ) { 153 | $.each( existingConstructor._childConstructors, function( i, child ) { 154 | var childPrototype = child.prototype; 155 | 156 | // redefine the child widget using the same prototype that was 157 | // originally used, but inherit from the new version of the base 158 | $.widget( childPrototype.namespace + "." + childPrototype.widgetName, constructor, child._proto ); 159 | }); 160 | // remove the list of existing child constructors from the old constructor 161 | // so the old child constructors can be garbage collected 162 | delete existingConstructor._childConstructors; 163 | } else { 164 | base._childConstructors.push( constructor ); 165 | } 166 | 167 | $.widget.bridge( name, constructor ); 168 | 169 | return constructor; 170 | }; 171 | 172 | $.widget.extend = function( target ) { 173 | var input = widget_slice.call( arguments, 1 ), 174 | inputIndex = 0, 175 | inputLength = input.length, 176 | key, 177 | value; 178 | for ( ; inputIndex < inputLength; inputIndex++ ) { 179 | for ( key in input[ inputIndex ] ) { 180 | value = input[ inputIndex ][ key ]; 181 | if ( input[ inputIndex ].hasOwnProperty( key ) && value !== undefined ) { 182 | // Clone objects 183 | if ( $.isPlainObject( value ) ) { 184 | target[ key ] = $.isPlainObject( target[ key ] ) ? 185 | $.widget.extend( {}, target[ key ], value ) : 186 | // Don't extend strings, arrays, etc. with objects 187 | $.widget.extend( {}, value ); 188 | // Copy everything else by reference 189 | } else { 190 | target[ key ] = value; 191 | } 192 | } 193 | } 194 | } 195 | return target; 196 | }; 197 | 198 | $.widget.bridge = function( name, object ) { 199 | var fullName = object.prototype.widgetFullName || name; 200 | $.fn[ name ] = function( options ) { 201 | var isMethodCall = typeof options === "string", 202 | args = widget_slice.call( arguments, 1 ), 203 | returnValue = this; 204 | 205 | if ( isMethodCall ) { 206 | this.each(function() { 207 | var methodValue, 208 | instance = $.data( this, fullName ); 209 | if ( options === "instance" ) { 210 | returnValue = instance; 211 | return false; 212 | } 213 | if ( !instance ) { 214 | return $.error( "cannot call methods on " + name + " prior to initialization; " + 215 | "attempted to call method '" + options + "'" ); 216 | } 217 | if ( !$.isFunction( instance[options] ) || options.charAt( 0 ) === "_" ) { 218 | return $.error( "no such method '" + options + "' for " + name + " widget instance" ); 219 | } 220 | methodValue = instance[ options ].apply( instance, args ); 221 | if ( methodValue !== instance && methodValue !== undefined ) { 222 | returnValue = methodValue && methodValue.jquery ? 223 | returnValue.pushStack( methodValue.get() ) : 224 | methodValue; 225 | return false; 226 | } 227 | }); 228 | } else { 229 | 230 | // Allow multiple hashes to be passed on init 231 | if ( args.length ) { 232 | options = $.widget.extend.apply( null, [ options ].concat(args) ); 233 | } 234 | 235 | this.each(function() { 236 | var instance = $.data( this, fullName ); 237 | if ( instance ) { 238 | instance.option( options || {} ); 239 | if ( instance._init ) { 240 | instance._init(); 241 | } 242 | } else { 243 | $.data( this, fullName, new object( options, this ) ); 244 | } 245 | }); 246 | } 247 | 248 | return returnValue; 249 | }; 250 | }; 251 | 252 | $.Widget = function( /* options, element */ ) {}; 253 | $.Widget._childConstructors = []; 254 | 255 | $.Widget.prototype = { 256 | widgetName: "widget", 257 | widgetEventPrefix: "", 258 | defaultElement: "
", 259 | options: { 260 | disabled: false, 261 | 262 | // callbacks 263 | create: null 264 | }, 265 | _createWidget: function( options, element ) { 266 | element = $( element || this.defaultElement || this )[ 0 ]; 267 | this.element = $( element ); 268 | this.uuid = widget_uuid++; 269 | this.eventNamespace = "." + this.widgetName + this.uuid; 270 | 271 | this.bindings = $(); 272 | this.hoverable = $(); 273 | this.focusable = $(); 274 | 275 | if ( element !== this ) { 276 | $.data( element, this.widgetFullName, this ); 277 | this._on( true, this.element, { 278 | remove: function( event ) { 279 | if ( event.target === element ) { 280 | this.destroy(); 281 | } 282 | } 283 | }); 284 | this.document = $( element.style ? 285 | // element within the document 286 | element.ownerDocument : 287 | // element is window or document 288 | element.document || element ); 289 | this.window = $( this.document[0].defaultView || this.document[0].parentWindow ); 290 | } 291 | 292 | this.options = $.widget.extend( {}, 293 | this.options, 294 | this._getCreateOptions(), 295 | options ); 296 | 297 | this._create(); 298 | this._trigger( "create", null, this._getCreateEventData() ); 299 | this._init(); 300 | }, 301 | _getCreateOptions: $.noop, 302 | _getCreateEventData: $.noop, 303 | _create: $.noop, 304 | _init: $.noop, 305 | 306 | destroy: function() { 307 | this._destroy(); 308 | // we can probably remove the unbind calls in 2.0 309 | // all event bindings should go through this._on() 310 | this.element 311 | .unbind( this.eventNamespace ) 312 | .removeData( this.widgetFullName ) 313 | // support: jquery <1.6.3 314 | // http://bugs.jquery.com/ticket/9413 315 | .removeData( $.camelCase( this.widgetFullName ) ); 316 | this.widget() 317 | .unbind( this.eventNamespace ) 318 | .removeAttr( "aria-disabled" ) 319 | .removeClass( 320 | this.widgetFullName + "-disabled " + 321 | "ui-state-disabled" ); 322 | 323 | // clean up events and states 324 | this.bindings.unbind( this.eventNamespace ); 325 | this.hoverable.removeClass( "ui-state-hover" ); 326 | this.focusable.removeClass( "ui-state-focus" ); 327 | }, 328 | _destroy: $.noop, 329 | 330 | widget: function() { 331 | return this.element; 332 | }, 333 | 334 | option: function( key, value ) { 335 | var options = key, 336 | parts, 337 | curOption, 338 | i; 339 | 340 | if ( arguments.length === 0 ) { 341 | // don't return a reference to the internal hash 342 | return $.widget.extend( {}, this.options ); 343 | } 344 | 345 | if ( typeof key === "string" ) { 346 | // handle nested keys, e.g., "foo.bar" => { foo: { bar: ___ } } 347 | options = {}; 348 | parts = key.split( "." ); 349 | key = parts.shift(); 350 | if ( parts.length ) { 351 | curOption = options[ key ] = $.widget.extend( {}, this.options[ key ] ); 352 | for ( i = 0; i < parts.length - 1; i++ ) { 353 | curOption[ parts[ i ] ] = curOption[ parts[ i ] ] || {}; 354 | curOption = curOption[ parts[ i ] ]; 355 | } 356 | key = parts.pop(); 357 | if ( arguments.length === 1 ) { 358 | return curOption[ key ] === undefined ? null : curOption[ key ]; 359 | } 360 | curOption[ key ] = value; 361 | } else { 362 | if ( arguments.length === 1 ) { 363 | return this.options[ key ] === undefined ? null : this.options[ key ]; 364 | } 365 | options[ key ] = value; 366 | } 367 | } 368 | 369 | this._setOptions( options ); 370 | 371 | return this; 372 | }, 373 | _setOptions: function( options ) { 374 | var key; 375 | 376 | for ( key in options ) { 377 | this._setOption( key, options[ key ] ); 378 | } 379 | 380 | return this; 381 | }, 382 | _setOption: function( key, value ) { 383 | this.options[ key ] = value; 384 | 385 | if ( key === "disabled" ) { 386 | this.widget() 387 | .toggleClass( this.widgetFullName + "-disabled", !!value ); 388 | 389 | // If the widget is becoming disabled, then nothing is interactive 390 | if ( value ) { 391 | this.hoverable.removeClass( "ui-state-hover" ); 392 | this.focusable.removeClass( "ui-state-focus" ); 393 | } 394 | } 395 | 396 | return this; 397 | }, 398 | 399 | enable: function() { 400 | return this._setOptions({ disabled: false }); 401 | }, 402 | disable: function() { 403 | return this._setOptions({ disabled: true }); 404 | }, 405 | 406 | _on: function( suppressDisabledCheck, element, handlers ) { 407 | var delegateElement, 408 | instance = this; 409 | 410 | // no suppressDisabledCheck flag, shuffle arguments 411 | if ( typeof suppressDisabledCheck !== "boolean" ) { 412 | handlers = element; 413 | element = suppressDisabledCheck; 414 | suppressDisabledCheck = false; 415 | } 416 | 417 | // no element argument, shuffle and use this.element 418 | if ( !handlers ) { 419 | handlers = element; 420 | element = this.element; 421 | delegateElement = this.widget(); 422 | } else { 423 | element = delegateElement = $( element ); 424 | this.bindings = this.bindings.add( element ); 425 | } 426 | 427 | $.each( handlers, function( event, handler ) { 428 | function handlerProxy() { 429 | // allow widgets to customize the disabled handling 430 | // - disabled as an array instead of boolean 431 | // - disabled class as method for disabling individual parts 432 | if ( !suppressDisabledCheck && 433 | ( instance.options.disabled === true || 434 | $( this ).hasClass( "ui-state-disabled" ) ) ) { 435 | return; 436 | } 437 | return ( typeof handler === "string" ? instance[ handler ] : handler ) 438 | .apply( instance, arguments ); 439 | } 440 | 441 | // copy the guid so direct unbinding works 442 | if ( typeof handler !== "string" ) { 443 | handlerProxy.guid = handler.guid = 444 | handler.guid || handlerProxy.guid || $.guid++; 445 | } 446 | 447 | var match = event.match( /^([\w:-]*)\s*(.*)$/ ), 448 | eventName = match[1] + instance.eventNamespace, 449 | selector = match[2]; 450 | if ( selector ) { 451 | delegateElement.delegate( selector, eventName, handlerProxy ); 452 | } else { 453 | element.bind( eventName, handlerProxy ); 454 | } 455 | }); 456 | }, 457 | 458 | _off: function( element, eventName ) { 459 | eventName = (eventName || "").split( " " ).join( this.eventNamespace + " " ) + 460 | this.eventNamespace; 461 | element.unbind( eventName ).undelegate( eventName ); 462 | 463 | // Clear the stack to avoid memory leaks (#10056) 464 | this.bindings = $( this.bindings.not( element ).get() ); 465 | this.focusable = $( this.focusable.not( element ).get() ); 466 | this.hoverable = $( this.hoverable.not( element ).get() ); 467 | }, 468 | 469 | _delay: function( handler, delay ) { 470 | function handlerProxy() { 471 | return ( typeof handler === "string" ? instance[ handler ] : handler ) 472 | .apply( instance, arguments ); 473 | } 474 | var instance = this; 475 | return setTimeout( handlerProxy, delay || 0 ); 476 | }, 477 | 478 | _hoverable: function( element ) { 479 | this.hoverable = this.hoverable.add( element ); 480 | this._on( element, { 481 | mouseenter: function( event ) { 482 | $( event.currentTarget ).addClass( "ui-state-hover" ); 483 | }, 484 | mouseleave: function( event ) { 485 | $( event.currentTarget ).removeClass( "ui-state-hover" ); 486 | } 487 | }); 488 | }, 489 | 490 | _focusable: function( element ) { 491 | this.focusable = this.focusable.add( element ); 492 | this._on( element, { 493 | focusin: function( event ) { 494 | $( event.currentTarget ).addClass( "ui-state-focus" ); 495 | }, 496 | focusout: function( event ) { 497 | $( event.currentTarget ).removeClass( "ui-state-focus" ); 498 | } 499 | }); 500 | }, 501 | 502 | _trigger: function( type, event, data ) { 503 | var prop, orig, 504 | callback = this.options[ type ]; 505 | 506 | data = data || {}; 507 | event = $.Event( event ); 508 | event.type = ( type === this.widgetEventPrefix ? 509 | type : 510 | this.widgetEventPrefix + type ).toLowerCase(); 511 | // the original event may come from any element 512 | // so we need to reset the target on the new event 513 | event.target = this.element[ 0 ]; 514 | 515 | // copy original event properties over to the new event 516 | orig = event.originalEvent; 517 | if ( orig ) { 518 | for ( prop in orig ) { 519 | if ( !( prop in event ) ) { 520 | event[ prop ] = orig[ prop ]; 521 | } 522 | } 523 | } 524 | 525 | this.element.trigger( event, data ); 526 | return !( $.isFunction( callback ) && 527 | callback.apply( this.element[0], [ event ].concat( data ) ) === false || 528 | event.isDefaultPrevented() ); 529 | } 530 | }; 531 | 532 | $.each( { show: "fadeIn", hide: "fadeOut" }, function( method, defaultEffect ) { 533 | $.Widget.prototype[ "_" + method ] = function( element, options, callback ) { 534 | if ( typeof options === "string" ) { 535 | options = { effect: options }; 536 | } 537 | var hasOptions, 538 | effectName = !options ? 539 | method : 540 | options === true || typeof options === "number" ? 541 | defaultEffect : 542 | options.effect || defaultEffect; 543 | options = options || {}; 544 | if ( typeof options === "number" ) { 545 | options = { duration: options }; 546 | } 547 | hasOptions = !$.isEmptyObject( options ); 548 | options.complete = callback; 549 | if ( options.delay ) { 550 | element.delay( options.delay ); 551 | } 552 | if ( hasOptions && $.effects && $.effects.effect[ effectName ] ) { 553 | element[ method ]( options ); 554 | } else if ( effectName !== method && element[ effectName ] ) { 555 | element[ effectName ]( options.duration, options.easing, callback ); 556 | } else { 557 | element.queue(function( next ) { 558 | $( this )[ method ](); 559 | if ( callback ) { 560 | callback.call( element[ 0 ] ); 561 | } 562 | next(); 563 | }); 564 | } 565 | }; 566 | }); 567 | 568 | var widget = $.widget; 569 | 570 | 571 | 572 | })); 573 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from ckanext.cloudstorage.storage import ResourceCloudStorage 4 | 5 | 6 | def use_secure_urls(): 7 | return all([ 8 | ResourceCloudStorage.use_secure_urls.fget(None), 9 | # Currently implemented just AWS version 10 | 'S3' in ResourceCloudStorage.driver_name.fget(None) 11 | ]) 12 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/logic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TkTech/ckanext-cloudstorage/ae83a6714505ba81417343989b2be2e7cabae854/ckanext/cloudstorage/logic/__init__.py -------------------------------------------------------------------------------- /ckanext/cloudstorage/logic/action/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TkTech/ckanext-cloudstorage/ae83a6714505ba81417343989b2be2e7cabae854/ckanext/cloudstorage/logic/action/__init__.py -------------------------------------------------------------------------------- /ckanext/cloudstorage/logic/action/multipart.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import logging 4 | import datetime 5 | 6 | from pylons import config 7 | from sqlalchemy.orm.exc import NoResultFound 8 | import ckan.model as model 9 | import ckan.lib.helpers as h 10 | import ckan.plugins.toolkit as toolkit 11 | 12 | from ckanext.cloudstorage.storage import ResourceCloudStorage 13 | from ckanext.cloudstorage.model import MultipartUpload, MultipartPart 14 | from werkzeug.datastructures import FileStorage as FlaskFileStorage 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | 19 | def _get_underlying_file(wrapper): 20 | if isinstance(wrapper, FlaskFileStorage): 21 | return wrapper.stream 22 | return wrapper.file 23 | 24 | 25 | def _get_max_multipart_lifetime(): 26 | value = float(config.get('ckanext.cloudstorage.max_multipart_lifetime', 7)) 27 | return datetime.timedelta(value) 28 | 29 | 30 | def _get_object_url(uploader, name): 31 | return '/' + uploader.container_name + '/' + name 32 | 33 | 34 | def _delete_multipart(upload, uploader): 35 | resp = uploader.driver.connection.request( 36 | _get_object_url(uploader, upload.name) + '?uploadId=' + upload.id, 37 | method='DELETE' 38 | ) 39 | if not resp.success(): 40 | raise toolkit.ValidationError(resp.error) 41 | 42 | upload.delete() 43 | upload.commit() 44 | return resp 45 | 46 | 47 | def _save_part_info(n, etag, upload): 48 | try: 49 | part = model.Session.query(MultipartPart).filter( 50 | MultipartPart.n == n, 51 | MultipartPart.upload == upload).one() 52 | except NoResultFound: 53 | part = MultipartPart(n, etag, upload) 54 | else: 55 | part.etag = etag 56 | part.save() 57 | return part 58 | 59 | 60 | def check_multipart(context, data_dict): 61 | """Check whether unfinished multipart upload already exists. 62 | 63 | :param context: 64 | :param data_dict: dict with required `id` 65 | :returns: None or dict with `upload` - existing multipart upload info 66 | :rtype: NoneType or dict 67 | 68 | """ 69 | 70 | h.check_access('cloudstorage_check_multipart', data_dict) 71 | id = toolkit.get_or_bust(data_dict, 'id') 72 | try: 73 | upload = model.Session.query(MultipartUpload).filter_by( 74 | resource_id=id).one() 75 | except NoResultFound: 76 | return 77 | upload_dict = upload.as_dict() 78 | upload_dict['parts'] = model.Session.query(MultipartPart).filter( 79 | MultipartPart.upload == upload).count() 80 | return {'upload': upload_dict} 81 | 82 | 83 | def initiate_multipart(context, data_dict): 84 | """Initiate new Multipart Upload. 85 | 86 | :param context: 87 | :param data_dict: dict with required keys: 88 | id: resource's id 89 | name: filename 90 | size: filesize 91 | 92 | :returns: MultipartUpload info 93 | :rtype: dict 94 | 95 | """ 96 | 97 | h.check_access('cloudstorage_initiate_multipart', data_dict) 98 | id, name, size = toolkit.get_or_bust(data_dict, ['id', 'name', 'size']) 99 | user_id = None 100 | if context['auth_user_obj']: 101 | user_id = context['auth_user_obj'].id 102 | 103 | uploader = ResourceCloudStorage({'multipart_name': name}) 104 | res_name = uploader.path_from_filename(id, name) 105 | 106 | upload_object = MultipartUpload.by_name(res_name) 107 | 108 | if upload_object is not None: 109 | _delete_multipart(upload_object, uploader) 110 | upload_object = None 111 | 112 | if upload_object is None: 113 | for old_upload in model.Session.query(MultipartUpload).filter_by( 114 | resource_id=id): 115 | _delete_multipart(old_upload, uploader) 116 | 117 | _rindex = res_name.rfind('/') 118 | if ~_rindex: 119 | try: 120 | name_prefix = res_name[:_rindex] 121 | for cloud_object in uploader.container.iterate_objects(): 122 | if cloud_object.name.startswith(name_prefix): 123 | log.info('Removing cloud object: %s' % cloud_object) 124 | cloud_object.delete() 125 | except Exception as e: 126 | log.exception('[delete from cloud] %s' % e) 127 | 128 | resp = uploader.driver.connection.request( 129 | _get_object_url(uploader, res_name) + '?uploads', 130 | method='POST' 131 | ) 132 | if not resp.success(): 133 | raise toolkit.ValidationError(resp.error) 134 | try: 135 | upload_id = resp.object.find( 136 | '{%s}UploadId' % resp.object.nsmap[None]).text 137 | except AttributeError: 138 | upload_id_list = filter( 139 | lambda e: e.tag.endswith('UploadId'), 140 | resp.object.getchildren() 141 | ) 142 | upload_id = upload_id_list[0].text 143 | upload_object = MultipartUpload(upload_id, id, res_name, size, name, user_id) 144 | 145 | upload_object.save() 146 | return upload_object.as_dict() 147 | 148 | 149 | def upload_multipart(context, data_dict): 150 | h.check_access('cloudstorage_upload_multipart', data_dict) 151 | upload_id, part_number, part_content = toolkit.get_or_bust( 152 | data_dict, ['uploadId', 'partNumber', 'upload']) 153 | 154 | uploader = ResourceCloudStorage({}) 155 | upload = model.Session.query(MultipartUpload).get(upload_id) 156 | 157 | resp = uploader.driver.connection.request( 158 | _get_object_url( 159 | uploader, upload.name) + '?partNumber={0}&uploadId={1}'.format( 160 | part_number, upload_id), 161 | method='PUT', 162 | data=bytearray(_get_underlying_file(part_content).read()) 163 | ) 164 | if resp.status != 200: 165 | raise toolkit.ValidationError('Upload failed: part %s' % part_number) 166 | 167 | _save_part_info(part_number, resp.headers['etag'], upload) 168 | return { 169 | 'partNumber': part_number, 170 | 'ETag': resp.headers['etag'] 171 | } 172 | 173 | 174 | def finish_multipart(context, data_dict): 175 | """Called after all parts had been uploaded. 176 | 177 | Triggers call to `_commit_multipart` which will convert separate uploaded 178 | parts into single file 179 | 180 | :param context: 181 | :param data_dict: dict with required key `uploadId` - id of Multipart Upload that should be finished 182 | :returns: None 183 | :rtype: NoneType 184 | 185 | """ 186 | 187 | h.check_access('cloudstorage_finish_multipart', data_dict) 188 | upload_id = toolkit.get_or_bust(data_dict, 'uploadId') 189 | save_action = data_dict.get('save_action', False) 190 | upload = model.Session.query(MultipartUpload).get(upload_id) 191 | chunks = [ 192 | (part.n, part.etag) 193 | for part in model.Session.query(MultipartPart).filter_by( 194 | upload_id=upload_id).order_by(MultipartPart.n) 195 | ] 196 | uploader = ResourceCloudStorage({}) 197 | try: 198 | obj = uploader.container.get_object(upload.name) 199 | obj.delete() 200 | except Exception: 201 | pass 202 | uploader.driver._commit_multipart( 203 | _get_object_url(uploader, upload.name), 204 | upload_id, 205 | chunks) 206 | upload.delete() 207 | upload.commit() 208 | 209 | if save_action and save_action == "go-metadata": 210 | try: 211 | res_dict = toolkit.get_action('resource_show')( 212 | context.copy(), {'id': data_dict.get('id')}) 213 | pkg_dict = toolkit.get_action('package_show')( 214 | context.copy(), {'id': res_dict['package_id']}) 215 | if pkg_dict['state'] == 'draft': 216 | toolkit.get_action('package_patch')( 217 | dict(context.copy(), allow_state_change=True), 218 | dict(id=pkg_dict['id'], state='active') 219 | ) 220 | except Exception as e: 221 | log.error(e) 222 | return {'commited': True} 223 | 224 | 225 | def abort_multipart(context, data_dict): 226 | h.check_access('cloudstorage_abort_multipart', data_dict) 227 | id = toolkit.get_or_bust(data_dict, ['id']) 228 | uploader = ResourceCloudStorage({}) 229 | 230 | resource_uploads = MultipartUpload.resource_uploads(id) 231 | 232 | aborted = [] 233 | for upload in resource_uploads: 234 | _delete_multipart(upload, uploader) 235 | 236 | aborted.append(upload.id) 237 | 238 | model.Session.commit() 239 | 240 | return aborted 241 | 242 | 243 | def clean_multipart(context, data_dict): 244 | """Clean old multipart uploads. 245 | 246 | :param context: 247 | :param data_dict: 248 | :returns: dict with: 249 | removed - amount of removed uploads. 250 | total - total amount of expired uploads. 251 | errors - list of errors raised during deletion. Appears when 252 | `total` and `removed` are different. 253 | :rtype: dict 254 | 255 | """ 256 | 257 | h.check_access('cloudstorage_clean_multipart', data_dict) 258 | uploader = ResourceCloudStorage({}) 259 | delta = _get_max_multipart_lifetime() 260 | oldest_allowed = datetime.datetime.utcnow() - delta 261 | 262 | uploads_to_remove = model.Session.query(MultipartUpload).filter( 263 | MultipartUpload.initiated < oldest_allowed 264 | ) 265 | 266 | result = { 267 | 'removed': 0, 268 | 'total': uploads_to_remove.count(), 269 | 'errors': [] 270 | } 271 | 272 | for upload in uploads_to_remove: 273 | try: 274 | _delete_multipart(upload, uploader) 275 | except toolkit.ValidationError as e: 276 | result['errors'].append(e.error_summary) 277 | else: 278 | result['removed'] += 1 279 | 280 | return result 281 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/logic/auth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TkTech/ckanext-cloudstorage/ae83a6714505ba81417343989b2be2e7cabae854/ckanext/cloudstorage/logic/auth/__init__.py -------------------------------------------------------------------------------- /ckanext/cloudstorage/logic/auth/multipart.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from ckan.logic import check_access 4 | 5 | 6 | def initiate_multipart(context, data_dict): 7 | return {'success': check_access('resource_create', context, data_dict)} 8 | 9 | 10 | def upload_multipart(context, data_dict): 11 | return {'success': check_access('resource_create', context, data_dict)} 12 | 13 | 14 | def finish_multipart(context, data_dict): 15 | return {'success': check_access('resource_create', context, data_dict)} 16 | 17 | 18 | def abort_multipart(context, data_dict): 19 | return {'success': check_access('resource_create', context, data_dict)} 20 | 21 | 22 | def check_multipart(context, data_dict): 23 | return {'success': check_access('resource_create', context, data_dict)} 24 | 25 | 26 | def clean_multipart(context, data_dict): 27 | return {'success': False} 28 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from sqlalchemy.ext.declarative import declarative_base 4 | from sqlalchemy.orm import relationship, backref 5 | import ckan.model as model 6 | from sqlalchemy import ( 7 | Column, 8 | UnicodeText, 9 | DateTime, 10 | ForeignKey, 11 | Integer, 12 | Numeric 13 | ) 14 | from datetime import datetime 15 | import ckan.model.meta as meta 16 | from ckan.model.domain_object import DomainObject 17 | 18 | Base = declarative_base() 19 | metadata = Base.metadata 20 | 21 | 22 | def drop_tables(): 23 | metadata.drop_all(model.meta.engine) 24 | 25 | 26 | def create_tables(): 27 | metadata.create_all(model.meta.engine) 28 | 29 | 30 | class MultipartPart(Base, DomainObject): 31 | __tablename__ = 'cloudstorage_multipart_part' 32 | 33 | def __init__(self, n, etag, upload): 34 | self.n = n 35 | self.etag = etag 36 | self.upload = upload 37 | 38 | n = Column(Integer, primary_key=True) 39 | etag = Column(UnicodeText, primary_key=True) 40 | upload_id = Column( 41 | UnicodeText, ForeignKey('cloudstorage_multipart_upload.id'), 42 | primary_key=True 43 | ) 44 | upload = relationship( 45 | 'MultipartUpload', 46 | backref=backref('parts', cascade='delete, delete-orphan'), 47 | single_parent=True) 48 | 49 | 50 | class MultipartUpload(Base, DomainObject): 51 | __tablename__ = 'cloudstorage_multipart_upload' 52 | 53 | def __init__(self, id, resource_id, name, size, original_name, user_id): 54 | self.id = id 55 | self.resource_id = resource_id 56 | self.name = name 57 | self.size = size 58 | self.original_name = original_name 59 | self.user_id = user_id 60 | 61 | @classmethod 62 | def resource_uploads(cls, resource_id): 63 | query = meta.Session.query(cls).filter_by( 64 | resource_id=resource_id 65 | ) 66 | return query 67 | 68 | id = Column(UnicodeText, primary_key=True) 69 | resource_id = Column(UnicodeText) 70 | name = Column(UnicodeText) 71 | initiated = Column(DateTime, default=datetime.utcnow) 72 | size = Column(Numeric) 73 | original_name = Column(UnicodeText) 74 | user_id = Column(UnicodeText) 75 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/plugin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from ckan import plugins 4 | from routes.mapper import SubMapper 5 | import os.path 6 | from ckanext.cloudstorage import storage 7 | from ckanext.cloudstorage import helpers 8 | import ckanext.cloudstorage.logic.action.multipart as m_action 9 | import ckanext.cloudstorage.logic.auth.multipart as m_auth 10 | 11 | 12 | class CloudStoragePlugin(plugins.SingletonPlugin): 13 | plugins.implements(plugins.IUploader) 14 | plugins.implements(plugins.IRoutes, inherit=True) 15 | plugins.implements(plugins.IConfigurable) 16 | plugins.implements(plugins.IConfigurer) 17 | plugins.implements(plugins.IActions) 18 | plugins.implements(plugins.ITemplateHelpers) 19 | plugins.implements(plugins.IAuthFunctions) 20 | plugins.implements(plugins.IResourceController, inherit=True) 21 | 22 | # IConfigurer 23 | 24 | def update_config(self, config): 25 | plugins.toolkit.add_template_directory(config, 'templates') 26 | plugins.toolkit.add_resource('fanstatic/scripts', 'cloudstorage-js') 27 | 28 | # ITemplateHelpers 29 | 30 | def get_helpers(self): 31 | return dict( 32 | cloudstorage_use_secure_urls=helpers.use_secure_urls 33 | ) 34 | 35 | def configure(self, config): 36 | 37 | required_keys = ( 38 | 'ckanext.cloudstorage.driver', 39 | 'ckanext.cloudstorage.driver_options', 40 | 'ckanext.cloudstorage.container_name' 41 | ) 42 | 43 | for rk in required_keys: 44 | if config.get(rk) is None: 45 | raise RuntimeError( 46 | 'Required configuration option {0} not found.'.format( 47 | rk 48 | ) 49 | ) 50 | 51 | def get_resource_uploader(self, data_dict): 52 | # We provide a custom Resource uploader. 53 | return storage.ResourceCloudStorage(data_dict) 54 | 55 | def get_uploader(self, upload_to, old_filename=None): 56 | # We don't provide misc-file storage (group images for example) 57 | # Returning None here will use the default Uploader. 58 | return None 59 | 60 | def before_map(self, map): 61 | sm = SubMapper( 62 | map, 63 | controller='ckanext.cloudstorage.controller:StorageController' 64 | ) 65 | 66 | # Override the resource download controllers so we can do our 67 | # lookup with libcloud. 68 | with sm: 69 | sm.connect( 70 | 'resource_download', 71 | '/dataset/{id}/resource/{resource_id}/download', 72 | action='resource_download' 73 | ) 74 | sm.connect( 75 | 'resource_download', 76 | '/dataset/{id}/resource/{resource_id}/download/{filename}', 77 | action='resource_download' 78 | ) 79 | 80 | return map 81 | 82 | # IActions 83 | 84 | def get_actions(self): 85 | return { 86 | 'cloudstorage_initiate_multipart': m_action.initiate_multipart, 87 | 'cloudstorage_upload_multipart': m_action.upload_multipart, 88 | 'cloudstorage_finish_multipart': m_action.finish_multipart, 89 | 'cloudstorage_abort_multipart': m_action.abort_multipart, 90 | 'cloudstorage_check_multipart': m_action.check_multipart, 91 | 'cloudstorage_clean_multipart': m_action.clean_multipart, 92 | } 93 | 94 | # IAuthFunctions 95 | 96 | def get_auth_functions(self): 97 | return { 98 | 'cloudstorage_initiate_multipart': m_auth.initiate_multipart, 99 | 'cloudstorage_upload_multipart': m_auth.upload_multipart, 100 | 'cloudstorage_finish_multipart': m_auth.finish_multipart, 101 | 'cloudstorage_abort_multipart': m_auth.abort_multipart, 102 | 'cloudstorage_check_multipart': m_auth.check_multipart, 103 | 'cloudstorage_clean_multipart': m_auth.clean_multipart, 104 | } 105 | 106 | # IResourceController 107 | 108 | def before_delete(self, context, resource, resources): 109 | # let's get all info about our resource. It somewhere in resources 110 | # but if there is some possibility that it isn't(magic?) we have 111 | # `else` clause 112 | 113 | for res in resources: 114 | if res['id'] == resource['id']: 115 | break 116 | else: 117 | return 118 | # just ignore simple links 119 | if res['url_type'] != 'upload': 120 | return 121 | 122 | # we don't want to change original item from resources, just in case 123 | # someone will use it in another `before_delete`. So, let's copy it 124 | # and add `clear_upload` flag 125 | res_dict = dict(res.items() + [('clear_upload', True)]) 126 | 127 | uploader = self.get_resource_uploader(res_dict) 128 | 129 | # to be on the safe side, let's check existence of container 130 | container = getattr(uploader, 'container', None) 131 | if container is None: 132 | return 133 | 134 | # and now uploader removes our file. 135 | uploader.upload(resource['id']) 136 | 137 | # and all other files linked to this resource 138 | if not uploader.leave_files: 139 | upload_path = os.path.dirname( 140 | uploader.path_from_filename( 141 | resource['id'], 142 | 'fake-name' 143 | ) 144 | ) 145 | 146 | for old_file in uploader.container.iterate_objects(): 147 | if old_file.name.startswith(upload_path): 148 | old_file.delete() 149 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/storage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import cgi 4 | import mimetypes 5 | import os.path 6 | import urlparse 7 | from ast import literal_eval 8 | from datetime import datetime, timedelta 9 | from tempfile import SpooledTemporaryFile 10 | 11 | from pylons import config 12 | from ckan import model 13 | from ckan.lib import munge 14 | import ckan.plugins as p 15 | 16 | from libcloud.storage.types import Provider, ObjectDoesNotExistError 17 | from libcloud.storage.providers import get_driver 18 | 19 | 20 | from werkzeug.datastructures import FileStorage as FlaskFileStorage 21 | ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage) 22 | 23 | 24 | def _get_underlying_file(wrapper): 25 | if isinstance(wrapper, FlaskFileStorage): 26 | return wrapper.stream 27 | return wrapper.file 28 | 29 | 30 | class CloudStorage(object): 31 | def __init__(self): 32 | self.driver = get_driver( 33 | getattr( 34 | Provider, 35 | self.driver_name 36 | ) 37 | )(**self.driver_options) 38 | self._container = None 39 | 40 | def path_from_filename(self, rid, filename): 41 | raise NotImplemented 42 | 43 | @property 44 | def container(self): 45 | """ 46 | Return the currently configured libcloud container. 47 | """ 48 | if self._container is None: 49 | self._container = self.driver.get_container( 50 | container_name=self.container_name 51 | ) 52 | 53 | return self._container 54 | 55 | @property 56 | def driver_options(self): 57 | """ 58 | A dictionary of options ckanext-cloudstorage has been configured to 59 | pass to the apache-libcloud driver. 60 | """ 61 | return literal_eval(config['ckanext.cloudstorage.driver_options']) 62 | 63 | @property 64 | def driver_name(self): 65 | """ 66 | The name of the driver (ex: AZURE_BLOBS, S3) that ckanext-cloudstorage 67 | is configured to use. 68 | 69 | 70 | .. note:: 71 | 72 | This value is used to lookup the apache-libcloud driver to use 73 | based on the Provider enum. 74 | """ 75 | return config['ckanext.cloudstorage.driver'] 76 | 77 | @property 78 | def container_name(self): 79 | """ 80 | The name of the container (also called buckets on some providers) 81 | ckanext-cloudstorage is configured to use. 82 | """ 83 | return config['ckanext.cloudstorage.container_name'] 84 | 85 | @property 86 | def use_secure_urls(self): 87 | """ 88 | `True` if ckanext-cloudstroage is configured to generate secure 89 | one-time URLs to resources, `False` otherwise. 90 | """ 91 | return p.toolkit.asbool( 92 | config.get('ckanext.cloudstorage.use_secure_urls', False) 93 | ) 94 | 95 | @property 96 | def leave_files(self): 97 | """ 98 | `True` if ckanext-cloudstorage is configured to leave files on the 99 | provider instead of removing them when a resource/package is deleted, 100 | otherwise `False`. 101 | """ 102 | return p.toolkit.asbool( 103 | config.get('ckanext.cloudstorage.leave_files', False) 104 | ) 105 | 106 | @property 107 | def can_use_advanced_azure(self): 108 | """ 109 | `True` if the `azure-storage` module is installed and 110 | ckanext-cloudstorage has been configured to use Azure, otherwise 111 | `False`. 112 | """ 113 | # Are we even using Azure? 114 | if self.driver_name == 'AZURE_BLOBS': 115 | try: 116 | # Yes? Is the azure-storage package available? 117 | from azure import storage 118 | # Shut the linter up. 119 | assert storage 120 | return True 121 | except ImportError: 122 | pass 123 | 124 | return False 125 | 126 | @property 127 | def can_use_advanced_aws(self): 128 | """ 129 | `True` if the `boto` module is installed and ckanext-cloudstorage has 130 | been configured to use Amazon S3, otherwise `False`. 131 | """ 132 | # Are we even using AWS? 133 | if 'S3' in self.driver_name: 134 | try: 135 | # Yes? Is the boto package available? 136 | import boto 137 | # Shut the linter up. 138 | assert boto 139 | return True 140 | except ImportError: 141 | pass 142 | 143 | return False 144 | 145 | @property 146 | def guess_mimetype(self): 147 | """ 148 | `True` if ckanext-cloudstorage is configured to guess mime types, 149 | `False` otherwise. 150 | """ 151 | return p.toolkit.asbool( 152 | config.get('ckanext.cloudstorage.guess_mimetype', False) 153 | ) 154 | 155 | 156 | class ResourceCloudStorage(CloudStorage): 157 | def __init__(self, resource): 158 | """ 159 | Support for uploading resources to any storage provider 160 | implemented by the apache-libcloud library. 161 | 162 | :param resource: The resource dict. 163 | """ 164 | super(ResourceCloudStorage, self).__init__() 165 | 166 | self.filename = None 167 | self.old_filename = None 168 | self.file = None 169 | self.resource = resource 170 | 171 | upload_field_storage = resource.pop('upload', None) 172 | self._clear = resource.pop('clear_upload', None) 173 | multipart_name = resource.pop('multipart_name', None) 174 | 175 | # Check to see if a file has been provided 176 | if isinstance(upload_field_storage, (ALLOWED_UPLOAD_TYPES)): 177 | self.filename = munge.munge_filename(upload_field_storage.filename) 178 | self.file_upload = _get_underlying_file(upload_field_storage) 179 | resource['url'] = self.filename 180 | resource['url_type'] = 'upload' 181 | elif multipart_name and self.can_use_advanced_aws: 182 | # This means that file was successfully uploaded and stored 183 | # at cloud. 184 | # Currently implemented just AWS version 185 | resource['url'] = munge.munge_filename(multipart_name) 186 | resource['url_type'] = 'upload' 187 | elif self._clear and resource.get('id'): 188 | # Apparently, this is a created-but-not-commited resource whose 189 | # file upload has been canceled. We're copying the behaviour of 190 | # ckaenxt-s3filestore here. 191 | old_resource = model.Session.query( 192 | model.Resource 193 | ).get( 194 | resource['id'] 195 | ) 196 | 197 | self.old_filename = old_resource.url 198 | resource['url_type'] = '' 199 | 200 | def path_from_filename(self, rid, filename): 201 | """ 202 | Returns a bucket path for the given resource_id and filename. 203 | 204 | :param rid: The resource ID. 205 | :param filename: The unmunged resource filename. 206 | """ 207 | return os.path.join( 208 | 'resources', 209 | rid, 210 | munge.munge_filename(filename) 211 | ) 212 | 213 | def upload(self, id, max_size=10): 214 | """ 215 | Complete the file upload, or clear an existing upload. 216 | 217 | :param id: The resource_id. 218 | :param max_size: Ignored. 219 | """ 220 | if self.filename: 221 | if self.can_use_advanced_azure: 222 | from azure.storage import blob as azure_blob 223 | from azure.storage.blob.models import ContentSettings 224 | 225 | blob_service = azure_blob.BlockBlobService( 226 | self.driver_options['key'], 227 | self.driver_options['secret'] 228 | ) 229 | content_settings = None 230 | if self.guess_mimetype: 231 | content_type, _ = mimetypes.guess_type(self.filename) 232 | if content_type: 233 | content_settings = ContentSettings( 234 | content_type=content_type 235 | ) 236 | 237 | return blob_service.create_blob_from_stream( 238 | container_name=self.container_name, 239 | blob_name=self.path_from_filename( 240 | id, 241 | self.filename 242 | ), 243 | stream=self.file_upload, 244 | content_settings=content_settings 245 | ) 246 | else: 247 | 248 | # TODO: This might not be needed once libcloud is upgraded 249 | if isinstance(self.file_upload, SpooledTemporaryFile): 250 | self.file_upload.next = self.file_upload.next() 251 | 252 | self.container.upload_object_via_stream( 253 | self.file_upload, 254 | object_name=self.path_from_filename( 255 | id, 256 | self.filename 257 | ) 258 | ) 259 | 260 | elif self._clear and self.old_filename and not self.leave_files: 261 | # This is only set when a previously-uploaded file is replace 262 | # by a link. We want to delete the previously-uploaded file. 263 | try: 264 | self.container.delete_object( 265 | self.container.get_object( 266 | self.path_from_filename( 267 | id, 268 | self.old_filename 269 | ) 270 | ) 271 | ) 272 | except ObjectDoesNotExistError: 273 | # It's possible for the object to have already been deleted, or 274 | # for it to not yet exist in a committed state due to an 275 | # outstanding lease. 276 | return 277 | 278 | def get_url_from_filename(self, rid, filename, content_type=None): 279 | """ 280 | Retrieve a publically accessible URL for the given resource_id 281 | and filename. 282 | 283 | .. note:: 284 | 285 | Works for Azure and any libcloud driver that implements 286 | support for get_object_cdn_url (ex: AWS S3). 287 | 288 | :param rid: The resource ID. 289 | :param filename: The resource filename. 290 | :param content_type: Optionally a Content-Type header. 291 | 292 | :returns: Externally accessible URL or None. 293 | """ 294 | # Find the key the file *should* be stored at. 295 | path = self.path_from_filename(rid, filename) 296 | 297 | # If advanced azure features are enabled, generate a temporary 298 | # shared access link instead of simply redirecting to the file. 299 | if self.can_use_advanced_azure and self.use_secure_urls: 300 | from azure.storage import blob as azure_blob 301 | 302 | blob_service = azure_blob.BlockBlobService( 303 | self.driver_options['key'], 304 | self.driver_options['secret'] 305 | ) 306 | 307 | return blob_service.make_blob_url( 308 | container_name=self.container_name, 309 | blob_name=path, 310 | sas_token=blob_service.generate_blob_shared_access_signature( 311 | container_name=self.container_name, 312 | blob_name=path, 313 | expiry=datetime.utcnow() + timedelta(hours=1), 314 | permission=azure_blob.BlobPermissions.READ 315 | ) 316 | ) 317 | elif self.can_use_advanced_aws and self.use_secure_urls: 318 | from boto.s3.connection import S3Connection 319 | s3_connection = S3Connection( 320 | self.driver_options['key'], 321 | self.driver_options['secret'] 322 | ) 323 | 324 | generate_url_params = {"expires_in": 60 * 60, 325 | "method": "GET", 326 | "bucket": self.container_name, 327 | "query_auth": True, 328 | "key": path} 329 | if content_type: 330 | generate_url_params['headers'] = {"Content-Type": content_type} 331 | 332 | return s3_connection.generate_url(**generate_url_params) 333 | 334 | # Find the object for the given key. 335 | obj = self.container.get_object(path) 336 | if obj is None: 337 | return 338 | 339 | # Not supported by all providers! 340 | try: 341 | return self.driver.get_object_cdn_url(obj) 342 | except NotImplementedError: 343 | if 'S3' in self.driver_name: 344 | return urlparse.urljoin( 345 | 'https://' + self.driver.connection.host, 346 | '{container}/{path}'.format( 347 | container=self.container_name, 348 | path=path 349 | ) 350 | ) 351 | # This extra 'url' property isn't documented anywhere, sadly. 352 | # See azure_blobs.py:_xml_to_object for more. 353 | elif 'url' in obj.extra: 354 | return obj.extra['url'] 355 | raise 356 | 357 | @property 358 | def package(self): 359 | return model.Package.get(self.resource['package_id']) 360 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/templates/cloudstorage/snippets/multipart_module.html: -------------------------------------------------------------------------------- 1 |
8 | {{ parent() }} 9 | 10 |
11 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/templates/package/new_resource.html: -------------------------------------------------------------------------------- 1 | {% ckan_extends %} 2 | 3 | 4 | {% block form %} 5 | 6 | {% snippet 'cloudstorage/snippets/multipart_module.html', pkg_name=pkg_name, parent=super %} 7 | 8 | {% endblock %} 9 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/templates/package/new_resource_not_draft.html: -------------------------------------------------------------------------------- 1 | {% ckan_extends %} 2 | 3 | 4 | {% block form %} 5 | 6 | {% snippet 'cloudstorage/snippets/multipart_module.html', pkg_name=pkg_name, parent=super %} 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/templates/package/resource_edit.html: -------------------------------------------------------------------------------- 1 | {% ckan_extends %} 2 | 3 | 4 | {% block form %} 5 | 6 | {% snippet 'cloudstorage/snippets/multipart_module.html', pkg_name=pkg.name, parent=super %} 7 | 8 | {% endblock %} 9 | -------------------------------------------------------------------------------- /ckanext/cloudstorage/templates/page.html: -------------------------------------------------------------------------------- 1 | {% ckan_extends %} 2 | 3 | {% block scripts %} 4 | {{ super() }} 5 | {% resource 'cloudstorage-js/main' %} 6 | 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from setuptools import setup, find_packages 4 | 5 | setup( 6 | name='ckanext-cloudstorage', 7 | version='0.1.1', 8 | description='Cloud storage for CKAN', 9 | classifiers=[], 10 | keywords='', 11 | author='Tyler Kennedy', 12 | author_email='tk@tkte.ch', 13 | url='http://github.com/open-data/ckanext-cloudstorage', 14 | license='MIT', 15 | packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 16 | namespace_packages=['ckanext'], 17 | include_package_data=True, 18 | zip_safe=False, 19 | install_requires=[ 20 | 'apache-libcloud==1.5', 21 | 'ckanapi>=1.0,<5' 22 | ], 23 | entry_points=( 24 | """ 25 | [ckan.plugins] 26 | cloudstorage=ckanext.cloudstorage.plugin:CloudStoragePlugin 27 | 28 | [paste.paster_command] 29 | cloudstorage=ckanext.cloudstorage.cli:PasterCommand 30 | """ 31 | ), 32 | ) 33 | --------------------------------------------------------------------------------