├── .gitignore ├── CHANGELOG ├── LICENSE ├── README.md ├── example ├── __init__.py ├── app.py ├── data │ └── hello.txt └── templates │ ├── index.html │ └── view.html ├── flask_cloudy.py ├── requirements-dev.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── config.py ├── data │ ├── hello.js │ └── hello.txt └── test_cloudy.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | __pycache__* 3 | 4 | # Packages 5 | *.egg 6 | *.egg-info 7 | dist 8 | build 9 | eggs 10 | parts 11 | bin 12 | var 13 | sdist 14 | develop-eggs 15 | .installed.cfg 16 | MANIFEST 17 | 18 | # PyCharm 19 | .idea/* 20 | .idea/libraries/sass_stdlib.xml 21 | 22 | # Distribution 23 | dist/* 24 | 25 | # Mac stuff 26 | .DS_Store 27 | 28 | docs/_build* 29 | 30 | .test_config 31 | 32 | .tox 33 | .cache 34 | 35 | tests/* 36 | !tests/__init__.py 37 | !tests/config.py 38 | !tests/test_cloudy.py 39 | 40 | tests/data/* 41 | !tests/data/hello.txt 42 | !tests/data/hello.js 43 | 44 | tests/container_1/* 45 | !tests/container_1/empty 46 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 1.1.0 2 | - fixed dependencies 3 | 1.0.0 4 | - Storage.upload now accepts a url, that will first download the file on local then upload it to storage 5 | - Added args random_name(bool) in Storage.upload. If True and Name is None, it will create a uuid4 name. 6 | `name` always takes precedence 7 | - Added Object.info which returns a dict of the data. Which can be saved in a database 8 | - Removed shortuuid in favor of uuid4 9 | - (backwards compatible) rename 'allowed_extensions' to `extensions`, 10 | but will still try to fall back to allowed extensions if None 11 | 0.15 12 | - Added Object.full_path to return the full path of the local storage object 13 | - remove Object.short_url which was deprecated 14 | 15 | 0.14.0 16 | - Added the context manager 'use' to temporarily use a different container on the same driver 17 | 18 | 0.13.2 19 | - Flask required version 0.10.1 + 20 | 21 | 0.13.1 22 | - Set default storage to True 23 | - Thanks to https://github.com/wassname for the contributions] 24 | 25 | 0.13.0 26 | - Upgrade apache-libcloud to 0.20.0 27 | - Fixed object being False 28 | - Thanks to https://github.com/alexa-infra for the contributions 29 | 30 | 0.12.0 31 | - Fixed prefix in Storage.upload that assumed the prefix is a directory 32 | Add a slash / at the end of prefix to make it a directory, or it will 33 | just append it to the name 34 | 35 | 0.11.0 36 | - Removed flask_cloudy.Object.short_url now use flask_cloudy.Object.url 37 | - Added flask_cloudy.Object.full_url to have the domain for local storage 38 | 39 | 0.10.0 40 | - Removed LOCAL_PATH configuration. Use CONTAINER as the LOCAL_PATH 41 | - Change config prefix to STORAGE_* 42 | 43 | 0.6.0 44 | - More pythonic 45 | - implement __contains__ to look for an item in the storage. `if object_name in storage` 46 | - rename Storage:objet to Storage:get(). 47 | 48 | 0.5.1 49 | - Fixed typo 50 | 51 | 0.5.0 52 | - object_path return the full path of the object when on local 53 | - get_url() can return short url for local file instead of the full domain one 54 | - rename Storage:get_object to Storage:object 55 | 56 | 0.4.0 57 | - Added: provider_name, container_name, and local_path, object_path to object 58 | 59 | 0.3.1 60 | - Added the config 'CLOUDSTORAGE_SERVE_FILES_URL_SECURE' in flask to serve 61 | files over https 62 | 63 | 0.3.0 64 | - Serve local files through Python 65 | - When flask FileStorage type is being uploaded, use the stream method 66 | - Object:get_url() will now return the appropriate url 67 | 68 | 0.2.0 69 | - Add extension of original file on upload if object_name doesn't have an extension 70 | - Use the original file name as object name, if object name is None 71 | 72 | 0.1.0 73 | - First -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Mardix 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flask-Cloudy 2 | 3 | 4 | ## About 5 | 6 | A Flask extension to **access, upload, download, save and delete** files on cloud storage providers such as: 7 | AWS S3, Google Storage, Microsoft Azure, Rackspace Cloudfiles, and even Local file system. 8 | 9 | For local file storage, it also provides a flask endpoint to access the files. 10 | 11 | 12 | Version: 1.x.x 13 | 14 | --- 15 | 16 | ## TLDR: Quick Example 17 | ```py 18 | from flask import Flask, request 19 | from flask_cloudy import Storage 20 | 21 | app = Flask(__name__) 22 | 23 | # Update the config 24 | app.config.update({ 25 | "STORAGE_PROVIDER": "LOCAL", # Can also be S3, GOOGLE_STORAGE, etc... 26 | "STORAGE_KEY": "", 27 | "STORAGE_SECRET": "", 28 | "STORAGE_CONTAINER": "./", # a directory path for local, bucket name of cloud 29 | "STORAGE_SERVER": True, 30 | "STORAGE_SERVER_URL": "/files" # The url endpoint to access files on LOCAL provider 31 | }) 32 | 33 | # Setup storage 34 | storage = Storage() 35 | storage.init_app(app) 36 | 37 | @app.route("/upload", methods=["POST", "GET"]) 38 | def upload(): 39 | if request.method == "POST": 40 | file = request.files.get("file") 41 | my_upload = storage.upload(file) 42 | 43 | # some useful properties 44 | name = my_upload.name 45 | extension = my_upload.extension 46 | size = my_upload.size 47 | url = my_upload.url 48 | 49 | return url 50 | 51 | # Pretending the file uploaded is "my-picture.jpg" 52 | # it will return a url in the format: http://domain.com/files/my-picture.jpg 53 | 54 | 55 | # A download endpoint, to download the file 56 | @app.route("/download/") 57 | def download(object_name): 58 | my_object = storage.get(object_name) 59 | if my_object: 60 | download_url = my_object.download() 61 | return download_url 62 | else: 63 | abort(404, "File doesn't exist") 64 | ``` 65 | --- 66 | 67 | Go to the "example" directory to get a workable flask-cloud example 68 | 69 | 70 | --- 71 | 72 | 73 | ### Features: 74 | 75 | - Browse files 76 | 77 | - Upload files 78 | 79 | - Download files 80 | 81 | - Delete files 82 | 83 | - Serve files via http 84 | 85 | 86 | ### Supported storage: 87 | 88 | - AWS S3 89 | 90 | - Google Storage 91 | 92 | - Microsoft Azure 93 | 94 | - Rackspace CloudFiles 95 | 96 | - Local (for local file system) 97 | 98 | 99 | **Dependecies:** (They will be installed upon setup) 100 | 101 | - Flask 102 | 103 | - Apache-Libcloud 104 | 105 | --- 106 | 107 | ## Install & Config 108 | 109 | pip install flask-cloudy 110 | 111 | --- 112 | 113 | (To use it as standalone, refer to API documentaion below) 114 | 115 | ## Config for Flask 116 | 117 | Within your Flask application's settings you can provide the following settings to control 118 | the behavior of Flask-Cloudy 119 | 120 | 121 | **- STORAGE_PROVIDER** (str) 122 | 123 | - LOCAL 124 | - S3 125 | - S3_US_WEST 126 | - S3_US_WEST_OREGON 127 | - S3_EU_WEST 128 | - S3_AP_SOUTHEAST 129 | - S3_AP_NORTHEAST 130 | - GOOGLE_STORAGE 131 | - AZURE_BLOBS 132 | - CLOUDFILES 133 | 134 | 135 | **- STORAGE_KEY** (str) 136 | 137 | The access key of the cloud storage provider 138 | 139 | None for LOCAL 140 | 141 | **- STORAGE_SECRET** (str) 142 | 143 | The access secret key of the cloud storage provider 144 | 145 | None for LOCAL 146 | 147 | **- STORAGE_CONTAINER** (str) 148 | 149 | The *BUCKET NAME* for cloud storage providers 150 | 151 | For *LOCAL* provider, this is the local directory path 152 | 153 | 154 | **STORAGE_ALLOWED_EXTENSIONS** (list) 155 | 156 | List of all extensions to allow 157 | 158 | Example: ["png", "jpg", "jpeg", "mp3"] 159 | 160 | **STORAGE_SERVER** (bool) 161 | 162 | For *LOCAL* provider only. 163 | 164 | True to expose the files in the container so they can be accessed 165 | 166 | Default: *True* 167 | 168 | **STORAGE_SERVER_URL** (str) 169 | 170 | For *LOCAL* provider only. 171 | 172 | The endpoint to access the files from the local storage. 173 | 174 | Default: */files* 175 | 176 | --- 177 | 178 | ## API Documention 179 | 180 | Flask-Cloudy is a wrapper around Apache-Libcloud, the Storage class gives you access to Driver and Container of Apache-Libcloud. 181 | 182 | *Lexicon:* 183 | 184 | Object: A file or a file path. 185 | 186 | Container: The main directory, or a bucket name containing all the objects 187 | 188 | Provider: The method 189 | 190 | Storage: 191 | 192 | ### flask_cloudy.Storage 193 | 194 | The **Storage** class allows you to access, upload, get an object from the Storage. 195 | 196 | #### Storage(provider, key=None, secret=None, container=None, allowed_extensions=None) 197 | 198 | - provider: the storage provider: 199 | 200 | - LOCAL 201 | - S3 202 | - S3_US_WEST 203 | - S3_US_WEST_OREGON 204 | - S3_EU_WEST 205 | - S3_AP_SOUTHEAST 206 | - S3_AP_NORTHEAST 207 | - GOOGLE_STORAGE 208 | - AZURE_BLOBS 209 | - CLOUDFILES 210 | 211 | - key: The access key of the cloud storage. None when provider is LOCAL 212 | 213 | - secret: The secret access key of the cloud storage. None when provider is LOCAL 214 | 215 | - container: 216 | 217 | - For cloud storage, use the **BUCKET NAME** 218 | 219 | - For LOCAL provider, it's the directory path where to access the files 220 | 221 | - allowed_extensions: List of extensions to upload to upload 222 | 223 | 224 | #### Storage.init_app(app) 225 | 226 | To initiate the Storage via Flask config. 227 | 228 | It will also setup a server endpoint when STORAGE_PROVIDER == LOCAL 229 | 230 | ```py 231 | 232 | from flask import Flask, request 233 | from flask_cloudy import Storage 234 | 235 | app = Flask(__name__) 236 | 237 | # Update the config 238 | app.config.update({ 239 | "STORAGE_PROVIDER": "LOCAL", # Can also be S3, GOOGLE_STORAGE, etc... 240 | "STORAGE_KEY": "", 241 | "STORAGE_SECRET": "", 242 | "STORAGE_CONTAINER": "./", # a directory path for local, bucket name of cloud 243 | "STORAGE_SERVER": True, 244 | "STORAGE_SERVER_URL": "/files" 245 | }) 246 | 247 | # Setup storage 248 | storage = Storage() 249 | storage.init_app(app) 250 | 251 | @app.route("/upload", methods=["POST", "GET"]): 252 | def upload(): 253 | if request.method == "POST": 254 | file = request.files.get("file") 255 | my_upload = storage.upload(file) 256 | 257 | # some useful properties 258 | name = my_upload.name 259 | extension = my_upload.extension 260 | size = my_upload.size 261 | url = my_upload.url 262 | 263 | return url 264 | 265 | # Pretending the file uploaded is "my-picture.jpg" 266 | # it will return a url in the format: http://domain.com/files/my-picture.jpg 267 | ``` 268 | 269 | 270 | #### Storage.get(object_name) 271 | 272 | Get an object in the storage by name, relative to the container. 273 | 274 | It will return an instance of **flask_cloudy.Object** 275 | 276 | - object_name: The name of the object. 277 | 278 | Some valid object names, they can contains slashes to indicate it's a directory 279 | 280 | 281 | - file.txt 282 | 283 | - my_dir/file.txt 284 | 285 | - my_dir/sub_dir/file.txt 286 | 287 | . 288 | ```py 289 | storage = Storage(provider, key, secret, container) 290 | object_name = "hello.txt" 291 | my_object = storage.get(object_name) 292 | ``` 293 | 294 | 295 | #### Storage.upload(file, name=None, prefix=None, extension=[], overwrite=Flase, public=False, random_name=False) 296 | 297 | To save or upload a file in the container 298 | 299 | - file: the string of the file location or a file object 300 | 301 | - name: to give the file a new name 302 | 303 | - prefix: a name to add in front of the file name. Add a slash at the end of 304 | prefix to make it a directory otherwise it will just append it to the name 305 | 306 | - extensions: list of extensions 307 | 308 | - overwrite: If True it will overwrite existing files, otherwise it will add a uuid in the file name to make it unique 309 | 310 | - public: Bool - To set the **acl** to *public-read* when True, *private* when False 311 | 312 | - random_name: Bool - To randomly create a unique name if `name` is None 313 | 314 | . 315 | ```py 316 | storage = Storage(provider, key, secret, container) 317 | my_file = "my_dir/readme.md" 318 | ``` 319 | 320 | **1) Upload file + file name is the name of the uploaded file ** 321 | ```py 322 | storage.upload(my_file) 323 | ``` 324 | 325 | **2) Upload file + file name is now `new_readme`. It will will keep the extension of the original file** 326 | ```py 327 | storage.upload(my_file, name="new_readme") 328 | ``` 329 | The uploaded file will be named: **new_readme.md** 330 | 331 | 332 | **3) Upload file to a different path using `prefix`** 333 | 334 | ```py 335 | storage.upload(my_file, name="new_readme", prefix="my_dir/") 336 | ``` 337 | 338 | now the filename becomes **my_dir/new_readme.md** 339 | 340 | On LOCAL it will create the directory *my_dir* if it doesn't exist. 341 | 342 | ```py 343 | storage.upload(my_file, name="new_readme", prefix="my_new_path-") 344 | ``` 345 | 346 | now the filename becomes **my_new_path-new_readme.md** 347 | 348 | ATTENTION: If you want the file to be place in a subdirectory, `prefix` must have the trailing slash 349 | 350 | 351 | **4a.) Public upload** 352 | ```py 353 | storage.upload(my_file, public=True) 354 | ``` 355 | 356 | **4b.) Private upload** 357 | ```py 358 | storage.upload(my_file, public=False) 359 | ``` 360 | **5) Upload + random name** 361 | ```py 362 | storage.upload(my_file, random_name=True) 363 | ``` 364 | **6) Upload with external url*** 365 | 366 | You can upload an item from the internet directly to your storage 367 | ```py 368 | storage.upload("http://the.site.path.com/abc.png") 369 | ``` 370 | It will save the image to your storage 371 | 372 | 373 | #### Storage.create(object_name, size=0, hash=None, extra=None, metda_data=None) 374 | 375 | Explicitly create an object that may exist already. Usually, when paramameters (name, size, hash, etc...) are already saved, let's say in the database, and you want Storage to manipulate the file. 376 | ```py 377 | storage = Storage(provider, key, secret, container) 378 | existing_name = "holla.txt" 379 | existing_size = "8000" # in bytes 380 | new_object = storage.create(object_name=existing_name, size=existing_size) 381 | 382 | # Now I can do 383 | url = new_object.url 384 | size = len(new_object) 385 | ``` 386 | 387 | #### Storage.use(container) 388 | 389 | A context manager to temporarily use a different container on the same provider 390 | ``` 391 | storage = Storage(provider, key, secret, container) 392 | 393 | with storage.use(another_container_name) as s3: 394 | s3.upload(newfile) 395 | ``` 396 | In the example above, it will upload the `newfile` to the new container name 397 | 398 | 399 | *It's Pythonic!!!* 400 | 401 | #### Iterate through all the objects in the container 402 | 403 | Each object is an instance on **flask_cloudy.Object** 404 | ```py 405 | storage = Storage(provider, key, secret, container) 406 | for obj in storage: 407 | print(obj.name) 408 | ``` 409 | #### Get the total objects in the container 410 | ```py 411 | storage = Storage(provider, key, secret, container) 412 | total_items = len(storage) 413 | ``` 414 | #### Check to see if an object exists in the container 415 | ```py 416 | storage = Storage(provider, key, secret, container) 417 | my_file = "hello.txt" 418 | 419 | if my_file in storage: 420 | print("File is in the storage") 421 | ``` 422 | --- 423 | 424 | 425 | ### flask_cloudy.Object 426 | 427 | The class **Object** is an entity of an object in the container. 428 | 429 | Usually, you will get a cloud object by accessing an object in the container. 430 | ```py 431 | storage = Storage(provider, key, secret, container) 432 | my_object = storage.get("my_object.txt") 433 | ``` 434 | Properties: 435 | 436 | #### Object.name 437 | 438 | The name of the object 439 | 440 | 441 | #### Object.size 442 | 443 | The size in bytes of the object 444 | 445 | 446 | #### Object.extension 447 | 448 | The extension of the object 449 | 450 | 451 | #### Object.url 452 | 453 | Return the url of the object 454 | 455 | On LOCAL, it will return the url without the domain name ( ie: /files/my-file.jpg ) 456 | 457 | For cloud providers it will return the full url 458 | 459 | #### Object.full_url 460 | 461 | Returns the full url of the object 462 | 463 | Specially for LOCAL provider, it will return the url with the domain. 464 | 465 | For cloud providers, it will return the full url just like **Object.url** 466 | 467 | 468 | #### Object.secure_url 469 | 470 | Return a secured url, with **https://** 471 | 472 | 473 | #### Object.path 474 | 475 | The path of the object relative to the container 476 | 477 | #### Object.full_path 478 | 479 | For Local, it will show the full path of the object, otherwise it just returns 480 | the Object.path 481 | 482 | 483 | #### Object.provider_name 484 | 485 | The provider name: ie: Local, S3,... 486 | 487 | 488 | #### Object.type 489 | 490 | The type of the object, ie: IMAGE, AUDIO, TEXT,... OTHER 491 | 492 | 493 | #### Object.info 494 | 495 | Returns a dict of the object name, extension, url, etc. This can be saved in a DB 496 | 497 | Methods: 498 | 499 | #### Object.save_to(destination, name=None, overwrite=False, delete_on_failure=True) 500 | 501 | To save the object to a local path 502 | 503 | - destination: The directory to save the object to 504 | 505 | - name: To rename the file in the local directory. Do not put the extension of the file, it will append automatically 506 | 507 | - overwrite: bool - To overwrite the file if it exists 508 | 509 | - delete_on_failure: bool - To delete the file it fails to save 510 | 511 | . 512 | ```py 513 | storage = Storage(provider, key, secret, container) 514 | my_object = storage.get("my_object.txt") 515 | my_new_path = "/my/new/path" 516 | my_new_file = my_object.save_to(my_new_path) 517 | 518 | print(my_new_file) # Will print -> /my/new/path/my_object.txt 519 | ``` 520 | 521 | #### Object.download_url(timeout=60, name=None) 522 | 523 | Return a URL that triggers the browser download of the file. On cloud providers it will return a signed url. 524 | 525 | - timeout: int - The time in seconds to give access to the url 526 | 527 | - name: str - for LOCAL only, to rename the file being downloaded 528 | 529 | . 530 | ```py 531 | storage = Storage(provider, key, secret, container) 532 | my_object = storage.get("my_object.txt") 533 | download_url = my_object.download_url() 534 | 535 | # or with flask 536 | 537 | @app.route("/download/"): 538 | def download(object_name): 539 | my_object = storage.get(object_name) 540 | if my_object: 541 | download_url = my_object.download_url() 542 | return redirect(download_url) 543 | else: 544 | abort(404, "File doesn't exist") 545 | ``` 546 | 547 | --- 548 | 549 | I hope you find this library useful, enjoy! 550 | 551 | 552 | Mardix :) 553 | 554 | --- 555 | 556 | License: MIT - Copyright 2017 Mardix 557 | 558 | -------------------------------------------------------------------------------- /example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mardix/flask-cloudy/cf9f6f074e92af1d98f99fdf6ca458e29099e4b1/example/__init__.py -------------------------------------------------------------------------------- /example/app.py: -------------------------------------------------------------------------------- 1 | 2 | from flask import Flask, request, render_template, redirect, abort, url_for 3 | from flask_cloudy import Storage 4 | 5 | app = Flask(__name__) 6 | 7 | app.config.update({ 8 | "STORAGE_PROVIDER": "LOCAL", 9 | "STORAGE_CONTAINER": "./data", 10 | "STORAGE_KEY": "", 11 | "STORAGE_SECRET": "", 12 | "STORAGE_SERVER": True 13 | }) 14 | 15 | storage = Storage() 16 | storage.init_app(app) 17 | 18 | @app.route("/") 19 | def index(): 20 | 21 | return render_template("index.html", storage=storage) 22 | 23 | @app.route("/view/") 24 | def view(object_name): 25 | obj = storage.get(object_name) 26 | print obj.name 27 | return render_template("view.html", obj=obj) 28 | 29 | @app.route("/upload", methods=["POST"]) 30 | def upload(): 31 | file = request.files.get("file") 32 | my_object = storage.upload(file) 33 | return redirect(url_for("view", object_name=my_object.name)) 34 | 35 | 36 | if __name__ == "__main__": 37 | app.run(debug=True, port=5000) -------------------------------------------------------------------------------- /example/data/hello.txt: -------------------------------------------------------------------------------- 1 | Hello World! 2 | 3 | from flask_cloud import Storage -------------------------------------------------------------------------------- /example/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Flask-Cloudy 6 | 7 | 8 | 9 |

Flask-Cloudy

10 | 11 | 12 |
13 | Select image to upload: 14 |
15 | 16 |
17 | 18 |
19 | 20 |

List of files available on the storage:

21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | {% for obj in storage %} 29 | 30 | 31 | 32 | 33 | {% endfor %} 34 | 35 | 36 |
NameSize
{{ obj.name }}{{ obj.size }} bytes
37 | 38 | 39 | -------------------------------------------------------------------------------- /example/templates/view.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Flask-Cloudy 6 | 7 | 8 | 9 |

Flask-Cloudy: View File

10 | <- Home 11 |

12 | 13 | Name: {{ obj.name }}

14 | Size: {{ obj.size }} bytes

15 | 16 | Short url: {{ obj.short_url }}

17 | 18 | View file: {{ obj.url }}

19 | 20 | {% set download_url = obj.download_url() %} 21 | Download: {{ download_url }}

22 | 23 | -------------------------------------------------------------------------------- /flask_cloudy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Flask-Cloudy 3 | """ 4 | 5 | import os 6 | import re 7 | import datetime 8 | import base64 9 | import hmac 10 | import hashlib 11 | import warnings 12 | from contextlib import contextmanager 13 | import copy 14 | from werkzeug.utils import secure_filename 15 | from werkzeug.datastructures import FileStorage 16 | from importlib import import_module 17 | from flask import send_file, abort, url_for 18 | from flask import request as flask_request 19 | import uuid 20 | from libcloud.storage.types import Provider, ObjectDoesNotExistError 21 | from libcloud.storage.providers import DRIVERS, get_driver 22 | from libcloud.storage.base import Object as BaseObject, StorageDriver 23 | from libcloud.storage.drivers import local 24 | from six.moves.urllib.parse import urlparse, urlunparse, urljoin, urlencode 25 | from six.moves.urllib import request 26 | from six import string_types 27 | import slugify 28 | 29 | 30 | SERVER_ENDPOINT = "FLASK_CLOUDY_SERVER" 31 | 32 | EXTENSIONS = { 33 | "TEXT": ["txt", "md"], 34 | "DOCUMENT": ["rtf", "odf", "ods", "gnumeric", "abw", "doc", "docx", "xls", "xlsx"], 35 | "IMAGE": ["jpg", "jpeg", "jpe", "png", "gif", "svg", "bmp", "webp"], 36 | "AUDIO": ["wav", "mp3", "aac", "ogg", "oga", "flac"], 37 | "DATA": ["csv", "ini", "json", "plist", "xml", "yaml", "yml"], 38 | "SCRIPT": ["js", "php", "pl", "py", "rb", "sh"], 39 | "ARCHIVE": ["gz", "bz2", "zip", "tar", "tgz", "txz", "7z"] 40 | } 41 | 42 | ALL_EXTENSIONS = EXTENSIONS["TEXT"] \ 43 | + EXTENSIONS["DOCUMENT"] \ 44 | + EXTENSIONS["IMAGE"] \ 45 | + EXTENSIONS["AUDIO"] \ 46 | + EXTENSIONS["DATA"] \ 47 | + EXTENSIONS["ARCHIVE"] 48 | 49 | URL_REGEXP = re.compile(r'^(http|https|ftp|ftps)://') 50 | 51 | class InvalidExtensionError(Exception): 52 | pass 53 | 54 | def get_file_name(filename): 55 | """ 56 | Return the filename without the path 57 | :param filename: 58 | :return: str 59 | """ 60 | return os.path.basename(filename) 61 | 62 | def get_file_extension(filename): 63 | """ 64 | Return a file extension 65 | :param filename: 66 | :return: str 67 | """ 68 | return os.path.splitext(filename)[1][1:].lower() 69 | 70 | def get_file_extension_type(filename): 71 | """ 72 | Return the group associated to the file 73 | :param filename: 74 | :return: str 75 | """ 76 | ext = get_file_extension(filename) 77 | if ext: 78 | for name, group in EXTENSIONS.items(): 79 | if ext in group: 80 | return name 81 | return "OTHER" 82 | 83 | def get_driver_class(provider): 84 | """ 85 | Return the driver class 86 | :param provider: str - provider name 87 | :return: 88 | """ 89 | if "." in provider: 90 | parts = provider.split('.') 91 | kls = parts.pop() 92 | path = '.'.join(parts) 93 | module = import_module(path) 94 | if not hasattr(module, kls): 95 | raise ImportError('{0} provider not found at {1}'.format( 96 | kls, 97 | path)) 98 | driver = getattr(module, kls) 99 | else: 100 | driver = getattr(Provider, provider.upper()) 101 | return get_driver(driver) 102 | 103 | def get_provider_name(driver): 104 | """ 105 | Return the provider name from the driver class 106 | :param driver: obj 107 | :return: str 108 | """ 109 | kls = driver.__class__.__name__ 110 | for d, prop in DRIVERS.items(): 111 | if prop[1] == kls: 112 | return d 113 | return None 114 | 115 | 116 | class Storage(object): 117 | container = None 118 | driver = None 119 | config = {} 120 | 121 | TEXT = EXTENSIONS["TEXT"] 122 | DOCUMENT = EXTENSIONS["DOCUMENT"] 123 | IMAGE = EXTENSIONS["IMAGE"] 124 | AUDIO = EXTENSIONS["AUDIO"] 125 | DATA = EXTENSIONS["DATA"] 126 | SCRIPT = EXTENSIONS["SCRIPT"] 127 | ARCHIVE = EXTENSIONS["ARCHIVE"] 128 | 129 | allowed_extensions = TEXT + DOCUMENT + IMAGE + AUDIO + DATA 130 | 131 | _kw = {} 132 | 133 | def __init__(self, 134 | provider=None, 135 | key=None, 136 | secret=None, 137 | container=None, 138 | allowed_extensions=None, 139 | app=None, 140 | **kwargs): 141 | 142 | """ 143 | Initiate the storage 144 | :param provider: str - provider name 145 | :param key: str - provider key 146 | :param secret: str - provider secret 147 | :param container: str - the name of the container (bucket or a dir name if local) 148 | :param allowed_extensions: list - extensions allowed for upload 149 | :param app: object - Flask instance 150 | :param kwargs: any other params will pass to the provider initialization 151 | :return: 152 | """ 153 | 154 | if app: 155 | self.init_app(app) 156 | 157 | if provider: 158 | # Hold the params that were passed 159 | self._kw = { 160 | "provider": provider, 161 | "key": key, 162 | "secret": secret, 163 | "container": container, 164 | "allowed_extensions": allowed_extensions, 165 | "app": app 166 | } 167 | self._kw.update(kwargs) 168 | 169 | if allowed_extensions: 170 | self.allowed_extensions = allowed_extensions 171 | 172 | kwparams = { 173 | "key": key, 174 | "secret": secret 175 | } 176 | 177 | if "local" in provider.lower(): 178 | kwparams["key"] = container 179 | container = "" 180 | 181 | kwparams.update(kwargs) 182 | 183 | self.driver = get_driver_class(provider)(**kwparams) 184 | if not isinstance(self.driver, StorageDriver): 185 | raise AttributeError("Invalid Driver") 186 | 187 | self.container = self.driver.get_container(container) 188 | 189 | def __iter__(self): 190 | """ 191 | ie: `for item in storage` 192 | Iterate over all the objects in the container 193 | :return: generator 194 | """ 195 | for obj in self.container.iterate_objects(): 196 | yield Object(obj=obj) 197 | 198 | def __len__(self): 199 | """ 200 | ie: `len(storage)` 201 | Return the total objects in the container 202 | :return: int 203 | """ 204 | return len(self.container.list_objects()) 205 | 206 | def __contains__(self, object_name): 207 | """ 208 | ie: `if name in storage` or `if name not in storage` 209 | Test if object exists 210 | :param object_name: the object name 211 | :return bool: 212 | """ 213 | try: 214 | self.driver.get_object(self.container.name, object_name) 215 | return True 216 | except ObjectDoesNotExistError: 217 | return False 218 | 219 | def init_app(self, app): 220 | """ 221 | To initiate with Flask 222 | :param app: Flask object 223 | :return: 224 | """ 225 | provider = app.config.get("STORAGE_PROVIDER", None) 226 | key = app.config.get("STORAGE_KEY", None) 227 | secret = app.config.get("STORAGE_SECRET", None) 228 | container = app.config.get("STORAGE_CONTAINER", None) 229 | allowed_extensions = app.config.get("STORAGE_ALLOWED_EXTENSIONS", None) 230 | serve_files = app.config.get("STORAGE_SERVER", True) 231 | serve_files_url = app.config.get("STORAGE_SERVER_URL", "files") 232 | 233 | self.config["serve_files"] = serve_files 234 | self.config["serve_files_url"] = serve_files_url 235 | 236 | if not provider: 237 | raise ValueError("'STORAGE_PROVIDER' is missing") 238 | 239 | if provider.upper() == "LOCAL": 240 | if not os.path.isdir(container): 241 | raise IOError("Local Container (directory) '%s' is not a " 242 | "directory or doesn't exist for LOCAL provider" % container) 243 | 244 | self.__init__(provider=provider, 245 | key=key, 246 | secret=secret, 247 | container=container, 248 | allowed_extensions=allowed_extensions) 249 | 250 | self._register_file_server(app) 251 | 252 | @contextmanager 253 | def use(self, container): 254 | """ 255 | A context manager to temporarily use a different container on the same driver 256 | :param container: str - the name of the container (bucket or a dir name if local) 257 | :yield: Storage 258 | """ 259 | kw = self._kw.copy() 260 | kw["container"] = container 261 | s = Storage(**kw) 262 | yield s 263 | del s 264 | 265 | def get(self, object_name): 266 | """ 267 | Return an object or None if it doesn't exist 268 | :param object_name: 269 | :return: Object 270 | """ 271 | if object_name in self: 272 | return Object(obj=self.container.get_object(object_name)) 273 | return None 274 | 275 | def create(self, object_name, size=0, hash=None, extra=None, meta_data=None): 276 | """ 277 | create a new object 278 | :param object_name: 279 | :param size: 280 | :param hash: 281 | :param extra: 282 | :param meta_data: 283 | :return: Object 284 | """ 285 | obj = BaseObject(container=self.container, 286 | driver=self.driver, 287 | name=object_name, 288 | size=size, 289 | hash=hash, 290 | extra=extra, 291 | meta_data=meta_data) 292 | return Object(obj=obj) 293 | 294 | def upload(self, 295 | file, 296 | name=None, 297 | prefix=None, 298 | extensions=None, 299 | overwrite=False, 300 | public=False, 301 | random_name=False, 302 | **kwargs): 303 | """ 304 | To upload file 305 | :param file: FileStorage object or string location 306 | :param name: The name of the object. 307 | :param prefix: A prefix for the object. Can be in the form of directory tree 308 | :param extensions: list of extensions to allow. If empty, it will use all extension. 309 | :param overwrite: bool - To overwrite if file exists 310 | :param public: bool - To set acl to private or public-read. Having acl in kwargs will override it 311 | :param random_name - If True and Name is None it will create a random name. 312 | Otherwise it will use the file name. `name` will always take precedence 313 | :param kwargs: extra params: ie: acl, meta_data etc. 314 | :return: Object 315 | """ 316 | tmp_file = None 317 | try: 318 | if "acl" not in kwargs: 319 | kwargs["acl"] = "public-read" if public else "private" 320 | extra = kwargs 321 | 322 | # It seems like this is a url, we'll try to download it first 323 | if isinstance(file, string_types) and re.match(URL_REGEXP, file): 324 | tmp_file = self._download_from_url(file) 325 | file = tmp_file 326 | 327 | # Create a random name 328 | if not name and random_name: 329 | name = uuid.uuid4().hex 330 | 331 | # coming from a flask, or upload object 332 | if isinstance(file, FileStorage): 333 | extension = get_file_extension(file.filename) 334 | if not name: 335 | fname = get_file_name(file.filename).split("." + extension)[0] 336 | name = slugify.slugify(fname) 337 | else: 338 | extension = get_file_extension(file) 339 | if not name: 340 | name = get_file_name(file) 341 | 342 | if len(get_file_extension(name).strip()) == 0: 343 | name += "." + extension 344 | 345 | name = name.strip("/").strip() 346 | 347 | if isinstance(self.driver, local.LocalStorageDriver): 348 | name = secure_filename(name) 349 | 350 | if prefix: 351 | name = prefix.lstrip("/") + name 352 | 353 | if not overwrite: 354 | name = self._safe_object_name(name) 355 | 356 | # For backwards compatibility, kwargs now holds `allowed_extensions` 357 | allowed_extensions = extensions or kwargs.get("allowed_extensions") 358 | if not allowed_extensions: 359 | allowed_extensions = self.allowed_extensions 360 | if extension.lower() not in allowed_extensions: 361 | raise InvalidExtensionError("Invalid file extension: '.%s' " % extension) 362 | 363 | if isinstance(file, FileStorage): 364 | obj = self.container.upload_object_via_stream(iterator=file.stream, 365 | object_name=name, 366 | extra=extra) 367 | else: 368 | obj = self.container.upload_object(file_path=file, 369 | object_name=name, 370 | extra=extra) 371 | return Object(obj=obj) 372 | except Exception as e: 373 | raise e 374 | finally: 375 | if tmp_file and os.path.isfile(tmp_file): 376 | os.remove(tmp_file) 377 | 378 | def _download_from_url(self, url): 379 | """ 380 | Download a url and return the tmp path 381 | :param url: 382 | :return: 383 | """ 384 | ext = get_file_extension(url) 385 | if "?" in url: 386 | ext = get_file_extension(os.path.splitext(url.split("?")[0])) 387 | filepath = "/tmp/%s.%s" % (uuid.uuid4().hex, ext) 388 | request.urlretrieve(url, filepath) 389 | return filepath 390 | 391 | def _safe_object_name(self, object_name): 392 | """ Add a UUID if to a object name if it exists. To prevent overwrites 393 | :param object_name: 394 | :return str: 395 | """ 396 | extension = get_file_extension(object_name) 397 | file_name = os.path.splitext(object_name)[0] 398 | while object_name in self: 399 | nuid = uuid.uuid4().hex 400 | object_name = "%s__%s.%s" % (file_name, nuid, extension) 401 | return object_name 402 | 403 | def _register_file_server(self, app): 404 | """ 405 | File server 406 | Only local files can be served 407 | It's recommended to serve static files through NGINX instead of Python 408 | Use this for development only 409 | :param app: Flask app instance 410 | 411 | """ 412 | if isinstance(self.driver, local.LocalStorageDriver) \ 413 | and self.config["serve_files"]: 414 | server_url = self.config["serve_files_url"].strip("/").strip() 415 | if server_url: 416 | url = "/%s/" % server_url 417 | 418 | @app.route(url, endpoint=SERVER_ENDPOINT) 419 | def files_server(object_name): 420 | obj = self.get(object_name) 421 | if obj is not None: 422 | dl = flask_request.args.get("dl") 423 | name = flask_request.args.get("name", obj.name) 424 | 425 | if get_file_extension(name) != obj.extension: 426 | name += ".%s" % obj.extension 427 | 428 | _url = obj.get_cdn_url() 429 | return send_file(_url, 430 | as_attachment=True if dl else False, 431 | attachment_filename=name, 432 | conditional=True) 433 | else: 434 | abort(404) 435 | else: 436 | warnings.warn("Flask-Cloudy can't serve files. 'STORAGE_SERVER_FILES_URL' is not set") 437 | 438 | 439 | class Object(object): 440 | """ 441 | The object file 442 | 443 | @property 444 | name 445 | size 446 | hash 447 | extra 448 | meta_data 449 | 450 | driver 451 | container 452 | 453 | @method 454 | download() use save_to() instead 455 | delete() 456 | """ 457 | 458 | _obj = None 459 | 460 | def __init__(self, obj, **kwargs): 461 | self._obj = obj 462 | self._kwargs = kwargs 463 | 464 | def __getattr__(self, item): 465 | return getattr(self._obj, item) 466 | 467 | def __len__(self): 468 | return self.size 469 | 470 | @property 471 | def info(self): 472 | """ 473 | Return all the info of this object 474 | :return: dict 475 | """ 476 | return { 477 | "name": self.name, 478 | "size": self.size, 479 | "extension": self.extension, 480 | "url": self.url, 481 | "full_url": self.full_url, 482 | "type": self.type, 483 | "path": self.path, 484 | "provider_name": self.provider_name 485 | } 486 | 487 | def get_url(self, secure=False, longurl=False): 488 | """ 489 | Return the url 490 | :param secure: bool - To use https 491 | :param longurl: bool - On local, reference the local path with the domain 492 | ie: http://site.com/files/object.png otherwise /files/object.png 493 | :return: str 494 | """ 495 | driver_name = self.driver.name.lower() 496 | try: 497 | # Currently only Cloudfiles and Local supports it 498 | url = self._obj.get_cdn_url() 499 | if "local" in driver_name: 500 | url = url_for(SERVER_ENDPOINT, 501 | object_name=self.name, 502 | _external=longurl) 503 | except NotImplementedError as e: 504 | object_path = '%s/%s' % (self.container.name, self.name) 505 | if 's3' in driver_name: 506 | base_url = 'http://%s' % self.driver.connection.host 507 | url = urljoin(base_url, object_path) 508 | elif 'google' in driver_name: 509 | url = urljoin('http://storage.googleapis.com', object_path) 510 | elif 'azure' in driver_name: 511 | base_url = ('http://%s.blob.core.windows.net' % self.driver.key) 512 | url = urljoin(base_url, object_path) 513 | else: 514 | raise e 515 | 516 | if secure: 517 | if 'cloudfiles' in driver_name: 518 | parsed_url = urlparse(url) 519 | if parsed_url.scheme != 'http': 520 | return url 521 | split_netloc = parsed_url.netloc.split('.') 522 | split_netloc[1] = 'ssl' 523 | url = urlunparse( 524 | 'https', 525 | '.'.join(split_netloc), 526 | parsed_url.path, 527 | parsed_url.params, parsed_url.query, 528 | parsed_url.fragment 529 | ) 530 | if ('s3' in driver_name or 531 | 'google' in driver_name or 532 | 'azure' in driver_name): 533 | url = url.replace('http://', 'https://') 534 | return url 535 | 536 | @property 537 | def url(self): 538 | """ 539 | Returns the url of the object. 540 | For Local it will return it without the domain name 541 | :return: str 542 | """ 543 | return self.get_url() 544 | 545 | @property 546 | def full_url(self): 547 | """ 548 | Returns the full url with the domain, specially for Local storage 549 | :return: str 550 | """ 551 | return self.get_url(longurl=True) 552 | 553 | 554 | @property 555 | def secure_url(self): 556 | """ 557 | Return the full url with https 558 | :return: 559 | """ 560 | return self.get_url(secure=True, longurl=True) 561 | 562 | @property 563 | def extension(self): 564 | """ 565 | Return the extension of the object 566 | :return: 567 | """ 568 | return get_file_extension(self.name) 569 | 570 | @property 571 | def type(self): 572 | """ 573 | Return the object type (IMAGE, AUDIO,...) or OTHER 574 | :return: 575 | """ 576 | return get_file_extension_type(self.name) 577 | 578 | @property 579 | def provider_name(self): 580 | """ 581 | Return the provider name 582 | :return: str 583 | """ 584 | return get_provider_name(self.driver) 585 | 586 | @property 587 | def path(self): 588 | """ 589 | Return the object path 590 | :return: str 591 | """ 592 | return "%s/%s" % (self.container.name, self.name) 593 | 594 | @property 595 | def full_path(self): 596 | """ 597 | Return the full path of the local object 598 | If not local, it will return self.path 599 | :return: str 600 | """ 601 | if "local" in self.driver.name.lower(): 602 | return "%s/%s" % self.container.key, self.path 603 | return self.path 604 | 605 | def save_to(self, destination, name=None, overwrite=False, delete_on_failure=True): 606 | """ 607 | To save the object in a local path 608 | :param destination: str - The directory to save the object to 609 | :param name: str - To rename the file name. Do not add extesion 610 | :param overwrite: 611 | :param delete_on_failure: 612 | :return: The new location of the file or None 613 | """ 614 | if not os.path.isdir(destination): 615 | raise IOError("'%s' is not a valid directory") 616 | 617 | obj_path = "%s/%s" % (destination, self._obj.name) 618 | if name: 619 | obj_path = "%s/%s.%s" % (destination, name, self.extension) 620 | 621 | file = self._obj.download(obj_path, 622 | overwrite_existing=overwrite, 623 | delete_on_failure=delete_on_failure) 624 | return obj_path if file else None 625 | 626 | def download_url(self, timeout=60, name=None): 627 | """ 628 | Trigger a browse download 629 | :param timeout: int - Time in seconds to expire the download 630 | :param name: str - for LOCAL only, to rename the file being downloaded 631 | :return: str 632 | """ 633 | if "local" in self.driver.name.lower(): 634 | return url_for(SERVER_ENDPOINT, 635 | object_name=self.name, 636 | dl=1, 637 | name=name, 638 | _external=True) 639 | else: 640 | driver_name = self.driver.name.lower() 641 | expires = (datetime.datetime.now() 642 | + datetime.timedelta(seconds=timeout)).strftime("%s") 643 | 644 | if 's3' in driver_name or 'google' in driver_name: 645 | 646 | s2s = "GET\n\n\n{expires}\n/{object_name}"\ 647 | .format(expires=expires, object_name=self.path) 648 | h = hmac.new(self.driver.secret.encode('utf-8'), s2s.encode('utf-8'), hashlib.sha1) 649 | s = base64.encodestring(h.digest()).strip() 650 | _keyIdName = "AWSAccessKeyId" if "s3" in driver_name else "GoogleAccessId" 651 | params = { 652 | _keyIdName: self.driver.key, 653 | "Expires": expires, 654 | "Signature": s 655 | } 656 | urlkv = urlencode(params) 657 | return "%s?%s" % (self.secure_url, urlkv) 658 | 659 | elif 'cloudfiles' in driver_name: 660 | return self.driver.ex_get_object_temp_url(self._obj, 661 | method="GET", 662 | timeout=expires) 663 | else: 664 | raise NotImplemented("This provider '%s' doesn't support or " 665 | "doesn't have a signed url " 666 | "implemented yet" % self.provider_name) 667 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Flask-Cloudy 3 | 4 | A wrapper around Apache-Libcloud to upload and save files on cloud storage 5 | providers such as: AWS S3, Google Storage, Microsoft Azure, Rackspace Cloudfiles, 6 | and even on local storage through a Flask application. 7 | (It can be used as standalone) 8 | 9 | Supported storage: 10 | 11 | - AWS S3 12 | - Google Storage 13 | - Microsoft Azure 14 | - Rackspace CloudFiles 15 | - Local 16 | 17 | """ 18 | 19 | from setuptools import setup, find_packages 20 | 21 | __NAME__ = "Flask-Cloudy" 22 | __version__ = "1.1.0" 23 | __author__ = "Mardix" 24 | __license__ = "MIT" 25 | __copyright__ = "2017" 26 | 27 | setup( 28 | name=__NAME__, 29 | version=__version__, 30 | license=__license__, 31 | author=__author__, 32 | author_email='mardix@github.com', 33 | description="Flask-Cloudy is a simple flask extension and standalone library to upload and save files on S3, Google storage or other Cloud Storages", 34 | long_description=__doc__, 35 | url='https://github.com/mardix/flask-cloudy/', 36 | download_url='http://github.com/mardix/flask-cloudy/tarball/master', 37 | py_modules=['flask_cloudy'], 38 | include_package_data=True, 39 | packages=find_packages(), 40 | install_requires=[ 41 | "Flask", 42 | "apache-libcloud", 43 | "lockfile", 44 | "six", 45 | 'python-slugify' 46 | ], 47 | 48 | keywords=["flask", "s3", "aws", "cloudfiles", "storage", "azure", "google", "cloudy"], 49 | platforms='any', 50 | classifiers=[ 51 | 'Environment :: Web Environment', 52 | 'Intended Audience :: Developers', 53 | 'License :: OSI Approved :: BSD License', 54 | 'Operating System :: OS Independent', 55 | 'Programming Language :: Python', 56 | 'Programming Language :: Python :: 2', 57 | 'Programming Language :: Python :: 2.6', 58 | 'Programming Language :: Python :: 2.7', 59 | 'Programming Language :: Python :: 3', 60 | 'Programming Language :: Python :: 3.3', 61 | 'Programming Language :: Python :: 3.4', 62 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 63 | 'Topic :: Software Development :: Libraries :: Python Modules' 64 | ], 65 | zip_safe=False 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mardix/flask-cloudy/cf9f6f074e92af1d98f99fdf6ca458e29099e4b1/tests/__init__.py -------------------------------------------------------------------------------- /tests/config.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | PROVIDER = "S3" 4 | KEY = "" 5 | SECRET = "" 6 | CONTAINER = "yoredis.com" 7 | 8 | # FOR LOCAL 9 | PROVIDER = "LOCAL" 10 | CONTAINER = "container_1" 11 | CONTAINER2 = "container_2" -------------------------------------------------------------------------------- /tests/data/hello.js: -------------------------------------------------------------------------------- 1 | // This is the javascript file 2 | -------------------------------------------------------------------------------- /tests/data/hello.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mardix/flask-cloudy/cf9f6f074e92af1d98f99fdf6ca458e29099e4b1/tests/data/hello.txt -------------------------------------------------------------------------------- /tests/test_cloudy.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | from libcloud.storage.base import (StorageDriver, Container) 4 | from flask_cloudy import (get_file_extension, 5 | get_file_extension_type, 6 | get_file_name, 7 | get_driver_class, 8 | get_provider_name, 9 | Storage, 10 | Object, 11 | InvalidExtensionError) 12 | from tests import config 13 | 14 | CWD = os.path.dirname(__file__) 15 | 16 | CONTAINER = "%s/%s" % (CWD, config.CONTAINER) if config.PROVIDER == "LOCAL" else config.CONTAINER 17 | CONTAINER2 = "%s/%s" % (CWD, config.CONTAINER2) if config.PROVIDER == "LOCAL" else config.CONTAINER2 18 | 19 | class App(object): 20 | config = dict( 21 | STORAGE_PROVIDER=config.PROVIDER, 22 | STORAGE_KEY=config.KEY, 23 | STORAGE_SECRET=config.SECRET, 24 | STORAGE_CONTAINER=CONTAINER, 25 | STORAGE_SERVER=False, 26 | STORAGE_ALLOWED_EXTENSIONS=[]) 27 | 28 | 29 | def test_get_file_extension(): 30 | filename = "hello.jpg" 31 | assert get_file_extension(filename) == "jpg" 32 | 33 | def test_get_file_extension_type(): 34 | filename = "hello.mp3" 35 | assert get_file_extension_type(filename) == "AUDIO" 36 | 37 | def test_get_file_name(): 38 | filename = "/dir1/dir2/dir3/hello.jpg" 39 | assert get_file_name(filename) == "hello.jpg" 40 | 41 | def test_get_provider_name(): 42 | class GoogleStorageDriver(object): 43 | pass 44 | driver = GoogleStorageDriver() 45 | assert get_provider_name(driver) == "google_storage" 46 | 47 | #--- 48 | 49 | app = App() 50 | 51 | def app_storage(): 52 | return Storage(app=App()) 53 | 54 | def test_get_driver_class(): 55 | driver = get_driver_class("S3") 56 | assert isinstance(driver, type) 57 | 58 | def test_driver(): 59 | storage = app_storage() 60 | assert isinstance(storage.driver, StorageDriver) 61 | 62 | def test_container(): 63 | storage = app_storage() 64 | assert isinstance(storage.container, Container) 65 | 66 | def test_flask_app(): 67 | storage = app_storage() 68 | assert isinstance(storage.driver, StorageDriver) 69 | 70 | def test_iter(): 71 | storage = app_storage() 72 | l = [o for o in storage] 73 | assert isinstance(l, list) 74 | 75 | def test_storage_object_not_exists(): 76 | object_name = "hello.png" 77 | storage = app_storage() 78 | assert object_name not in storage 79 | 80 | def test_storage_object(): 81 | object_name = "hello.txt" 82 | storage = app_storage() 83 | o = storage.create(object_name) 84 | assert isinstance(o, Object) 85 | 86 | def test_object_type_extension(): 87 | object_name = "hello.jpg" 88 | storage = app_storage() 89 | o = storage.create(object_name) 90 | assert o.type == "IMAGE" 91 | assert o.extension == "jpg" 92 | 93 | def test_object_provider_name(): 94 | object_name = "hello.jpg" 95 | storage = app_storage() 96 | o = storage.create(object_name) 97 | assert o.provider_name == config.PROVIDER.lower() 98 | 99 | def test_object_object_path(): 100 | object_name = "hello.jpg" 101 | storage = app_storage() 102 | o = storage.create(object_name) 103 | p = "%s/%s" % (o.container.name, o.name) 104 | assert o.path.endswith(p) 105 | 106 | def test_storage_upload_invalid(): 107 | storage = app_storage() 108 | object_name = "my-js/hello.js" 109 | with pytest.raises(InvalidExtensionError): 110 | storage.upload(CWD + "/data/hello.js", name=object_name) 111 | 112 | def test_storage_upload_ovewrite(): 113 | storage = app_storage() 114 | object_name = "my-txt-hello.txt" 115 | o = storage.upload(CWD + "/data/hello.txt", name=object_name, overwrite=True) 116 | assert isinstance(o, Object) 117 | assert o.name == object_name 118 | 119 | def test_storage_get(): 120 | storage = app_storage() 121 | object_name = "my-txt-helloIII.txt" 122 | o = storage.upload(CWD + "/data/hello.txt", name=object_name, overwrite=True) 123 | o2 = storage.get(o.name) 124 | assert isinstance(o2, Object) 125 | 126 | def test_storage_get_none(): 127 | storage = app_storage() 128 | o2 = storage.get("idonexist") 129 | assert o2 is None 130 | 131 | def test_storage_upload(): 132 | storage = app_storage() 133 | object_name = "my-txt-hello2.txt" 134 | storage.upload(CWD + "/data/hello.txt", name=object_name) 135 | o = storage.upload(CWD + "/data/hello.txt", name=object_name) 136 | assert isinstance(o, Object) 137 | assert o.name != object_name 138 | 139 | def test_storage_upload_use_filename_name(): 140 | storage = app_storage() 141 | object_name = "hello.js" 142 | o = storage.upload(CWD + "/data/hello.js", overwrite=True, extensions=["js"]) 143 | assert o.name == object_name 144 | 145 | def test_storage_upload_append_extension(): 146 | storage = app_storage() 147 | object_name = "my-txt-hello-hello" 148 | o = storage.upload(CWD + "/data/hello.txt", object_name, overwrite=True) 149 | assert get_file_extension(o.name) == "txt" 150 | 151 | def test_storage_upload_with_prefix(): 152 | storage = app_storage() 153 | object_name = "my-txt-hello-hello" 154 | prefix = "dir1/dir2/dir3/" 155 | full_name = "%s%s.%s" % (prefix, object_name, "txt") 156 | o = storage.upload(CWD + "/data/hello.txt", name=object_name, prefix=prefix, overwrite=True) 157 | assert full_name in storage 158 | assert o.name == full_name 159 | 160 | 161 | def test_save_to(): 162 | storage = app_storage() 163 | object_name = "my-txt-hello-to-save.txt" 164 | o = storage.upload(CWD + "/data/hello.txt", name=object_name) 165 | file = o.save_to(CWD + "/data", overwrite=True) 166 | file2 = o.save_to(CWD + "/data", name="my_new_file", overwrite=True) 167 | assert os.path.isfile(file) 168 | assert file2 == CWD + "/data/my_new_file.txt" 169 | 170 | def test_delete(): 171 | storage = app_storage() 172 | object_name = "my-txt-hello-to-delete.txt" 173 | o = storage.upload(CWD + "/data/hello.txt", name=object_name) 174 | assert object_name in storage 175 | o.delete() 176 | assert object_name not in storage 177 | 178 | def test_use(): 179 | storage = app_storage() 180 | object_name = "my-txt-hello-to-save-with-use.txt" 181 | f = CWD + "/data/hello.txt" 182 | with storage.use(CONTAINER2) as s2: 183 | assert isinstance(s2.container, Container) 184 | o = s2.upload(f, "hello.txt") 185 | assert isinstance(o, Object) 186 | o1 = storage.upload(f, name=object_name) 187 | assert isinstance(o1, Object) 188 | assert o1.name == object_name 189 | 190 | def test_werkzeug_upload(): 191 | try: 192 | import werkzeug 193 | except ImportError: 194 | return 195 | storage = app_storage() 196 | object_name = "my-txt-hello.txt" 197 | filepath = CWD + "/data/hello.txt" 198 | file = None 199 | with open(filepath, 'rb') as fp: 200 | file = werkzeug.datastructures.FileStorage(fp) 201 | file.filename = object_name 202 | o = storage.upload(file, overwrite=True) 203 | assert isinstance(o, Object) 204 | assert o.name == object_name 205 | 206 | 207 | def test_random(): 208 | storage = app_storage() 209 | o = storage.upload(CWD + "/data/hello.js", overwrite=True, extensions=["js"], random_name=True) 210 | assert len(o.name) == 32 + 3 # 3 extensions 211 | 212 | def test_upload_image_from_url(): 213 | storage = app_storage() 214 | # Gooole logo: G 215 | url = "https://yt3.ggpht.com/-v0soe-ievYE/AAAAAAAAAAI/AAAAAAAAAAA/OixOH_h84Po/s900-c-k-no-mo-rj-c0xffffff/photo.jpg" 216 | o = storage.upload(url) 217 | assert isinstance(o, Object) 218 | 219 | 220 | # def test_object_info(): 221 | # object_name = "hello.jpg" 222 | # storage = app_storage() 223 | # o = storage.create(object_name) 224 | # assert isinstance(o.info, dict) 225 | # 226 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # content of: tox.ini , put in same dir as setup.py 2 | [tox] 3 | envlist = py26,py27 4 | [testenv] 5 | deps=pytest 6 | commands=py.test --------------------------------------------------------------------------------